From b2fd5744fb8103e728c3eb3c900de825b9a021cb Mon Sep 17 00:00:00 2001 From: Nick Lozon Date: Tue, 12 Dec 2023 14:39:33 -0500 Subject: [PATCH 01/15] perform intersection on specified hosts --- lib/kamal/cli/accessory.rb | 62 +++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 47bc1a06..b9cfee1f 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -5,11 +5,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base if name == "all" KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) } else - with_accessory(name) do |accessory| + with_accessory(name) do |accessory, hosts| directories(name) upload(name) - on(accessory.hosts) do + on(hosts) do execute *KAMAL.registry.login if login execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug execute *accessory.run @@ -22,8 +22,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "upload [NAME]", "Upload accessory files to host", hide: true def upload(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do accessory.files.each do |(local, remote)| accessory.ensure_local_file_present(local) @@ -39,8 +39,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "directories [NAME]", "Create accessory directories on host", hide: true def directories(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do accessory.directories.keys.each do |host_path| execute *accessory.make_directory(host_path) end @@ -55,8 +55,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base if name == "all" KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) } else - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *KAMAL.registry.login end @@ -71,8 +71,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "start [NAME]", "Start existing accessory container on host" def start(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug execute *accessory.start end @@ -83,8 +83,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "stop [NAME]", "Stop existing accessory container on host" def stop(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug execute *accessory.stop, raise_on_non_zero_exit: false end @@ -107,8 +107,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base if name == "all" KAMAL.accessory_names.each { |accessory_name| details(accessory_name) } else - with_accessory(name) do |accessory| - on(accessory.hosts) { puts capture_with_info(*accessory.info) } + with_accessory(name) do |accessory, hosts| + on(hosts) { puts capture_with_info(*accessory.info) } end end end @@ -117,7 +117,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)" option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one" def exec(name, cmd) - with_accessory(name) do |accessory| + with_accessory(name) do |accessory, hosts| case when options[:interactive] && options[:reuse] say "Launching interactive command with via SSH from existing container...", :magenta @@ -129,14 +129,14 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base when options[:reuse] say "Launching command from existing container...", :magenta - on(accessory.hosts) do + on(hosts) do execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug capture_with_info(*accessory.execute_in_existing_container(cmd)) end else say "Launching command from new container...", :magenta - on(accessory.hosts) do + on(hosts) do execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug capture_with_info(*accessory.execute_in_new_container(cmd)) end @@ -150,12 +150,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)" def logs(name) - with_accessory(name) do |accessory| + with_accessory(name) do |accessory, hosts| grep = options[:grep] if options[:follow] run_locally do - info "Following logs on #{accessory.hosts}..." + info "Following logs on #{hosts}..." info accessory.follow_logs(grep: grep) exec accessory.follow_logs(grep: grep) end @@ -163,7 +163,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base since = options[:since] lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set - on(accessory.hosts) do + on(hosts) do puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep)) end end @@ -192,8 +192,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "remove_container [NAME]", "Remove accessory container from host", hide: true def remove_container(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug execute *accessory.remove_container end @@ -204,8 +204,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "remove_image [NAME]", "Remove accessory image from host", hide: true def remove_image(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug execute *accessory.remove_image end @@ -216,8 +216,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true def remove_service_directory(name) mutating do - with_accessory(name) do |accessory| - on(accessory.hosts) do + with_accessory(name) do |accessory, hosts| + on(hosts) do execute *accessory.remove_service_directory end end @@ -227,7 +227,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base private def with_accessory(name) if accessory = KAMAL.accessory(name) - yield accessory + yield accessory, accessory_hosts(accessory) else error_on_missing_accessory(name) end @@ -240,4 +240,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base "No accessory by the name of '#{name}'" + (options ? " (options: #{options.to_sentence})" : "") end + + def accessory_hosts(accessory) + if KAMAL.specific_hosts&.any? + KAMAL.specific_hosts & accessory.hosts + else + accessory.hosts + end + end end From f6a9d54902675a55d9725cadcef49512b85ffa05 Mon Sep 17 00:00:00 2001 From: Nick Lozon Date: Tue, 12 Dec 2023 15:07:29 -0500 Subject: [PATCH 02/15] unit test --- test/cli/accessory_test.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index c4cf4828..eb40847a 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -148,6 +148,30 @@ class CliAccessoryTest < CliTestCase assert_match "rm -rf app-mysql", run_command("remove_service_directory", "mysql") end + test "hosts param respected" do + Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") + + run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output| + assert_match /docker login.*on 1.1.1.1/, output + refute_match /docker login.*on 1.1.1.2/, output + assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output + refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output + end + end + + test "hosts param intersects hosts" do + Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") + + run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output| + assert_match /docker login.*on 1.1.1.1/, output + refute_match /docker login.*on 1.1.1.3/, output + assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output + refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output + end + end + private def run_command(*command) stdouted { Kamal::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } From f8d8319c2f4abb909d41732a605a25ea8b6445ad Mon Sep 17 00:00:00 2001 From: Nick Lozon Date: Tue, 12 Dec 2023 15:37:12 -0500 Subject: [PATCH 03/15] better test description --- test/cli/accessory_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index eb40847a..d8f3fc63 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -160,7 +160,7 @@ class CliAccessoryTest < CliTestCase end end - test "hosts param intersects hosts" do + test "hosts param intersected with configuration" do Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis") Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") From 1a3dd52af435ffc9fde5549f48c76c1630dea142 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 9 Jan 2024 10:47:02 +0000 Subject: [PATCH 04/15] Rails 7.2 compatible Rubies 1. Add Ruby 2.7 specific Gemfile that uses an older version of nokogiri 2. Rails edge doesn't support Ruby 2.7.0, so exclude it. 3. Add Ruby 3.3 4. Update Gemfile.lock to test against Rails 7.1.2 as it's the latest version. 5. Remove continue-on-error from the matrix and always set to true --- .github/workflows/ci.yml | 18 +++++- Gemfile.lock | 119 ++++++++++++++++++++++++-------------- gemfiles/ruby_2.7.gemfile | 6 ++ 3 files changed, 95 insertions(+), 48 deletions(-) create mode 100644 gemfiles/ruby_2.7.gemfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8094c05..44fd5fb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: CI -on: +on: push: branches: - main @@ -12,13 +12,25 @@ jobs: - "2.7" - "3.1" - "3.2" + - "3.3" gemfile: - Gemfile + - gemfiles/ruby_2.7.gemfile - gemfiles/rails_edge.gemfile - continue-on-error: [false] + exclude: + - ruby-version: "2.7" + gemfile: Gemfile + - ruby-version: "2.7" + gemfile: gemfiles/rails_edge.gemfile + - ruby-version: "3.1" + gemfile: gemfiles/ruby_2.7.gemfile + - ruby-version: "3.2" + gemfile: gemfiles/ruby_2.7.gemfile + - ruby-version: "3.3" + gemfile: gemfiles/ruby_2.7.gemfile name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }} runs-on: ubuntu-latest - continue-on-error: ${{ matrix.continue-on-error }} + continue-on-error: true env: BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }} steps: diff --git a/Gemfile.lock b/Gemfile.lock index aba3b3e5..31068493 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -15,82 +15,111 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (7.0.4.3) - actionview (= 7.0.4.3) - activesupport (= 7.0.4.3) - rack (~> 2.0, >= 2.2.0) + actionpack (7.1.2) + actionview (= 7.1.2) + activesupport (= 7.1.2) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4) + rack-session (>= 1.0.1) rack-test (>= 0.6.3) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.2.0) - actionview (7.0.4.3) - activesupport (= 7.0.4.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + actionview (7.1.2) + activesupport (= 7.1.2) builder (~> 3.1) - erubi (~> 1.4) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.1, >= 1.2.0) - activesupport (7.0.4.3) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activesupport (7.1.2) + base64 + bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb i18n (>= 1.6, < 2) minitest (>= 5.1) + mutex_m tzinfo (~> 2.0) + base64 (0.2.0) bcrypt_pbkdf (1.1.0) + bigdecimal (3.1.5) builder (3.2.4) concurrent-ruby (1.2.2) + connection_pool (2.4.1) crass (1.0.6) - debug (1.7.2) - irb (>= 1.5.0) - reline (>= 0.3.1) + debug (1.9.1) + irb (~> 1.10) + reline (>= 0.3.8) dotenv (2.8.1) + drb (2.2.0) + ruby2_keywords ed25519 (1.3.0) erubi (1.12.0) - i18n (1.12.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) - io-console (0.6.0) - irb (1.6.3) - reline (>= 0.3.0) - loofah (2.20.0) + io-console (0.7.1) + irb (1.11.0) + rdoc + reline (>= 0.3.8) + loofah (2.22.0) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - method_source (1.0.0) - minitest (5.18.0) - mocha (2.0.2) + nokogiri (>= 1.12.0) + minitest (5.20.0) + mocha (2.1.0) ruby2_keywords (>= 0.0.5) + mutex_m (0.2.0) net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) - net-ssh (7.1.0) - nokogiri (1.14.2-arm64-darwin) + net-ssh (7.2.1) + nokogiri (1.16.0-arm64-darwin) racc (~> 1.4) - nokogiri (1.14.2-x86_64-darwin) + nokogiri (1.16.0-x86_64-darwin) racc (~> 1.4) - nokogiri (1.14.2-x86_64-linux) + nokogiri (1.16.0-x86_64-linux) racc (~> 1.4) - racc (1.6.2) - rack (2.2.6.4) + psych (5.1.2) + stringio + racc (1.7.3) + rack (3.0.8) + rack-session (2.0.0) + rack (>= 3.0.0) rack-test (2.1.0) rack (>= 1.3) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rackup (2.1.0) + rack (>= 3) + webrick (~> 1.8) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.5.0) - loofah (~> 2.19, >= 2.19.1) - railties (7.0.4.3) - actionpack (= 7.0.4.3) - activesupport (= 7.0.4.3) - method_source + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (7.1.2) + actionpack (= 7.1.2) + activesupport (= 7.1.2) + irb + rackup (>= 1.0.0) rake (>= 12.2) - thor (~> 1.0) - zeitwerk (~> 2.5) - rake (13.0.6) - reline (0.3.3) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) + rake (13.1.0) + rdoc (6.6.2) + psych (>= 4.0.0) + reline (0.4.2) io-console (~> 0.5) ruby2_keywords (0.0.5) - sshkit (1.21.4) + sshkit (1.21.7) + mutex_m net-scp (>= 1.1.2) net-ssh (>= 2.8.0) - thor (1.2.1) + stringio (3.1.0) + thor (1.3.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - zeitwerk (2.6.7) + webrick (1.8.1) + zeitwerk (2.6.12) PLATFORMS arm64-darwin diff --git a/gemfiles/ruby_2.7.gemfile b/gemfiles/ruby_2.7.gemfile new file mode 100644 index 00000000..1463b323 --- /dev/null +++ b/gemfiles/ruby_2.7.gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' +git_source(:github) { |repo| "https://github.com/#{repo}.git" } + +gemspec path: "../" + +gem "nokogiri", "~> 1.15.0" From 68eb549795b6184a149e7bd22700f19d82899db8 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 9 Jan 2024 11:35:10 +0000 Subject: [PATCH 05/15] Update to actions/checkout@v4 to silence node warning --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44fd5fb9..67316a75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: env: BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install Ruby uses: ruby/setup-ruby@v1 From c5ae54d7d4c11438dfbf90d757b9c9f003d30887 Mon Sep 17 00:00:00 2001 From: Matthew Kent Date: Mon, 8 Jan 2024 11:02:16 -0800 Subject: [PATCH 06/15] Add a missing base64 require. Also, prepare for the moving of base64 from default to a bundled gem in ruby 3.4. --- Gemfile.lock | 1 + kamal.gemspec | 1 + lib/kamal/commands/lock.rb | 1 + 3 files changed, 3 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index 31068493..575a9021 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,6 +3,7 @@ PATH specs: kamal (1.3.0) activesupport (>= 7.0) + base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) concurrent-ruby (~> 1.2) dotenv (~> 2.8) diff --git a/kamal.gemspec b/kamal.gemspec index 8dcaa6ea..55f18ce9 100644 --- a/kamal.gemspec +++ b/kamal.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |spec| spec.add_dependency "ed25519", "~> 1.2" spec.add_dependency "bcrypt_pbkdf", "~> 1.0" spec.add_dependency "concurrent-ruby", "~> 1.2" + spec.add_dependency "base64", "~> 0.2" spec.add_development_dependency "debug" spec.add_development_dependency "mocha" diff --git a/lib/kamal/commands/lock.rb b/lib/kamal/commands/lock.rb index 82340714..607bd79d 100644 --- a/lib/kamal/commands/lock.rb +++ b/lib/kamal/commands/lock.rb @@ -1,5 +1,6 @@ require "active_support/duration" require "time" +require "base64" class Kamal::Commands::Lock < Kamal::Commands::Base def acquire(message, version) From 2a8924b53c9b18dd2e42d5ad4f0f18754bca19e4 Mon Sep 17 00:00:00 2001 From: Matthew Kent Date: Mon, 8 Jan 2024 13:32:34 -0800 Subject: [PATCH 07/15] Address Net::SSH::HostKeyMismatch seen locally between bin/test runs. --- test/integration/docker/deployer/setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/docker/deployer/setup.sh b/test/integration/docker/deployer/setup.sh index 50af91bf..a2c9c6e8 100755 --- a/test/integration/docker/deployer/setup.sh +++ b/test/integration/docker/deployer/setup.sh @@ -21,3 +21,7 @@ install_kamal push_image_to_registry_4443 nginx 1-alpine-slim push_image_to_registry_4443 traefik v2.9 push_image_to_registry_4443 busybox 1.36.0 + +# .ssh is on a shared volume that persists between runs. Clean it up as the +# churn of temporary vm IPs can eventually create conflicts. +rm -f /root/.ssh/known_hosts From 9c681d4a386eba38c817f5a49bb489c94a4d7ca0 Mon Sep 17 00:00:00 2001 From: Matthew Kent Date: Mon, 8 Jan 2024 13:34:03 -0800 Subject: [PATCH 08/15] Be a bit more patient during tests. Seeing reasonably consistent local failures at 20 seconds. --- test/integration/integration_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb index 950e41ed..7d620438 100644 --- a/test/integration/integration_test.rb +++ b/test/integration/integration_test.rb @@ -103,7 +103,7 @@ class IntegrationTest < ActiveSupport::TestCase assert_equal "200", code end - def wait_for_healthy(timeout: 20) + def wait_for_healthy(timeout: 30) timeout_at = Time.now + timeout while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0" if timeout_at < Time.now From aca7796e9d6a2621bac1ff54f64342d0e9ab1e76 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 10 Jan 2024 08:56:34 +0000 Subject: [PATCH 09/15] Bump version for 1.3.1 --- Gemfile.lock | 2 +- lib/kamal/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 575a9021..68e0225d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kamal (1.3.0) + kamal (1.3.1) activesupport (>= 7.0) base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) diff --git a/lib/kamal/version.rb b/lib/kamal/version.rb index e9f06ea9..59756e50 100644 --- a/lib/kamal/version.rb +++ b/lib/kamal/version.rb @@ -1,3 +1,3 @@ module Kamal - VERSION = "1.3.0" + VERSION = "1.3.1" end From 5a09aa12ba97f92fff56b871690a15e10f9bec6c Mon Sep 17 00:00:00 2001 From: dhh Date: Wed, 10 Jan 2024 13:00:48 -0800 Subject: [PATCH 10/15] Allow local builds using a different arch than native --- lib/kamal/commands/builder/multiarch.rb | 10 +++++++++- test/commands/builder_test.rb | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/kamal/commands/builder/multiarch.rb b/lib/kamal/commands/builder/multiarch.rb index 458c4ac6..2f9e4d19 100644 --- a/lib/kamal/commands/builder/multiarch.rb +++ b/lib/kamal/commands/builder/multiarch.rb @@ -10,7 +10,7 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base def push docker :buildx, :build, "--push", - "--platform", "linux/amd64,linux/arm64", + "--platform", platform_names, "--builder", builder_name, *build_options, build_context @@ -26,4 +26,12 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base def builder_name "kamal-#{config.service}-multiarch" end + + def platform_names + if local_arch + "linux/#{local_arch}" + else + "linux/amd64,linux/arm64" + end + end end diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index c0256113..22ed8575 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -37,6 +37,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase builder.push.join(" ") end + test "target multiarch local when arch is set" do + builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } }) + assert_equal "multiarch", builder.name + assert_equal \ + "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .", + builder.push.join(" ") + end + test "target native remote when only remote is set" do builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } }) assert_equal "native/remote", builder.name From c984db152f239c6c2407554d0310ca226e77f637 Mon Sep 17 00:00:00 2001 From: Juan Aparicio Date: Thu, 11 Jan 2024 17:00:13 -0300 Subject: [PATCH 11/15] require missing net/scp dependency --- lib/kamal/sshkit_with_ext.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/kamal/sshkit_with_ext.rb b/lib/kamal/sshkit_with_ext.rb index eb8f359b..e0c62c3a 100644 --- a/lib/kamal/sshkit_with_ext.rb +++ b/lib/kamal/sshkit_with_ext.rb @@ -1,5 +1,6 @@ require "sshkit" require "sshkit/dsl" +require "net/scp" require "active_support/core_ext/hash/deep_merge" require "json" From 77e72e34ceedc3c141a063f4844c1cc6f747e1db Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Tue, 13 Feb 2024 16:00:02 +0400 Subject: [PATCH 12/15] Bumped default Traefik image to 2.10 --- lib/kamal/commands/traefik.rb | 2 +- test/integration/docker/deployer/app/config/deploy.yml | 2 +- test/integration/docker/deployer/setup.sh | 2 +- test/integration/main_test.rb | 2 +- test/integration/traefik_test.rb | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/kamal/commands/traefik.rb b/lib/kamal/commands/traefik.rb index c6ec8c8d..bb8f048b 100644 --- a/lib/kamal/commands/traefik.rb +++ b/lib/kamal/commands/traefik.rb @@ -1,7 +1,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base delegate :argumentize, :optionize, to: Kamal::Utils - DEFAULT_IMAGE = "traefik:v2.9" + DEFAULT_IMAGE = "traefik:v2.10" CONTAINER_PORT = 80 DEFAULT_ARGS = { 'log.level' => 'DEBUG' diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index ec070f6c..0ac5bb2a 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -24,7 +24,7 @@ traefik: args: accesslog: true accesslog.format: json - image: registry:4443/traefik:v2.9 + image: registry:4443/traefik:v2.10 accessories: busybox: image: registry:4443/busybox:1.36.0 diff --git a/test/integration/docker/deployer/setup.sh b/test/integration/docker/deployer/setup.sh index a2c9c6e8..0cd511d9 100755 --- a/test/integration/docker/deployer/setup.sh +++ b/test/integration/docker/deployer/setup.sh @@ -19,7 +19,7 @@ push_image_to_registry_4443() { install_kamal push_image_to_registry_4443 nginx 1-alpine-slim -push_image_to_registry_4443 traefik v2.9 +push_image_to_registry_4443 traefik v2.10 push_image_to_registry_4443 busybox 1.36.0 # .ssh is on a shared volume that persists between runs. Clean it up as the diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index a0364fb2..856781cb 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -32,7 +32,7 @@ class MainTest < IntegrationTest assert_match /Traefik Host: vm2/, details assert_match /App Host: vm1/, details assert_match /App Host: vm2/, details - assert_match /traefik:v2.9/, details + assert_match /traefik:v2.10/, details assert_match /registry:4443\/app:#{first_version}/, details audit = kamal :audit, capture: true diff --git a/test/integration/traefik_test.rb b/test/integration/traefik_test.rb index ed8a956f..6f735612 100644 --- a/test/integration/traefik_test.rb +++ b/test/integration/traefik_test.rb @@ -52,11 +52,11 @@ class TraefikTest < IntegrationTest private def assert_traefik_running - assert_match /traefik:v2.9 "\/entrypoint.sh/, traefik_details + assert_match /traefik:v2.10 "\/entrypoint.sh/, traefik_details end def assert_traefik_not_running - refute_match /traefik:v2.9 "\/entrypoint.sh/, traefik_details + refute_match /traefik:v2.10 "\/entrypoint.sh/, traefik_details end def traefik_details From b411356409928753744eeaa1b45a1d3262d54ed5 Mon Sep 17 00:00:00 2001 From: Krzysztof Adamski Date: Thu, 15 Feb 2024 11:12:18 +0100 Subject: [PATCH 13/15] Allow for Custom Accessory Service Name --- lib/kamal/configuration/accessory.rb | 2 +- test/commands/accessory_test.rb | 5 +++-- test/configuration/accessory_test.rb | 2 ++ test/integration/docker/deployer/app/config/deploy.yml | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 8fdcd227..d26c6c85 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -8,7 +8,7 @@ class Kamal::Configuration::Accessory end def service_name - "#{config.service}-#{name}" + specifics["service"] || "#{config.service}-#{name}" end def image diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 687c7fe5..92ed1211 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -34,6 +34,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase ] }, "busybox" => { + "service" => "custom-busybox", "image" => "busybox:latest", "host" => "1.1.1.7" } @@ -57,7 +58,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:redis).run.join(" ") assert_equal \ - "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest", new_command(:busybox).run.join(" ") end @@ -65,7 +66,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest", new_command(:busybox).run.join(" ") end diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index 522dc631..4b56be51 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -49,6 +49,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase } }, "monitoring" => { + "service" => "custom-monitoring", "image" => "monitoring:latest", "roles" => [ "web" ], "port" => "4321:4321", @@ -72,6 +73,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase test "service name" do assert_equal "app-mysql", @config.accessory(:mysql).service_name assert_equal "app-redis", @config.accessory(:redis).service_name + assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name end test "port" do diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index ec070f6c..c1f6f7c5 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -27,6 +27,7 @@ traefik: image: registry:4443/traefik:v2.9 accessories: busybox: + service: custom-busybox image: registry:4443/busybox:1.36.0 cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done' roles: From 1c2a45817af81597d634c2576d178675fa869a14 Mon Sep 17 00:00:00 2001 From: Krzysztof Adamski Date: Wed, 6 Sep 2023 15:37:22 +0200 Subject: [PATCH 14/15] Supports Passing SSH Args to Build Options --- lib/kamal/commands/builder/base.rb | 8 ++++++-- lib/kamal/configuration/builder.rb | 4 ++++ test/commands/builder_test.rb | 8 ++++++++ test/configuration/builder_test.rb | 10 ++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/kamal/commands/builder/base.rb b/lib/kamal/commands/builder/base.rb index 8723fa45..f710807c 100644 --- a/lib/kamal/commands/builder/base.rb +++ b/lib/kamal/commands/builder/base.rb @@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base class BuilderError < StandardError; end delegate :argumentize, to: Kamal::Utils - delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config + delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config def clean docker :image, :rm, "--force", config.absolute_image @@ -14,7 +14,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base end def build_options - [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ] + [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ] end def build_context @@ -60,6 +60,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base end end + def build_ssh + argumentize "--ssh", ssh if ssh.present? + end + def builder_config config.builder end diff --git a/lib/kamal/configuration/builder.rb b/lib/kamal/configuration/builder.rb index c7d81922..9daa3138 100644 --- a/lib/kamal/configuration/builder.rb +++ b/lib/kamal/configuration/builder.rb @@ -81,6 +81,10 @@ class Kamal::Configuration::Builder end end + def ssh + @options["ssh"] + end + private def valid? if @options["cache"] && @options["cache"]["type"] diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index 22ed8575..51bb837c 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -111,6 +111,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase builder.push.join(" ") end + test "build with ssh agent socket" do + builder = new_builder_command(builder: { "ssh" => 'default=$SSH_AUTH_SOCK' }) + + assert_equal \ + "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --ssh default=$SSH_AUTH_SOCK", + builder.target.build_options.join(" ") + end + test "validate image" do assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the `service` label\" && exit 1)", new_builder_command.validate_image.join(" ") end diff --git a/test/configuration/builder_test.rb b/test/configuration/builder_test.rb index da30d915..4f4bbc40 100644 --- a/test/configuration/builder_test.rb +++ b/test/configuration/builder_test.rb @@ -148,4 +148,14 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase assert_equal "..", @config_with_builder_option.builder.context end + + test "ssh" do + assert_nil @config.builder.ssh + end + + test "setting ssh params" do + @deploy_with_builder_option[:builder] = { "ssh" => 'default=$SSH_AUTH_SOCK' } + + assert_equal 'default=$SSH_AUTH_SOCK', @config_with_builder_option.builder.ssh + end end From 6892abb4be2ebf7fd43bfea13856d09f82dc4511 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 15 Jan 2024 14:17:43 +0000 Subject: [PATCH 15/15] Config the number of containers to keep By default we keep 5 containers around for rollback. The containers don't take much space, but the images for them can. Make the number of containers to retain configurable, either in the config with the `retain_containers` setting on the command line with the `--retain` option. --- lib/kamal/cli/prune.rb | 8 ++++++-- lib/kamal/commands/prune.rb | 4 ++-- lib/kamal/configuration.rb | 12 +++++++++++- test/cli/prune_test.rb | 9 +++++++++ test/commands/prune_test.rb | 6 +++++- test/configuration_test.rb | 10 +++++++++- 6 files changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/kamal/cli/prune.rb b/lib/kamal/cli/prune.rb index 236c7d55..498e4ec4 100644 --- a/lib/kamal/cli/prune.rb +++ b/lib/kamal/cli/prune.rb @@ -18,12 +18,16 @@ class Kamal::Cli::Prune < Kamal::Cli::Base end end - desc "containers", "Prune all stopped containers, except the last 5" + desc "containers", "Prune all stopped containers, except the last n (default 5)" + option :retain, type: :numeric, default: nil, desc: "Number of containers to retain" def containers + retain = options.fetch(:retain, KAMAL.config.retain_containers) + raise "retain must be at least 1" if retain < 1 + mutating do on(KAMAL.hosts) do execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug - execute *KAMAL.prune.app_containers + execute *KAMAL.prune.app_containers(retain: retain) execute *KAMAL.prune.healthcheck_containers end end diff --git a/lib/kamal/commands/prune.rb b/lib/kamal/commands/prune.rb index f9f37b24..40f7dfc4 100644 --- a/lib/kamal/commands/prune.rb +++ b/lib/kamal/commands/prune.rb @@ -13,10 +13,10 @@ class Kamal::Commands::Prune < Kamal::Commands::Base "while read image tag; do docker rmi $tag; done" end - def app_containers(keep_last: 5) + def app_containers(retain:) pipe \ docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters), - "tail -n +#{keep_last + 1}", + "tail -n +#{retain + 1}", "while read container_id; do docker rm $container_id; done" end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index e5ecbb53..a3b074d5 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -127,6 +127,10 @@ class Kamal::Configuration raw_config.require_destination end + def retain_containers + raw_config.retain_containers || 5 + end + def volume_args if raw_config.volumes.present? @@ -218,7 +222,7 @@ class Kamal::Configuration def valid? - ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version + ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid end def to_h @@ -291,6 +295,12 @@ class Kamal::Configuration true end + def ensure_retain_containers_valid + raise ArgumentError, "Must retain at least 1 container" if retain_containers < 1 + + true + end + def role_names raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort diff --git a/test/cli/prune_test.rb b/test/cli/prune_test.rb index 77c3ebd4..11acc469 100644 --- a/test/cli/prune_test.rb +++ b/test/cli/prune_test.rb @@ -20,6 +20,15 @@ class CliPruneTest < CliTestCase assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output end + + run_command("containers", "--retain", "10").tap do |output| + assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +11 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output + assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output + end + + assert_raises(RuntimeError, "retain must be at least 1") do + run_command("containers", "--retain", "0") + end end private diff --git a/test/commands/prune_test.rb b/test/commands/prune_test.rb index 00db4ccc..c4a56a9a 100644 --- a/test/commands/prune_test.rb +++ b/test/commands/prune_test.rb @@ -23,7 +23,11 @@ class CommandsPruneTest < ActiveSupport::TestCase test "app containers" do assert_equal \ "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", - new_command.app_containers.join(" ") + new_command.app_containers(retain: 5).join(" ") + + assert_equal \ + "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +4 | while read container_id; do docker rm $container_id; done", + new_command.app_containers(retain: 3).join(" ") end test "healthcheck containers" do diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 392b6afb..fbc87a91 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -299,7 +299,7 @@ class ConfigurationTest < ActiveSupport::TestCase assert_equal "alternate_web", config.primary_role assert_equal "1.1.1.4", config.primary_host - assert config.role(:alternate_web).primary? + assert config.role(:alternate_web).primary? assert config.role(:alternate_web).running_traefik? end @@ -309,4 +309,12 @@ class ConfigurationTest < ActiveSupport::TestCase end assert_match /bar isn't defined/, error.message end + + test "retain_containers" do + assert_equal 5, @config.retain_containers + config = Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 2)) + assert_equal 2, config.retain_containers + + assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) } + end end