Seed docker mirrors by pulling once per mirror first

Find the first registry mirror on each host. If we find any, pull the
images on one host per mirror, then do the remainder concurrently.

The initial pulls will seed the mirrors ensuring that we pull the image
from Docker Hub once each.

This works best if there is only one mirror on each host.
This commit is contained in:
Donal McBreen
2024-07-11 13:05:32 +01:00
parent 1573cebadf
commit 1369c46a83
6 changed files with 81 additions and 6 deletions

View File

@@ -59,11 +59,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
desc "pull", "Pull app image from registry onto servers"
def pull
on(KAMAL.hosts) do
execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
execute *KAMAL.builder.pull
execute *KAMAL.builder.validate_image
if (first_hosts = mirror_hosts).any?
#  Pull on a single host per mirror first to seed them
say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
pull_on_hosts(first_hosts)
say "Pulling image on remaining hosts...", :magenta
pull_on_hosts(KAMAL.hosts - first_hosts)
else
pull_on_hosts(KAMAL.hosts)
end
end
@@ -131,4 +134,28 @@ class Kamal::Cli::Build < Kamal::Cli::Base
end
end
end
def mirror_hosts
if KAMAL.hosts.many?
mirror_hosts = Concurrent::Hash.new
on(KAMAL.hosts) do |host|
first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
mirror_hosts[first_mirror] ||= host if first_mirror
rescue SSHKit::Command::Failed => e
raise unless e.message =~ /error calling index: reflect: slice index out of range/
end
mirror_hosts.values
else
[]
end
end
def pull_on_hosts(hosts)
on(hosts) do
execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
execute *KAMAL.builder.pull
execute *KAMAL.builder.validate_image
end
end
end

View File

@@ -2,7 +2,7 @@ require "active_support/core_ext/string/filters"
class Kamal::Commands::Builder < Kamal::Commands::Base
delegate :create, :remove, :push, :clean, :pull, :info, :context_hosts, :config_context_hosts, :validate_image,
to: :target
:first_mirror, to: :target
include Clone

View File

@@ -40,6 +40,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
[]
end
def first_mirror
docker(:info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
end
private
def build_tags
[ "-t", config.absolute_image, "-t", config.latest_image ]