diff --git a/lib/mrsk/configuration/builder.rb b/lib/mrsk/configuration/builder.rb
index 46c12cb4..309240fb 100644
--- a/lib/mrsk/configuration/builder.rb
+++ b/lib/mrsk/configuration/builder.rb
@@ -96,12 +96,20 @@ class Mrsk::Configuration::Builder
       @options["cache"]&.fetch("image", nil) || "#{@image}-build-cache"
     end
 
+    def cache_image_ref
+      if @server.present?
+        "#{@server}/#{cache_image}"
+      else
+        cache_image
+      end
+    end
+
     def cache_from_config_for_gha
       "type=gha"
     end
 
     def cache_from_config_for_registry
-      [ "type=registry", "ref=#{@server}/#{cache_image}" ].compact.join(",")
+      [ "type=registry", "ref=#{cache_image_ref}" ].compact.join(",")
     end
 
     def cache_to_config_for_gha
@@ -109,6 +117,6 @@ class Mrsk::Configuration::Builder
     end
 
     def cache_to_config_for_registry
-      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{@server}/#{cache_image}" ].compact.join(",")
+      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
     end
 end
diff --git a/test/configuration/builder_test.rb b/test/configuration/builder_test.rb
index 1b355187..b81cc93a 100644
--- a/test/configuration/builder_test.rb
+++ b/test/configuration/builder_test.rb
@@ -98,15 +98,23 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   test "setting registry cache" do
     @deploy_with_builder_option[:builder] = { "cache" => { "type" => "registry", "options" => "mode=max,image-manifest=true,oci-mediatypes=true" } }
 
-    assert_equal "type=registry,ref=/dhh/app-build-cache", @config_with_builder_option.builder.cache_from
-    assert_equal "type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=/dhh/app-build-cache", @config_with_builder_option.builder.cache_to
+    assert_equal "type=registry,ref=dhh/app-build-cache", @config_with_builder_option.builder.cache_from
+    assert_equal "type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=dhh/app-build-cache", @config_with_builder_option.builder.cache_to
+  end
+
+  test "setting registry cache when using a custom registry" do
+    @config_with_builder_option.registry["server"] = "registry.example.com"
+    @deploy_with_builder_option[:builder] = { "cache" => { "type" => "registry", "options" => "mode=max,image-manifest=true,oci-mediatypes=true" } }
+
+    assert_equal "type=registry,ref=registry.example.com/dhh/app-build-cache", @config_with_builder_option.builder.cache_from
+    assert_equal "type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=registry.example.com/dhh/app-build-cache", @config_with_builder_option.builder.cache_to
   end
 
   test "setting registry cache with image" do
     @deploy_with_builder_option[:builder] = { "cache" => { "type" => "registry", "image" => "mrsk", "options" => "mode=max" } }
 
-    assert_equal "type=registry,ref=/mrsk", @config_with_builder_option.builder.cache_from
-    assert_equal "type=registry,mode=max,ref=/mrsk", @config_with_builder_option.builder.cache_to
+    assert_equal "type=registry,ref=mrsk", @config_with_builder_option.builder.cache_from
+    assert_equal "type=registry,mode=max,ref=mrsk", @config_with_builder_option.builder.cache_to
   end
 
   test "args" do