feat: adds support for both client-side and server-side remote URL uploads fetching (#10004)

### What?

The `pasteURL` feature for Upload fields has been updated to support
both **client-side** and **server-side** URL fetching. Previously, users
could only paste URLs from the same domain as their Payload instance
(internal) or public domains, which led to **CORS** errors when trying
to fetch files from external URLs.

Now, users can choose between **client-side fetching** (default) and
**server-side fetching** using the new `pasteURL` option in the Upload
collection config.

### How?

- By default, Payload will attempt to fetch the file client-side
directly in the browser.
- To enable server-side fetching, you can configure the new `pasteURL`
option with an `allowList` of trusted domains.
- The new `/api/:collectionSlug/paste-url` endpoint is used to fetch
files server-side and stream them back to the browser.

#### Example

```
import type { CollectionConfig } from 'payload'

export const Media: CollectionConfig = {
  slug: 'media',
  upload: {
    // pasteURL: false, // Can now disable the pasteURL option entirely by passing "false".
    pasteURL: {
      allowList: [
        {
          hostname: 'payloadcms.com', // required
          pathname: '',
          port: '',
          protocol: 'https', // defaults to https - options: "https" | "http"
          search: ''
        },
        {
          hostname: 'example.com',
          pathname: '/images/*',
        },
      ],
    },
  },
}
```

### Why

This update provides more flexibility for users to paste URLs into
Upload fields without running into **CORS errors** and allows Payload to
securely fetch files from trusted domains.

packages/payload/src/collections/endpoints/getFileFromURL.ts

@@ -0,0 +1,81 @@
+import type { PayloadHandler } from '../../config/types.js'
+
+import executeAccess from '../../auth/executeAccess.js'
+import { APIError } from '../../errors/APIError.js'
+import { Forbidden } from '../../errors/Forbidden.js'
+import { getRequestCollectionWithID } from '../../utilities/getRequestEntity.js'
+import { isURLAllowed } from '../../utilities/isURLAllowed.js'
+
+// If doc id is provided, it means we are updating the doc
+// /:collectionSlug/paste-url/:doc-id?src=:fileUrl
+
+// If doc id is not provided, it means we are creating a new doc
+// /:collectionSlug/paste-url?src=:fileUrl
+
+export const getFileFromURLHandler: PayloadHandler = async (req) => {
+  const { id, collection } = getRequestCollectionWithID(req, { optionalID: true })
+
+  if (!req.user) {
+    throw new Forbidden(req.t)
+  }
+
+  const config = collection?.config
+
+  if (id) {
+    // updating doc
+    const accessResult = await executeAccess({ req }, config.access.update)
+    if (!accessResult) {
+      throw new Forbidden(req.t)
+    }
+  } else {
+    // creating doc
+    const accessResult = await executeAccess({ req }, config.access?.create)
+    if (!accessResult) {
+      throw new Forbidden(req.t)
+    }
+  }
+  try {
+    if (!req.url) {
+      throw new APIError('Request URL is missing.', 400)
+    }
+
+    const { searchParams } = new URL(req.url)
+    const src = searchParams.get('src')
+
+    if (!src || typeof src !== 'string') {
+      throw new APIError('A valid URL string is required.', 400)
+    }
+
+    const validatedUrl = new URL(src)
+
+    if (
+      typeof config.upload?.pasteURL === 'object' &&
+      !isURLAllowed(validatedUrl.href, config.upload.pasteURL.allowList)
+    ) {
+      throw new APIError(`The provided URL (${validatedUrl.href}) is not allowed.`, 400)
+    }
+
+    // Fetch the file with no compression
+    const response = await fetch(validatedUrl.href, {
+      headers: {
+        'Accept-Encoding': 'identity',
+      },
+    })
+
+    if (!response.ok) {
+      throw new APIError(`Failed to fetch file from ${validatedUrl.href}`, response.status)
+    }
+
+    const decodedFileName = decodeURIComponent(validatedUrl.pathname.split('/').pop() || '')
+
+    return new Response(response.body, {
+      headers: {
+        'Content-Disposition': `attachment; filename="${decodedFileName}"`,
+        'Content-Length': response.headers.get('content-length') || '',
+        'Content-Type': response.headers.get('content-type') || 'application/octet-stream',
+      },
+    })
+  } catch (error) {
+    throw new APIError(`Error fetching file: ${error.message}`, 500)
+  }
+}

packages/payload/src/collections/endpoints/index.ts

@@ -13,6 +13,7 @@ import { findByIDHandler } from './findByID.js'
 import { findVersionByIDHandler } from './findVersionByID.js'
 import { findVersionsHandler } from './findVersions.js'
 import { getFileHandler } from './getFile.js'
+import { getFileFromURLHandler } from './getFileFromURL.js'
 import { previewHandler } from './preview.js'
 import { restoreVersionHandler } from './restoreVersion.js'
 import { updateHandler } from './update.js'
@@ -46,6 +47,11 @@ export const defaultCollectionEndpoints: Endpoint[] = [
       method: 'post',
       path: '/access/:id?',
     },
+    {
+      handler: getFileFromURLHandler,
+      method: 'get',
+      path: '/paste-url/:id?',
+    },
     {
       handler: findVersionsHandler,
       method: 'get',

packages/payload/src/uploads/types.ts

@@ -83,6 +83,14 @@ export type ImageSize = {
 
 export type GetAdminThumbnail = (args: { doc: Record<string, unknown> }) => false | null | string
 
+export type AllowList = Array<{
+  hostname: string
+  pathname?: string
+  port?: string
+  protocol?: 'http' | 'https'
+  search?: string
+}>
+
 export type UploadConfig = {
   /**
    * The adapter name to use for uploads. Used for storage adapter telemetry.
@@ -175,6 +183,17 @@ export type UploadConfig = {
    * @default undefined
    */
   modifyResponseHeaders?: ({ headers }: { headers: Headers }) => Headers
+  /**
+   * Controls the behavior of pasting/uploading files from URLs.
+   * If set to `false`, fetching from remote URLs is disabled.
+   * If an allowList is provided, server-side fetching will be enabled for specified URLs.
+   * @default true (client-side fetching enabled)
+   */
+  pasteURL?:
+    | {
+        allowList: AllowList
+      }
+    | false
   /**
    * Sharp resize options for the original image.
    * @link https://sharp.pixelplumbing.com/api-resize#resize

packages/payload/src/utilities/isURLAllowed.ts

@@ -0,0 +1,35 @@
+import type { AllowList } from '../uploads/types.js'
+
+export const isURLAllowed = (url: string, allowList: AllowList): boolean => {
+  try {
+    const parsedUrl = new URL(url)
+
+    return allowList.some((allowItem) => {
+      return Object.entries(allowItem).every(([key, value]) => {
+        // Skip undefined or null values
+        if (!value) {
+          return true
+        }
+        // Compare protocol with colon
+        if (key === 'protocol') {
+          return typeof value === 'string' && parsedUrl.protocol === `${value}:`
+        }
+
+        if (key === 'pathname') {
+          // Convert wildcards to a regex
+          const regexPattern = value
+            .replace(/\*\*/g, '.*') // Match any path
+            .replace(/\*/g, '[^/]*') // Match any part of a path segment
+            .replace(/\/$/, '(/)?') // Allow optional trailing slash
+          const regex = new RegExp(`^${regexPattern}$`)
+          return regex.test(parsedUrl.pathname)
+        }
+
+        // Default comparison for all other properties (hostname, port, search)
+        return parsedUrl[key as keyof URL] === value
+      })
+    })
+  } catch {
+    return false // If the URL is invalid, deny by default
+  }
+}

packages/ui/src/elements/Upload/index.tsx

@@ -2,13 +2,14 @@
 import type { FormState, SanitizedCollectionConfig, UploadEdits } from 'payload'
 
 import { isImage } from 'payload/shared'
-import React, { useCallback, useEffect, useRef, useState } from 'react'
+import React, { Fragment, useCallback, useEffect, useRef, useState } from 'react'
 import { toast } from 'sonner'
 
 import { FieldError } from '../../fields/FieldError/index.js'
 import { fieldBaseClass } from '../../fields/shared/index.js'
 import { useForm, useFormProcessing } from '../../forms/Form/index.js'
 import { useField } from '../../forms/useField/index.js'
+import { useConfig } from '../../providers/Config/index.js'
 import { useDocumentInfo } from '../../providers/DocumentInfo/index.js'
 import { EditDepthProvider } from '../../providers/EditDepth/index.js'
 import { useTranslation } from '../../providers/Translation/index.js'
@@ -18,8 +19,8 @@ import { Drawer, DrawerToggler } from '../Drawer/index.js'
 import { Dropzone } from '../Dropzone/index.js'
 import { EditUpload } from '../EditUpload/index.js'
 import { FileDetails } from '../FileDetails/index.js'
-import { PreviewSizes } from '../PreviewSizes/index.js'
 import './index.scss'
+import { PreviewSizes } from '../PreviewSizes/index.js'
 import { Thumbnail } from '../Thumbnail/index.js'
 
 const baseClass = 'file-field'
@@ -92,10 +93,17 @@ export type UploadProps = {
 export const Upload: React.FC<UploadProps> = (props) => {
   const { collectionSlug, customActions, initialState, onChange, uploadConfig } = props
 
+  const {
+    config: {
+      routes: { api },
+      serverURL,
+    },
+  } = useConfig()
+
   const { t } = useTranslation()
   const { setModified } = useForm()
   const { resetUploadEdits, updateUploadEdits, uploadEdits } = useUploadEdits()
-  const { docPermissions, savedDocumentData, setUploadStatus } = useDocumentInfo()
+  const { id, docPermissions, savedDocumentData, setUploadStatus } = useDocumentInfo()
   const isFormSubmitting = useFormProcessing()
   const { errorMessage, setValue, showError, value } = useField<File>({
     path: 'file',
@@ -111,6 +119,9 @@ export const Upload: React.FC<UploadProps> = (props) => {
   const urlInputRef = useRef<HTMLInputElement>(null)
   const inputRef = useRef<HTMLInputElement>(null)
 
+  const useServerSideFetch =
+    typeof uploadConfig?.pasteURL === 'object' && uploadConfig.pasteURL.allowList?.length > 0
+
   const handleFileChange = useCallback(
     (newFile: File) => {
       if (newFile instanceof File) {
@@ -174,23 +185,53 @@ export const Upload: React.FC<UploadProps> = (props) => {
   )
 
   const handleUrlSubmit = async () => {
-    if (fileUrl) {
-      setUploadStatus('uploading')
-      try {
-        const response = await fetch(fileUrl)
-        const data = await response.blob()
+    if (!fileUrl || uploadConfig?.pasteURL === false) {
+      return
+    }
 
-        // Extract the file name from the URL
-        const fileName = fileUrl.split('/').pop()
+    setUploadStatus('uploading')
+    try {
+      // Attempt client-side fetch
+      const clientResponse = await fetch(fileUrl)
 
-        // Create a new File object from the Blob data
-        const file = new File([data], fileName, { type: data.type })
-        handleFileChange(file)
-        setUploadStatus('idle')
-      } catch (e) {
-        toast.error(e.message)
-        setUploadStatus('failed')
+      if (!clientResponse.ok) {
+        throw new Error(`Fetch failed with status: ${clientResponse.status}`)
       }
+
+      const blob = await clientResponse.blob()
+      const fileName = decodeURIComponent(fileUrl.split('/').pop() || '')
+      const file = new File([blob], fileName, { type: blob.type })
+
+      handleFileChange(file)
+      setUploadStatus('idle')
+      return // Exit if client-side fetch succeeds
+    } catch (_clientError) {
+      if (!useServerSideFetch) {
+        // If server-side fetch is not enabled, show client-side error
+        toast.error('Failed to fetch the file.')
+        setUploadStatus('failed')
+        return
+      }
+    }
+
+    // Attempt server-side fetch if client-side fetch fails and useServerSideFetch is true
+    try {
+      const pasteURL = `/${collectionSlug}/paste-url${id ? `/${id}?` : '?'}src=${encodeURIComponent(fileUrl)}`
+      const serverResponse = await fetch(`${serverURL}${api}${pasteURL}`)
+
+      if (!serverResponse.ok) {
+        throw new Error(`Fetch failed with status: ${serverResponse.status}`)
+      }
+
+      const blob = await serverResponse.blob()
+      const fileName = decodeURIComponent(fileUrl.split('/').pop() || '')
+      const file = new File([blob], fileName, { type: blob.type })
+
+      handleFileChange(file)
+      setUploadStatus('idle')
+    } catch (_serverError) {
+      toast.error('The provided URL is not allowed.')
+      setUploadStatus('failed')
     }
   }
 
@@ -272,16 +313,20 @@ export const Upload: React.FC<UploadProps> = (props) => {
                     ref={inputRef}
                     type="file"
                   />
-                  <span className={`${baseClass}__orText`}>{t('general:or')}</span>
-                  <Button
-                    buttonStyle="pill"
-                    onClick={() => {
-                      setShowUrlInput(true)
-                    }}
-                    size="small"
-                  >
-                    {t('upload:pasteURL')}
-                  </Button>
+                  {uploadConfig?.pasteURL !== false && (
+                    <Fragment>
+                      <span className={`${baseClass}__orText`}>{t('general:or')}</span>
+                      <Button
+                        buttonStyle="pill"
+                        onClick={() => {
+                          setShowUrlInput(true)
+                        }}
+                        size="small"
+                      >
+                        {t('upload:pasteURL')}
+                      </Button>
+                    </Fragment>
+                  )}
                 </div>
 
                 <p className={`${baseClass}__dragAndDropText`}>

packages/ui/src/utilities/isURLAllowed.ts

@@ -0,0 +1,34 @@
+import type { AllowList } from 'payload'
+
+export const isURLAllowed = (url: string, allowList: AllowList): boolean => {
+  try {
+    const parsedUrl = new URL(url)
+
+    return allowList.some((allowItem) => {
+      return Object.entries(allowItem).every(([key, value]) => {
+        // Skip undefined or null values
+        if (!value) {
+          return true
+        }
+        // Compare protocol with colon
+        if (key === 'protocol') {
+          return typeof value === 'string' && parsedUrl.protocol === `${value}:`
+        }
+
+        if (key === 'pathname') {
+          // Convert wildcards to a regex
+          const regexPattern = value
+            .replace(/\*\*/g, '.*') // Match any path
+            .replace(/\*/g, '[^/]*') // Match any part of a path segment
+          const regex = new RegExp(`^${regexPattern}$`)
+          return regex.test(parsedUrl.pathname)
+        }
+
+        // Default comparison for all other properties (hostname, port, search)
+        return parsedUrl[key as keyof URL] === value
+      })
+    })
+  } catch {
+    return false // If the URL is invalid, deny by default
+  }
+}