feat(storage-s3): presigned URLs for file downloads (#12307)

Adds pre-signed URLs support file downloads with the S3 adapter. Can be
enabled per-collection:
```ts
s3Storage({
  collections: {
    media: { signedDownloads: true }, // or { signedDownloads: { expiresIn: 3600 }} for custom expiresIn (default 7200)
  },
  bucket: process.env.S3_BUCKET,
  config: {
    credentials: {
      accessKeyId: process.env.S3_ACCESS_KEY_ID,
      secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
    },
    endpoint: process.env.S3_ENDPOINT,
    forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true',
    region: process.env.S3_REGION,
  },
}),
```

The main use case is when you care about the Payload access control (so
you don't want to use `disablePayloadAccessControl: true` but you don't
want your files to be served through Payload (which can affect
performance with large videos for example).
This feature instead generates a signed URL (after verifying the access
control) and redirects you directly to the S3 provider.

This is an addition to https://github.com/payloadcms/payload/pull/11382
which added pre-signed URLs for file uploads.

test/storage-s3/collections/MediaWithSignedDownloads.ts

@@ -0,0 +1,9 @@
+import type { CollectionConfig } from 'payload'
+
+import { mediaWithSignedDownloadsSlug } from '../shared.js'
+
+export const MediaWithSignedDownloads: CollectionConfig = {
+  slug: mediaWithSignedDownloadsSlug,
+  upload: true,
+  fields: [],
+}

test/storage-s3/config.ts

@@ -7,8 +7,9 @@ import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js'
 import { devUser } from '../credentials.js'
 import { Media } from './collections/Media.js'
 import { MediaWithPrefix } from './collections/MediaWithPrefix.js'
+import { MediaWithSignedDownloads } from './collections/MediaWithSignedDownloads.js'
 import { Users } from './collections/Users.js'
-import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js'
+import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js'
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
 
@@ -25,7 +26,7 @@ export default buildConfigWithDefaults({
       baseDir: path.resolve(dirname),
     },
   },
-  collections: [Media, MediaWithPrefix, Users],
+  collections: [Media, MediaWithPrefix, MediaWithSignedDownloads, Users],
   onInit: async (payload) => {
     await payload.create({
       collection: 'users',
@@ -42,6 +43,9 @@ export default buildConfigWithDefaults({
         [mediaWithPrefixSlug]: {
           prefix,
         },
+        [mediaWithSignedDownloadsSlug]: {
+          signedDownloads: true,
+        },
       },
       bucket: process.env.S3_BUCKET,
       config: {

test/storage-s3/int.spec.ts

@@ -4,12 +4,16 @@ import * as AWS from '@aws-sdk/client-s3'
 import path from 'path'
 import { fileURLToPath } from 'url'
 
+import type { NextRESTClient } from '../helpers/NextRESTClient.js'
+
 import { initPayloadInt } from '../helpers/initPayloadInt.js'
-import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js'
+import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js'
 
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
 
+let restClient: NextRESTClient
+
 let payload: Payload
 
 describe('@payloadcms/storage-s3', () => {
@@ -17,7 +21,7 @@ describe('@payloadcms/storage-s3', () => {
   let client: AWS.S3Client
 
   beforeAll(async () => {
-    ;({ payload } = await initPayloadInt(dirname))
+    ;({ payload, restClient } = await initPayloadInt(dirname))
     TEST_BUCKET = process.env.S3_BUCKET
 
     client = new AWS.S3({
@@ -77,15 +81,38 @@ describe('@payloadcms/storage-s3', () => {
     expect(upload.url).toEqual(`/api/${mediaWithPrefixSlug}/file/${String(upload.filename)}`)
   })
 
+  it('can download with signed downloads', async () => {
+    await payload.create({
+      collection: mediaWithSignedDownloadsSlug,
+      data: {},
+      filePath: path.resolve(dirname, '../uploads/image.png'),
+    })
+
+    const response = await restClient.GET(`/${mediaWithSignedDownloadsSlug}/file/image.png`)
+    expect(response.status).toBe(302)
+    const url = response.headers.get('Location')
+    expect(url).toBeDefined()
+    expect(url!).toContain(`/${TEST_BUCKET}/image.png`)
+    expect(new URLSearchParams(url!).get('x-id')).toBe('GetObject')
+    const file = await fetch(url!)
+    expect(file.headers.get('Content-Type')).toBe('image/png')
+  })
+
   describe('R2', () => {
     it.todo('can upload')
   })
 
   async function createTestBucket() {
-    const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET }))
+    try {
+      const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET }))
 
-    if (makeBucketRes.$metadata.httpStatusCode !== 200) {
-      throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`)
+      if (makeBucketRes.$metadata.httpStatusCode !== 200) {
+        throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`)
+      }
+    } catch (e) {
+      if (e instanceof AWS.BucketAlreadyOwnedByYou) {
+        console.log('Bucket already exists')
+      }
     }
   }
 
@@ -96,7 +123,9 @@ describe('@payloadcms/storage-s3', () => {
       }),
     )
 
-    if (!listedObjects?.Contents?.length) return
+    if (!listedObjects?.Contents?.length) {
+      return
+    }
 
     const deleteParams = {
       Bucket: TEST_BUCKET,

test/storage-s3/payload-types.ts

@@ -69,6 +69,7 @@ export interface Config {
   collections: {
     media: Media;
     'media-with-prefix': MediaWithPrefix;
+    'media-with-signed-downloads': MediaWithSignedDownload;
     users: User;
     'payload-locked-documents': PayloadLockedDocument;
     'payload-preferences': PayloadPreference;
@@ -78,6 +79,7 @@ export interface Config {
   collectionsSelect: {
     media: MediaSelect<false> | MediaSelect<true>;
     'media-with-prefix': MediaWithPrefixSelect<false> | MediaWithPrefixSelect<true>;
+    'media-with-signed-downloads': MediaWithSignedDownloadsSelect<false> | MediaWithSignedDownloadsSelect<true>;
     users: UsersSelect<false> | UsersSelect<true>;
     'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
     'payload-preferences': PayloadPreferencesSelect<false> | PayloadPreferencesSelect<true>;
@@ -171,6 +173,24 @@ export interface MediaWithPrefix {
   focalX?: number | null;
   focalY?: number | null;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "media-with-signed-downloads".
+ */
+export interface MediaWithSignedDownload {
+  id: string;
+  updatedAt: string;
+  createdAt: string;
+  url?: string | null;
+  thumbnailURL?: string | null;
+  filename?: string | null;
+  mimeType?: string | null;
+  filesize?: number | null;
+  width?: number | null;
+  height?: number | null;
+  focalX?: number | null;
+  focalY?: number | null;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users".
@@ -203,6 +223,10 @@ export interface PayloadLockedDocument {
         relationTo: 'media-with-prefix';
         value: string | MediaWithPrefix;
       } | null)
+    | ({
+        relationTo: 'media-with-signed-downloads';
+        value: string | MediaWithSignedDownload;
+      } | null)
     | ({
         relationTo: 'users';
         value: string | User;
@@ -309,6 +333,23 @@ export interface MediaWithPrefixSelect<T extends boolean = true> {
   focalX?: T;
   focalY?: T;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "media-with-signed-downloads_select".
+ */
+export interface MediaWithSignedDownloadsSelect<T extends boolean = true> {
+  updatedAt?: T;
+  createdAt?: T;
+  url?: T;
+  thumbnailURL?: T;
+  filename?: T;
+  mimeType?: T;
+  filesize?: T;
+  width?: T;
+  height?: T;
+  focalX?: T;
+  focalY?: T;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users_select".

test/storage-s3/shared.ts

@@ -1,3 +1,5 @@
 export const mediaSlug = 'media'
 export const mediaWithPrefixSlug = 'media-with-prefix'
 export const prefix = 'test-prefix'
+
+export const mediaWithSignedDownloadsSlug = 'media-with-signed-downloads'