From 800c424777e5281b9a2ddc6e2701cc445b1d6300 Mon Sep 17 00:00:00 2001 From: Sasha <64744993+r1tsuu@users.noreply.github.com> Date: Mon, 5 May 2025 23:16:14 +0300 Subject: [PATCH] feat(storage-s3): presigned URLs for file downloads (#12307) Adds pre-signed URLs support file downloads with the S3 adapter. Can be enabled per-collection: ```ts s3Storage({ collections: { media: { signedDownloads: true }, // or { signedDownloads: { expiresIn: 3600 }} for custom expiresIn (default 7200) }, bucket: process.env.S3_BUCKET, config: { credentials: { accessKeyId: process.env.S3_ACCESS_KEY_ID, secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, }, endpoint: process.env.S3_ENDPOINT, forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true', region: process.env.S3_REGION, }, }), ``` The main use case is when you care about the Payload access control (so you don't want to use `disablePayloadAccessControl: true` but you don't want your files to be served through Payload (which can affect performance with large videos for example). This feature instead generates a signed URL (after verifying the access control) and redirects you directly to the S3 provider. This is an addition to https://github.com/payloadcms/payload/pull/11382 which added pre-signed URLs for file uploads. --- docs/upload/storage-adapters.mdx | 1 + packages/storage-s3/src/index.ts | 45 +++++++++++++++++-- packages/storage-s3/src/staticHandler.ts | 28 +++++++++++- .../collections/MediaWithSignedDownloads.ts | 9 ++++ test/storage-s3/config.ts | 8 +++- test/storage-s3/int.spec.ts | 41 ++++++++++++++--- test/storage-s3/payload-types.ts | 41 +++++++++++++++++ test/storage-s3/shared.ts | 2 + 8 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 test/storage-s3/collections/MediaWithSignedDownloads.ts diff --git a/docs/upload/storage-adapters.mdx b/docs/upload/storage-adapters.mdx index 35c29d9d64..3d4f3639a2 100644 --- a/docs/upload/storage-adapters.mdx +++ b/docs/upload/storage-adapters.mdx @@ -84,6 +84,7 @@ pnpm add @payloadcms/storage-s3 - The `config` object can be any [`S3ClientConfig`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3) object (from [`@aws-sdk/client-s3`](https://github.com/aws/aws-sdk-js-v3)). _This is highly dependent on your AWS setup_. Check the AWS documentation for more information. - When enabled, this package will automatically set `disableLocalStorage` to `true` for each collection. - When deploying to Vercel, server uploads are limited with 4.5MB. Set `clientUploads` to `true` to do uploads directly on the client. You must allow CORS PUT method for the bucket to your website. +- Configure `signedDownloads` (either globally of per-collection in `collections`) to use [presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) for files downloading. This can improve performance for large files (like videos) while still respecting your access control. ```ts import { s3Storage } from '@payloadcms/storage-s3' diff --git a/packages/storage-s3/src/index.ts b/packages/storage-s3/src/index.ts index b45b881f49..b9a0b850a6 100644 --- a/packages/storage-s3/src/index.ts +++ b/packages/storage-s3/src/index.ts @@ -12,6 +12,8 @@ import * as AWS from '@aws-sdk/client-s3' import { cloudStoragePlugin } from '@payloadcms/plugin-cloud-storage' import { initClientUploads } from '@payloadcms/plugin-cloud-storage/utilities' +import type { SignedDownloadsConfig } from './staticHandler.js' + import { getGenerateSignedURLHandler } from './generateSignedURL.js' import { getGenerateURL } from './generateURL.js' import { getHandleDelete } from './handleDelete.js' @@ -24,6 +26,7 @@ export type S3StorageOptions = { */ acl?: 'private' | 'public-read' + /** * Bucket name to upload files to. * @@ -39,8 +42,15 @@ export type S3StorageOptions = { /** * Collection options to apply the S3 adapter to. */ - collections: Partial | true>> - + collections: Partial< + Record< + UploadCollectionSlug, + | ({ + signedDownloads?: SignedDownloadsConfig + } & Omit) + | true + > + > /** * AWS S3 client configuration. Highly dependent on your AWS setup. * @@ -61,6 +71,10 @@ export type S3StorageOptions = { * Default: true */ enabled?: boolean + /** + * Use pre-signed URLs for files downloading. Can be overriden per-collection. + */ + signedDownloads?: SignedDownloadsConfig } type S3StoragePlugin = (storageS3Args: S3StorageOptions) => Plugin @@ -158,9 +172,27 @@ export const s3Storage: S3StoragePlugin = function s3StorageInternal( getStorageClient: () => AWS.S3, - { acl, bucket, clientUploads, config = {} }: S3StorageOptions, + { + acl, + bucket, + clientUploads, + collections, + config = {}, + signedDownloads: topLevelSignedDownloads, + }: S3StorageOptions, ): Adapter { return ({ collection, prefix }): GeneratedAdapter => { + const collectionStorageConfig = collections[collection.slug] + + let signedDownloads: null | SignedDownloadsConfig = + typeof collectionStorageConfig === 'object' + ? (collectionStorageConfig.signedDownloads ?? false) + : null + + if (signedDownloads === null) { + signedDownloads = topLevelSignedDownloads ?? null + } + return { name: 's3', clientUploads, @@ -173,7 +205,12 @@ function s3StorageInternal( getStorageClient, prefix, }), - staticHandler: getHandler({ bucket, collection, getStorageClient }), + staticHandler: getHandler({ + bucket, + collection, + getStorageClient, + signedDownloads: signedDownloads ?? false, + }), } } } diff --git a/packages/storage-s3/src/staticHandler.ts b/packages/storage-s3/src/staticHandler.ts index 0c01319c14..7f328c1072 100644 --- a/packages/storage-s3/src/staticHandler.ts +++ b/packages/storage-s3/src/staticHandler.ts @@ -3,13 +3,23 @@ import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types' import type { CollectionConfig } from 'payload' import type { Readable } from 'stream' +import { GetObjectCommand } from '@aws-sdk/client-s3' +import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities' import path from 'path' +export type SignedDownloadsConfig = + | { + /** @default 7200 */ + expiresIn?: number + } + | boolean + interface Args { bucket: string collection: CollectionConfig getStorageClient: () => AWS.S3 + signedDownloads?: SignedDownloadsConfig } // Type guard for NodeJS.Readable streams @@ -40,7 +50,12 @@ const streamToBuffer = async (readableStream: any) => { return Buffer.concat(chunks) } -export const getHandler = ({ bucket, collection, getStorageClient }: Args): StaticHandler => { +export const getHandler = ({ + bucket, + collection, + getStorageClient, + signedDownloads, +}: Args): StaticHandler => { return async (req, { params: { clientUploadContext, filename } }) => { let object: AWS.GetObjectOutput | undefined = undefined try { @@ -48,6 +63,17 @@ export const getHandler = ({ bucket, collection, getStorageClient }: Args): Stat const key = path.posix.join(prefix, filename) + if (signedDownloads && !clientUploadContext) { + const command = new GetObjectCommand({ Bucket: bucket, Key: key }) + const signedUrl = await getSignedUrl( + // @ts-expect-error mismatch versions + getStorageClient(), + command, + typeof signedDownloads === 'object' ? signedDownloads : { expiresIn: 7200 }, + ) + return Response.redirect(signedUrl) + } + object = await getStorageClient().getObject({ Bucket: bucket, Key: key, diff --git a/test/storage-s3/collections/MediaWithSignedDownloads.ts b/test/storage-s3/collections/MediaWithSignedDownloads.ts new file mode 100644 index 0000000000..d4318caea1 --- /dev/null +++ b/test/storage-s3/collections/MediaWithSignedDownloads.ts @@ -0,0 +1,9 @@ +import type { CollectionConfig } from 'payload' + +import { mediaWithSignedDownloadsSlug } from '../shared.js' + +export const MediaWithSignedDownloads: CollectionConfig = { + slug: mediaWithSignedDownloadsSlug, + upload: true, + fields: [], +} diff --git a/test/storage-s3/config.ts b/test/storage-s3/config.ts index 8bef0ea42b..59327fda9f 100644 --- a/test/storage-s3/config.ts +++ b/test/storage-s3/config.ts @@ -7,8 +7,9 @@ import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' import { devUser } from '../credentials.js' import { Media } from './collections/Media.js' import { MediaWithPrefix } from './collections/MediaWithPrefix.js' +import { MediaWithSignedDownloads } from './collections/MediaWithSignedDownloads.js' import { Users } from './collections/Users.js' -import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) @@ -25,7 +26,7 @@ export default buildConfigWithDefaults({ baseDir: path.resolve(dirname), }, }, - collections: [Media, MediaWithPrefix, Users], + collections: [Media, MediaWithPrefix, MediaWithSignedDownloads, Users], onInit: async (payload) => { await payload.create({ collection: 'users', @@ -42,6 +43,9 @@ export default buildConfigWithDefaults({ [mediaWithPrefixSlug]: { prefix, }, + [mediaWithSignedDownloadsSlug]: { + signedDownloads: true, + }, }, bucket: process.env.S3_BUCKET, config: { diff --git a/test/storage-s3/int.spec.ts b/test/storage-s3/int.spec.ts index ed898fac60..91b533ef97 100644 --- a/test/storage-s3/int.spec.ts +++ b/test/storage-s3/int.spec.ts @@ -4,12 +4,16 @@ import * as AWS from '@aws-sdk/client-s3' import path from 'path' import { fileURLToPath } from 'url' +import type { NextRESTClient } from '../helpers/NextRESTClient.js' + import { initPayloadInt } from '../helpers/initPayloadInt.js' -import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) +let restClient: NextRESTClient + let payload: Payload describe('@payloadcms/storage-s3', () => { @@ -17,7 +21,7 @@ describe('@payloadcms/storage-s3', () => { let client: AWS.S3Client beforeAll(async () => { - ;({ payload } = await initPayloadInt(dirname)) + ;({ payload, restClient } = await initPayloadInt(dirname)) TEST_BUCKET = process.env.S3_BUCKET client = new AWS.S3({ @@ -77,15 +81,38 @@ describe('@payloadcms/storage-s3', () => { expect(upload.url).toEqual(`/api/${mediaWithPrefixSlug}/file/${String(upload.filename)}`) }) + it('can download with signed downloads', async () => { + await payload.create({ + collection: mediaWithSignedDownloadsSlug, + data: {}, + filePath: path.resolve(dirname, '../uploads/image.png'), + }) + + const response = await restClient.GET(`/${mediaWithSignedDownloadsSlug}/file/image.png`) + expect(response.status).toBe(302) + const url = response.headers.get('Location') + expect(url).toBeDefined() + expect(url!).toContain(`/${TEST_BUCKET}/image.png`) + expect(new URLSearchParams(url!).get('x-id')).toBe('GetObject') + const file = await fetch(url!) + expect(file.headers.get('Content-Type')).toBe('image/png') + }) + describe('R2', () => { it.todo('can upload') }) async function createTestBucket() { - const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET })) + try { + const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET })) - if (makeBucketRes.$metadata.httpStatusCode !== 200) { - throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`) + if (makeBucketRes.$metadata.httpStatusCode !== 200) { + throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`) + } + } catch (e) { + if (e instanceof AWS.BucketAlreadyOwnedByYou) { + console.log('Bucket already exists') + } } } @@ -96,7 +123,9 @@ describe('@payloadcms/storage-s3', () => { }), ) - if (!listedObjects?.Contents?.length) return + if (!listedObjects?.Contents?.length) { + return + } const deleteParams = { Bucket: TEST_BUCKET, diff --git a/test/storage-s3/payload-types.ts b/test/storage-s3/payload-types.ts index 7311e4ebb9..0f7d9fb900 100644 --- a/test/storage-s3/payload-types.ts +++ b/test/storage-s3/payload-types.ts @@ -69,6 +69,7 @@ export interface Config { collections: { media: Media; 'media-with-prefix': MediaWithPrefix; + 'media-with-signed-downloads': MediaWithSignedDownload; users: User; 'payload-locked-documents': PayloadLockedDocument; 'payload-preferences': PayloadPreference; @@ -78,6 +79,7 @@ export interface Config { collectionsSelect: { media: MediaSelect | MediaSelect; 'media-with-prefix': MediaWithPrefixSelect | MediaWithPrefixSelect; + 'media-with-signed-downloads': MediaWithSignedDownloadsSelect | MediaWithSignedDownloadsSelect; users: UsersSelect | UsersSelect; 'payload-locked-documents': PayloadLockedDocumentsSelect | PayloadLockedDocumentsSelect; 'payload-preferences': PayloadPreferencesSelect | PayloadPreferencesSelect; @@ -171,6 +173,24 @@ export interface MediaWithPrefix { focalX?: number | null; focalY?: number | null; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "media-with-signed-downloads". + */ +export interface MediaWithSignedDownload { + id: string; + updatedAt: string; + createdAt: string; + url?: string | null; + thumbnailURL?: string | null; + filename?: string | null; + mimeType?: string | null; + filesize?: number | null; + width?: number | null; + height?: number | null; + focalX?: number | null; + focalY?: number | null; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "users". @@ -203,6 +223,10 @@ export interface PayloadLockedDocument { relationTo: 'media-with-prefix'; value: string | MediaWithPrefix; } | null) + | ({ + relationTo: 'media-with-signed-downloads'; + value: string | MediaWithSignedDownload; + } | null) | ({ relationTo: 'users'; value: string | User; @@ -309,6 +333,23 @@ export interface MediaWithPrefixSelect { focalX?: T; focalY?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "media-with-signed-downloads_select". + */ +export interface MediaWithSignedDownloadsSelect { + updatedAt?: T; + createdAt?: T; + url?: T; + thumbnailURL?: T; + filename?: T; + mimeType?: T; + filesize?: T; + width?: T; + height?: T; + focalX?: T; + focalY?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "users_select". diff --git a/test/storage-s3/shared.ts b/test/storage-s3/shared.ts index 7d74b323ae..27653dab7d 100644 --- a/test/storage-s3/shared.ts +++ b/test/storage-s3/shared.ts @@ -1,3 +1,5 @@ export const mediaSlug = 'media' export const mediaWithPrefixSlug = 'media-with-prefix' export const prefix = 'test-prefix' + +export const mediaWithSignedDownloadsSlug = 'media-with-signed-downloads'