diff --git a/packages/plugin-cloud-storage/docker-compose.yml b/packages/plugin-cloud-storage/docker-compose.yml index 550571250..8c1ffa784 100644 --- a/packages/plugin-cloud-storage/docker-compose.yml +++ b/packages/plugin-cloud-storage/docker-compose.yml @@ -15,7 +15,7 @@ services: - '/var/run/docker.sock:/var/run/docker.sock' azure-storage: - image: mcr.microsoft.com/azure-storage/azurite:3.18.0 + image: mcr.microsoft.com/azure-storage/azurite:latest platform: linux/amd64 restart: always command: 'azurite --loose --blobHost 0.0.0.0 --tableHost 0.0.0.0 --queueHost 0.0.0.0' diff --git a/packages/storage-azure/.eslintignore b/packages/storage-azure/.eslintignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-azure/.eslintignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-azure/.eslintrc.cjs b/packages/storage-azure/.eslintrc.cjs new file mode 100644 index 000000000..d6b3a476b --- /dev/null +++ b/packages/storage-azure/.eslintrc.cjs @@ -0,0 +1,7 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + parserOptions: { + project: ['./tsconfig.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/packages/storage-azure/.prettierignore b/packages/storage-azure/.prettierignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-azure/.prettierignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-azure/.swcrc b/packages/storage-azure/.swcrc new file mode 100644 index 000000000..14463f4b0 --- /dev/null +++ b/packages/storage-azure/.swcrc @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "sourceMaps": true, + "jsc": { + "target": "esnext", + "parser": { + "syntax": "typescript", + "tsx": true, + "dts": true + } + }, + "module": { + "type": "es6" + } +} diff --git a/packages/storage-azure/LICENSE.md b/packages/storage-azure/LICENSE.md new file mode 100644 index 000000000..05e80b2b4 --- /dev/null +++ b/packages/storage-azure/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018-2022 Payload CMS, LLC +Portions Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/storage-azure/README.md b/packages/storage-azure/README.md new file mode 100644 index 000000000..081cfd4c2 --- /dev/null +++ b/packages/storage-azure/README.md @@ -0,0 +1 @@ +# Azure Storage diff --git a/packages/storage-azure/package.json b/packages/storage-azure/package.json new file mode 100644 index 000000000..f30c7f1a7 --- /dev/null +++ b/packages/storage-azure/package.json @@ -0,0 +1,61 @@ +{ + "name": "@payloadcms/storage-azure", + "version": "0.0.0", + "description": "Payload storage adapter for Azure Blob Storage", + "repository": { + "type": "git", + "url": "https://github.com/payloadcms/payload.git", + "directory": "packages/storage-azure" + }, + "license": "MIT", + "homepage": "https://payloadcms.com", + "author": "Payload CMS, Inc.", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "scripts": { + "build": "pnpm build:swc && pnpm build:types", + "build:swc": "swc ./src -d ./dist --config-file .swcrc", + "build:types": "tsc --emitDeclarationOnly --outDir dist", + "build:clean": "find . \\( -type d \\( -name build -o -name dist -o -name .cache \\) -o -type f -name tsconfig.tsbuildinfo \\) -exec rm -rf {} + && pnpm build", + "clean": "rimraf {dist,*.tsbuildinfo}", + "prepublishOnly": "pnpm clean && pnpm turbo build" + }, + "exports": { + ".": { + "import": "./src/index.ts", + "require": "./src/index.ts", + "types": "./src/index.ts" + } + }, + "publishConfig": { + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + } + }, + "dependencies": { + "@azure/abort-controller": "^1.1.0", + "@azure/storage-blob": "^12.11.0", + "@payloadcms/plugin-cloud-storage": "workspace:*", + "range-parser": "^1.2.1" + }, + "devDependencies": { + "@types/range-parser": "^1.2.7", + "payload": "workspace:*" + }, + "peerDependencies": { + "payload": "workspace:*" + }, + "engines": { + "node": ">=18.20.2" + }, + "files": [ + "dist" + ] +} diff --git a/packages/storage-azure/src/generateURL.ts b/packages/storage-azure/src/generateURL.ts new file mode 100644 index 000000000..5fc703c9c --- /dev/null +++ b/packages/storage-azure/src/generateURL.ts @@ -0,0 +1,14 @@ +import type { GenerateURL } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +interface Args { + baseURL: string + containerName: string +} + +export const getGenerateURL = + ({ baseURL, containerName }: Args): GenerateURL => + ({ filename, prefix = '' }) => { + return `${baseURL}/${containerName}/${path.posix.join(prefix, filename)}` + } diff --git a/packages/storage-azure/src/handleDelete.ts b/packages/storage-azure/src/handleDelete.ts new file mode 100644 index 000000000..2ebaeebc7 --- /dev/null +++ b/packages/storage-azure/src/handleDelete.ts @@ -0,0 +1,17 @@ +import type { ContainerClient } from '@azure/storage-blob' +import type { HandleDelete } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import path from 'path' + +interface Args { + collection: CollectionConfig + getStorageClient: () => ContainerClient +} + +export const getHandleDelete = ({ getStorageClient }: Args): HandleDelete => { + return async ({ doc: { prefix = '' }, filename }) => { + const blockBlobClient = getStorageClient().getBlockBlobClient(path.posix.join(prefix, filename)) + await blockBlobClient.deleteIfExists() + } +} diff --git a/packages/storage-azure/src/handleUpload.ts b/packages/storage-azure/src/handleUpload.ts new file mode 100644 index 000000000..3b48a1a2d --- /dev/null +++ b/packages/storage-azure/src/handleUpload.ts @@ -0,0 +1,42 @@ +import type { ContainerClient } from '@azure/storage-blob' +import type { HandleUpload } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { AbortController } from '@azure/abort-controller' +import fs from 'fs' +import path from 'path' +import { Readable } from 'stream' + +interface Args { + collection: CollectionConfig + getStorageClient: () => ContainerClient + prefix?: string +} + +const multipartThreshold = 1024 * 1024 * 50 // 50MB +export const getHandleUpload = ({ getStorageClient, prefix = '' }: Args): HandleUpload => { + return async ({ data, file }) => { + const fileKey = path.posix.join(data.prefix || prefix, file.filename) + + const blockBlobClient = getStorageClient().getBlockBlobClient(fileKey) + + // when there are no temp files, or the upload is less than the threshold size, do not stream files + if (!file.tempFilePath && file.buffer.length > 0 && file.buffer.length < multipartThreshold) { + await blockBlobClient.upload(file.buffer, file.buffer.byteLength, { + blobHTTPHeaders: { blobContentType: file.mimeType }, + }) + + return data + } + + const fileBufferOrStream: Readable = file.tempFilePath + ? fs.createReadStream(file.tempFilePath) + : Readable.from(file.buffer) + + await blockBlobClient.uploadStream(fileBufferOrStream, 4 * 1024 * 1024, 4, { + abortSignal: AbortController.timeout(30 * 60 * 1000), + }) + + return data + } +} diff --git a/packages/storage-azure/src/index.ts b/packages/storage-azure/src/index.ts new file mode 100644 index 000000000..4334f8781 --- /dev/null +++ b/packages/storage-azure/src/index.ts @@ -0,0 +1,106 @@ +import type { ContainerClient } from '@azure/storage-blob' +import type { + Adapter, + PluginOptions as CloudStoragePluginOptions, + CollectionOptions, + GeneratedAdapter, +} from '@payloadcms/plugin-cloud-storage/types' +import type { Config, Plugin } from 'payload/config' + +import { BlobServiceClient } from '@azure/storage-blob' +import { cloudStorage } from '@payloadcms/plugin-cloud-storage' + +import { getGenerateURL } from './generateURL.js' +import { getHandleDelete } from './handleDelete.js' +import { getHandleUpload } from './handleUpload.js' +import { getHandler } from './staticHandler.js' + +export type AzureStorageOptions = { + allowContainerCreate: boolean + baseURL: string + /** + * Collection options to apply the Azure Blob adapter to. + */ + collections: Record | true> + + /** + * Azure Blob storage connection string + */ + connectionString: string + + /** + * Azure Blob storage container name + */ + containerName: string + + /** + * Whether or not to enable the plugin + * + * Default: true + */ + enabled?: boolean +} + +type AzureStoragePlugin = (azureStorageArgs: AzureStorageOptions) => Plugin + +export const azureStorage: AzureStoragePlugin = + (azureStorageOptions: AzureStorageOptions) => + (incomingConfig: Config): Config => { + if (azureStorageOptions.enabled === false) { + return incomingConfig + } + + const adapter = azureStorageInternal(azureStorageOptions) + + // Add adapter to each collection option object + const collectionsWithAdapter: CloudStoragePluginOptions['collections'] = Object.entries( + azureStorageOptions.collections, + ).reduce( + (acc, [slug, collOptions]) => ({ + ...acc, + [slug]: { + ...(collOptions === true ? {} : collOptions), + adapter, + }, + }), + {} as Record, + ) + + return cloudStorage({ + collections: collectionsWithAdapter, + })(incomingConfig) + } + +function azureStorageInternal({ + allowContainerCreate, + baseURL, + connectionString, + containerName, +}: AzureStorageOptions): Adapter { + let storageClient: ContainerClient | null = null + const getStorageClient = () => { + if (storageClient) return storageClient + + const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString) + storageClient = blobServiceClient.getContainerClient(containerName) + return storageClient + } + + const createContainerIfNotExists = () => { + void getStorageClient().createIfNotExists({ access: 'blob' }) + } + + return ({ collection, prefix }): GeneratedAdapter => { + return { + generateURL: getGenerateURL({ baseURL, containerName }), + handleDelete: getHandleDelete({ collection, getStorageClient }), + handleUpload: getHandleUpload({ + collection, + getStorageClient, + prefix, + }), + staticHandler: getHandler({ collection, getStorageClient }), + ...(allowContainerCreate && { onInit: createContainerIfNotExists }), + } + } +} diff --git a/packages/storage-azure/src/staticHandler.ts b/packages/storage-azure/src/staticHandler.ts new file mode 100644 index 000000000..ca328a720 --- /dev/null +++ b/packages/storage-azure/src/staticHandler.ts @@ -0,0 +1,61 @@ +import type { ContainerClient } from '@azure/storage-blob' +import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities' +import path from 'path' + +import { getRangeFromHeader } from './utils/getRangeFromHeader.js' + +interface Args { + collection: CollectionConfig + getStorageClient: () => ContainerClient +} + +export const getHandler = ({ collection, getStorageClient }: Args): StaticHandler => { + return async (req, { params: { filename } }) => { + try { + const prefix = await getFilePrefix({ collection, filename, req }) + const blockBlobClient = getStorageClient().getBlockBlobClient( + path.posix.join(prefix, filename), + ) + + const { end, start } = await getRangeFromHeader( + blockBlobClient, + String(req.headers.get('range')), + ) + + const blob = await blockBlobClient.download(start, end) + // eslint-disable-next-line no-underscore-dangle + const response = blob._response + + // Manually create a ReadableStream for the web from a Node.js stream. + const readableStream = new ReadableStream({ + start(controller) { + const nodeStream = blob.readableStreamBody + if (!nodeStream) { + throw new Error('No readable stream body') + } + + nodeStream.on('data', (chunk) => { + controller.enqueue(new Uint8Array(chunk)) + }) + nodeStream.on('end', () => { + controller.close() + }) + nodeStream.on('error', (err) => { + controller.error(err) + }) + }, + }) + + return new Response(readableStream, { + headers: response.headers.rawHeaders(), + status: response.status, + }) + } catch (err: unknown) { + req.payload.logger.error(err) + return new Response('Internal Server Error', { status: 500 }) + } + } +} diff --git a/packages/storage-azure/src/utils/getRangeFromHeader.ts b/packages/storage-azure/src/utils/getRangeFromHeader.ts new file mode 100644 index 000000000..6cf9f42bc --- /dev/null +++ b/packages/storage-azure/src/utils/getRangeFromHeader.ts @@ -0,0 +1,26 @@ +import type { BlockBlobClient } from '@azure/storage-blob' + +import parseRange from 'range-parser' + +export const getRangeFromHeader = async ( + blockBlobClient: BlockBlobClient, + rangeHeader?: string, +): Promise<{ end: number | undefined; start: number }> => { + const fullRange = { end: undefined, start: 0 } + + if (!rangeHeader) { + return fullRange + } + + const size = await blockBlobClient.getProperties().then((props) => props.contentLength) + if (size === undefined) { + return fullRange + } + + const range = parseRange(size, rangeHeader) + if (range === -1 || range === -2 || range.type !== 'bytes' || range.length !== 1) { + return fullRange + } + + return range[0] +} diff --git a/packages/storage-azure/tsconfig.json b/packages/storage-azure/tsconfig.json new file mode 100644 index 000000000..5880d1adc --- /dev/null +++ b/packages/storage-azure/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, // Make sure typescript knows that this module depends on their references + "noEmit": false /* Do not emit outputs. */, + "emitDeclarationOnly": true, + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + "rootDir": "./src" /* Specify the root folder within your source files. */, + "strict": true, + }, + "exclude": [ + "dist", + "node_modules", + ], + "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.d.ts", "src/**/*.json"], + "references": [ + { "path": "../payload" }, + { "path": "../plugin-cloud-storage" }, + ] +} diff --git a/packages/storage-gcs/.eslintignore b/packages/storage-gcs/.eslintignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-gcs/.eslintignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-gcs/.eslintrc.cjs b/packages/storage-gcs/.eslintrc.cjs new file mode 100644 index 000000000..d6b3a476b --- /dev/null +++ b/packages/storage-gcs/.eslintrc.cjs @@ -0,0 +1,7 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + parserOptions: { + project: ['./tsconfig.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/packages/storage-gcs/.prettierignore b/packages/storage-gcs/.prettierignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-gcs/.prettierignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-gcs/.swcrc b/packages/storage-gcs/.swcrc new file mode 100644 index 000000000..14463f4b0 --- /dev/null +++ b/packages/storage-gcs/.swcrc @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "sourceMaps": true, + "jsc": { + "target": "esnext", + "parser": { + "syntax": "typescript", + "tsx": true, + "dts": true + } + }, + "module": { + "type": "es6" + } +} diff --git a/packages/storage-gcs/LICENSE.md b/packages/storage-gcs/LICENSE.md new file mode 100644 index 000000000..05e80b2b4 --- /dev/null +++ b/packages/storage-gcs/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018-2022 Payload CMS, LLC +Portions Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/storage-gcs/README.md b/packages/storage-gcs/README.md new file mode 100644 index 000000000..1740cad35 --- /dev/null +++ b/packages/storage-gcs/README.md @@ -0,0 +1 @@ +# Nodemailer Email Adapter diff --git a/packages/storage-gcs/package.json b/packages/storage-gcs/package.json new file mode 100644 index 000000000..1c94c8644 --- /dev/null +++ b/packages/storage-gcs/package.json @@ -0,0 +1,58 @@ +{ + "name": "@payloadcms/storage-gcs", + "version": "0.0.0", + "description": "Payload storage adapter for Google Cloud Storage", + "repository": { + "type": "git", + "url": "https://github.com/payloadcms/payload.git", + "directory": "packages/storage-gcs" + }, + "license": "MIT", + "homepage": "https://payloadcms.com", + "author": "Payload CMS, Inc.", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "scripts": { + "build": "pnpm build:swc && pnpm build:types", + "build:swc": "swc ./src -d ./dist --config-file .swcrc", + "build:types": "tsc --emitDeclarationOnly --outDir dist", + "build:clean": "find . \\( -type d \\( -name build -o -name dist -o -name .cache \\) -o -type f -name tsconfig.tsbuildinfo \\) -exec rm -rf {} + && pnpm build", + "clean": "rimraf {dist,*.tsbuildinfo}", + "prepublishOnly": "pnpm clean && pnpm turbo build" + }, + "exports": { + ".": { + "import": "./src/index.ts", + "require": "./src/index.ts", + "types": "./src/index.ts" + } + }, + "publishConfig": { + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + } + }, + "dependencies": { + "@payloadcms/plugin-cloud-storage": "workspace:*", + "@google-cloud/storage": "^7.7.0" + }, + "devDependencies": { + "payload": "workspace:*" + }, + "peerDependencies": { + "payload": "workspace:*" + }, + "engines": { + "node": ">=18.20.2" + }, + "files": [ + "dist" + ] +} diff --git a/packages/storage-gcs/src/generateURL.ts b/packages/storage-gcs/src/generateURL.ts new file mode 100644 index 000000000..1d5e25c1a --- /dev/null +++ b/packages/storage-gcs/src/generateURL.ts @@ -0,0 +1,17 @@ +import type { Storage } from '@google-cloud/storage' +import type { GenerateURL } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +interface Args { + bucket: string + getStorageClient: () => Storage +} + +export const getGenerateURL = + ({ bucket, getStorageClient }: Args): GenerateURL => + ({ filename, prefix = '' }) => { + return decodeURIComponent( + getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename)).publicUrl(), + ) + } diff --git a/packages/storage-gcs/src/handleDelete.ts b/packages/storage-gcs/src/handleDelete.ts new file mode 100644 index 000000000..d2688875c --- /dev/null +++ b/packages/storage-gcs/src/handleDelete.ts @@ -0,0 +1,17 @@ +import type { Storage } from '@google-cloud/storage' +import type { HandleDelete } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +interface Args { + bucket: string + getStorageClient: () => Storage +} + +export const getHandleDelete = ({ bucket, getStorageClient }: Args): HandleDelete => { + return async ({ doc: { prefix = '' }, filename }) => { + await getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename)).delete({ + ignoreNotFound: true, + }) + } +} diff --git a/packages/storage-gcs/src/handleUpload.ts b/packages/storage-gcs/src/handleUpload.ts new file mode 100644 index 000000000..3b7cfe4a5 --- /dev/null +++ b/packages/storage-gcs/src/handleUpload.ts @@ -0,0 +1,37 @@ +import type { Storage } from '@google-cloud/storage' +import type { HandleUpload } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import path from 'path' + +interface Args { + acl?: 'Private' | 'Public' + bucket: string + collection: CollectionConfig + getStorageClient: () => Storage + prefix?: string +} + +export const getHandleUpload = ({ + acl, + bucket, + getStorageClient, + prefix = '', +}: Args): HandleUpload => { + return async ({ data, file }) => { + const fileKey = path.posix.join(data.prefix || prefix, file.filename) + + const gcsFile = getStorageClient().bucket(bucket).file(fileKey) + await gcsFile.save(file.buffer, { + metadata: { + contentType: file.mimeType, + }, + }) + + if (acl) { + await gcsFile[`make${acl}`]() + } + + return data + } +} diff --git a/packages/storage-gcs/src/index.ts b/packages/storage-gcs/src/index.ts new file mode 100644 index 000000000..e7fe6f462 --- /dev/null +++ b/packages/storage-gcs/src/index.ts @@ -0,0 +1,97 @@ +import type { StorageOptions } from '@google-cloud/storage' +import type { + Adapter, + PluginOptions as CloudStoragePluginOptions, + CollectionOptions, + GeneratedAdapter, +} from '@payloadcms/plugin-cloud-storage/types' +import type { Config, Plugin } from 'payload/config' + +import { Storage } from '@google-cloud/storage' +import { cloudStorage } from '@payloadcms/plugin-cloud-storage' + +import { getGenerateURL } from './generateURL.js' +import { getHandleDelete } from './handleDelete.js' +import { getHandleUpload } from './handleUpload.js' +import { getHandler } from './staticHandler.js' + +export interface GcsStorageOptions { + acl?: 'Private' | 'Public' + + /** + * The name of the bucket to use. + */ + bucket: string + /** + * Collection options to apply the S3 adapter to. + */ + collections: Record | true> + /** + * Whether or not to enable the plugin + * + * Default: true + */ + enabled?: boolean + + /** + * Google Cloud Storage client configuration. + * + * @see https://github.com/googleapis/nodejs-storage + */ + options: StorageOptions +} + +type GcsStoragePlugin = (gcsStorageArgs: GcsStorageOptions) => Plugin + +export const gcsStorage: GcsStoragePlugin = + (gcsStorageOptions: GcsStorageOptions) => + (incomingConfig: Config): Config => { + if (gcsStorageOptions.enabled === false) { + return incomingConfig + } + + const adapter = gcsStorageInternal(gcsStorageOptions) + + // Add adapter to each collection option object + const collectionsWithAdapter: CloudStoragePluginOptions['collections'] = Object.entries( + gcsStorageOptions.collections, + ).reduce( + (acc, [slug, collOptions]) => ({ + ...acc, + [slug]: { + ...(collOptions === true ? {} : collOptions), + adapter, + }, + }), + {} as Record, + ) + + return cloudStorage({ + collections: collectionsWithAdapter, + })(incomingConfig) + } + +function gcsStorageInternal({ acl, bucket, options }: GcsStorageOptions): Adapter { + return ({ collection, prefix }): GeneratedAdapter => { + let storageClient: Storage | null = null + + const getStorageClient = (): Storage => { + if (storageClient) return storageClient + storageClient = new Storage(options) + return storageClient + } + + return { + generateURL: getGenerateURL({ bucket, getStorageClient }), + handleDelete: getHandleDelete({ bucket, getStorageClient }), + handleUpload: getHandleUpload({ + acl, + bucket, + collection, + getStorageClient, + prefix, + }), + staticHandler: getHandler({ bucket, collection, getStorageClient }), + } + } +} diff --git a/packages/storage-gcs/src/staticHandler.ts b/packages/storage-gcs/src/staticHandler.ts new file mode 100644 index 000000000..bc6b5610f --- /dev/null +++ b/packages/storage-gcs/src/staticHandler.ts @@ -0,0 +1,51 @@ +import type { Storage } from '@google-cloud/storage' +import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities' +import path from 'path' + +interface Args { + bucket: string + collection: CollectionConfig + getStorageClient: () => Storage +} + +export const getHandler = ({ bucket, collection, getStorageClient }: Args): StaticHandler => { + return async (req, { params: { filename } }) => { + try { + const prefix = await getFilePrefix({ collection, filename, req }) + const file = getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename)) + + const [metadata] = await file.getMetadata() + + // Manually create a ReadableStream for the web from a Node.js stream. + const readableStream = new ReadableStream({ + start(controller) { + const nodeStream = file.createReadStream() + nodeStream.on('data', (chunk) => { + controller.enqueue(new Uint8Array(chunk)) + }) + nodeStream.on('end', () => { + controller.close() + }) + nodeStream.on('error', (err) => { + controller.error(err) + }) + }, + }) + + return new Response(readableStream, { + headers: new Headers({ + 'Content-Length': String(metadata.size), + 'Content-Type': String(metadata.contentType), + ETag: String(metadata.etag), + }), + status: 200, + }) + } catch (err: unknown) { + req.payload.logger.error(err) + return new Response('Internal Server Error', { status: 500 }) + } + } +} diff --git a/packages/storage-gcs/tsconfig.json b/packages/storage-gcs/tsconfig.json new file mode 100644 index 000000000..5880d1adc --- /dev/null +++ b/packages/storage-gcs/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, // Make sure typescript knows that this module depends on their references + "noEmit": false /* Do not emit outputs. */, + "emitDeclarationOnly": true, + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + "rootDir": "./src" /* Specify the root folder within your source files. */, + "strict": true, + }, + "exclude": [ + "dist", + "node_modules", + ], + "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.d.ts", "src/**/*.json"], + "references": [ + { "path": "../payload" }, + { "path": "../plugin-cloud-storage" }, + ] +} diff --git a/packages/storage-s3/.eslintignore b/packages/storage-s3/.eslintignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-s3/.eslintignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-s3/.eslintrc.cjs b/packages/storage-s3/.eslintrc.cjs new file mode 100644 index 000000000..d6b3a476b --- /dev/null +++ b/packages/storage-s3/.eslintrc.cjs @@ -0,0 +1,7 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + parserOptions: { + project: ['./tsconfig.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/packages/storage-s3/.prettierignore b/packages/storage-s3/.prettierignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-s3/.prettierignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-s3/.swcrc b/packages/storage-s3/.swcrc new file mode 100644 index 000000000..14463f4b0 --- /dev/null +++ b/packages/storage-s3/.swcrc @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "sourceMaps": true, + "jsc": { + "target": "esnext", + "parser": { + "syntax": "typescript", + "tsx": true, + "dts": true + } + }, + "module": { + "type": "es6" + } +} diff --git a/packages/storage-s3/LICENSE.md b/packages/storage-s3/LICENSE.md new file mode 100644 index 000000000..05e80b2b4 --- /dev/null +++ b/packages/storage-s3/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018-2022 Payload CMS, LLC +Portions Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/storage-s3/README.md b/packages/storage-s3/README.md new file mode 100644 index 000000000..1740cad35 --- /dev/null +++ b/packages/storage-s3/README.md @@ -0,0 +1 @@ +# Nodemailer Email Adapter diff --git a/packages/storage-s3/docker-compose.yml b/packages/storage-s3/docker-compose.yml new file mode 100644 index 000000000..7d9aba936 --- /dev/null +++ b/packages/storage-s3/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.2' +services: + localstack: + image: localstack/localstack:latest + container_name: localstack_demo + ports: + - '4563-4599:4563-4599' + - '8055:8080' + environment: + - SERVICES=s3 + - DEBUG=1 + - DATA_DIR=/tmp/localstack/data + volumes: + - './.localstack:/var/lib/localstack' + - '/var/run/docker.sock:/var/run/docker.sock' diff --git a/packages/storage-s3/package.json b/packages/storage-s3/package.json new file mode 100644 index 000000000..b9efc49a5 --- /dev/null +++ b/packages/storage-s3/package.json @@ -0,0 +1,59 @@ +{ + "name": "@payloadcms/storage-s3", + "version": "0.0.0", + "description": "Payload storage adapter for Amazon S3", + "repository": { + "type": "git", + "url": "https://github.com/payloadcms/payload.git", + "directory": "packages/storage-s3" + }, + "license": "MIT", + "homepage": "https://payloadcms.com", + "author": "Payload CMS, Inc.", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "scripts": { + "build": "pnpm build:swc && pnpm build:types", + "build:swc": "swc ./src -d ./dist --config-file .swcrc", + "build:types": "tsc --emitDeclarationOnly --outDir dist", + "build:clean": "find . \\( -type d \\( -name build -o -name dist -o -name .cache \\) -o -type f -name tsconfig.tsbuildinfo \\) -exec rm -rf {} + && pnpm build", + "clean": "rimraf {dist,*.tsbuildinfo}", + "prepublishOnly": "pnpm clean && pnpm turbo build" + }, + "exports": { + ".": { + "import": "./src/index.ts", + "require": "./src/index.ts", + "types": "./src/index.ts" + } + }, + "publishConfig": { + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + } + }, + "dependencies": { + "@payloadcms/plugin-cloud-storage": "workspace:*" + }, + "devDependencies": { + "@aws-sdk/client-s3": "^3.525.0", + "@aws-sdk/lib-storage": "^3.525.0", + "payload": "workspace:*" + }, + "peerDependencies": { + "payload": "workspace:*" + }, + "engines": { + "node": ">=18.20.2" + }, + "files": [ + "dist" + ] +} diff --git a/packages/storage-s3/src/generateURL.ts b/packages/storage-s3/src/generateURL.ts new file mode 100644 index 000000000..358502e7e --- /dev/null +++ b/packages/storage-s3/src/generateURL.ts @@ -0,0 +1,16 @@ +import type * as AWS from '@aws-sdk/client-s3' +import type { GenerateURL } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +interface Args { + bucket: string + config: AWS.S3ClientConfig +} + +export const getGenerateURL = + ({ bucket, config: { endpoint } }: Args): GenerateURL => + ({ filename, prefix = '' }) => { + const stringifiedEndpoint = typeof endpoint === 'string' ? endpoint : endpoint?.toString() + return `${stringifiedEndpoint}/${bucket}/${path.posix.join(prefix, filename)}` + } diff --git a/packages/storage-s3/src/handleDelete.ts b/packages/storage-s3/src/handleDelete.ts new file mode 100644 index 000000000..cfbc258da --- /dev/null +++ b/packages/storage-s3/src/handleDelete.ts @@ -0,0 +1,18 @@ +import type * as AWS from '@aws-sdk/client-s3' +import type { HandleDelete } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +interface Args { + bucket: string + getStorageClient: () => AWS.S3 +} + +export const getHandleDelete = ({ bucket, getStorageClient }: Args): HandleDelete => { + return async ({ doc: { prefix = '' }, filename }) => { + await getStorageClient().deleteObject({ + Bucket: bucket, + Key: path.posix.join(prefix, filename), + }) + } +} diff --git a/packages/storage-s3/src/handleUpload.ts b/packages/storage-s3/src/handleUpload.ts new file mode 100644 index 000000000..ce74a2403 --- /dev/null +++ b/packages/storage-s3/src/handleUpload.ts @@ -0,0 +1,61 @@ +import type * as AWS from '@aws-sdk/client-s3' +import type { HandleUpload } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { Upload } from '@aws-sdk/lib-storage' +import fs from 'fs' +import path from 'path' + +interface Args { + acl?: 'private' | 'public-read' + bucket: string + collection: CollectionConfig + getStorageClient: () => AWS.S3 + prefix?: string +} + +const multipartThreshold = 1024 * 1024 * 50 // 50MB + +export const getHandleUpload = ({ + acl, + bucket, + getStorageClient, + prefix = '', +}: Args): HandleUpload => { + return async ({ data, file }) => { + const fileKey = path.posix.join(data.prefix || prefix, file.filename) + + const fileBufferOrStream = file.tempFilePath + ? fs.createReadStream(file.tempFilePath) + : file.buffer + + if (file.buffer.length > 0 && file.buffer.length < multipartThreshold) { + await getStorageClient().putObject({ + ACL: acl, + Body: fileBufferOrStream, + Bucket: bucket, + ContentType: file.mimeType, + Key: fileKey, + }) + + return data + } + + const parallelUploadS3 = new Upload({ + client: getStorageClient(), + params: { + ACL: acl, + Body: fileBufferOrStream, + Bucket: bucket, + ContentType: file.mimeType, + Key: fileKey, + }, + partSize: multipartThreshold, + queueSize: 4, + }) + + await parallelUploadS3.done() + + return data + } +} diff --git a/packages/storage-s3/src/index.ts b/packages/storage-s3/src/index.ts new file mode 100644 index 000000000..b26cfa6d7 --- /dev/null +++ b/packages/storage-s3/src/index.ts @@ -0,0 +1,109 @@ +import type { + Adapter, + PluginOptions as CloudStoragePluginOptions, + CollectionOptions, + GeneratedAdapter, +} from '@payloadcms/plugin-cloud-storage/types' +import type { Config, Plugin } from 'payload/config' + +import * as AWS from '@aws-sdk/client-s3' +import { cloudStorage } from '@payloadcms/plugin-cloud-storage' + +import { getGenerateURL } from './generateURL.js' +import { getHandleDelete } from './handleDelete.js' +import { getHandleUpload } from './handleUpload.js' +import { getHandler } from './staticHandler.js' + +export type S3StorageOptions = { + /** + * Access control list for uploaded files. + */ + + acl?: 'private' | 'public-read' + /** + * Bucket name to upload files to. + * + * Must follow [AWS S3 bucket naming conventions](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html). + */ + + bucket: string + + /** + * Collection options to apply the S3 adapter to. + */ + collections: Record | true> + /** + * AWS S3 client configuration. Highly dependent on your AWS setup. + * + * [AWS.S3ClientConfig Docs](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html) + */ + config: AWS.S3ClientConfig + + /** + * Whether or not to disable local storage + * + * @default true + */ + disableLocalStorage?: boolean + + /** + * Whether or not to enable the plugin + * + * Default: true + */ + enabled?: boolean +} + +type S3StoragePlugin = (storageS3Args: S3StorageOptions) => Plugin + +export const s3Storage: S3StoragePlugin = + (s3StorageOptions: S3StorageOptions) => + (incomingConfig: Config): Config => { + if (s3StorageOptions.enabled === false) { + return incomingConfig + } + + const adapter = s3StorageInternal(s3StorageOptions) + + // Add adapter to each collection option object + const collectionsWithAdapter: CloudStoragePluginOptions['collections'] = Object.entries( + s3StorageOptions.collections, + ).reduce( + (acc, [slug, collOptions]) => ({ + ...acc, + [slug]: { + ...(collOptions === true ? {} : collOptions), + adapter, + }, + }), + {} as Record, + ) + + return cloudStorage({ + collections: collectionsWithAdapter, + })(incomingConfig) + } + +function s3StorageInternal({ acl, bucket, config = {} }: S3StorageOptions): Adapter { + return ({ collection, prefix }): GeneratedAdapter => { + let storageClient: AWS.S3 | null = null + const getStorageClient: () => AWS.S3 = () => { + if (storageClient) return storageClient + storageClient = new AWS.S3(config) + return storageClient + } + + return { + generateURL: getGenerateURL({ bucket, config }), + handleDelete: getHandleDelete({ bucket, getStorageClient }), + handleUpload: getHandleUpload({ + acl, + bucket, + collection, + getStorageClient, + prefix, + }), + staticHandler: getHandler({ bucket, collection, getStorageClient }), + } + } +} diff --git a/packages/storage-s3/src/staticHandler.ts b/packages/storage-s3/src/staticHandler.ts new file mode 100644 index 000000000..f904cdaf9 --- /dev/null +++ b/packages/storage-s3/src/staticHandler.ts @@ -0,0 +1,54 @@ +import type * as AWS from '@aws-sdk/client-s3' +import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities' +import path from 'path' + +interface Args { + bucket: string + collection: CollectionConfig + getStorageClient: () => AWS.S3 +} + +// Convert a stream into a promise that resolves with a Buffer +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const streamToBuffer = async (readableStream: any) => { + const chunks = [] + for await (const chunk of readableStream) { + chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) + } + return Buffer.concat(chunks) +} + +export const getHandler = ({ bucket, collection, getStorageClient }: Args): StaticHandler => { + return async (req, { params: { filename } }) => { + try { + const prefix = await getFilePrefix({ collection, filename, req }) + + const object = await getStorageClient().getObject({ + Bucket: bucket, + Key: path.posix.join(prefix, filename), + }) + + if (!object.Body) { + return new Response(null, { status: 404, statusText: 'Not Found' }) + } + + const bodyBuffer = await streamToBuffer(object.Body) + + return new Response(bodyBuffer, { + headers: new Headers({ + 'Accept-Ranges': String(object.AcceptRanges), + 'Content-Length': String(object.ContentLength), + 'Content-Type': String(object.ContentType), + ETag: String(object.ETag), + }), + status: 200, + }) + } catch (err) { + req.payload.logger.error(err) + return new Response('Internal Server Error', { status: 500 }) + } + } +} diff --git a/packages/storage-s3/tsconfig.json b/packages/storage-s3/tsconfig.json new file mode 100644 index 000000000..5880d1adc --- /dev/null +++ b/packages/storage-s3/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, // Make sure typescript knows that this module depends on their references + "noEmit": false /* Do not emit outputs. */, + "emitDeclarationOnly": true, + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + "rootDir": "./src" /* Specify the root folder within your source files. */, + "strict": true, + }, + "exclude": [ + "dist", + "node_modules", + ], + "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.d.ts", "src/**/*.json"], + "references": [ + { "path": "../payload" }, + { "path": "../plugin-cloud-storage" }, + ] +} diff --git a/packages/storage-vercel-blob/.eslintignore b/packages/storage-vercel-blob/.eslintignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-vercel-blob/.eslintignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-vercel-blob/.eslintrc.cjs b/packages/storage-vercel-blob/.eslintrc.cjs new file mode 100644 index 000000000..d6b3a476b --- /dev/null +++ b/packages/storage-vercel-blob/.eslintrc.cjs @@ -0,0 +1,7 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + parserOptions: { + project: ['./tsconfig.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/packages/storage-vercel-blob/.prettierignore b/packages/storage-vercel-blob/.prettierignore new file mode 100644 index 000000000..247f3f12d --- /dev/null +++ b/packages/storage-vercel-blob/.prettierignore @@ -0,0 +1,10 @@ +.tmp +**/.git +**/.hg +**/.pnp.* +**/.svn +**/.yarn/** +**/build +**/dist/** +**/node_modules +**/temp diff --git a/packages/storage-vercel-blob/.swcrc b/packages/storage-vercel-blob/.swcrc new file mode 100644 index 000000000..14463f4b0 --- /dev/null +++ b/packages/storage-vercel-blob/.swcrc @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "sourceMaps": true, + "jsc": { + "target": "esnext", + "parser": { + "syntax": "typescript", + "tsx": true, + "dts": true + } + }, + "module": { + "type": "es6" + } +} diff --git a/packages/storage-vercel-blob/LICENSE.md b/packages/storage-vercel-blob/LICENSE.md new file mode 100644 index 000000000..05e80b2b4 --- /dev/null +++ b/packages/storage-vercel-blob/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018-2022 Payload CMS, LLC +Portions Copyright (c) Meta Platforms, Inc. and affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/storage-vercel-blob/README.md b/packages/storage-vercel-blob/README.md new file mode 100644 index 000000000..8c7948fed --- /dev/null +++ b/packages/storage-vercel-blob/README.md @@ -0,0 +1 @@ +# Vercel Blob Storage diff --git a/packages/storage-vercel-blob/package.json b/packages/storage-vercel-blob/package.json new file mode 100644 index 000000000..43647b260 --- /dev/null +++ b/packages/storage-vercel-blob/package.json @@ -0,0 +1,58 @@ +{ + "name": "@payloadcms/storage-vercel-blob", + "version": "0.0.0", + "description": "Payload storage adapter for Vercel Blob Storage", + "repository": { + "type": "git", + "url": "https://github.com/payloadcms/payload.git", + "directory": "packages/storage-vercel-blob" + }, + "license": "MIT", + "homepage": "https://payloadcms.com", + "author": "Payload CMS, Inc.", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "scripts": { + "build": "pnpm build:swc && pnpm build:types", + "build:swc": "swc ./src -d ./dist --config-file .swcrc", + "build:types": "tsc --emitDeclarationOnly --outDir dist", + "build:clean": "find . \\( -type d \\( -name build -o -name dist -o -name .cache \\) -o -type f -name tsconfig.tsbuildinfo \\) -exec rm -rf {} + && pnpm build", + "clean": "rimraf {dist,*.tsbuildinfo}", + "prepublishOnly": "pnpm clean && pnpm turbo build" + }, + "exports": { + ".": { + "import": "./src/index.ts", + "require": "./src/index.ts", + "types": "./src/index.ts" + } + }, + "publishConfig": { + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + } + } + }, + "dependencies": { + "@payloadcms/plugin-cloud-storage": "workspace:*" + }, + "devDependencies": { + "@vercel/blob": "^0.22.3", + "payload": "workspace:*" + }, + "peerDependencies": { + "payload": "workspace:*" + }, + "engines": { + "node": ">=18.20.2" + }, + "files": [ + "dist" + ] +} diff --git a/packages/storage-vercel-blob/src/generateURL.ts b/packages/storage-vercel-blob/src/generateURL.ts new file mode 100644 index 000000000..fcd3ae98f --- /dev/null +++ b/packages/storage-vercel-blob/src/generateURL.ts @@ -0,0 +1,14 @@ +import type { GenerateURL } from '@payloadcms/plugin-cloud-storage/types' + +import path from 'path' + +type GenerateUrlArgs = { + baseUrl: string + prefix?: string +} + +export const getGenerateUrl = ({ baseUrl }: GenerateUrlArgs): GenerateURL => { + return ({ filename, prefix = '' }) => { + return `${baseUrl}/${path.posix.join(prefix, filename)}` + } +} diff --git a/packages/storage-vercel-blob/src/handleDelete.ts b/packages/storage-vercel-blob/src/handleDelete.ts new file mode 100644 index 000000000..2fbc4bcf4 --- /dev/null +++ b/packages/storage-vercel-blob/src/handleDelete.ts @@ -0,0 +1,19 @@ +import type { HandleDelete } from '@payloadcms/plugin-cloud-storage/types' + +import { del } from '@vercel/blob' +import path from 'path' + +type HandleDeleteArgs = { + baseUrl: string + prefix?: string + token: string +} + +export const getHandleDelete = ({ baseUrl, token }: HandleDeleteArgs): HandleDelete => { + return async ({ doc: { prefix = '' }, filename }) => { + const fileUrl = `${baseUrl}/${path.posix.join(prefix, filename)}` + const deletedBlob = await del(fileUrl, { token }) + + return deletedBlob + } +} diff --git a/packages/storage-vercel-blob/src/handleUpload.ts b/packages/storage-vercel-blob/src/handleUpload.ts new file mode 100644 index 000000000..e5b0afb32 --- /dev/null +++ b/packages/storage-vercel-blob/src/handleUpload.ts @@ -0,0 +1,39 @@ +import type { HandleUpload } from '@payloadcms/plugin-cloud-storage/types' + +import { put } from '@vercel/blob' +import path from 'path' + +import type { VercelBlobStorageOptions } from './index.js' + +type HandleUploadArgs = Omit & { + baseUrl: string + prefix?: string +} + +export const getHandleUpload = ({ + access = 'public', + addRandomSuffix, + baseUrl, + cacheControlMaxAge, + prefix = '', + token, +}: HandleUploadArgs): HandleUpload => { + return async ({ data, file: { buffer, filename, mimeType } }) => { + const fileKey = path.posix.join(data.prefix || prefix, filename) + + const result = await put(fileKey, buffer, { + access, + addRandomSuffix, + cacheControlMaxAge, + contentType: mimeType, + token, + }) + + // Get filename with suffix from returned url + if (addRandomSuffix) { + data.filename = result.url.replace(`${baseUrl}/`, '') + } + + return data + } +} diff --git a/packages/storage-vercel-blob/src/index.ts b/packages/storage-vercel-blob/src/index.ts new file mode 100644 index 000000000..cd98b515a --- /dev/null +++ b/packages/storage-vercel-blob/src/index.ts @@ -0,0 +1,133 @@ +import type { + PluginOptions as CloudStoragePluginOptions, + CollectionOptions, +} from '@payloadcms/plugin-cloud-storage/types' +import type { Adapter, GeneratedAdapter } from '@payloadcms/plugin-cloud-storage/types' +import type { Config, Plugin } from 'payload/config' + +import { cloudStorage } from '@payloadcms/plugin-cloud-storage' + +import { getGenerateUrl } from './generateURL.js' +import { getHandleDelete } from './handleDelete.js' +import { getHandleUpload } from './handleUpload.js' +import { getStaticHandler } from './staticHandler.js' + +export type VercelBlobStorageOptions = { + /** + * Access control level + * + * @default 'public' + */ + access?: 'public' + + /** + * Add a random suffix to the uploaded file name + * + * @default false + */ + addRandomSuffix?: boolean + + /** + * Cache-Control max-age in seconds + * + * @default 31536000 (1 year) + */ + cacheControlMaxAge?: number + + /** + * Collections to apply the Vercel Blob adapter to + */ + collections: Record | true> + + /** + * Whether or not to enable the plugin + * + * Default: true + */ + enabled?: boolean + + /** + * Vercel Blob storage read/write token + * + * Usually process.env.BLOB_READ_WRITE_TOKEN set by Vercel + */ + token: string +} + +const defaultUploadOptions: Partial = { + access: 'public', + addRandomSuffix: false, + cacheControlMaxAge: 60 * 60 * 24 * 365, // 1 year + enabled: true, +} + +type VercelBlobStoragePlugin = (vercelBlobStorageOpts: VercelBlobStorageOptions) => Plugin + +export const vercelBlobStorage: VercelBlobStoragePlugin = + (options: VercelBlobStorageOptions) => + (incomingConfig: Config): Config => { + if (options.enabled === false) { + return incomingConfig + } + + if (!options.token) { + throw new Error('The token argument is required for the Vercel Blob adapter.') + } + + // Parse storeId from token + const storeId = options.token.match(/^vercel_blob_rw_([a-z\d]+)_[a-z\d]+$/i)?.[1]?.toLowerCase() + + if (!storeId) { + throw new Error( + 'Invalid token format for Vercel Blob adapter. Should be vercel_blob_rw__.', + ) + } + + const optionsWithDefaults = { + ...defaultUploadOptions, + ...options, + } + + const baseUrl = `https://${storeId}.${optionsWithDefaults.access}.blob.vercel-storage.com` + + const adapter = vercelBlobStorageInternal({ ...optionsWithDefaults, baseUrl }) + + // Add adapter to each collection option object + const collectionsWithAdapter: CloudStoragePluginOptions['collections'] = Object.entries( + options.collections, + ).reduce( + (acc, [slug, collOptions]) => ({ + ...acc, + [slug]: { + ...(collOptions === true ? {} : collOptions), + adapter, + }, + }), + {} as Record, + ) + + return cloudStorage({ + collections: collectionsWithAdapter, + })(incomingConfig) + } + +function vercelBlobStorageInternal( + options: VercelBlobStorageOptions & { baseUrl: string }, +): Adapter { + return ({ collection, prefix }): GeneratedAdapter => { + const { access, addRandomSuffix, baseUrl, cacheControlMaxAge, token } = options + return { + generateURL: getGenerateUrl({ baseUrl, prefix }), + handleDelete: getHandleDelete({ baseUrl, prefix, token: options.token }), + handleUpload: getHandleUpload({ + access, + addRandomSuffix, + baseUrl, + cacheControlMaxAge, + prefix, + token, + }), + staticHandler: getStaticHandler({ baseUrl, token }, collection), + } + } +} diff --git a/packages/storage-vercel-blob/src/staticHandler.ts b/packages/storage-vercel-blob/src/staticHandler.ts new file mode 100644 index 000000000..de7077386 --- /dev/null +++ b/packages/storage-vercel-blob/src/staticHandler.ts @@ -0,0 +1,51 @@ +import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types' +import type { CollectionConfig } from 'payload/types' + +import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities' +import { head } from '@vercel/blob' +import path from 'path' + +type StaticHandlerArgs = { + baseUrl: string + token: string +} + +export const getStaticHandler = ( + { baseUrl, token }: StaticHandlerArgs, + collection: CollectionConfig, +): StaticHandler => { + return async (req, { params: { filename } }) => { + try { + const prefix = await getFilePrefix({ collection, filename, req }) + + const fileUrl = `${baseUrl}/${path.posix.join(prefix, filename)}` + + const blobMetadata = await head(fileUrl, { token }) + if (!blobMetadata) { + return new Response(null, { status: 404, statusText: 'Not Found' }) + } + + const { contentDisposition, contentType, size } = blobMetadata + const response = await fetch(fileUrl) + const blob = await response.blob() + + if (!blob) { + return new Response(null, { status: 204, statusText: 'No Content' }) + } + + const bodyBuffer = await blob.arrayBuffer() + + return new Response(bodyBuffer, { + headers: new Headers({ + 'Content-Disposition': contentDisposition, + 'Content-Length': String(size), + 'Content-Type': contentType, + }), + status: 200, + }) + } catch (err: unknown) { + req.payload.logger.error({ err, msg: 'Unexpected error in staticHandler' }) + return new Response('Internal Server Error', { status: 500 }) + } + } +} diff --git a/packages/storage-vercel-blob/tsconfig.json b/packages/storage-vercel-blob/tsconfig.json new file mode 100644 index 000000000..5880d1adc --- /dev/null +++ b/packages/storage-vercel-blob/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, // Make sure typescript knows that this module depends on their references + "noEmit": false /* Do not emit outputs. */, + "emitDeclarationOnly": true, + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + "rootDir": "./src" /* Specify the root folder within your source files. */, + "strict": true, + }, + "exclude": [ + "dist", + "node_modules", + ], + "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.d.ts", "src/**/*.json"], + "references": [ + { "path": "../payload" }, + { "path": "../plugin-cloud-storage" }, + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9e408e406..592fcbeed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1381,6 +1381,70 @@ importers: specifier: workspace:* version: link:../payload + packages/storage-azure: + dependencies: + '@azure/abort-controller': + specifier: ^1.1.0 + version: 1.1.0 + '@azure/storage-blob': + specifier: ^12.11.0 + version: 12.17.0 + '@payloadcms/plugin-cloud-storage': + specifier: workspace:* + version: link:../plugin-cloud-storage + range-parser: + specifier: ^1.2.1 + version: 1.2.1 + devDependencies: + '@types/range-parser': + specifier: ^1.2.7 + version: 1.2.7 + payload: + specifier: workspace:* + version: link:../payload + + packages/storage-gcs: + dependencies: + '@google-cloud/storage': + specifier: ^7.7.0 + version: 7.9.0 + '@payloadcms/plugin-cloud-storage': + specifier: workspace:* + version: link:../plugin-cloud-storage + devDependencies: + payload: + specifier: workspace:* + version: link:../payload + + packages/storage-s3: + dependencies: + '@payloadcms/plugin-cloud-storage': + specifier: workspace:* + version: link:../plugin-cloud-storage + devDependencies: + '@aws-sdk/client-s3': + specifier: ^3.525.0 + version: 3.550.0 + '@aws-sdk/lib-storage': + specifier: ^3.525.0 + version: 3.550.0(@aws-sdk/client-s3@3.550.0) + payload: + specifier: workspace:* + version: link:../payload + + packages/storage-vercel-blob: + dependencies: + '@payloadcms/plugin-cloud-storage': + specifier: workspace:* + version: link:../plugin-cloud-storage + devDependencies: + '@vercel/blob': + specifier: ^0.22.3 + version: 0.22.3 + payload: + specifier: workspace:* + version: link:../payload + packages/translations: devDependencies: '@payloadcms/eslint-config': @@ -1591,6 +1655,18 @@ importers: '@payloadcms/richtext-slate': specifier: workspace:* version: link:../packages/richtext-slate + '@payloadcms/storage-azure': + specifier: workspace:* + version: link:../packages/storage-azure + '@payloadcms/storage-gcs': + specifier: workspace:* + version: link:../packages/storage-gcs + '@payloadcms/storage-s3': + specifier: workspace:* + version: link:../packages/storage-s3 + '@payloadcms/storage-vercel-blob': + specifier: workspace:* + version: link:../packages/storage-vercel-blob '@payloadcms/translations': specifier: workspace:* version: link:../packages/translations @@ -2352,21 +2428,19 @@ packages: dependencies: tslib: 2.6.2 - /@azure/abort-controller@2.1.1: - resolution: {integrity: sha512-NhzeNm5zu2fPlwGXPUjzsRCRuPx5demaZyNcyNYJDqpa/Sbxzvo/RYt9IwUaAOnDW5+r7J9UOE6f22TQnb9nhQ==} + /@azure/abort-controller@2.1.2: + resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} engines: {node: '>=18.0.0'} dependencies: tslib: 2.6.2 - dev: true /@azure/core-auth@1.7.1: resolution: {integrity: sha512-dyeQwvgthqs/SlPVQbZQetpslXceHd4i5a7M/7z/lGEAVwnSluabnQOjF2/dk/hhWgMISusv1Ytp4mQ8JNy62A==} engines: {node: '>=18.0.0'} dependencies: - '@azure/abort-controller': 2.1.1 + '@azure/abort-controller': 2.1.2 '@azure/core-util': 1.8.1 tslib: 2.6.2 - dev: true /@azure/core-http@3.0.4: resolution: {integrity: sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ==} @@ -2388,24 +2462,21 @@ packages: xml2js: 0.5.0 transitivePeerDependencies: - encoding - dev: true /@azure/core-lro@2.7.1: resolution: {integrity: sha512-kXSlrNHOCTVZMxpXNRqzgh9/j4cnNXU5Hf2YjMyjddRhCXFiFRzmNaqwN+XO9rGTsCOIaaG7M67zZdyliXZG9g==} engines: {node: '>=18.0.0'} dependencies: - '@azure/abort-controller': 2.1.1 + '@azure/abort-controller': 2.1.2 '@azure/core-util': 1.8.1 '@azure/logger': 1.1.1 tslib: 2.6.2 - dev: true /@azure/core-paging@1.6.1: resolution: {integrity: sha512-3tKIQXSU3mlN+ITz0m2pXLnKK3oQ6/EVcW8ud011Iq+M0rx6Wnm7NUEpoMeOAEedeKlPtemrQzO6YWoDR71O5w==} engines: {node: '>=18.0.0'} dependencies: tslib: 2.6.2 - dev: true /@azure/core-tracing@1.0.0-preview.13: resolution: {integrity: sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ==} @@ -2413,22 +2484,19 @@ packages: dependencies: '@opentelemetry/api': 1.8.0 tslib: 2.6.2 - dev: true /@azure/core-util@1.8.1: resolution: {integrity: sha512-L3voj0StUdJ+YKomvwnTv7gHzguJO+a6h30pmmZdRprJCM+RJlGMPxzuh4R7lhQu1jNmEtaHX5wvTgWLDAmbGQ==} engines: {node: '>=18.0.0'} dependencies: - '@azure/abort-controller': 2.1.1 + '@azure/abort-controller': 2.1.2 tslib: 2.6.2 - dev: true /@azure/logger@1.1.1: resolution: {integrity: sha512-/+4TtokaGgC+MnThdf6HyIH9Wrjp+CnCn3Nx3ggevN7FFjjNyjqg0yLlc2i9S+Z2uAzI8GYOo35Nzb1MhQ89MA==} engines: {node: '>=18.0.0'} dependencies: tslib: 2.6.2 - dev: true /@azure/storage-blob@12.17.0: resolution: {integrity: sha512-sM4vpsCpcCApagRW5UIjQNlNylo02my2opgp0Emi8x888hZUvJ3dN69Oq20cEGXkMUWnoCrBaB0zyS3yeB87sQ==} @@ -2444,7 +2512,6 @@ packages: tslib: 2.6.2 transitivePeerDependencies: - encoding - dev: true /@babel/code-frame@7.24.2: resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==} @@ -3845,17 +3912,14 @@ packages: dependencies: arrify: 2.0.1 extend: 3.0.2 - dev: true /@google-cloud/projectify@4.0.0: resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==} engines: {node: '>=14.0.0'} - dev: true /@google-cloud/promisify@4.0.0: resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==} engines: {node: '>=14'} - dev: true /@google-cloud/storage@7.9.0: resolution: {integrity: sha512-PlFl7g3r91NmXtZHXsSEfTZES5ysD3SSBWmX4iBdQ2TFH7tN/Vn/IhnVELCHtgh1vc+uYPZ7XvRYaqtDCdghIA==} @@ -3881,7 +3945,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /@hapi/hoek@9.3.0: resolution: {integrity: sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==} @@ -4756,7 +4819,6 @@ packages: /@opentelemetry/api@1.8.0: resolution: {integrity: sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==} engines: {node: '>=8.0.0'} - dev: true /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} @@ -5587,7 +5649,6 @@ packages: /@tootallnate/once@2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - dev: true /@trysound/sax@0.2.0: resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} @@ -5666,7 +5727,6 @@ packages: /@types/caseless@0.12.5: resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} - dev: true /@types/command-exists@1.2.3: resolution: {integrity: sha512-PpbaE2XWLaWYboXD6k70TcXO/OdOyyRFq5TVpmlUELNxdkkmXU9fkImNosmXU1DtsNrqdUgWd/nJQYXgwmtdXQ==} @@ -5995,7 +6055,6 @@ packages: dependencies: '@types/node': 20.12.5 form-data: 3.0.1 - dev: true /@types/node@20.12.5: resolution: {integrity: sha512-BD+BjQ9LS/D8ST9p5uqBxghlN+S42iuNxjsUGjeZobe/ciXzk2qb1B6IXc6AnRLS+yFJRpN2IPEHMzwspfDJNw==} @@ -6132,7 +6191,6 @@ packages: '@types/node': 20.12.5 '@types/tough-cookie': 4.0.5 form-data: 2.5.1 - dev: true /@types/responselike@1.0.3: resolution: {integrity: sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==} @@ -6174,13 +6232,11 @@ packages: /@types/tough-cookie@4.0.5: resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - dev: true /@types/tunnel@0.0.3: resolution: {integrity: sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==} dependencies: '@types/node': 20.12.5 - dev: true /@types/uuid@8.3.4: resolution: {integrity: sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==} @@ -6782,7 +6838,6 @@ packages: debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color - dev: true /agent-base@7.1.1: resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} @@ -6791,7 +6846,6 @@ packages: debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color - dev: true /aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} @@ -7039,7 +7093,6 @@ packages: /arrify@2.0.1: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} - dev: true /ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -7055,7 +7108,6 @@ packages: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} dependencies: retry: 0.13.1 - dev: true /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -7243,7 +7295,6 @@ packages: /bignumber.js@9.1.2: resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} - dev: true /bin-check@4.1.0: resolution: {integrity: sha512-b6weQyEUKsDGFlACWSIOfveEnImkJyK/FGW6FAG42loyoquvjdtOIqO6yBFzHyqyVVhNgNkQxxx09SFLK28YnA==} @@ -7744,7 +7795,6 @@ packages: engines: {node: '>= 0.6'} dependencies: mime-db: 1.52.0 - dev: true /compute-scroll-into-view@1.0.20: resolution: {integrity: sha512-UCB0ioiyj8CRjtrvaceBLqqhZCVP+1B8+NWQhmdsm0VXOJtobBCf1dBQmebCCo34qZmUwZfIH2MZLqNHazrfjg==} @@ -8799,7 +8849,6 @@ packages: inherits: 2.0.4 readable-stream: 3.6.2 stream-shift: 1.0.3 - dev: true /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -8860,7 +8909,6 @@ packages: /ent@2.2.0: resolution: {integrity: sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==} - dev: true /entities@4.5.0: resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} @@ -9635,7 +9683,6 @@ packages: /extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - dev: true /fast-base64-decode@1.0.0: resolution: {integrity: sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==} @@ -9685,7 +9732,6 @@ packages: hasBin: true dependencies: strnum: 1.0.5 - dev: true /fastest-levenshtein@1.0.16: resolution: {integrity: sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==} @@ -9915,7 +9961,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /form-data@3.0.1: resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} @@ -9924,7 +9969,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} @@ -10020,7 +10064,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /gcp-metadata@6.1.0: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} @@ -10031,7 +10074,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} @@ -10246,7 +10288,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} @@ -10330,7 +10371,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /gzip-size@6.0.0: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} @@ -10480,7 +10520,6 @@ packages: debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color - dev: true /http-status@1.6.2: resolution: {integrity: sha512-oUExvfNckrpTpDazph7kNG8sQi5au3BeTo0idaZFXEhTaJKu7GNJCLHI0rYY2wljm548MSTM+Ljj/c6anqu2zQ==} @@ -10502,7 +10541,6 @@ packages: debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color - dev: true /https-proxy-agent@7.0.4: resolution: {integrity: sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==} @@ -10512,7 +10550,6 @@ packages: debug: 4.3.4(supports-color@5.5.0) transitivePeerDependencies: - supports-color - dev: true /human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} @@ -11595,7 +11632,6 @@ packages: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} dependencies: bignumber.js: 9.1.2 - dev: true /json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -11713,7 +11749,6 @@ packages: buffer-equal-constant-time: 1.0.1 ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 - dev: true /jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} @@ -11727,7 +11762,6 @@ packages: dependencies: jwa: 2.0.0 safe-buffer: 5.2.1 - dev: true /jwt-decode@4.0.0: resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} @@ -12109,7 +12143,6 @@ packages: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} hasBin: true - dev: true /mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} @@ -14505,12 +14538,10 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} - dev: true /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -14610,7 +14641,6 @@ packages: /sax@1.3.0: resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} - dev: true /saxes@6.0.0: resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} @@ -15068,11 +15098,9 @@ packages: resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} dependencies: stubs: 3.0.0 - dev: true /stream-shift@1.0.3: resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} - dev: true /streamsearch@1.1.0: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} @@ -15251,7 +15279,6 @@ packages: /stubs@3.0.0: resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} - dev: true /styled-jsx@5.1.1(@babel/core@7.24.4)(react@18.2.0): resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} @@ -15417,7 +15444,6 @@ packages: transitivePeerDependencies: - encoding - supports-color - dev: true /temp-dir@2.0.0: resolution: {integrity: sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==} @@ -15756,7 +15782,6 @@ packages: /tunnel@0.0.6: resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} - dev: true /turbo-darwin-64@1.13.2: resolution: {integrity: sha512-CCSuD8CfmtncpohCuIgq7eAzUas0IwSbHfI8/Q3vKObTdXyN8vAo01gwqXjDGpzG9bTEVedD0GmLbD23dR0MLA==} @@ -16058,7 +16083,6 @@ packages: /uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true - dev: true /uuid@9.0.0: resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} @@ -16416,12 +16440,10 @@ packages: dependencies: sax: 1.3.0 xmlbuilder: 11.0.1 - dev: true /xmlbuilder@11.0.1: resolution: {integrity: sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==} engines: {node: '>=4.0'} - dev: true /xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} diff --git a/test/package.json b/test/package.json index 4b768f6c9..009063699 100644 --- a/test/package.json +++ b/test/package.json @@ -31,6 +31,10 @@ "@payloadcms/plugin-stripe": "workspace:*", "@payloadcms/richtext-lexical": "workspace:*", "@payloadcms/richtext-slate": "workspace:*", + "@payloadcms/storage-azure": "workspace:*", + "@payloadcms/storage-gcs": "workspace:*", + "@payloadcms/storage-s3": "workspace:*", + "@payloadcms/storage-vercel-blob": "workspace:*", "@payloadcms/translations": "workspace:*", "@payloadcms/ui": "workspace:*", "comment-json": "^4.2.3", diff --git a/test/plugin-cloud-storage/config.ts b/test/plugin-cloud-storage/config.ts index d93f32dd1..118e0d471 100644 --- a/test/plugin-cloud-storage/config.ts +++ b/test/plugin-cloud-storage/config.ts @@ -13,6 +13,7 @@ import { Media } from './collections/Media.js' import { MediaWithPrefix } from './collections/MediaWithPrefix.js' import { Users } from './collections/Users.js' import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +import { createTestBucket } from './utils.js' let adapter: Adapter let uploadOptions @@ -22,10 +23,6 @@ dotenv.config({ path: path.resolve(process.cwd(), './test/plugin-cloud-storage/.env.emulated'), }) -console.log( - `Using plugin-cloud-storage adapter: ${process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER}`, -) - if (process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER === 'azure') { adapter = azureBlobStorageAdapter({ allowContainerCreate: process.env.AZURE_STORAGE_ALLOW_CONTAINER_CREATE === 'true', @@ -114,6 +111,12 @@ export default buildConfigWithDefaults({ password: devUser.password, }, }) + + await createTestBucket() + + payload.logger.info( + `Using plugin-cloud-storage adapter: ${process.env.PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER}`, + ) }, plugins: [ cloudStorage({ diff --git a/test/plugin-cloud-storage/int.spec.ts b/test/plugin-cloud-storage/int.spec.ts index b420c71a2..9cfea43d3 100644 --- a/test/plugin-cloud-storage/int.spec.ts +++ b/test/plugin-cloud-storage/int.spec.ts @@ -8,6 +8,7 @@ import { describeIfInCIOrHasLocalstack } from '../helpers.js' import { initPayloadInt } from '../helpers/initPayloadInt.js' import configPromise from './config.js' import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +import { clearTestBucket, createTestBucket } from './utils.js' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) @@ -25,7 +26,7 @@ describe('@payloadcms/plugin-cloud-storage', () => { } }) - const TEST_BUCKET = 'payload-bucket' + const TEST_BUCKET = process.env.S3_BUCKET let client: AWS.S3Client describeIfInCIOrHasLocalstack()('plugin-cloud-storage', () => { @@ -42,11 +43,11 @@ describe('@payloadcms/plugin-cloud-storage', () => { }) await createTestBucket() - await clearTestBucket() + await clearTestBucket(client) }) afterEach(async () => { - await clearTestBucket() + await clearTestBucket(client) }) it('can upload', async () => { @@ -63,7 +64,7 @@ describe('@payloadcms/plugin-cloud-storage', () => { uploadId: upload.id, }) - expect(upload.url).toEqual(`/api/${mediaSlug}/file/${upload.filename as string}`) + expect(upload.url).toEqual(`/api/${mediaSlug}/file/${String(upload.filename)}`) }) it('can upload with prefix', async () => { @@ -97,38 +98,6 @@ describe('@payloadcms/plugin-cloud-storage', () => { it.todo('can upload') }) - async function createTestBucket() { - const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET })) - - if (makeBucketRes.$metadata.httpStatusCode !== 200) { - throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`) - } - } - - async function clearTestBucket() { - const listedObjects = await client.send( - new AWS.ListObjectsV2Command({ - Bucket: TEST_BUCKET, - }), - ) - - if (!listedObjects?.Contents?.length) return - - const deleteParams = { - Bucket: TEST_BUCKET, - Delete: { Objects: [] }, - } - - listedObjects.Contents.forEach(({ Key }) => { - deleteParams.Delete.Objects.push({ Key }) - }) - - const deleteResult = await client.send(new AWS.DeleteObjectsCommand(deleteParams)) - if (deleteResult.Errors?.length) { - throw new Error(JSON.stringify(deleteResult.Errors)) - } - } - async function verifyUploads({ collectionSlug, uploadId, diff --git a/test/plugin-cloud-storage/utils.ts b/test/plugin-cloud-storage/utils.ts new file mode 100644 index 000000000..d7ea7c5cf --- /dev/null +++ b/test/plugin-cloud-storage/utils.ts @@ -0,0 +1,48 @@ +import * as AWS from '@aws-sdk/client-s3' + +const getS3Client = () => { + return new AWS.S3({ + endpoint: process.env.S3_ENDPOINT, + forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true', + region: process.env.S3_REGION, + credentials: { + accessKeyId: process.env.S3_ACCESS_KEY_ID, + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, + }, + }) +} + +export async function createTestBucket(bucketName?: string) { + const client = getS3Client() + const makeBucketRes = await client.send( + new AWS.CreateBucketCommand({ Bucket: bucketName || process.env.S3_BUCKET }), + ) + + if (makeBucketRes.$metadata.httpStatusCode !== 200) { + throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`) + } +} + +export async function clearTestBucket(client: AWS.S3Client, bucketName?: string) { + const listedObjects = await client.send( + new AWS.ListObjectsV2Command({ + Bucket: bucketName || process.env.S3_BUCKET, + }), + ) + + if (!listedObjects?.Contents?.length) return + + const deleteParams = { + Bucket: bucketName || process.env.S3_BUCKET, + Delete: { Objects: [] }, + } + + listedObjects.Contents.forEach(({ Key }) => { + deleteParams.Delete.Objects.push({ Key }) + }) + + const deleteResult = await client.send(new AWS.DeleteObjectsCommand(deleteParams)) + if (deleteResult.Errors?.length) { + throw new Error(JSON.stringify(deleteResult.Errors)) + } +} diff --git a/test/storage-azure/.eslintrc.cjs b/test/storage-azure/.eslintrc.cjs new file mode 100644 index 000000000..39a96642f --- /dev/null +++ b/test/storage-azure/.eslintrc.cjs @@ -0,0 +1,8 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + ignorePatterns: ['payload-types.ts'], + parserOptions: { + project: ['./tsconfig.eslint.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/test/storage-azure/collections/Media.ts b/test/storage-azure/collections/Media.ts new file mode 100644 index 000000000..4d9acc2a6 --- /dev/null +++ b/test/storage-azure/collections/Media.ts @@ -0,0 +1,34 @@ +import type { CollectionConfig } from 'payload/types' + +export const Media: CollectionConfig = { + slug: 'media', + upload: { + disableLocalStorage: true, + resizeOptions: { + position: 'center', + width: 200, + height: 200, + }, + imageSizes: [ + { + height: 400, + width: 400, + crop: 'center', + name: 'square', + }, + { + width: 900, + height: 450, + crop: 'center', + name: 'sixteenByNineMedium', + }, + ], + }, + fields: [ + { + name: 'alt', + label: 'Alt Text', + type: 'text', + }, + ], +} diff --git a/test/storage-azure/collections/MediaWithPrefix.ts b/test/storage-azure/collections/MediaWithPrefix.ts new file mode 100644 index 000000000..cab50b8ba --- /dev/null +++ b/test/storage-azure/collections/MediaWithPrefix.ts @@ -0,0 +1,9 @@ +import type { CollectionConfig } from 'payload/types' + +export const MediaWithPrefix: CollectionConfig = { + slug: 'media-with-prefix', + upload: { + disableLocalStorage: false, + }, + fields: [], +} diff --git a/test/storage-azure/collections/Users.ts b/test/storage-azure/collections/Users.ts new file mode 100644 index 000000000..a62115927 --- /dev/null +++ b/test/storage-azure/collections/Users.ts @@ -0,0 +1,16 @@ +import type { CollectionConfig } from 'payload/types' + +export const Users: CollectionConfig = { + slug: 'users', + auth: true, + admin: { + useAsTitle: 'email', + }, + access: { + read: () => true, + }, + fields: [ + // Email added by default + // Add more fields as needed + ], +} diff --git a/test/storage-azure/config.ts b/test/storage-azure/config.ts new file mode 100644 index 000000000..56b931ec8 --- /dev/null +++ b/test/storage-azure/config.ts @@ -0,0 +1,48 @@ +import { azureStorage } from '@payloadcms/storage-azure' +import dotenv from 'dotenv' +import { fileURLToPath } from 'node:url' +import path from 'path' + +import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' +import { devUser } from '../credentials.js' +import { Media } from './collections/Media.js' +import { MediaWithPrefix } from './collections/MediaWithPrefix.js' +import { Users } from './collections/Users.js' +import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +let uploadOptions + +// Load config to work with emulated services +dotenv.config({ + path: path.resolve(dirname, '../plugin-cloud-storage/.env.emulated'), +}) + +export default buildConfigWithDefaults({ + collections: [Media, MediaWithPrefix, Users], + onInit: async (payload) => { + await payload.create({ + collection: 'users', + data: { + email: devUser.email, + password: devUser.password, + }, + }) + }, + plugins: [ + azureStorage({ + collections: { + [mediaSlug]: true, + [mediaWithPrefixSlug]: { + prefix, + }, + }, + allowContainerCreate: process.env.AZURE_STORAGE_ALLOW_CONTAINER_CREATE === 'true', + baseURL: process.env.AZURE_STORAGE_ACCOUNT_BASEURL, + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, + containerName: process.env.AZURE_STORAGE_CONTAINER_NAME, + }), + ], + upload: uploadOptions, +}) diff --git a/test/storage-azure/shared.ts b/test/storage-azure/shared.ts new file mode 100644 index 000000000..7d74b323a --- /dev/null +++ b/test/storage-azure/shared.ts @@ -0,0 +1,3 @@ +export const mediaSlug = 'media' +export const mediaWithPrefixSlug = 'media-with-prefix' +export const prefix = 'test-prefix' diff --git a/test/storage-azure/tsconfig.eslint.json b/test/storage-azure/tsconfig.eslint.json new file mode 100644 index 000000000..b34cc7afb --- /dev/null +++ b/test/storage-azure/tsconfig.eslint.json @@ -0,0 +1,13 @@ +{ + // extend your base config to share compilerOptions, etc + //"extends": "./tsconfig.json", + "compilerOptions": { + // ensure that nobody can accidentally use this config for a build + "noEmit": true + }, + "include": [ + // whatever paths you intend to lint + "./**/*.ts", + "./**/*.tsx" + ] +} diff --git a/test/storage-gcs/.eslintrc.cjs b/test/storage-gcs/.eslintrc.cjs new file mode 100644 index 000000000..39a96642f --- /dev/null +++ b/test/storage-gcs/.eslintrc.cjs @@ -0,0 +1,8 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + ignorePatterns: ['payload-types.ts'], + parserOptions: { + project: ['./tsconfig.eslint.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/test/storage-gcs/collections/Media.ts b/test/storage-gcs/collections/Media.ts new file mode 100644 index 000000000..4d9acc2a6 --- /dev/null +++ b/test/storage-gcs/collections/Media.ts @@ -0,0 +1,34 @@ +import type { CollectionConfig } from 'payload/types' + +export const Media: CollectionConfig = { + slug: 'media', + upload: { + disableLocalStorage: true, + resizeOptions: { + position: 'center', + width: 200, + height: 200, + }, + imageSizes: [ + { + height: 400, + width: 400, + crop: 'center', + name: 'square', + }, + { + width: 900, + height: 450, + crop: 'center', + name: 'sixteenByNineMedium', + }, + ], + }, + fields: [ + { + name: 'alt', + label: 'Alt Text', + type: 'text', + }, + ], +} diff --git a/test/storage-gcs/collections/MediaWithPrefix.ts b/test/storage-gcs/collections/MediaWithPrefix.ts new file mode 100644 index 000000000..cab50b8ba --- /dev/null +++ b/test/storage-gcs/collections/MediaWithPrefix.ts @@ -0,0 +1,9 @@ +import type { CollectionConfig } from 'payload/types' + +export const MediaWithPrefix: CollectionConfig = { + slug: 'media-with-prefix', + upload: { + disableLocalStorage: false, + }, + fields: [], +} diff --git a/test/storage-gcs/collections/Users.ts b/test/storage-gcs/collections/Users.ts new file mode 100644 index 000000000..a62115927 --- /dev/null +++ b/test/storage-gcs/collections/Users.ts @@ -0,0 +1,16 @@ +import type { CollectionConfig } from 'payload/types' + +export const Users: CollectionConfig = { + slug: 'users', + auth: true, + admin: { + useAsTitle: 'email', + }, + access: { + read: () => true, + }, + fields: [ + // Email added by default + // Add more fields as needed + ], +} diff --git a/test/storage-gcs/config.ts b/test/storage-gcs/config.ts new file mode 100644 index 000000000..47b8cb897 --- /dev/null +++ b/test/storage-gcs/config.ts @@ -0,0 +1,49 @@ +import { gcsStorage } from '@payloadcms/storage-gcs' +import dotenv from 'dotenv' +import { fileURLToPath } from 'node:url' +import path from 'path' + +import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' +import { devUser } from '../credentials.js' +import { Media } from './collections/Media.js' +import { MediaWithPrefix } from './collections/MediaWithPrefix.js' +import { Users } from './collections/Users.js' +import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +let uploadOptions + +// Load config to work with emulated services +dotenv.config({ + path: path.resolve(dirname, '../plugin-cloud-storage/.env.emulated'), +}) + +export default buildConfigWithDefaults({ + collections: [Media, MediaWithPrefix, Users], + onInit: async (payload) => { + await payload.create({ + collection: 'users', + data: { + email: devUser.email, + password: devUser.password, + }, + }) + }, + plugins: [ + gcsStorage({ + collections: { + [mediaSlug]: true, + [mediaWithPrefixSlug]: { + prefix, + }, + }, + bucket: process.env.GCS_BUCKET, + options: { + apiEndpoint: process.env.GCS_ENDPOINT, + projectId: process.env.GCS_PROJECT_ID, + }, + }), + ], + upload: uploadOptions, +}) diff --git a/test/storage-gcs/shared.ts b/test/storage-gcs/shared.ts new file mode 100644 index 000000000..7d74b323a --- /dev/null +++ b/test/storage-gcs/shared.ts @@ -0,0 +1,3 @@ +export const mediaSlug = 'media' +export const mediaWithPrefixSlug = 'media-with-prefix' +export const prefix = 'test-prefix' diff --git a/test/storage-gcs/tsconfig.eslint.json b/test/storage-gcs/tsconfig.eslint.json new file mode 100644 index 000000000..b34cc7afb --- /dev/null +++ b/test/storage-gcs/tsconfig.eslint.json @@ -0,0 +1,13 @@ +{ + // extend your base config to share compilerOptions, etc + //"extends": "./tsconfig.json", + "compilerOptions": { + // ensure that nobody can accidentally use this config for a build + "noEmit": true + }, + "include": [ + // whatever paths you intend to lint + "./**/*.ts", + "./**/*.tsx" + ] +} diff --git a/test/storage-s3/.env.emulated b/test/storage-s3/.env.emulated new file mode 100644 index 000000000..84e6b5045 --- /dev/null +++ b/test/storage-s3/.env.emulated @@ -0,0 +1,32 @@ +# Sample creds for working locally with docker-compose + +MONGODB_URI=mongodb://localhost/payload-plugin-cloud-storage +PAYLOAD_PUBLIC_SERVER_URL=http://localhost:3000 +PAYLOAD_SECRET=45ligj345ligj4wl5igj4lw5igj45ligj45wlijl +PAYLOAD_CONFIG_PATH=config.ts + +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1; +AZURE_STORAGE_CONTAINER_NAME=az-media +AZURE_STORAGE_ALLOW_CONTAINER_CREATE=true +AZURE_STORAGE_ACCOUNT_BASEURL=http://localhost:10000/devstoreaccount1 + +S3_ENDPOINT=http://localhost:4566 +S3_ACCESS_KEY_ID=payloadAccessKey +S3_SECRET_ACCESS_KEY=alwiejglaiwhewlihgawe +S3_BUCKET=payload-bucket +S3_FORCE_PATH_STYLE=true +S3_REGION=us-east-1 + +GCS_ENDPOINT=http://localhost:4443 +GCS_PROJECT_ID=test +GCS_BUCKET=payload-bucket + +R2_ENDPOINT=https://cloudflare-generated-domain.r2.cloudflarestorage.com +R2_REGION=auto +R2_ACCESS_KEY_ID=access-key-id +R2_SECRET_ACCESS_KEY=secret-access-key +R2_BUCKET=payload-bucket +R2_FORCE_PATH_STYLE= + +PAYLOAD_DROP_DATABASE=true +PAYLOAD_PUBLIC_CLOUD_STORAGE_ADAPTER=s3 diff --git a/test/storage-s3/.eslintrc.cjs b/test/storage-s3/.eslintrc.cjs new file mode 100644 index 000000000..39a96642f --- /dev/null +++ b/test/storage-s3/.eslintrc.cjs @@ -0,0 +1,8 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + ignorePatterns: ['payload-types.ts'], + parserOptions: { + project: ['./tsconfig.eslint.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/test/storage-s3/collections/Media.ts b/test/storage-s3/collections/Media.ts new file mode 100644 index 000000000..4d9acc2a6 --- /dev/null +++ b/test/storage-s3/collections/Media.ts @@ -0,0 +1,34 @@ +import type { CollectionConfig } from 'payload/types' + +export const Media: CollectionConfig = { + slug: 'media', + upload: { + disableLocalStorage: true, + resizeOptions: { + position: 'center', + width: 200, + height: 200, + }, + imageSizes: [ + { + height: 400, + width: 400, + crop: 'center', + name: 'square', + }, + { + width: 900, + height: 450, + crop: 'center', + name: 'sixteenByNineMedium', + }, + ], + }, + fields: [ + { + name: 'alt', + label: 'Alt Text', + type: 'text', + }, + ], +} diff --git a/test/storage-s3/collections/MediaWithPrefix.ts b/test/storage-s3/collections/MediaWithPrefix.ts new file mode 100644 index 000000000..cab50b8ba --- /dev/null +++ b/test/storage-s3/collections/MediaWithPrefix.ts @@ -0,0 +1,9 @@ +import type { CollectionConfig } from 'payload/types' + +export const MediaWithPrefix: CollectionConfig = { + slug: 'media-with-prefix', + upload: { + disableLocalStorage: false, + }, + fields: [], +} diff --git a/test/storage-s3/collections/Users.ts b/test/storage-s3/collections/Users.ts new file mode 100644 index 000000000..a62115927 --- /dev/null +++ b/test/storage-s3/collections/Users.ts @@ -0,0 +1,16 @@ +import type { CollectionConfig } from 'payload/types' + +export const Users: CollectionConfig = { + slug: 'users', + auth: true, + admin: { + useAsTitle: 'email', + }, + access: { + read: () => true, + }, + fields: [ + // Email added by default + // Add more fields as needed + ], +} diff --git a/test/storage-s3/config.ts b/test/storage-s3/config.ts new file mode 100644 index 000000000..6cc72116f --- /dev/null +++ b/test/storage-s3/config.ts @@ -0,0 +1,54 @@ +import { s3Storage } from '@payloadcms/storage-s3' +import dotenv from 'dotenv' +import { fileURLToPath } from 'node:url' +import path from 'path' + +import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' +import { devUser } from '../credentials.js' +import { Media } from './collections/Media.js' +import { MediaWithPrefix } from './collections/MediaWithPrefix.js' +import { Users } from './collections/Users.js' +import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +let uploadOptions + +// Load config to work with emulated services +dotenv.config({ + path: path.resolve(dirname, '../plugin-cloud-storage/.env.emulated'), +}) + +export default buildConfigWithDefaults({ + collections: [Media, MediaWithPrefix, Users], + onInit: async (payload) => { + await payload.create({ + collection: 'users', + data: { + email: devUser.email, + password: devUser.password, + }, + }) + }, + plugins: [ + s3Storage({ + collections: { + [mediaSlug]: true, + [mediaWithPrefixSlug]: { + prefix, + }, + }, + bucket: process.env.S3_BUCKET, + config: { + credentials: { + accessKeyId: process.env.S3_ACCESS_KEY_ID, + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, + }, + endpoint: process.env.S3_ENDPOINT, + forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true', + region: process.env.S3_REGION, + }, + }), + ], + upload: uploadOptions, +}) diff --git a/test/storage-s3/int.spec.ts b/test/storage-s3/int.spec.ts new file mode 100644 index 000000000..2209fbe32 --- /dev/null +++ b/test/storage-s3/int.spec.ts @@ -0,0 +1,155 @@ +import type { Payload } from 'payload' + +import * as AWS from '@aws-sdk/client-s3' +import path from 'path' +import { fileURLToPath } from 'url' + +import { initPayloadInt } from '../helpers/initPayloadInt.js' +import configPromise from './config.js' +import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' + +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +let payload: Payload + +describe('@payloadcms/storage-s3', () => { + const TEST_BUCKET = process.env.S3_BUCKET + let client: AWS.S3Client + + beforeAll(async () => { + ;({ payload } = await initPayloadInt(configPromise)) + + client = new AWS.S3({ + endpoint: process.env.S3_ENDPOINT, + forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true', + region: process.env.S3_REGION, + credentials: { + accessKeyId: process.env.S3_ACCESS_KEY_ID, + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY, + }, + }) + + await createTestBucket() + await clearTestBucket() + }) + + afterAll(async () => { + if (typeof payload.db.destroy === 'function') { + await payload.db.destroy() + } + }) + afterEach(async () => { + await clearTestBucket() + }) + + it('can upload', async () => { + const upload = await payload.create({ + collection: mediaSlug, + data: {}, + filePath: path.resolve(dirname, '../uploads/image.png'), + }) + + expect(upload.id).toBeTruthy() + + await verifyUploads({ + collectionSlug: mediaSlug, + uploadId: upload.id, + }) + + expect(upload.url).toEqual(`/api/${mediaSlug}/file/${String(upload.filename)}`) + }) + + it('can upload with prefix', async () => { + const upload = await payload.create({ + collection: mediaWithPrefixSlug, + data: {}, + filePath: path.resolve(dirname, '../uploads/image.png'), + }) + + expect(upload.id).toBeTruthy() + + await verifyUploads({ + collectionSlug: mediaWithPrefixSlug, + uploadId: upload.id, + prefix, + }) + expect(upload.url).toEqual(`/api/${mediaWithPrefixSlug}/file/${String(upload.filename)}`) + }) + + describe('R2', () => { + it.todo('can upload') + }) + + async function createTestBucket() { + const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET })) + + if (makeBucketRes.$metadata.httpStatusCode !== 200) { + throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`) + } + } + + async function clearTestBucket() { + const listedObjects = await client.send( + new AWS.ListObjectsV2Command({ + Bucket: TEST_BUCKET, + }), + ) + + if (!listedObjects?.Contents?.length) return + + const deleteParams = { + Bucket: TEST_BUCKET, + Delete: { Objects: [] }, + } + + listedObjects.Contents.forEach(({ Key }) => { + deleteParams.Delete.Objects.push({ Key }) + }) + + const deleteResult = await client.send(new AWS.DeleteObjectsCommand(deleteParams)) + if (deleteResult.Errors?.length) { + throw new Error(JSON.stringify(deleteResult.Errors)) + } + } + + async function verifyUploads({ + collectionSlug, + uploadId, + prefix = '', + }: { + collectionSlug: string + prefix?: string + uploadId: number | string + }) { + const uploadData = (await payload.findByID({ + collection: collectionSlug, + id: uploadId, + })) as unknown as { filename: string; sizes: Record } + + const fileKeys = Object.keys(uploadData.sizes || {}).map((key) => { + const rawFilename = uploadData.sizes[key].filename + return prefix ? `${prefix}/${rawFilename}` : rawFilename + }) + + fileKeys.push(`${prefix ? `${prefix}/` : ''}${uploadData.filename}`) + try { + for (const key of fileKeys) { + const { $metadata } = await client.send( + new AWS.HeadObjectCommand({ Bucket: TEST_BUCKET, Key: key }), + ) + + if ($metadata.httpStatusCode !== 200) { + console.error('Error verifying uploads', key, $metadata) + throw new Error(`Error verifying uploads: ${key}, ${$metadata.httpStatusCode}`) + } + + // Verify each size was properly uploaded + expect($metadata.httpStatusCode).toBe(200) + } + } catch (error: unknown) { + console.error('Error verifying uploads:', fileKeys, error) + throw error + } + } +}) diff --git a/test/storage-s3/shared.ts b/test/storage-s3/shared.ts new file mode 100644 index 000000000..7d74b323a --- /dev/null +++ b/test/storage-s3/shared.ts @@ -0,0 +1,3 @@ +export const mediaSlug = 'media' +export const mediaWithPrefixSlug = 'media-with-prefix' +export const prefix = 'test-prefix' diff --git a/test/storage-s3/tsconfig.eslint.json b/test/storage-s3/tsconfig.eslint.json new file mode 100644 index 000000000..b34cc7afb --- /dev/null +++ b/test/storage-s3/tsconfig.eslint.json @@ -0,0 +1,13 @@ +{ + // extend your base config to share compilerOptions, etc + //"extends": "./tsconfig.json", + "compilerOptions": { + // ensure that nobody can accidentally use this config for a build + "noEmit": true + }, + "include": [ + // whatever paths you intend to lint + "./**/*.ts", + "./**/*.tsx" + ] +} diff --git a/test/storage-vercel-blob/.eslintrc.cjs b/test/storage-vercel-blob/.eslintrc.cjs new file mode 100644 index 000000000..39a96642f --- /dev/null +++ b/test/storage-vercel-blob/.eslintrc.cjs @@ -0,0 +1,8 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + ignorePatterns: ['payload-types.ts'], + parserOptions: { + project: ['./tsconfig.eslint.json'], + tsconfigRootDir: __dirname, + }, +} diff --git a/test/storage-vercel-blob/collections/Media.ts b/test/storage-vercel-blob/collections/Media.ts new file mode 100644 index 000000000..4d9acc2a6 --- /dev/null +++ b/test/storage-vercel-blob/collections/Media.ts @@ -0,0 +1,34 @@ +import type { CollectionConfig } from 'payload/types' + +export const Media: CollectionConfig = { + slug: 'media', + upload: { + disableLocalStorage: true, + resizeOptions: { + position: 'center', + width: 200, + height: 200, + }, + imageSizes: [ + { + height: 400, + width: 400, + crop: 'center', + name: 'square', + }, + { + width: 900, + height: 450, + crop: 'center', + name: 'sixteenByNineMedium', + }, + ], + }, + fields: [ + { + name: 'alt', + label: 'Alt Text', + type: 'text', + }, + ], +} diff --git a/test/storage-vercel-blob/collections/MediaWithPrefix.ts b/test/storage-vercel-blob/collections/MediaWithPrefix.ts new file mode 100644 index 000000000..cab50b8ba --- /dev/null +++ b/test/storage-vercel-blob/collections/MediaWithPrefix.ts @@ -0,0 +1,9 @@ +import type { CollectionConfig } from 'payload/types' + +export const MediaWithPrefix: CollectionConfig = { + slug: 'media-with-prefix', + upload: { + disableLocalStorage: false, + }, + fields: [], +} diff --git a/test/storage-vercel-blob/collections/Users.ts b/test/storage-vercel-blob/collections/Users.ts new file mode 100644 index 000000000..a62115927 --- /dev/null +++ b/test/storage-vercel-blob/collections/Users.ts @@ -0,0 +1,16 @@ +import type { CollectionConfig } from 'payload/types' + +export const Users: CollectionConfig = { + slug: 'users', + auth: true, + admin: { + useAsTitle: 'email', + }, + access: { + read: () => true, + }, + fields: [ + // Email added by default + // Add more fields as needed + ], +} diff --git a/test/storage-vercel-blob/config.ts b/test/storage-vercel-blob/config.ts new file mode 100644 index 000000000..40f566d87 --- /dev/null +++ b/test/storage-vercel-blob/config.ts @@ -0,0 +1,45 @@ +import { vercelBlobStorage } from '@payloadcms/storage-vercel-blob' +import dotenv from 'dotenv' +import { fileURLToPath } from 'node:url' +import path from 'path' + +import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' +import { devUser } from '../credentials.js' +import { Media } from './collections/Media.js' +import { MediaWithPrefix } from './collections/MediaWithPrefix.js' +import { Users } from './collections/Users.js' +import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js' +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +let uploadOptions + +// TODO: Load this into CI or have shared creds +dotenv.config({ + path: path.resolve(dirname, '.env'), +}) + +export default buildConfigWithDefaults({ + collections: [Media, MediaWithPrefix, Users], + onInit: async (payload) => { + await payload.create({ + collection: 'users', + data: { + email: devUser.email, + password: devUser.password, + }, + }) + }, + plugins: [ + vercelBlobStorage({ + collections: { + [mediaSlug]: true, + [mediaWithPrefixSlug]: { + prefix, + }, + }, + token: process.env.BLOB_READ_WRITE_TOKEN, + }), + ], + upload: uploadOptions, +}) diff --git a/test/storage-vercel-blob/shared.ts b/test/storage-vercel-blob/shared.ts new file mode 100644 index 000000000..7d74b323a --- /dev/null +++ b/test/storage-vercel-blob/shared.ts @@ -0,0 +1,3 @@ +export const mediaSlug = 'media' +export const mediaWithPrefixSlug = 'media-with-prefix' +export const prefix = 'test-prefix' diff --git a/test/storage-vercel-blob/tsconfig.eslint.json b/test/storage-vercel-blob/tsconfig.eslint.json new file mode 100644 index 000000000..b34cc7afb --- /dev/null +++ b/test/storage-vercel-blob/tsconfig.eslint.json @@ -0,0 +1,13 @@ +{ + // extend your base config to share compilerOptions, etc + //"extends": "./tsconfig.json", + "compilerOptions": { + // ensure that nobody can accidentally use this config for a build + "noEmit": true + }, + "include": [ + // whatever paths you intend to lint + "./**/*.ts", + "./**/*.tsx" + ] +}