From 82820312e8ceb61100007ef41e84b2315291e2a2 Mon Sep 17 00:00:00 2001 From: Jacob Fletcher Date: Thu, 11 Sep 2025 09:14:56 -0400 Subject: [PATCH] feat: expose multipart/form-data parsing options (#13766) When sending REST API requests with multipart/form-data, e.g. PATCH or POST within the admin panel, a request body larger than 1MB throws the following error: ``` Unterminated string in JSON at position... ``` This is because there are sensible defaults imposed by the HTML form data parser (currently using [busboy](https://github.com/fastify/busboy)). If your documents exceed this limit, you may run into this error when editing them within the admin panel. To support large documents over 1MB, use the new `bodyParser` property on the root config: ```ts import { buildConfig } from 'payload' const config = buildConfig({ // ... bodyParser: { limits: { fieldSize: 2 * 1024 * 1024, // This will allow requests containing up to 2MB of multipart/form-data } } } ``` --- - To see the specific tasks where the Asana app for GitHub is being used, see below: - https://app.asana.com/0/0/1211317005907885 --- packages/next/src/routes/rest/index.ts | 2 +- packages/payload/src/config/types.ts | 13 ++++- .../src/uploads/fetchAPI-multipart/index.ts | 16 ++++-- .../src/utilities/addDataAndFileToRequest.ts | 11 +++-- pnpm-lock.yaml | 3 ++ .../collections/LargeDocuments.ts | 19 +++++++ test/collections-rest/config.ts | 7 +++ test/collections-rest/int.spec.ts | 44 +++++++++++++++++ test/collections-rest/payload-types.ts | 49 +++++++++++++++++++ test/helpers/NextRESTClient.ts | 31 ++++++++---- test/helpers/getFormDataSize.ts | 16 ++++++ test/package.json | 1 + 12 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 test/collections-rest/collections/LargeDocuments.ts create mode 100644 test/helpers/getFormDataSize.ts diff --git a/packages/next/src/routes/rest/index.ts b/packages/next/src/routes/rest/index.ts index b3c601dc9..737b38542 100644 --- a/packages/next/src/routes/rest/index.ts +++ b/packages/next/src/routes/rest/index.ts @@ -14,7 +14,7 @@ const handlerBuilder = ): Promise => { const awaitedConfig = await config - // Add this endpoint only when using Next.js, still can be overriden. + // Add this endpoint only when using Next.js, still can be overridden. if ( initedOGEndpoint === false && !awaitedConfig.endpoints.some( diff --git a/packages/payload/src/config/types.ts b/packages/payload/src/config/types.ts index de05ac396..6f8761096 100644 --- a/packages/payload/src/config/types.ts +++ b/packages/payload/src/config/types.ts @@ -275,6 +275,7 @@ export type InitOptions = { disableOnInit?: boolean importMap?: ImportMap + /** * A function that is called immediately following startup that receives the Payload instance as it's only argument. */ @@ -980,6 +981,7 @@ export type Config = { /** The slug of a Collection that you want to be used to log in to the Admin dashboard. */ user?: string } + /** * Configure authentication-related Payload-wide settings. */ @@ -993,6 +995,15 @@ export type Config = { /** Custom Payload bin scripts can be injected via the config. */ bin?: BinScriptConfig[] blocks?: Block[] + /** + * Pass additional options to the parser used to process `multipart/form-data` requests. + * For example, a PATCH request containing HTML form data. + * For example, you may want to increase the `limits` imposed by the parser. + * Currently using @link {https://www.npmjs.com/package/busboy|busboy} under the hood. + * + * @experimental This property is experimental and may change in future releases. Use at your own discretion. + */ + bodyParser?: Partial /** * Manage the datamodel of your application * @@ -1024,10 +1035,8 @@ export type Config = { cors?: '*' | CORSConfig | string[] /** A whitelist array of URLs to allow Payload cookies to be accepted from as a form of CSRF protection. */ csrf?: string[] - /** Extension point to add your custom data. Server only. */ custom?: Record - /** Pass in a database adapter for use on this project. */ db: DatabaseAdapterResult /** Enable to expose more detailed error information. */ diff --git a/packages/payload/src/uploads/fetchAPI-multipart/index.ts b/packages/payload/src/uploads/fetchAPI-multipart/index.ts index 3ca52f004..9aae5f452 100644 --- a/packages/payload/src/uploads/fetchAPI-multipart/index.ts +++ b/packages/payload/src/uploads/fetchAPI-multipart/index.ts @@ -7,7 +7,7 @@ import { isEligibleRequest } from './isEligibleRequest.js' import { processMultipart } from './processMultipart.js' import { debugLog } from './utilities.js' -const DEFAULT_OPTIONS: FetchAPIFileUploadOptions = { +const DEFAULT_UPLOAD_OPTIONS: FetchAPIFileUploadOptions = { abortOnLimit: false, createParentPath: false, debug: false, @@ -53,16 +53,22 @@ type FetchAPIFileUpload = (args: { options?: FetchAPIFileUploadOptions request: Request }) => Promise -export const fetchAPIFileUpload: FetchAPIFileUpload = async ({ options, request }) => { - const uploadOptions: FetchAPIFileUploadOptions = { ...DEFAULT_OPTIONS, ...options } + +export const processMultipartFormdata: FetchAPIFileUpload = async ({ + options: incomingOptions, + request, +}) => { + const options: FetchAPIFileUploadOptions = { ...DEFAULT_UPLOAD_OPTIONS, ...incomingOptions } + if (!isEligibleRequest(request)) { - debugLog(uploadOptions, 'Request is not eligible for file upload!') + debugLog(options, 'Request is not eligible for file upload!') + return { error: new APIError('Request is not eligible for file upload', 500), fields: undefined!, files: undefined!, } } else { - return processMultipart({ options: uploadOptions, request }) + return processMultipart({ options, request }) } } diff --git a/packages/payload/src/utilities/addDataAndFileToRequest.ts b/packages/payload/src/utilities/addDataAndFileToRequest.ts index 5065d9631..5e6c25954 100644 --- a/packages/payload/src/utilities/addDataAndFileToRequest.ts +++ b/packages/payload/src/utilities/addDataAndFileToRequest.ts @@ -1,7 +1,7 @@ import type { PayloadRequest } from '../types/index.js' import { APIError } from '../errors/APIError.js' -import { fetchAPIFileUpload } from '../uploads/fetchAPI-multipart/index.js' +import { processMultipartFormdata } from '../uploads/fetchAPI-multipart/index.js' type AddDataAndFileToRequest = (req: PayloadRequest) => Promise @@ -24,12 +24,15 @@ export const addDataAndFileToRequest: AddDataAndFileToRequest = async (req) => { req.payload.logger.error(error) } finally { req.data = data - // @ts-expect-error + // @ts-expect-error attach json method to request req.json = () => Promise.resolve(data) } } else if (bodyByteSize && contentType?.includes('multipart/')) { - const { error, fields, files } = await fetchAPIFileUpload({ - options: payload.config.upload, + const { error, fields, files } = await processMultipartFormdata({ + options: { + ...(payload.config.bodyParser || {}), + ...(payload.config.upload || {}), + }, request: req as Request, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4dfa9303d..7f1398a50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2115,6 +2115,9 @@ importers: nodemailer: specifier: 6.9.16 version: 6.9.16 + object-to-formdata: + specifier: 4.5.1 + version: 4.5.1 payload: specifier: workspace:* version: link:../packages/payload diff --git a/test/collections-rest/collections/LargeDocuments.ts b/test/collections-rest/collections/LargeDocuments.ts new file mode 100644 index 000000000..92e265d42 --- /dev/null +++ b/test/collections-rest/collections/LargeDocuments.ts @@ -0,0 +1,19 @@ +import type { CollectionConfig } from 'payload' + +export const largeDocumentsCollectionSlug = 'large-documents' + +export const LargeDocuments: CollectionConfig = { + slug: largeDocumentsCollectionSlug, + fields: [ + { + name: 'array', + type: 'array', + fields: [ + { + name: 'text', + type: 'text', + }, + ], + }, + ], +} diff --git a/test/collections-rest/config.ts b/test/collections-rest/config.ts index 49bd6e654..b3b0abf45 100644 --- a/test/collections-rest/config.ts +++ b/test/collections-rest/config.ts @@ -6,6 +6,7 @@ import { APIError, type CollectionConfig, type Endpoint } from 'payload' import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' import { devUser } from '../credentials.js' +import { LargeDocuments } from './collections/LargeDocuments.js' export interface Relation { id: string @@ -280,7 +281,13 @@ export default buildConfigWithDefaults({ ], disableBulkEdit: true, }, + LargeDocuments, ], + bodyParser: { + limits: { + fieldSize: 2 * 1024 * 1024, // 2MB + }, + }, endpoints: [ { handler: async ({ payload }) => { diff --git a/test/collections-rest/int.spec.ts b/test/collections-rest/int.spec.ts index e233f4622..0f1fe04d4 100644 --- a/test/collections-rest/int.spec.ts +++ b/test/collections-rest/int.spec.ts @@ -1,6 +1,7 @@ import type { Payload, SanitizedCollectionConfig } from 'payload' import { randomBytes, randomUUID } from 'crypto' +import { serialize } from 'object-to-formdata' import path from 'path' import { APIError, NotFound } from 'payload' import { fileURLToPath } from 'url' @@ -9,7 +10,9 @@ import type { NextRESTClient } from '../helpers/NextRESTClient.js' import type { Relation } from './config.js' import type { Post } from './payload-types.js' +import { getFormDataSize } from '../helpers/getFormDataSize.js' import { initPayloadInt } from '../helpers/initPayloadInt.js' +import { largeDocumentsCollectionSlug } from './collections/LargeDocuments.js' import { customIdNumberSlug, customIdSlug, @@ -119,6 +122,47 @@ describe('collections-rest', () => { expect(doc.description).toEqual(description) // Check was not modified }) + it('can handle REST API requests with over 1mb of multipart/form-data', async () => { + const doc = await payload.create({ + collection: largeDocumentsCollectionSlug, + data: {}, + }) + + const arrayData = new Array(500).fill({ text: randomUUID().repeat(100) }) + + // Now use the REST API and attempt to PATCH the document with a payload over 1mb + const dataToSerialize: Record = { + _payload: JSON.stringify({ + title: 'Hello, world!', + // fill with long, random string of text to exceed 1mb + array: arrayData, + }), + } + + const formData: FormData = serialize(dataToSerialize, { + indices: true, + nullsAsUndefineds: false, + }) + + // Ensure the form data we are about to send is greater than the default limit (1mb) + // But less than the increased limit that we've set in the root config (2mb) + const docSize = getFormDataSize(formData) + expect(docSize).toBeGreaterThan(1 * 1024 * 1024) + expect(docSize).toBeLessThan(2 * 1024 * 1024) + + // This request should not fail with error: "Unterminated string in JSON at position..." + // This is because we set `bodyParser.limits.fieldSize` to 2mb in the root config + const res = await restClient + .PATCH(`/${largeDocumentsCollectionSlug}/${doc.id}?limit=1`, { + body: formData, + }) + .then((res) => res.json()) + + expect(res).not.toHaveProperty('errors') + expect(res.doc.id).toEqual(doc.id) + expect(res.doc.array[0].text).toEqual(arrayData[0].text) + }) + describe('Bulk operations', () => { it('should bulk update', async () => { for (let i = 0; i < 11; i++) { diff --git a/test/collections-rest/payload-types.ts b/test/collections-rest/payload-types.ts index 1a7fca99d..1efbf2ba4 100644 --- a/test/collections-rest/payload-types.ts +++ b/test/collections-rest/payload-types.ts @@ -76,6 +76,7 @@ export interface Config { 'error-on-hooks': ErrorOnHook; endpoints: Endpoint; 'disabled-bulk-edit-docs': DisabledBulkEditDoc; + 'large-documents': LargeDocument; users: User; 'payload-locked-documents': PayloadLockedDocument; 'payload-preferences': PayloadPreference; @@ -92,6 +93,7 @@ export interface Config { 'error-on-hooks': ErrorOnHooksSelect | ErrorOnHooksSelect; endpoints: EndpointsSelect | EndpointsSelect; 'disabled-bulk-edit-docs': DisabledBulkEditDocsSelect | DisabledBulkEditDocsSelect; + 'large-documents': LargeDocumentsSelect | LargeDocumentsSelect; users: UsersSelect | UsersSelect; 'payload-locked-documents': PayloadLockedDocumentsSelect | PayloadLockedDocumentsSelect; 'payload-preferences': PayloadPreferencesSelect | PayloadPreferencesSelect; @@ -259,6 +261,21 @@ export interface DisabledBulkEditDoc { updatedAt: string; createdAt: string; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "large-documents". + */ +export interface LargeDocument { + id: string; + array?: + | { + text?: string | null; + id?: string | null; + }[] + | null; + updatedAt: string; + createdAt: string; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "users". @@ -274,6 +291,13 @@ export interface User { hash?: string | null; loginAttempts?: number | null; lockUntil?: string | null; + sessions?: + | { + id: string; + createdAt?: string | null; + expiresAt: string; + }[] + | null; password?: string | null; } /** @@ -319,6 +343,10 @@ export interface PayloadLockedDocument { relationTo: 'disabled-bulk-edit-docs'; value: string | DisabledBulkEditDoc; } | null) + | ({ + relationTo: 'large-documents'; + value: string | LargeDocument; + } | null) | ({ relationTo: 'users'; value: string | User; @@ -471,6 +499,20 @@ export interface DisabledBulkEditDocsSelect { updatedAt?: T; createdAt?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "large-documents_select". + */ +export interface LargeDocumentsSelect { + array?: + | T + | { + text?: T; + id?: T; + }; + updatedAt?: T; + createdAt?: T; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "users_select". @@ -485,6 +527,13 @@ export interface UsersSelect { hash?: T; loginAttempts?: T; lockUntil?: T; + sessions?: + | T + | { + id?: T; + createdAt?: T; + expiresAt?: T; + }; } /** * This interface was referenced by `Config`'s JSON-Schema diff --git a/test/helpers/NextRESTClient.ts b/test/helpers/NextRESTClient.ts index 30c65eec5..9151df471 100644 --- a/test/helpers/NextRESTClient.ts +++ b/test/helpers/NextRESTClient.ts @@ -12,6 +12,7 @@ import { import * as qs from 'qs-esm' import { devUser } from '../credentials.js' +import { getFormDataSize } from './getFormDataSize.js' type ValidPath = `/${string}` type RequestOptions = { @@ -94,17 +95,26 @@ export class NextRESTClient { } private buildHeaders(options: FileArg & RequestInit & RequestOptions): Headers { - const defaultHeaders = { - 'Content-Type': 'application/json', + // Only set `Content-Type` to `application/json` if body is not `FormData` + const isFormData = + options && + typeof options.body !== 'undefined' && + typeof FormData !== 'undefined' && + options.body instanceof FormData + + const headers = new Headers(options.headers || {}) + + if (options?.file) { + headers.set('Content-Length', options.file.size.toString()) + } + + if (isFormData) { + headers.set('Content-Length', getFormDataSize(options.body as FormData).toString()) + } + + if (!isFormData && !headers.has('Content-Type')) { + headers.set('Content-Type', 'application/json') } - const headers = new Headers({ - ...(options?.file - ? { - 'Content-Length': options.file.size.toString(), - } - : defaultHeaders), - ...(options?.headers || {}), - }) if (options.auth !== false && this.token) { headers.set('Authorization', `JWT ${this.token}`) @@ -213,6 +223,7 @@ export class NextRESTClient { headers: this.buildHeaders(options), method: 'PATCH', }) + return this._PATCH(request, { params: Promise.resolve({ slug }) }) } diff --git a/test/helpers/getFormDataSize.ts b/test/helpers/getFormDataSize.ts new file mode 100644 index 000000000..431c7e4bb --- /dev/null +++ b/test/helpers/getFormDataSize.ts @@ -0,0 +1,16 @@ +export function getFormDataSize(formData: FormData) { + const blob = new Blob(formDataToArray(formData)) + return blob.size +} + +function formDataToArray(formData: FormData) { + const parts = [] + for (const [key, value] of formData.entries()) { + if (value instanceof File) { + parts.push(value) + } else { + parts.push(new Blob([value], { type: 'text/plain' })) + } + } + return parts +} diff --git a/test/package.json b/test/package.json index 63254767a..ea399adf4 100644 --- a/test/package.json +++ b/test/package.json @@ -83,6 +83,7 @@ "mongoose": "8.15.1", "next": "15.4.4", "nodemailer": "6.9.16", + "object-to-formdata": "4.5.1", "payload": "workspace:*", "pg": "8.16.3", "qs-esm": "7.0.2",