diff --git a/package.json b/package.json index 4b5579772a..50c65a8c26 100644 --- a/package.json +++ b/package.json @@ -238,6 +238,7 @@ "@types/pino-std-serializers": "^4.0.0", "@types/pluralize": "^0.0.29", "@types/prismjs": "^1.26.0", + "@types/probe-image-size": "^7.2.0", "@types/prop-types": "^15.7.5", "@types/qs": "^6.9.7", "@types/qs-middleware": "^1.0.1", diff --git a/src/collections/config/schema.ts b/src/collections/config/schema.ts index c1757c9116..b65802204e 100644 --- a/src/collections/config/schema.ts +++ b/src/collections/config/schema.ts @@ -145,6 +145,8 @@ const collectionSchema = joi.object().keys({ staticURL: joi.string(), staticDir: joi.string(), disableLocalStorage: joi.bool(), + useTempFiles: joi.bool(), + tempFileDir: joi.string(), adminThumbnail: joi.alternatives().try( joi.string(), joi.func(), diff --git a/src/collections/operations/create.ts b/src/collections/operations/create.ts index 7254113843..f091fec397 100644 --- a/src/collections/operations/create.ts +++ b/src/collections/operations/create.ts @@ -1,3 +1,6 @@ +import fs from 'fs'; +import { promisify } from 'util'; + import crypto from 'crypto'; import { Config as GeneratedTypes } from 'payload/generated-types'; import { MarkOptional } from 'ts-essentials'; @@ -18,6 +21,9 @@ import { afterChange } from '../../fields/hooks/afterChange'; import { afterRead } from '../../fields/hooks/afterRead'; import { generateFileData } from '../../uploads/generateFileData'; import { saveVersion } from '../../versions/saveVersion'; +import { mapAsync } from '../../utilities/mapAsync'; + +const unlinkFile = promisify(fs.unlink); export type Arguments = { collection: Collection @@ -304,6 +310,18 @@ async function create( }) || result; }, Promise.resolve()); + // Remove temp files if enabled, as express-fileupload does not do this automatically + if (config.upload?.useTempFiles && collectionConfig.upload) { + const { files } = req; + const fileArray = Array.isArray(files) ? files : [files]; + await mapAsync(fileArray, async ({ file }) => { + // Still need this check because this will not be populated if using local API + if (file.tempFilePath) { + await unlinkFile(file.tempFilePath); + } + }); + } + // ///////////////////////////////////// // Return results // ///////////////////////////////////// diff --git a/src/collections/operations/update.ts b/src/collections/operations/update.ts index 21542bd386..20154580c8 100644 --- a/src/collections/operations/update.ts +++ b/src/collections/operations/update.ts @@ -1,3 +1,6 @@ +import fs from 'fs'; +import { promisify } from 'util'; + import httpStatus from 'http-status'; import { Config as GeneratedTypes } from 'payload/generated-types'; import { MarkOptional } from 'ts-essentials'; @@ -16,6 +19,9 @@ import { afterChange } from '../../fields/hooks/afterChange'; import { afterRead } from '../../fields/hooks/afterRead'; import { generateFileData } from '../../uploads/generateFileData'; import { getLatestEntityVersion } from '../../versions/getLatestCollectionVersion'; +import { mapAsync } from '../../utilities/mapAsync'; + +const unlinkFile = promisify(fs.unlink); export type Arguments = { collection: Collection @@ -325,6 +331,17 @@ async function update( }) || result; }, Promise.resolve()); + // Remove temp files if enabled, as express-fileupload does not do this automatically + if (config.upload?.useTempFiles && collectionConfig.upload) { + const { files } = req; + const fileArray = Array.isArray(files) ? files : [files]; + await mapAsync(fileArray, async ({ file }) => { + // Still need this check because this will not be populated if using local API + if (file.tempFilePath) { + await unlinkFile(file.tempFilePath); + } + }); + } // ///////////////////////////////////// // Return results // ///////////////////////////////////// diff --git a/src/config/types.ts b/src/config/types.ts index afbe5a1033..facdf9bd77 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -1,7 +1,7 @@ import { Express, NextFunction, Response } from 'express'; import { DeepRequired } from 'ts-essentials'; import { Transporter } from 'nodemailer'; -import { Options } from 'express-fileupload'; +import { Options as ExpressFileUploadOptions } from 'express-fileupload'; import { Configuration } from 'webpack'; import SMTPConnection from 'nodemailer/lib/smtp-connection'; import GraphQL from 'graphql'; @@ -473,7 +473,7 @@ export type Config = { /** * Customize the handling of incoming file uploads for collections that have uploads enabled. */ - upload?: Options; + upload?: ExpressFileUploadOptions; /** * Translate your content to different languages/locales. * diff --git a/src/uploads/generateFileData.ts b/src/uploads/generateFileData.ts index ed5a03e948..7be68a8443 100644 --- a/src/uploads/generateFileData.ts +++ b/src/uploads/generateFileData.ts @@ -7,10 +7,10 @@ import { Collection } from '../collections/config/types'; import { SanitizedConfig } from '../config/types'; import { FileUploadError, MissingFile } from '../errors'; import { PayloadRequest } from '../express/types'; -import getImageSize, { ProbedImageSize } from './getImageSize'; +import getImageSize from './getImageSize'; import getSafeFileName from './getSafeFilename'; import resizeAndSave from './imageResizer'; -import { FileData, FileToSave } from './types'; +import { FileData, FileToSave, ProbedImageSize } from './types'; import canResizeImage from './canResizeImage'; import isImage from './isImage'; @@ -39,116 +39,126 @@ export const generateFileData = async ({ throwOnMissingFile, overwriteExistingFiles, }: Args): Result => { + if (!collectionConfig.upload) { + return { + data, + files: [], + }; + } + + const { file } = req.files || {}; + if (!file) { + if (throwOnMissingFile) throw new MissingFile(req.t); + + return { + data, + files: [], + }; + } + + const { staticDir, imageSizes, disableLocalStorage, resizeOptions, formatOptions } = collectionConfig.upload; + + let staticPath = staticDir; + if (staticDir.indexOf('/') !== 0) { + staticPath = path.resolve(config.paths.configDir, staticDir); + } + + if (!disableLocalStorage) { + mkdirp.sync(staticPath); + } + + let newData = data; const filesToSave: FileToSave[] = []; + const fileData: Partial = {}; + try { + const fileSupportsResize = canResizeImage(file.mimetype); + let fsSafeName: string; + let originalFile: Sharp | undefined; + let dimensions: ProbedImageSize | undefined; + let fileBuffer; + let ext; + let mime: string; - if (collectionConfig.upload) { - const fileData: Partial = {}; - - const { staticDir, imageSizes, disableLocalStorage, resizeOptions, formatOptions } = collectionConfig.upload; - - const { file } = req.files || {}; - - if (throwOnMissingFile && !file) { - throw new MissingFile(req.t); - } - - let staticPath = staticDir; - - if (staticDir.indexOf('/') !== 0) { - staticPath = path.resolve(config.paths.configDir, staticDir); - } - - if (!disableLocalStorage) { - mkdirp.sync(staticPath); - } - - if (file) { - try { - const shouldResize = canResizeImage(file.mimetype); - let fsSafeName: string; - let resized: Sharp | undefined; - let dimensions; - let fileBuffer; - let bufferInfo; - let ext; - let mime; - - if (shouldResize) { - if (resizeOptions) { - resized = sharp(file.data) - .resize(resizeOptions); - } - if (formatOptions) { - resized = (resized ?? sharp(file.data)).toFormat(formatOptions.format, formatOptions.options); - } - } - - if (isImage(file.mimetype)) { - dimensions = await getImageSize(file); - fileData.width = dimensions.width; - fileData.height = dimensions.height; - } - - if (resized) { - fileBuffer = await resized.toBuffer({ resolveWithObject: true }); - bufferInfo = await fromBuffer(fileBuffer.data); - - mime = bufferInfo.mime; - ext = bufferInfo.ext; - fileData.width = fileBuffer.info.width; - fileData.height = fileBuffer.info.height; - fileData.filesize = fileBuffer.data.length; - } else { - mime = file.mimetype; - fileData.filesize = file.size; - ext = file.name.split('.').pop(); - } - - if (mime === 'application/xml' && ext === 'svg') mime = 'image/svg+xml'; - fileData.mimeType = mime; - - const baseFilename = sanitize(file.name.substring(0, file.name.lastIndexOf('.')) || file.name); - fsSafeName = `${baseFilename}.${ext}`; - - if (!overwriteExistingFiles) { - fsSafeName = await getSafeFileName(Model, staticPath, `${baseFilename}.${ext}`); - } - - fileData.filename = fsSafeName; - - filesToSave.push({ - path: `${staticPath}/${fsSafeName}`, - buffer: fileBuffer?.data || file.data, - }); - - if (Array.isArray(imageSizes) && shouldResize) { - req.payloadUploadSizes = {}; - const { sizeData, sizesToSave } = await resizeAndSave({ - req, - file: file.data, - dimensions, - staticPath, - config: collectionConfig, - savedFilename: fsSafeName || file.name, - mimeType: fileData.mimeType, - }); - - fileData.sizes = sizeData; - filesToSave.push(...sizesToSave); - } - } catch (err) { - console.error(err); - throw new FileUploadError(req.t); + if (fileSupportsResize) { + if (file.tempFilePath) { + originalFile = sharp(file.tempFilePath); + } else { + originalFile = sharp(file.data); } - newData = { - ...newData, - ...fileData, - }; + if (resizeOptions) { + originalFile = originalFile + .resize(resizeOptions); + } + if (formatOptions) { + originalFile = originalFile.toFormat(formatOptions.format, formatOptions.options); + } } + + if (isImage(file.mimetype)) { + dimensions = await getImageSize(file); + fileData.width = dimensions.width; + fileData.height = dimensions.height; + } + + if (originalFile) { + fileBuffer = await originalFile.toBuffer({ resolveWithObject: true }); + ({ mime, ext } = await fromBuffer(fileBuffer.data)); + fileData.width = fileBuffer.info.width; + fileData.height = fileBuffer.info.height; + fileData.filesize = fileBuffer.data.length; + } else { + mime = file.mimetype; + fileData.filesize = file.size; + ext = file.name.split('.').pop(); + } + + // Adust SVG mime type. fromBuffer modifies it. + if (mime === 'application/xml' && ext === 'svg') mime = 'image/svg+xml'; + fileData.mimeType = mime; + + const baseFilename = sanitize(file.name.substring(0, file.name.lastIndexOf('.')) || file.name); + fsSafeName = `${baseFilename}.${ext}`; + + if (!overwriteExistingFiles) { + fsSafeName = await getSafeFileName(Model, staticPath, `${baseFilename}.${ext}`); + } + + fileData.filename = fsSafeName; + + // Original file + filesToSave.push({ + path: `${staticPath}/${fsSafeName}`, + buffer: fileBuffer?.data || file.data, + }); + + if (Array.isArray(imageSizes) && fileSupportsResize) { + req.payloadUploadSizes = {}; + + const { sizeData, sizesToSave } = await resizeAndSave({ + req, + file, + dimensions, + staticPath, + config: collectionConfig, + savedFilename: fsSafeName || file.name, + mimeType: fileData.mimeType, + }); + + fileData.sizes = sizeData; + filesToSave.push(...sizesToSave); + } + } catch (err) { + console.error(err); + throw new FileUploadError(req.t); } + newData = { + ...newData, + ...fileData, + }; + return { data: newData, files: filesToSave, diff --git a/src/uploads/getImageSize.ts b/src/uploads/getImageSize.ts index 4a24c86ce5..0f6fbd480e 100644 --- a/src/uploads/getImageSize.ts +++ b/src/uploads/getImageSize.ts @@ -1,13 +1,10 @@ -import { UploadedFile } from 'express-fileupload'; +import fs from 'fs'; import probeImageSize from 'probe-image-size'; +import { UploadedFile } from 'express-fileupload'; +import { ProbedImageSize } from './types'; -export type ProbedImageSize = { - width: number, - height: number, - type: string, - mime: string, -} - -export default async function (image: UploadedFile): Promise { - return probeImageSize.sync(image.data); +export default async function (file: UploadedFile): Promise { + return file.tempFilePath + ? probeImageSize(fs.createReadStream(file.tempFilePath)) + : probeImageSize.sync(file.data); } diff --git a/src/uploads/imageResizer.ts b/src/uploads/imageResizer.ts index 7b7a2568ea..e274a770af 100644 --- a/src/uploads/imageResizer.ts +++ b/src/uploads/imageResizer.ts @@ -1,3 +1,4 @@ +import { UploadedFile } from 'express-fileupload'; import { fromBuffer } from 'file-type'; import fs from 'fs'; import sanitize from 'sanitize-filename'; @@ -11,7 +12,7 @@ type Dimensions = { width?: number, height?: number } type Args = { req: PayloadRequest - file: Buffer + file: UploadedFile dimensions: Dimensions staticPath: string config: SanitizedCollectionConfig @@ -55,6 +56,8 @@ export default async function resizeAndSave({ const sizesToSave: FileToSave[] = []; const sizeData = {}; + const sharpInstance = sharp(file.tempFilePath || file.data); + const promises = imageSizes .map(async (desiredSize) => { if (!needsResize(desiredSize, dimensions)) { @@ -68,7 +71,7 @@ export default async function resizeAndSave({ }; return; } - let resized = sharp(file).resize(desiredSize); + let resized = sharpInstance.resize(desiredSize); if (desiredSize.formatOptions) { resized = resized.toFormat(desiredSize.formatOptions.format, desiredSize.formatOptions.options); diff --git a/src/uploads/types.ts b/src/uploads/types.ts index 784e09fe77..c5f1480b85 100644 --- a/src/uploads/types.ts +++ b/src/uploads/types.ts @@ -22,8 +22,16 @@ export type FileData = { width: number; height: number; sizes: FileSizes; + tempFilePath?: string; }; +export type ProbedImageSize = { + width: number, + height: number, + type: string, + mime: string, +} + /** * Params sent to the sharp toFormat() function * @link https://sharp.pixelplumbing.com/api-output#toformat