tabshift/payload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support large file uploads (#1981)

Browse Source 
* feat: support express-fileupload useTempFiles and tempFileDir

* feat: rework uploads to accommodate useTempFiles option

* fix: properly check versions config before handling version deletion

* chore: fix aspect ratio handling

* chore: bump probe-image-size

* chore: handle temp file resizing buffer

* chore: yarn.lock after reverting probe-image-size bump

* chore: get yarn.lock from master

* chore: clear temp files directly in operations instead of injecting a hook
This commit is contained in:
Elliot DeNolf
2023-02-01 15:59:53 -05:00
committed by GitHub
parent c1df7674b2
commit 12ed655881
9 changed files with 174 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/collections/config/schema.ts
View File

@@ -145,6 +145,8 @@ const collectionSchema = joi.object().keys({
staticURL: joi.string(),
staticDir: joi.string(),
disableLocalStorage: joi.bool(),
useTempFiles: joi.bool(),
tempFileDir: joi.string(),
adminThumbnail: joi.alternatives().try(
joi.string(),
joi.func(),

18
src/collections/operations/create.ts
View File

@@ -1,3 +1,6 @@
import fs from 'fs';
import { promisify } from 'util';
import crypto from 'crypto';
import { Config as GeneratedTypes } from 'payload/generated-types';
import { MarkOptional } from 'ts-essentials';
@@ -18,6 +21,9 @@ import { afterChange } from '../../fields/hooks/afterChange';
import { afterRead } from '../../fields/hooks/afterRead';
import { generateFileData } from '../../uploads/generateFileData';
import { saveVersion } from '../../versions/saveVersion';
import { mapAsync } from '../../utilities/mapAsync';
const unlinkFile = promisify(fs.unlink);
export type Arguments<T extends { [field: string | number | symbol]: unknown }> = {
collection: Collection
@@ -304,6 +310,18 @@ async function create<TSlug extends keyof GeneratedTypes['collections']>(
}) || result;
}, Promise.resolve());
// Remove temp files if enabled, as express-fileupload does not do this automatically
if (config.upload?.useTempFiles && collectionConfig.upload) {
const { files } = req;
const fileArray = Array.isArray(files) ? files : [files];
await mapAsync(fileArray, async ({ file }) => {
// Still need this check because this will not be populated if using local API
if (file.tempFilePath) {
await unlinkFile(file.tempFilePath);
}
});
}
// /////////////////////////////////////
// Return results
// /////////////////////////////////////

17
src/collections/operations/update.ts
View File

@@ -1,3 +1,6 @@
import fs from 'fs';
import { promisify } from 'util';
import httpStatus from 'http-status';
import { Config as GeneratedTypes } from 'payload/generated-types';
import { MarkOptional } from 'ts-essentials';
@@ -16,6 +19,9 @@ import { afterChange } from '../../fields/hooks/afterChange';
import { afterRead } from '../../fields/hooks/afterRead';
import { generateFileData } from '../../uploads/generateFileData';
import { getLatestEntityVersion } from '../../versions/getLatestCollectionVersion';
import { mapAsync } from '../../utilities/mapAsync';
const unlinkFile = promisify(fs.unlink);
export type Arguments<T extends { [field: string | number | symbol]: unknown }> = {
collection: Collection
@@ -325,6 +331,17 @@ async function update<TSlug extends keyof GeneratedTypes['collections']>(
}) || result;
}, Promise.resolve());
// Remove temp files if enabled, as express-fileupload does not do this automatically
if (config.upload?.useTempFiles && collectionConfig.upload) {
const { files } = req;
const fileArray = Array.isArray(files) ? files : [files];
await mapAsync(fileArray, async ({ file }) => {
// Still need this check because this will not be populated if using local API
if (file.tempFilePath) {
await unlinkFile(file.tempFilePath);
}
});
}
// /////////////////////////////////////
// Return results
// /////////////////////////////////////

4
src/config/types.ts
View File

@@ -1,7 +1,7 @@
import { Express, NextFunction, Response } from 'express';
import { DeepRequired } from 'ts-essentials';
import { Transporter } from 'nodemailer';
import { Options } from 'express-fileupload';
import { Options as ExpressFileUploadOptions } from 'express-fileupload';
import { Configuration } from 'webpack';
import SMTPConnection from 'nodemailer/lib/smtp-connection';
import GraphQL from 'graphql';
@@ -473,7 +473,7 @@ export type Config = {
/**
* Customize the handling of incoming file uploads for collections that have uploads enabled.
*/
upload?: Options;
upload?: ExpressFileUploadOptions;
/**
* Translate your content to different languages/locales.
*

218
src/uploads/generateFileData.ts
View File

@@ -7,10 +7,10 @@ import { Collection } from '../collections/config/types';
import { SanitizedConfig } from '../config/types';
import { FileUploadError, MissingFile } from '../errors';
import { PayloadRequest } from '../express/types';
import getImageSize, { ProbedImageSize } from './getImageSize';
import getImageSize from './getImageSize';
import getSafeFileName from './getSafeFilename';
import resizeAndSave from './imageResizer';
import { FileData, FileToSave } from './types';
import { FileData, FileToSave, ProbedImageSize } from './types';
import canResizeImage from './canResizeImage';
import isImage from './isImage';
@@ -39,116 +39,126 @@ export const generateFileData = async <T>({
throwOnMissingFile,
overwriteExistingFiles,
}: Args<T>): Result<T> => {
if (!collectionConfig.upload) {
return {
data,
files: [],
};
}
const { file } = req.files || {};
if (!file) {
if (throwOnMissingFile) throw new MissingFile(req.t);
return {
data,
files: [],
};
}
const { staticDir, imageSizes, disableLocalStorage, resizeOptions, formatOptions } = collectionConfig.upload;
let staticPath = staticDir;
if (staticDir.indexOf('/') !== 0) {
staticPath = path.resolve(config.paths.configDir, staticDir);
}
if (!disableLocalStorage) {
mkdirp.sync(staticPath);
}
let newData = data;
const filesToSave: FileToSave[] = [];
const fileData: Partial<FileData> = {};
try {
const fileSupportsResize = canResizeImage(file.mimetype);
let fsSafeName: string;
let originalFile: Sharp | undefined;
let dimensions: ProbedImageSize | undefined;
let fileBuffer;
let ext;
let mime: string;
if (collectionConfig.upload) {
const fileData: Partial<FileData> = {};
const { staticDir, imageSizes, disableLocalStorage, resizeOptions, formatOptions } = collectionConfig.upload;
const { file } = req.files || {};
if (throwOnMissingFile && !file) {
throw new MissingFile(req.t);
}
let staticPath = staticDir;
if (staticDir.indexOf('/') !== 0) {
staticPath = path.resolve(config.paths.configDir, staticDir);
}
if (!disableLocalStorage) {
mkdirp.sync(staticPath);
}
if (file) {
try {
const shouldResize = canResizeImage(file.mimetype);
let fsSafeName: string;
let resized: Sharp | undefined;
let dimensions;
let fileBuffer;
let bufferInfo;
let ext;
let mime;
if (shouldResize) {
if (resizeOptions) {
resized = sharp(file.data)
.resize(resizeOptions);
}
if (formatOptions) {
resized = (resized ?? sharp(file.data)).toFormat(formatOptions.format, formatOptions.options);
}
}
if (isImage(file.mimetype)) {
dimensions = await getImageSize(file);
fileData.width = dimensions.width;
fileData.height = dimensions.height;
}
if (resized) {
fileBuffer = await resized.toBuffer({ resolveWithObject: true });
bufferInfo = await fromBuffer(fileBuffer.data);
mime = bufferInfo.mime;
ext = bufferInfo.ext;
fileData.width = fileBuffer.info.width;
fileData.height = fileBuffer.info.height;
fileData.filesize = fileBuffer.data.length;
} else {
mime = file.mimetype;
fileData.filesize = file.size;
ext = file.name.split('.').pop();
}
if (mime === 'application/xml' && ext === 'svg') mime = 'image/svg+xml';
fileData.mimeType = mime;
const baseFilename = sanitize(file.name.substring(0, file.name.lastIndexOf('.')) || file.name);
fsSafeName = `${baseFilename}.${ext}`;
if (!overwriteExistingFiles) {
fsSafeName = await getSafeFileName(Model, staticPath, `${baseFilename}.${ext}`);
}
fileData.filename = fsSafeName;
filesToSave.push({
path: `${staticPath}/${fsSafeName}`,
buffer: fileBuffer?.data || file.data,
});
if (Array.isArray(imageSizes) && shouldResize) {
req.payloadUploadSizes = {};
const { sizeData, sizesToSave } = await resizeAndSave({
req,
file: file.data,
dimensions,
staticPath,
config: collectionConfig,
savedFilename: fsSafeName || file.name,
mimeType: fileData.mimeType,
});
fileData.sizes = sizeData;
filesToSave.push(...sizesToSave);
}
} catch (err) {
console.error(err);
throw new FileUploadError(req.t);
if (fileSupportsResize) {
if (file.tempFilePath) {
originalFile = sharp(file.tempFilePath);
} else {
originalFile = sharp(file.data);
}
newData = {
...newData,
...fileData,
};
if (resizeOptions) {
originalFile = originalFile
.resize(resizeOptions);
}
if (formatOptions) {
originalFile = originalFile.toFormat(formatOptions.format, formatOptions.options);
}
}
if (isImage(file.mimetype)) {
dimensions = await getImageSize(file);
fileData.width = dimensions.width;
fileData.height = dimensions.height;
}
if (originalFile) {
fileBuffer = await originalFile.toBuffer({ resolveWithObject: true });
({ mime, ext } = await fromBuffer(fileBuffer.data));
fileData.width = fileBuffer.info.width;
fileData.height = fileBuffer.info.height;
fileData.filesize = fileBuffer.data.length;
} else {
mime = file.mimetype;
fileData.filesize = file.size;
ext = file.name.split('.').pop();
}
// Adust SVG mime type. fromBuffer modifies it.
if (mime === 'application/xml' && ext === 'svg') mime = 'image/svg+xml';
fileData.mimeType = mime;
const baseFilename = sanitize(file.name.substring(0, file.name.lastIndexOf('.')) || file.name);
fsSafeName = `${baseFilename}.${ext}`;
if (!overwriteExistingFiles) {
fsSafeName = await getSafeFileName(Model, staticPath, `${baseFilename}.${ext}`);
}
fileData.filename = fsSafeName;
// Original file
filesToSave.push({
path: `${staticPath}/${fsSafeName}`,
buffer: fileBuffer?.data || file.data,
});
if (Array.isArray(imageSizes) && fileSupportsResize) {
req.payloadUploadSizes = {};
const { sizeData, sizesToSave } = await resizeAndSave({
req,
file,
dimensions,
staticPath,
config: collectionConfig,
savedFilename: fsSafeName || file.name,
mimeType: fileData.mimeType,
});
fileData.sizes = sizeData;
filesToSave.push(...sizesToSave);
}
} catch (err) {
console.error(err);
throw new FileUploadError(req.t);
}
newData = {
...newData,
...fileData,
};
return {
data: newData,
files: filesToSave,

17
src/uploads/getImageSize.ts
View File

@@ -1,13 +1,10 @@
import { UploadedFile } from 'express-fileupload';
import fs from 'fs';
import probeImageSize from 'probe-image-size';
import { UploadedFile } from 'express-fileupload';
import { ProbedImageSize } from './types';
export type ProbedImageSize = {
width: number,
height: number,
type: string,
mime: string,
}
export default async function (image: UploadedFile): Promise<ProbedImageSize> {
return probeImageSize.sync(image.data);
export default async function (file: UploadedFile): Promise<ProbedImageSize> {
return file.tempFilePath
? probeImageSize(fs.createReadStream(file.tempFilePath))
: probeImageSize.sync(file.data);
}

7
src/uploads/imageResizer.ts
View File

@@ -1,3 +1,4 @@
import { UploadedFile } from 'express-fileupload';
import { fromBuffer } from 'file-type';
import fs from 'fs';
import sanitize from 'sanitize-filename';
@@ -11,7 +12,7 @@ type Dimensions = { width?: number, height?: number }
type Args = {
req: PayloadRequest
file: Buffer
file: UploadedFile
dimensions: Dimensions
staticPath: string
config: SanitizedCollectionConfig
@@ -55,6 +56,8 @@ export default async function resizeAndSave({
const sizesToSave: FileToSave[] = [];
const sizeData = {};
const sharpInstance = sharp(file.tempFilePath || file.data);
const promises = imageSizes
.map(async (desiredSize) => {
if (!needsResize(desiredSize, dimensions)) {
@@ -68,7 +71,7 @@ export default async function resizeAndSave({
};
return;
}
let resized = sharp(file).resize(desiredSize);
let resized = sharpInstance.resize(desiredSize);
if (desiredSize.formatOptions) {
resized = resized.toFormat(desiredSize.formatOptions.format, desiredSize.formatOptions.options);

8
src/uploads/types.ts
View File

@@ -22,8 +22,16 @@ export type FileData = {
width: number;
height: number;
sizes: FileSizes;
tempFilePath?: string;
};
export type ProbedImageSize = {
width: number,
height: number,
type: string,
mime: string,
}
/**
* Params sent to the sharp toFormat() function
* @link https://sharp.pixelplumbing.com/api-output#toformat