From 5fc9f764062dbfd631a30329764306bca14cbab4 Mon Sep 17 00:00:00 2001 From: Elliot DeNolf Date: Tue, 13 Aug 2024 13:55:10 -0400 Subject: [PATCH] feat: filename compound index (#7651) Allow a compound index to be used for upload collections via a `filenameCompoundIndex` field. Previously, `filename` was always treated as unique. Usage: ```ts { slug: 'upload-field', upload: { // Slugs to include in compound index filenameCompoundIndex: ['filename', 'alt'], }, } ``` --- docs/upload/overview.mdx | 1 + .../src/models/buildCollectionSchema.ts | 12 ++ packages/db-postgres/src/init.ts | 16 +++ packages/db-postgres/src/schema/build.ts | 4 + packages/db-sqlite/src/init.ts | 16 ++- packages/db-sqlite/src/schema/build.ts | 4 + packages/payload/src/uploads/getBaseFields.ts | 9 +- packages/payload/src/uploads/types.ts | 4 + scripts/pack-all-to-dest.ts | 2 +- test/uploads/config.ts | 38 ++++++ test/uploads/payload-types.ts | 128 ++++++++++++------ 11 files changed, 186 insertions(+), 48 deletions(-) diff --git a/docs/upload/overview.mdx b/docs/upload/overview.mdx index 6372a71c6..a26a2afd0 100644 --- a/docs/upload/overview.mdx +++ b/docs/upload/overview.mdx @@ -97,6 +97,7 @@ _An asterisk denotes that an option is required._ | **`displayPreview`** | Enable displaying preview of the uploaded file in Upload fields related to this Collection. Can be locally overridden by `displayPreview` option in Upload field. [More](/docs/fields/upload#config-options). | | **`externalFileHeaderFilter`** | Accepts existing headers and returns the headers after filtering or modifying. | | **`filesRequiredOnCreate`** | Mandate file data on creation, default is true. | +| **`filenameCompoundIndex`** | Field slugs to use for a compount index instead of the default filename index. | **`focalPoint`** | Set to `false` to disable the focal point selection tool in the [Admin Panel](../admin/overview). The focal point selector is only available when `imageSizes` or `resizeOptions` are defined. [More](#crop-and-focal-point-selector) | | **`formatOptions`** | An object with `format` and `options` that are used with the Sharp image library to format the upload file. [More](https://sharp.pixelplumbing.com/api-output#toformat) | | **`handlers`** | Array of Request handlers to execute when fetching a file, if a handler returns a Response it will be sent to the client. Otherwise Payload will retrieve and send back the file. | diff --git a/packages/db-mongodb/src/models/buildCollectionSchema.ts b/packages/db-mongodb/src/models/buildCollectionSchema.ts index 19e72a07a..08eaf9d29 100644 --- a/packages/db-mongodb/src/models/buildCollectionSchema.ts +++ b/packages/db-mongodb/src/models/buildCollectionSchema.ts @@ -21,6 +21,18 @@ const buildCollectionSchema = ( }, }) + if (Array.isArray(collection.upload.filenameCompoundIndex)) { + const indexDefinition: Record = collection.upload.filenameCompoundIndex.reduce( + (acc, index) => { + acc[index] = 1 + return acc + }, + {}, + ) + + schema.index(indexDefinition, { unique: true }) + } + if (config.indexSortableFields && collection.timestamps !== false) { schema.index({ updatedAt: 1 }) schema.index({ createdAt: 1 }) diff --git a/packages/db-postgres/src/init.ts b/packages/db-postgres/src/init.ts index 6836f251d..724e376c9 100644 --- a/packages/db-postgres/src/init.ts +++ b/packages/db-postgres/src/init.ts @@ -1,9 +1,11 @@ import type { Init, SanitizedCollectionConfig } from 'payload' import { createTableName } from '@payloadcms/drizzle' +import { uniqueIndex } from 'drizzle-orm/pg-core' import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload' import toSnakeCase from 'to-snake-case' +import type { BaseExtraConfig } from './schema/build.js' import type { PostgresAdapter } from './types.js' import { buildTable } from './schema/build.js' @@ -34,8 +36,22 @@ export const init: Init = function init(this: PostgresAdapter) { this.payload.config.collections.forEach((collection: SanitizedCollectionConfig) => { const tableName = this.tableNameMap.get(toSnakeCase(collection.slug)) + const baseExtraConfig: BaseExtraConfig = {} + + if (collection.upload.filenameCompoundIndex) { + const indexName = `${tableName}_filename_compound_idx` + + baseExtraConfig.filename_compound_index = (cols) => { + const colsConstraint = collection.upload.filenameCompoundIndex.map((f) => { + return cols[f] + }) + return uniqueIndex(indexName).on(colsConstraint[0], ...colsConstraint.slice(1)) + } + } + buildTable({ adapter: this, + baseExtraConfig, disableNotNull: !!collection?.versions?.drafts, disableUnique: false, fields: collection.fields, diff --git a/packages/db-postgres/src/schema/build.ts b/packages/db-postgres/src/schema/build.ts index 0e814abff..52abfd72f 100644 --- a/packages/db-postgres/src/schema/build.ts +++ b/packages/db-postgres/src/schema/build.ts @@ -38,6 +38,10 @@ export type RelationMap = Map + /** + * After table is created, run these functions to add extra config to the table + * ie. indexes, multiple columns, etc + */ baseExtraConfig?: BaseExtraConfig buildNumbers?: boolean buildRelationships?: boolean diff --git a/packages/db-sqlite/src/init.ts b/packages/db-sqlite/src/init.ts index 14891c0b5..2a8cc7d40 100644 --- a/packages/db-sqlite/src/init.ts +++ b/packages/db-sqlite/src/init.ts @@ -1,11 +1,12 @@ -/* eslint-disable no-param-reassign */ import type { DrizzleAdapter } from '@payloadcms/drizzle/types' import type { Init, SanitizedCollectionConfig } from 'payload' import { createTableName } from '@payloadcms/drizzle' +import { uniqueIndex } from 'drizzle-orm/sqlite-core' import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload' import toSnakeCase from 'to-snake-case' +import type { BaseExtraConfig } from './schema/build.js' import type { SQLiteAdapter } from './types.js' import { buildTable } from './schema/build.js' @@ -37,6 +38,19 @@ export const init: Init = function init(this: SQLiteAdapter) { this.payload.config.collections.forEach((collection: SanitizedCollectionConfig) => { const tableName = this.tableNameMap.get(toSnakeCase(collection.slug)) + const baseExtraConfig: BaseExtraConfig = {} + + if (collection.upload.filenameCompoundIndex) { + const indexName = `${tableName}_filename_compound_idx` + + baseExtraConfig.filename_compound_index = (cols) => { + const colsConstraint = collection.upload.filenameCompoundIndex.map((f) => { + return cols[f] + }) + return uniqueIndex(indexName).on(colsConstraint[0], ...colsConstraint.slice(1)) + } + } + buildTable({ adapter: this, disableNotNull: !!collection?.versions?.drafts, diff --git a/packages/db-sqlite/src/schema/build.ts b/packages/db-sqlite/src/schema/build.ts index 163d8501d..7c1e50320 100644 --- a/packages/db-sqlite/src/schema/build.ts +++ b/packages/db-sqlite/src/schema/build.ts @@ -40,6 +40,10 @@ export type RelationMap = Map + /** + * After table is created, run these functions to add extra config to the table + * ie. indexes, multiple columns, etc + */ baseExtraConfig?: BaseExtraConfig buildNumbers?: boolean buildRelationships?: boolean diff --git a/packages/payload/src/uploads/getBaseFields.ts b/packages/payload/src/uploads/getBaseFields.ts index 801aff927..81473c9e2 100644 --- a/packages/payload/src/uploads/getBaseFields.ts +++ b/packages/payload/src/uploads/getBaseFields.ts @@ -111,7 +111,14 @@ export const getBaseUploadFields = ({ collection, config }: Options): Field[] => }, index: true, label: ({ t }) => t('upload:fileName'), - unique: true, + } + + // Only set unique: true if the collection does not have a compound index + if ( + collection.upload === true || + (typeof collection.upload === 'object' && !collection.upload.filenameCompoundIndex) + ) { + filename.unique = true } const url: Field = { diff --git a/packages/payload/src/uploads/types.ts b/packages/payload/src/uploads/types.ts index 0cdb7d83c..75df7e63f 100644 --- a/packages/payload/src/uploads/types.ts +++ b/packages/payload/src/uploads/types.ts @@ -118,6 +118,10 @@ export type UploadConfig = { * @default undefined */ externalFileHeaderFilter?: (headers: Record) => Record + /** + * Field slugs to use for a compount index instead of the default filename index. + */ + filenameCompoundIndex?: string[] /** * Require files to be uploaded when creating a document. * @default true diff --git a/scripts/pack-all-to-dest.ts b/scripts/pack-all-to-dest.ts index 473ac014c..ce29d47cd 100644 --- a/scripts/pack-all-to-dest.ts +++ b/scripts/pack-all-to-dest.ts @@ -59,7 +59,7 @@ async function main() { ${chalk.white.bold(filtered.map((p) => p.name).join('\n'))} `) - //execSync('pnpm build:all --output-logs=errors-only', { stdio: 'inherit' }) + execSync('pnpm build:all --output-logs=errors-only', { stdio: 'inherit' }) header(`\n 📦 Packing all packages to ${dest}...`) diff --git a/test/uploads/config.ts b/test/uploads/config.ts index 0e901b3bf..aa8dbc9c4 100644 --- a/test/uploads/config.ts +++ b/test/uploads/config.ts @@ -95,6 +95,36 @@ export default buildConfigWithDefaults({ staticDir: path.resolve(dirname, './media-gif'), }, }, + { + slug: 'filename-compound-index', + fields: [ + { + name: 'alt', + type: 'text', + admin: { + description: 'Alt text to be used for compound index', + }, + }, + ], + upload: { + filenameCompoundIndex: ['filename', 'alt'], + imageSizes: [ + { + name: 'small', + formatOptions: { format: 'gif', options: { quality: 90 } }, + height: 100, + width: 100, + }, + { + name: 'large', + formatOptions: { format: 'gif', options: { quality: 90 } }, + height: 1000, + width: 1000, + }, + ], + mimeTypes: ['image/*'], + }, + }, { slug: 'no-image-sizes', fields: [], @@ -787,6 +817,14 @@ export default buildConfigWithDefaults({ imageWithoutPreview3: uploadedImageWithoutPreview, }, }) + + await payload.create({ + collection: 'filename-compound-index', + data: { + alt: 'alt-1', + }, + file: imageFile, + }) }, serverURL: undefined, upload: { diff --git a/test/uploads/payload-types.ts b/test/uploads/payload-types.ts index 0e2a35d71..bb6d26739 100644 --- a/test/uploads/payload-types.ts +++ b/test/uploads/payload-types.ts @@ -14,6 +14,7 @@ export interface Config { relation: Relation; audio: Audio; 'gif-resize': GifResize; + 'filename-compound-index': FilenameCompoundIndex; 'no-image-sizes': NoImageSize; 'object-fit': ObjectFit; 'with-meta-data': WithMetaDatum; @@ -46,7 +47,7 @@ export interface Config { 'payload-migrations': PayloadMigration; }; db: { - defaultIDType: string; + defaultIDType: number; }; globals: {}; locale: null; @@ -77,9 +78,9 @@ export interface UserAuthOperations { * via the `definition` "relation". */ export interface Relation { - id: string; - image?: string | Media | null; - versionedImage?: string | Version | null; + id: number; + image?: number | Media | null; + versionedImage?: number | Version | null; updatedAt: string; createdAt: string; } @@ -88,7 +89,7 @@ export interface Relation { * via the `definition` "media". */ export interface Media { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -228,7 +229,7 @@ export interface Media { * via the `definition` "versions". */ export interface Version { - id: string; + id: number; title?: string | null; updatedAt: string; createdAt: string; @@ -248,8 +249,8 @@ export interface Version { * via the `definition` "audio". */ export interface Audio { - id: string; - audio?: string | Media | null; + id: number; + audio?: number | Media | null; updatedAt: string; createdAt: string; } @@ -258,7 +259,44 @@ export interface Audio { * via the `definition` "gif-resize". */ export interface GifResize { - id: string; + id: number; + updatedAt: string; + createdAt: string; + url?: string | null; + thumbnailURL?: string | null; + filename?: string | null; + mimeType?: string | null; + filesize?: number | null; + width?: number | null; + height?: number | null; + focalX?: number | null; + focalY?: number | null; + sizes?: { + small?: { + url?: string | null; + width?: number | null; + height?: number | null; + mimeType?: string | null; + filesize?: number | null; + filename?: string | null; + }; + large?: { + url?: string | null; + width?: number | null; + height?: number | null; + mimeType?: string | null; + filesize?: number | null; + filename?: string | null; + }; + }; +} +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "filename-compound-index". + */ +export interface FilenameCompoundIndex { + id: number; + alt?: string | null; updatedAt: string; createdAt: string; url?: string | null; @@ -294,7 +332,7 @@ export interface GifResize { * via the `definition` "no-image-sizes". */ export interface NoImageSize { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -312,7 +350,7 @@ export interface NoImageSize { * via the `definition` "object-fit". */ export interface ObjectFit { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -364,7 +402,7 @@ export interface ObjectFit { * via the `definition` "with-meta-data". */ export interface WithMetaDatum { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -392,7 +430,7 @@ export interface WithMetaDatum { * via the `definition` "without-meta-data". */ export interface WithoutMetaDatum { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -420,7 +458,7 @@ export interface WithoutMetaDatum { * via the `definition` "with-only-jpeg-meta-data". */ export interface WithOnlyJpegMetaDatum { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -448,7 +486,7 @@ export interface WithOnlyJpegMetaDatum { * via the `definition` "crop-only". */ export interface CropOnly { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -492,7 +530,7 @@ export interface CropOnly { * via the `definition` "focal-only". */ export interface FocalOnly { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -536,7 +574,7 @@ export interface FocalOnly { * via the `definition` "focal-no-sizes". */ export interface FocalNoSize { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -554,7 +592,7 @@ export interface FocalNoSize { * via the `definition` "animated-type-media". */ export interface AnimatedTypeMedia { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -606,7 +644,7 @@ export interface AnimatedTypeMedia { * via the `definition` "enlarge". */ export interface Enlarge { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -666,7 +704,7 @@ export interface Enlarge { * via the `definition` "reduce". */ export interface Reduce { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -718,7 +756,7 @@ export interface Reduce { * via the `definition` "media-trim". */ export interface MediaTrim { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -762,7 +800,7 @@ export interface MediaTrim { * via the `definition` "custom-file-name-media". */ export interface CustomFileNameMedia { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -790,7 +828,7 @@ export interface CustomFileNameMedia { * via the `definition` "unstored-media". */ export interface UnstoredMedia { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -808,7 +846,7 @@ export interface UnstoredMedia { * via the `definition` "externally-served-media". */ export interface ExternallyServedMedia { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -826,8 +864,8 @@ export interface ExternallyServedMedia { * via the `definition` "uploads-1". */ export interface Uploads1 { - id: string; - media?: string | Uploads2 | null; + id: number; + media?: number | Uploads2 | null; richText?: { root: { type: string; @@ -860,7 +898,7 @@ export interface Uploads1 { * via the `definition` "uploads-2". */ export interface Uploads2 { - id: string; + id: number; title?: string | null; updatedAt: string; createdAt: string; @@ -879,7 +917,7 @@ export interface Uploads2 { * via the `definition` "admin-thumbnail-function". */ export interface AdminThumbnailFunction { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -897,7 +935,7 @@ export interface AdminThumbnailFunction { * via the `definition` "admin-thumbnail-size". */ export interface AdminThumbnailSize { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -933,7 +971,7 @@ export interface AdminThumbnailSize { * via the `definition` "optional-file". */ export interface OptionalFile { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -951,7 +989,7 @@ export interface OptionalFile { * via the `definition` "required-file". */ export interface RequiredFile { - id: string; + id: number; updatedAt: string; createdAt: string; url?: string | null; @@ -969,7 +1007,7 @@ export interface RequiredFile { * via the `definition` "custom-upload-field". */ export interface CustomUploadField { - id: string; + id: number; alt?: string | null; updatedAt: string; createdAt: string; @@ -988,7 +1026,7 @@ export interface CustomUploadField { * via the `definition` "media-with-relation-preview". */ export interface MediaWithRelationPreview { - id: string; + id: number; title?: string | null; updatedAt: string; createdAt: string; @@ -1007,7 +1045,7 @@ export interface MediaWithRelationPreview { * via the `definition` "media-without-relation-preview". */ export interface MediaWithoutRelationPreview { - id: string; + id: number; title?: string | null; updatedAt: string; createdAt: string; @@ -1026,13 +1064,13 @@ export interface MediaWithoutRelationPreview { * via the `definition` "relation-preview". */ export interface RelationPreview { - id: string; - imageWithPreview1?: string | MediaWithRelationPreview | null; - imageWithPreview2?: string | MediaWithRelationPreview | null; - imageWithoutPreview1?: string | MediaWithRelationPreview | null; - imageWithoutPreview2?: string | MediaWithoutRelationPreview | null; - imageWithPreview3?: string | MediaWithoutRelationPreview | null; - imageWithoutPreview3?: string | MediaWithoutRelationPreview | null; + id: number; + imageWithPreview1?: number | MediaWithRelationPreview | null; + imageWithPreview2?: number | MediaWithRelationPreview | null; + imageWithoutPreview1?: number | MediaWithRelationPreview | null; + imageWithoutPreview2?: number | MediaWithoutRelationPreview | null; + imageWithPreview3?: number | MediaWithoutRelationPreview | null; + imageWithoutPreview3?: number | MediaWithoutRelationPreview | null; updatedAt: string; createdAt: string; } @@ -1041,7 +1079,7 @@ export interface RelationPreview { * via the `definition` "users". */ export interface User { - id: string; + id: number; updatedAt: string; createdAt: string; email: string; @@ -1058,10 +1096,10 @@ export interface User { * via the `definition` "payload-preferences". */ export interface PayloadPreference { - id: string; + id: number; user: { relationTo: 'users'; - value: string | User; + value: number | User; }; key?: string | null; value?: @@ -1081,7 +1119,7 @@ export interface PayloadPreference { * via the `definition` "payload-migrations". */ export interface PayloadMigration { - id: string; + id: number; name?: string | null; batch?: number | null; updatedAt: string;