feat: filename compound index (#7651)

Allow a compound index to be used for upload collections via a `filenameCompoundIndex` field. Previously, `filename` was always treated as unique. Usage: ```ts { slug: 'upload-field', upload: { // Slugs to include in compound index filenameCompoundIndex: ['filename', 'alt'], }, } ```
2024-08-13 13:55:10 -04:00
parent 6c0f99082b
commit 5fc9f76406
11 changed files with 186 additions and 48 deletions
--- a/docs/upload/overview.mdx
+++ b/docs/upload/overview.mdx
@@ -97,6 +97,7 @@ _An asterisk denotes that an option is required._
 | **`displayPreview`**           | Enable displaying preview of the uploaded file in Upload fields related to this Collection. Can be locally overridden by `displayPreview` option in Upload field. [More](/docs/fields/upload#config-options).                         |
 | **`externalFileHeaderFilter`** | Accepts existing headers and returns the headers after filtering or modifying.                                                                                                                                                        |
 | **`filesRequiredOnCreate`**    | Mandate file data on creation, default is true.                                                                                                                                                                                       |
+| **`filenameCompoundIndex`**    | Field slugs to use for a compount index instead of the default filename index.
 | **`focalPoint`**               | Set to `false` to disable the focal point selection tool in the [Admin Panel](../admin/overview). The focal point selector is only available when `imageSizes` or `resizeOptions` are defined. [More](#crop-and-focal-point-selector) |
 | **`formatOptions`**            | An object with `format` and `options` that are used with the Sharp image library to format the upload file. [More](https://sharp.pixelplumbing.com/api-output#toformat)                                                               |
 | **`handlers`**                 | Array of Request handlers to execute when fetching a file, if a handler returns a Response it will be sent to the client. Otherwise Payload will retrieve and send back the file.                                                     |
--- a/packages/db-mongodb/src/models/buildCollectionSchema.ts
+++ b/packages/db-mongodb/src/models/buildCollectionSchema.ts
@@ -21,6 +21,18 @@ const buildCollectionSchema = (
    },
  })

+  if (Array.isArray(collection.upload.filenameCompoundIndex)) {
+    const indexDefinition: Record<string, 1> = collection.upload.filenameCompoundIndex.reduce(
+      (acc, index) => {
+        acc[index] = 1
+        return acc
+      },
+      {},
+    )
+
+    schema.index(indexDefinition, { unique: true })
+  }
+
  if (config.indexSortableFields && collection.timestamps !== false) {
    schema.index({ updatedAt: 1 })
    schema.index({ createdAt: 1 })
--- a/packages/db-postgres/src/init.ts
+++ b/packages/db-postgres/src/init.ts
@@ -1,9 +1,11 @@
 import type { Init, SanitizedCollectionConfig } from 'payload'

 import { createTableName } from '@payloadcms/drizzle'
+import { uniqueIndex } from 'drizzle-orm/pg-core'
 import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload'
 import toSnakeCase from 'to-snake-case'

+import type { BaseExtraConfig } from './schema/build.js'
 import type { PostgresAdapter } from './types.js'

 import { buildTable } from './schema/build.js'
@@ -34,8 +36,22 @@ export const init: Init = function init(this: PostgresAdapter) {
  this.payload.config.collections.forEach((collection: SanitizedCollectionConfig) => {
    const tableName = this.tableNameMap.get(toSnakeCase(collection.slug))

+    const baseExtraConfig: BaseExtraConfig = {}
+
+    if (collection.upload.filenameCompoundIndex) {
+      const indexName = `${tableName}_filename_compound_idx`
+
+      baseExtraConfig.filename_compound_index = (cols) => {
+        const colsConstraint = collection.upload.filenameCompoundIndex.map((f) => {
+          return cols[f]
+        })
+        return uniqueIndex(indexName).on(colsConstraint[0], ...colsConstraint.slice(1))
+      }
+    }
+
    buildTable({
      adapter: this,
+      baseExtraConfig,
      disableNotNull: !!collection?.versions?.drafts,
      disableUnique: false,
      fields: collection.fields,
--- a/packages/db-postgres/src/schema/build.ts
+++ b/packages/db-postgres/src/schema/build.ts
@@ -38,6 +38,10 @@ export type RelationMap = Map<string, { localized: boolean; target: string; type
 type Args = {
  adapter: PostgresAdapter
  baseColumns?: Record<string, PgColumnBuilder>
+  /**
+   * After table is created, run these functions to add extra config to the table
+   * ie. indexes, multiple columns, etc
+   */
  baseExtraConfig?: BaseExtraConfig
  buildNumbers?: boolean
  buildRelationships?: boolean
--- a/packages/db-sqlite/src/init.ts
+++ b/packages/db-sqlite/src/init.ts
@@ -1,11 +1,12 @@
-/* eslint-disable no-param-reassign */
 import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
 import type { Init, SanitizedCollectionConfig } from 'payload'

 import { createTableName } from '@payloadcms/drizzle'
+import { uniqueIndex } from 'drizzle-orm/sqlite-core'
 import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload'
 import toSnakeCase from 'to-snake-case'

+import type { BaseExtraConfig } from './schema/build.js'
 import type { SQLiteAdapter } from './types.js'

 import { buildTable } from './schema/build.js'
@@ -37,6 +38,19 @@ export const init: Init = function init(this: SQLiteAdapter) {
  this.payload.config.collections.forEach((collection: SanitizedCollectionConfig) => {
    const tableName = this.tableNameMap.get(toSnakeCase(collection.slug))

+    const baseExtraConfig: BaseExtraConfig = {}
+
+    if (collection.upload.filenameCompoundIndex) {
+      const indexName = `${tableName}_filename_compound_idx`
+
+      baseExtraConfig.filename_compound_index = (cols) => {
+        const colsConstraint = collection.upload.filenameCompoundIndex.map((f) => {
+          return cols[f]
+        })
+        return uniqueIndex(indexName).on(colsConstraint[0], ...colsConstraint.slice(1))
+      }
+    }
+
    buildTable({
      adapter: this,
      disableNotNull: !!collection?.versions?.drafts,
--- a/packages/db-sqlite/src/schema/build.ts
+++ b/packages/db-sqlite/src/schema/build.ts
@@ -40,6 +40,10 @@ export type RelationMap = Map<string, { localized: boolean; target: string; type
 type Args = {
  adapter: SQLiteAdapter
  baseColumns?: Record<string, SQLiteColumnBuilder>
+  /**
+   * After table is created, run these functions to add extra config to the table
+   * ie. indexes, multiple columns, etc
+   */
  baseExtraConfig?: BaseExtraConfig
  buildNumbers?: boolean
  buildRelationships?: boolean
--- a/packages/payload/src/uploads/getBaseFields.ts
+++ b/packages/payload/src/uploads/getBaseFields.ts
@@ -111,7 +111,14 @@ export const getBaseUploadFields = ({ collection, config }: Options): Field[] =>
    },
    index: true,
    label: ({ t }) => t('upload:fileName'),
-    unique: true,
+  }
+
+  // Only set unique: true if the collection does not have a compound index
+  if (
+    collection.upload === true ||
+    (typeof collection.upload === 'object' && !collection.upload.filenameCompoundIndex)
+  ) {
+    filename.unique = true
  }

  const url: Field = {
--- a/packages/payload/src/uploads/types.ts
+++ b/packages/payload/src/uploads/types.ts
@@ -118,6 +118,10 @@ export type UploadConfig = {
   * @default undefined
   */
  externalFileHeaderFilter?: (headers: Record<string, string>) => Record<string, string>
+  /**
+   * Field slugs to use for a compount index instead of the default filename index.
+   */
+  filenameCompoundIndex?: string[]
  /**
   * Require files to be uploaded when creating a document.
   * @default true
--- a/scripts/pack-all-to-dest.ts
+++ b/scripts/pack-all-to-dest.ts
@@ -59,7 +59,7 @@ async function main() {
 ${chalk.white.bold(filtered.map((p) => p.name).join('\n'))}
 `)

-  //execSync('pnpm build:all --output-logs=errors-only', { stdio: 'inherit' })
+  execSync('pnpm build:all --output-logs=errors-only', { stdio: 'inherit' })

  header(`\n 📦 Packing all packages to ${dest}...`)

--- a/test/uploads/config.ts
+++ b/test/uploads/config.ts
@@ -95,6 +95,36 @@ export default buildConfigWithDefaults({
        staticDir: path.resolve(dirname, './media-gif'),
      },
    },
+    {
+      slug: 'filename-compound-index',
+      fields: [
+        {
+          name: 'alt',
+          type: 'text',
+          admin: {
+            description: 'Alt text to be used for compound index',
+          },
+        },
+      ],
+      upload: {
+        filenameCompoundIndex: ['filename', 'alt'],
+        imageSizes: [
+          {
+            name: 'small',
+            formatOptions: { format: 'gif', options: { quality: 90 } },
+            height: 100,
+            width: 100,
+          },
+          {
+            name: 'large',
+            formatOptions: { format: 'gif', options: { quality: 90 } },
+            height: 1000,
+            width: 1000,
+          },
+        ],
+        mimeTypes: ['image/*'],
+      },
+    },
    {
      slug: 'no-image-sizes',
      fields: [],
@@ -787,6 +817,14 @@ export default buildConfigWithDefaults({
        imageWithoutPreview3: uploadedImageWithoutPreview,
      },
    })
+
+    await payload.create({
+      collection: 'filename-compound-index',
+      data: {
+        alt: 'alt-1',
+      },
+      file: imageFile,
+    })
  },
  serverURL: undefined,
  upload: {
--- a/test/uploads/payload-types.ts
+++ b/test/uploads/payload-types.ts
@@ -14,6 +14,7 @@ export interface Config {
    relation: Relation;
    audio: Audio;
    'gif-resize': GifResize;
+    'filename-compound-index': FilenameCompoundIndex;
    'no-image-sizes': NoImageSize;
    'object-fit': ObjectFit;
    'with-meta-data': WithMetaDatum;
@@ -46,7 +47,7 @@ export interface Config {
    'payload-migrations': PayloadMigration;
  };
  db: {
-    defaultIDType: string;
+    defaultIDType: number;
  };
  globals: {};
  locale: null;
@@ -77,9 +78,9 @@ export interface UserAuthOperations {
 * via the `definition` "relation".
 */
 export interface Relation {
-  id: string;
-  image?: string | Media | null;
-  versionedImage?: string | Version | null;
+  id: number;
+  image?: number | Media | null;
+  versionedImage?: number | Version | null;
  updatedAt: string;
  createdAt: string;
 }
@@ -88,7 +89,7 @@ export interface Relation {
 * via the `definition` "media".
 */
 export interface Media {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -228,7 +229,7 @@ export interface Media {
 * via the `definition` "versions".
 */
 export interface Version {
-  id: string;
+  id: number;
  title?: string | null;
  updatedAt: string;
  createdAt: string;
@@ -248,8 +249,8 @@ export interface Version {
 * via the `definition` "audio".
 */
 export interface Audio {
-  id: string;
-  audio?: string | Media | null;
+  id: number;
+  audio?: number | Media | null;
  updatedAt: string;
  createdAt: string;
 }
@@ -258,7 +259,44 @@ export interface Audio {
 * via the `definition` "gif-resize".
 */
 export interface GifResize {
-  id: string;
+  id: number;
+  updatedAt: string;
+  createdAt: string;
+  url?: string | null;
+  thumbnailURL?: string | null;
+  filename?: string | null;
+  mimeType?: string | null;
+  filesize?: number | null;
+  width?: number | null;
+  height?: number | null;
+  focalX?: number | null;
+  focalY?: number | null;
+  sizes?: {
+    small?: {
+      url?: string | null;
+      width?: number | null;
+      height?: number | null;
+      mimeType?: string | null;
+      filesize?: number | null;
+      filename?: string | null;
+    };
+    large?: {
+      url?: string | null;
+      width?: number | null;
+      height?: number | null;
+      mimeType?: string | null;
+      filesize?: number | null;
+      filename?: string | null;
+    };
+  };
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "filename-compound-index".
+ */
+export interface FilenameCompoundIndex {
+  id: number;
+  alt?: string | null;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -294,7 +332,7 @@ export interface GifResize {
 * via the `definition` "no-image-sizes".
 */
 export interface NoImageSize {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -312,7 +350,7 @@ export interface NoImageSize {
 * via the `definition` "object-fit".
 */
 export interface ObjectFit {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -364,7 +402,7 @@ export interface ObjectFit {
 * via the `definition` "with-meta-data".
 */
 export interface WithMetaDatum {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -392,7 +430,7 @@ export interface WithMetaDatum {
 * via the `definition` "without-meta-data".
 */
 export interface WithoutMetaDatum {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -420,7 +458,7 @@ export interface WithoutMetaDatum {
 * via the `definition` "with-only-jpeg-meta-data".
 */
 export interface WithOnlyJpegMetaDatum {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -448,7 +486,7 @@ export interface WithOnlyJpegMetaDatum {
 * via the `definition` "crop-only".
 */
 export interface CropOnly {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -492,7 +530,7 @@ export interface CropOnly {
 * via the `definition` "focal-only".
 */
 export interface FocalOnly {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -536,7 +574,7 @@ export interface FocalOnly {
 * via the `definition` "focal-no-sizes".
 */
 export interface FocalNoSize {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -554,7 +592,7 @@ export interface FocalNoSize {
 * via the `definition` "animated-type-media".
 */
 export interface AnimatedTypeMedia {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -606,7 +644,7 @@ export interface AnimatedTypeMedia {
 * via the `definition` "enlarge".
 */
 export interface Enlarge {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -666,7 +704,7 @@ export interface Enlarge {
 * via the `definition` "reduce".
 */
 export interface Reduce {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -718,7 +756,7 @@ export interface Reduce {
 * via the `definition` "media-trim".
 */
 export interface MediaTrim {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -762,7 +800,7 @@ export interface MediaTrim {
 * via the `definition` "custom-file-name-media".
 */
 export interface CustomFileNameMedia {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -790,7 +828,7 @@ export interface CustomFileNameMedia {
 * via the `definition` "unstored-media".
 */
 export interface UnstoredMedia {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -808,7 +846,7 @@ export interface UnstoredMedia {
 * via the `definition` "externally-served-media".
 */
 export interface ExternallyServedMedia {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -826,8 +864,8 @@ export interface ExternallyServedMedia {
 * via the `definition` "uploads-1".
 */
 export interface Uploads1 {
-  id: string;
-  media?: string | Uploads2 | null;
+  id: number;
+  media?: number | Uploads2 | null;
  richText?: {
    root: {
      type: string;
@@ -860,7 +898,7 @@ export interface Uploads1 {
 * via the `definition` "uploads-2".
 */
 export interface Uploads2 {
-  id: string;
+  id: number;
  title?: string | null;
  updatedAt: string;
  createdAt: string;
@@ -879,7 +917,7 @@ export interface Uploads2 {
 * via the `definition` "admin-thumbnail-function".
 */
 export interface AdminThumbnailFunction {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -897,7 +935,7 @@ export interface AdminThumbnailFunction {
 * via the `definition` "admin-thumbnail-size".
 */
 export interface AdminThumbnailSize {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -933,7 +971,7 @@ export interface AdminThumbnailSize {
 * via the `definition` "optional-file".
 */
 export interface OptionalFile {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -951,7 +989,7 @@ export interface OptionalFile {
 * via the `definition` "required-file".
 */
 export interface RequiredFile {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -969,7 +1007,7 @@ export interface RequiredFile {
 * via the `definition` "custom-upload-field".
 */
 export interface CustomUploadField {
-  id: string;
+  id: number;
  alt?: string | null;
  updatedAt: string;
  createdAt: string;
@@ -988,7 +1026,7 @@ export interface CustomUploadField {
 * via the `definition` "media-with-relation-preview".
 */
 export interface MediaWithRelationPreview {
-  id: string;
+  id: number;
  title?: string | null;
  updatedAt: string;
  createdAt: string;
@@ -1007,7 +1045,7 @@ export interface MediaWithRelationPreview {
 * via the `definition` "media-without-relation-preview".
 */
 export interface MediaWithoutRelationPreview {
-  id: string;
+  id: number;
  title?: string | null;
  updatedAt: string;
  createdAt: string;
@@ -1026,13 +1064,13 @@ export interface MediaWithoutRelationPreview {
 * via the `definition` "relation-preview".
 */
 export interface RelationPreview {
-  id: string;
-  imageWithPreview1?: string | MediaWithRelationPreview | null;
-  imageWithPreview2?: string | MediaWithRelationPreview | null;
-  imageWithoutPreview1?: string | MediaWithRelationPreview | null;
-  imageWithoutPreview2?: string | MediaWithoutRelationPreview | null;
-  imageWithPreview3?: string | MediaWithoutRelationPreview | null;
-  imageWithoutPreview3?: string | MediaWithoutRelationPreview | null;
+  id: number;
+  imageWithPreview1?: number | MediaWithRelationPreview | null;
+  imageWithPreview2?: number | MediaWithRelationPreview | null;
+  imageWithoutPreview1?: number | MediaWithRelationPreview | null;
+  imageWithoutPreview2?: number | MediaWithoutRelationPreview | null;
+  imageWithPreview3?: number | MediaWithoutRelationPreview | null;
+  imageWithoutPreview3?: number | MediaWithoutRelationPreview | null;
  updatedAt: string;
  createdAt: string;
 }
@@ -1041,7 +1079,7 @@ export interface RelationPreview {
 * via the `definition` "users".
 */
 export interface User {
-  id: string;
+  id: number;
  updatedAt: string;
  createdAt: string;
  email: string;
@@ -1058,10 +1096,10 @@ export interface User {
 * via the `definition` "payload-preferences".
 */
 export interface PayloadPreference {
-  id: string;
+  id: number;
  user: {
    relationTo: 'users';
-    value: string | User;
+    value: number | User;
  };
  key?: string | null;
  value?:
@@ -1081,7 +1119,7 @@ export interface PayloadPreference {
 * via the `definition` "payload-migrations".
 */
 export interface PayloadMigration {
-  id: string;
+  id: number;
  name?: string | null;
  batch?: number | null;
  updatedAt: string;