chore(db-postgres): schema building implements collection indexes (#3429)

packages/db-postgres/src/init.ts

@@ -14,11 +14,7 @@ export const init: Init = async function init(this: PostgresAdapter) {
   if (this.payload.config.localization) {
     this.enums.enum__locales = pgEnum(
       '_locales',
-      // TODO: types out of sync with core, monorepo please
+      this.payload.config.localization.locales.map(({ code }) => code) as [string, ...string[]],
-      // this.payload.config.localization.localeCodes,
-      (this.payload.config.localization.locales as unknown as { code: string }[]).map(
-        ({ code }) => code,
-      ) as [string, ...string[]],
     )
   }
 
@@ -28,6 +24,7 @@ export const init: Init = async function init(this: PostgresAdapter) {
     buildTable({
       adapter: this,
       buildRelationships: true,
+      collectionIndexes: collection.indexes,
       disableUnique: false,
       fields: collection.fields,
       tableName,

packages/db-postgres/src/schema/build.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-param-reassign */
 import type { Relation } from 'drizzle-orm'
 import type { IndexBuilder, PgColumnBuilder, UniqueConstraintBuilder } from 'drizzle-orm/pg-core'
-import type { Field } from 'payload/types'
+import type { Field, SanitizedCollectionConfig } from 'payload/types'
 
 import { relations } from 'drizzle-orm'
 import {
@@ -27,6 +27,7 @@ type Args = {
   baseColumns?: Record<string, PgColumnBuilder>
   baseExtraConfig?: Record<string, (cols: GenericColumns) => IndexBuilder | UniqueConstraintBuilder>
   buildRelationships?: boolean
+  collectionIndexes?: SanitizedCollectionConfig['indexes']
   disableUnique: boolean
   fields: Field[]
   rootRelationsToBuild?: Map<string, string>
@@ -45,6 +46,7 @@ export const buildTable = ({
   baseColumns = {},
   baseExtraConfig = {},
   buildRelationships,
+  collectionIndexes = [],
   disableUnique = false,
   fields,
   rootRelationsToBuild,
@@ -96,6 +98,7 @@ export const buildTable = ({
   } = traverseFields({
     adapter,
     buildRelationships,
+    collectionIndexes,
     columns,
     disableUnique,
     fields,

packages/db-postgres/src/schema/createIndex.ts

@@ -5,13 +5,22 @@ import type { GenericColumn } from '../types'
 
 type CreateIndexArgs = {
   columnName: string
-  name: string
+  name: string | string[]
   unique?: boolean
 }
 
 export const createIndex = ({ name, columnName, unique }: CreateIndexArgs) => {
   return (table: { [x: string]: GenericColumn }) => {
-    if (unique) return uniqueIndex(`${columnName}_idx`).on(table[name])
+    let columns
-    return index(`${columnName}_idx`).on(table[name])
+    if (Array.isArray(name)) {
+      columns = name
+        .map((columnName) => table[columnName])
+        // exclude fields were included in compound indexes but do not exist on the table
+        .filter((col) => typeof col !== 'undefined')
+    } else {
+      columns = [table[name]]
+    }
+    if (unique) return uniqueIndex(`${columnName}_idx`).on(columns[0], ...columns.slice(1))
+    return index(`${columnName}_idx`).on(columns[0], ...columns.slice(1))
   }
 }

packages/db-postgres/src/schema/traverseFields.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-param-reassign */
 import type { Relation } from 'drizzle-orm'
 import type { IndexBuilder, PgColumnBuilder, UniqueConstraintBuilder } from 'drizzle-orm/pg-core'
-import type { Field, TabAsField } from 'payload/types'
+import type { Field, SanitizedCollectionConfig, TabAsField } from 'payload/types'
 
 import { relations } from 'drizzle-orm'
 import {
@@ -33,6 +33,7 @@ import { validateExistingBlockIsIdentical } from './validateExistingBlockIsIdent
 type Args = {
   adapter: PostgresAdapter
   buildRelationships: boolean
+  collectionIndexes: SanitizedCollectionConfig['indexes']
   columnPrefix?: string
   columns: Record<string, PgColumnBuilder>
   disableUnique?: boolean
@@ -61,6 +62,7 @@ type Result = {
 export const traverseFields = ({
   adapter,
   buildRelationships,
+  collectionIndexes,
   columnPrefix,
   columns,
   disableUnique = false,
@@ -109,6 +111,24 @@ export const traverseFields = ({
         targetIndexes = localesIndexes
       }
 
+      const collectionIndex = collectionIndexes
+        ? collectionIndexes.findIndex((index) => {
+            return Object.keys(index.fields).some((indexField) => indexField === fieldName)
+          })
+        : -1
+
+      if (collectionIndex > -1) {
+        const name = toSnakeCase(
+          `${Object.keys(collectionIndexes[collectionIndex].fields).join('_')}`,
+        )
+        targetIndexes[`${name}Idx`] = createIndex({
+          name: Object.keys(collectionIndexes[collectionIndex].fields),
+          columnName: name,
+          unique: collectionIndexes[collectionIndex].options.unique,
+        })
+        collectionIndexes.splice(collectionIndex)
+      }
+
       if (
         (field.unique || field.index) &&
         !['array', 'blocks', 'group', 'point', 'relationship', 'upload'].includes(field.type) &&
@@ -415,6 +435,7 @@ export const traverseFields = ({
           } = traverseFields({
             adapter,
             buildRelationships,
+            collectionIndexes,
             columnPrefix,
             columns,
             disableUnique,
@@ -448,6 +469,7 @@ export const traverseFields = ({
         } = traverseFields({
           adapter,
           buildRelationships,
+          collectionIndexes,
           columnPrefix: `${columnName}_`,
           columns,
           disableUnique,
@@ -482,6 +504,7 @@ export const traverseFields = ({
         } = traverseFields({
           adapter,
           buildRelationships,
+          collectionIndexes,
           columnPrefix,
           columns,
           disableUnique,
@@ -518,6 +541,7 @@ export const traverseFields = ({
         } = traverseFields({
           adapter,
           buildRelationships,
+          collectionIndexes,
           columnPrefix,
           columns,
           disableUnique,

test/fields/int.spec.ts

@@ -41,7 +41,7 @@ describe('Fields', () => {
     ;({ serverURL } = await initPayloadTest({ __dirname, init: { local: false } }))
     config = await configPromise
 
-    client = new RESTClient(config, { serverURL, defaultSlug: 'point-fields' })
+    client = new RESTClient(config, { defaultSlug: 'point-fields', serverURL })
     const graphQLURL = `${serverURL}${config.routes.api}${config.routes.graphQL}`
     graphQLClient = new GraphQLClient(graphQLURL)
     token = await client.login()
@@ -64,7 +64,7 @@ describe('Fields', () => {
     })
 
     it('should populate default values in beforeValidate hook', async () => {
-      const { fieldWithDefaultValue, dependentOnFieldWithDefaultValue } = await payload.create({
+      const { dependentOnFieldWithDefaultValue, fieldWithDefaultValue } = await payload.create({
         collection: 'text-fields',
         data: { text },
       })
@@ -89,10 +89,6 @@ describe('Fields', () => {
         depth: 0,
         where: {
           updatedAt: {
-            // TODO:
-            //  drizzle is not adjusting for timezones
-            //  tenMinutesAgo: "2023-08-29T15:49:39.897Z" UTC
-            //  doc.updatedAt: "2023-08-29T11:59:43.738Z" GMT -4
             greater_than_equal: tenMinutesAgo,
           },
         },
@@ -121,15 +117,15 @@ describe('Fields', () => {
     beforeAll(async () => {
       const { id } = await payload.create({
         collection: 'select-fields',
-        locale: 'en',
         data: {
           selectHasManyLocalized: ['one', 'two'],
         },
+        locale: 'en',
       })
       doc = await payload.findByID({
+        id,
         collection: 'select-fields',
         locale: 'all',
-        id,
       })
     })
 
@@ -146,8 +142,8 @@ describe('Fields', () => {
       })
 
       const updatedDoc = await payload.update({
-        collection: 'select-fields',
         id,
+        collection: 'select-fields',
         data: {
           select: 'one',
         },
@@ -245,15 +241,15 @@ describe('Fields', () => {
       const localizedHasMany = [5, 10]
       const { id } = await payload.create({
         collection: 'number-fields',
-        locale: 'en',
         data: {
           localizedHasMany,
         },
+        locale: 'en',
       })
       const localizedDoc = await payload.findByID({
+        id,
         collection: 'number-fields',
         locale: 'all',
-        id,
       })
 
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -284,81 +280,44 @@ describe('Fields', () => {
       it('should have indexes', () => {
         expect(definitions.text).toEqual(1)
       })
+
       it('should have unique indexes', () => {
         expect(definitions.uniqueText).toEqual(1)
         expect(options.uniqueText).toMatchObject({ unique: true })
       })
-      it('should have 2dsphere indexes on point fields', () => {
+
-        expect(definitions.point).toEqual('2dsphere')
-      })
-      it('should have 2dsphere indexes on point fields in groups', () => {
-        expect(definitions['group.point']).toEqual('2dsphere')
-      })
-      it('should have a sparse index on a unique localized field in a group', () => {
-        expect(definitions['group.localizedUnique.en']).toEqual(1)
-        expect(options['group.localizedUnique.en']).toMatchObject({ unique: true, sparse: true })
-        expect(definitions['group.localizedUnique.es']).toEqual(1)
-        expect(options['group.localizedUnique.es']).toMatchObject({ unique: true, sparse: true })
-      })
-      it('should have unique indexes in a collapsible', () => {
-        expect(definitions['collapsibleLocalizedUnique.en']).toEqual(1)
-        expect(options['collapsibleLocalizedUnique.en']).toMatchObject({
-          unique: true,
-          sparse: true,
-        })
-        expect(definitions.collapsibleTextUnique).toEqual(1)
-        expect(options.collapsibleTextUnique).toMatchObject({ unique: true })
-      })
       it('should have unique compound indexes', () => {
         expect(definitions.partOne).toEqual(1)
         expect(options.partOne).toMatchObject({
-          unique: true,
           name: 'compound-index',
           sparse: true,
+          unique: true,
         })
       })
-      it('should throw validation error saving on unique fields', async () => {
+
-        const data = {
+      it('should have 2dsphere indexes on point fields', () => {
-          text: 'a',
+        expect(definitions.point).toEqual('2dsphere')
-          uniqueText: 'a',
-        }
-        await payload.create({
-          collection: 'indexed-fields',
-          data,
-        })
-        expect(async () => {
-          const result = await payload.create({
-            collection: 'indexed-fields',
-            data,
-          })
-          return result.error
-        }).toBeDefined()
       })
-      it('should throw validation error saving on unique combined fields', async () => {
+
-        await payload.delete({ collection: 'indexed-fields', where: {} })
+      it('should have 2dsphere indexes on point fields in groups', () => {
-        const data1 = {
+        expect(definitions['group.point']).toEqual('2dsphere')
-          text: 'a',
+      })
-          uniqueText: 'a',
+
-          partOne: 'u',
+      it('should have a sparse index on a unique localized field in a group', () => {
-          partTwo: 'u',
+        expect(definitions['group.localizedUnique.en']).toEqual(1)
-        }
+        expect(options['group.localizedUnique.en']).toMatchObject({ sparse: true, unique: true })
-        const data2 = {
+        expect(definitions['group.localizedUnique.es']).toEqual(1)
-          text: 'b',
+        expect(options['group.localizedUnique.es']).toMatchObject({ sparse: true, unique: true })
-          uniqueText: 'b',
+      })
-          partOne: 'u',
+
-          partTwo: 'u',
+      it('should have unique indexes in a collapsible', () => {
-        }
+        expect(definitions['collapsibleLocalizedUnique.en']).toEqual(1)
-        await payload.create({
+        expect(options['collapsibleLocalizedUnique.en']).toMatchObject({
-          collection: 'indexed-fields',
+          sparse: true,
-          data: data1,
+          unique: true,
         })
-        expect(async () => {
+        expect(definitions.collapsibleTextUnique).toEqual(1)
-          const result = await payload.create({
+        expect(options.collapsibleTextUnique).toMatchObject({ unique: true })
-            collection: 'indexed-fields',
-            data: data2,
-          })
-          return result.error
-        }).toBeDefined()
       })
     })
 
@@ -386,9 +345,9 @@ describe('Fields', () => {
       it('should have version indexes from collection indexes', () => {
         expect(definitions['version.partOne']).toEqual(1)
         expect(options['version.partOne']).toMatchObject({
-          unique: true,
           name: 'compound-index',
           sparse: true,
+          unique: true,
         })
       })
     })
@@ -424,9 +383,9 @@ describe('Fields', () => {
         doc = await payload.create({
           collection: 'point-fields',
           data: {
-            point,
-            localized,
             group,
+            localized,
+            point,
           },
         })
 
@@ -440,9 +399,9 @@ describe('Fields', () => {
           payload.create({
             collection: 'point-fields',
             data: {
-              point,
-              localized,
               group,
+              localized,
+              point,
             },
           }),
         ).rejects.toThrow(Error)
@@ -463,6 +422,52 @@ describe('Fields', () => {
     })
   }
 
+  describe('unique indexes', () => {
+    it('should throw validation error saving on unique fields', async () => {
+      const data = {
+        text: 'a',
+        uniqueText: 'a',
+      }
+      await payload.create({
+        collection: 'indexed-fields',
+        data,
+      })
+      expect(async () => {
+        const result = await payload.create({
+          collection: 'indexed-fields',
+          data,
+        })
+        return result.error
+      }).toBeDefined()
+    })
+    it('should throw validation error saving on unique combined fields', async () => {
+      await payload.delete({ collection: 'indexed-fields', where: {} })
+      const data1 = {
+        partOne: 'u',
+        partTwo: 'u',
+        text: 'a',
+        uniqueText: 'a',
+      }
+      const data2 = {
+        partOne: 'u',
+        partTwo: 'u',
+        text: 'b',
+        uniqueText: 'b',
+      }
+      await payload.create({
+        collection: 'indexed-fields',
+        data: data1,
+      })
+      expect(async () => {
+        const result = await payload.create({
+          collection: 'indexed-fields',
+          data: data2,
+        })
+        return result.error
+      }).toBeDefined()
+    })
+  })
+
   describe('array', () => {
     let doc
     const collection = arrayFieldsSlug
@@ -499,26 +504,26 @@ describe('Fields', () => {
       })
 
       const enDoc = await payload.update({
-        collection,
         id,
-        locale: 'en',
+        collection,
         data: {
           localized: [{ text: enText }],
         },
+        locale: 'en',
       })
 
       const esDoc = await payload.update({
-        collection,
         id,
-        locale: 'es',
+        collection,
         data: {
           localized: [{ text: esText }],
         },
+        locale: 'es',
       })
 
       const allLocales = (await payload.findByID({
-        collection,
         id,
+        collection,
         locale: 'all',
       })) as unknown as { localized: { en: unknown; es: unknown } }
 
@@ -569,8 +574,8 @@ describe('Fields', () => {
 
     it('should create with localized text inside a named tab', async () => {
       document = await payload.findByID({
-        collection: tabsSlug,
         id: document.id,
+        collection: tabsSlug,
         locale: 'all',
       })
       expect(document.localizedTab.en.text).toStrictEqual(localizedTextValue)
@@ -578,8 +583,8 @@ describe('Fields', () => {
 
     it('should allow access control on a named tab', async () => {
       document = await payload.findByID({
-        collection: tabsSlug,
         id: document.id,
+        collection: tabsSlug,
         overrideAccess: false,
       })
       expect(document.accessControlTab).toBeUndefined()
@@ -735,8 +740,8 @@ describe('Fields', () => {
       expect(jsonFieldsDoc.json.state).toEqual({})
 
       const updatedJsonFieldsDoc = await payload.update({
-        collection: 'json-fields',
         id: jsonFieldsDoc.id,
+        collection: 'json-fields',
         data: {
           json: {
             state: {},
@@ -800,8 +805,8 @@ describe('Fields', () => {
       expect(nodes).toBeDefined()
       const child = nodes.flatMap((n) => n.children).find((c) => c.doc)
       expect(child).toMatchObject({
-        type: 'link',
         linkType: 'internal',
+        type: 'link',
       })
       expect(child.doc.relationTo).toEqual('array-fields')