### What?
Improves querying performance of the Local API, reduces copying overhead
### How?
Adds `flattenedFields` property for sanitized collection/global config
which contains fields in database schema structure.
For example, It removes rows / collapsible / unnamed tabs and merges
them to the parent `fields`, ignores UI fields, named tabs are added as
`type: 'tab'`.
This simplifies code in places like Drizzle `transform/traverseFields`.
Also, now we can avoid calling `flattenTopLevelFields` which adds some
overhead and use `collection.flattenedFields` / `field.flattenedFields`.
By refactoring `configToJSONSchema.ts` with `flattenedFields` it also
1. Fixes https://github.com/payloadcms/payload/issues/9467
2. Fixes types for UI fields were generated for `select`
Removes this deep copying for each `where` query constraint
58ac784425/packages/payload/src/database/queryValidation/validateQueryPaths.ts (L69-L73)
which potentially can add overhead if you have a large collection/global
config
UPD:
The overhead is even much more than in the benchmark below if you have
Lexical with blocks.
Benchmark in `relationships/int.spec.ts`:
```ts
const now = Date.now()
for (let i = 0; i < 10; i++) {
const now = Date.now()
for (let i = 0; i < 300; i++) {
const query = await payload.find({
collection: 'chained',
where: {
'relation.relation.name': {
equals: 'third',
},
and: [
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
{
'relation.relation.name': {
equals: 'third',
},
},
],
},
})
}
payload.logger.info(`#${i + 1} ${Date.now() - now}`)
}
payload.logger.info(`Total ${Date.now() - now}`)
```
Before:
```
[02:11:48] INFO: #1 3682
[02:11:50] INFO: #2 2199
[02:11:54] INFO: #3 3483
[02:11:56] INFO: #4 2516
[02:11:59] INFO: #5 2467
[02:12:01] INFO: #6 1987
[02:12:03] INFO: #7 1986
[02:12:05] INFO: #8 2375
[02:12:07] INFO: #9 2040
[02:12:09] INFO: #10 1920
[PASS] Relationships > Querying > Nested Querying > should allow querying two levels deep (24667ms)
[02:12:09] INFO: Total 24657
```
After:
```
[02:12:36] INFO: #1 2113
[02:12:38] INFO: #2 1854
[02:12:40] INFO: #3 1700
[02:12:42] INFO: #4 1797
[02:12:44] INFO: #5 2121
[02:12:46] INFO: #6 1774
[02:12:47] INFO: #7 1670
[02:12:49] INFO: #8 1610
[02:12:50] INFO: #9 1596
[02:12:52] INFO: #10 1576
[PASS] Relationships > Querying > Nested Querying > should allow querying two levels deep (17828ms)
[02:12:52] INFO: Total 17818
```
545 lines
16 KiB
TypeScript
545 lines
16 KiB
TypeScript
import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
|
|
import type { Relation } from 'drizzle-orm'
|
|
import type {
|
|
AnySQLiteColumn,
|
|
ForeignKeyBuilder,
|
|
IndexBuilder,
|
|
SQLiteColumnBuilder,
|
|
SQLiteTableWithColumns,
|
|
UniqueConstraintBuilder,
|
|
} from 'drizzle-orm/sqlite-core'
|
|
import type { FlattenedField } from 'payload'
|
|
|
|
import { buildIndexName, createTableName } from '@payloadcms/drizzle'
|
|
import { relations, sql } from 'drizzle-orm'
|
|
import {
|
|
foreignKey,
|
|
index,
|
|
integer,
|
|
numeric,
|
|
sqliteTable,
|
|
text,
|
|
unique,
|
|
} from 'drizzle-orm/sqlite-core'
|
|
import toSnakeCase from 'to-snake-case'
|
|
|
|
import type { GenericColumns, GenericTable, IDType, SQLiteAdapter } from '../types.js'
|
|
|
|
import { createIndex } from './createIndex.js'
|
|
import { getIDColumn } from './getIDColumn.js'
|
|
import { setColumnID } from './setColumnID.js'
|
|
import { traverseFields } from './traverseFields.js'
|
|
|
|
export type BaseExtraConfig = Record<
|
|
string,
|
|
(cols: {
|
|
[x: string]: AnySQLiteColumn
|
|
}) => ForeignKeyBuilder | IndexBuilder | UniqueConstraintBuilder
|
|
>
|
|
|
|
export type RelationMap = Map<
|
|
string,
|
|
{
|
|
localized: boolean
|
|
relationName?: string
|
|
target: string
|
|
type: 'many' | 'one'
|
|
}
|
|
>
|
|
|
|
type Args = {
|
|
adapter: SQLiteAdapter
|
|
baseColumns?: Record<string, SQLiteColumnBuilder>
|
|
/**
|
|
* After table is created, run these functions to add extra config to the table
|
|
* ie. indexes, multiple columns, etc
|
|
*/
|
|
baseExtraConfig?: BaseExtraConfig
|
|
buildNumbers?: boolean
|
|
buildRelationships?: boolean
|
|
disableNotNull: boolean
|
|
disableRelsTableUnique?: boolean
|
|
disableUnique: boolean
|
|
fields: FlattenedField[]
|
|
locales?: [string, ...string[]]
|
|
rootRelationships?: Set<string>
|
|
rootRelationsToBuild?: RelationMap
|
|
rootTableIDColType?: IDType
|
|
rootTableName?: string
|
|
rootUniqueRelationships?: Set<string>
|
|
tableName: string
|
|
timestamps?: boolean
|
|
versions: boolean
|
|
/**
|
|
* Tracks whether or not this table is built
|
|
* from the result of a localized array or block field at some point
|
|
*/
|
|
withinLocalizedArrayOrBlock?: boolean
|
|
}
|
|
|
|
type Result = {
|
|
hasLocalizedManyNumberField: boolean
|
|
hasLocalizedManyTextField: boolean
|
|
hasLocalizedRelationshipField: boolean
|
|
hasManyNumberField: 'index' | boolean
|
|
hasManyTextField: 'index' | boolean
|
|
relationsToBuild: RelationMap
|
|
}
|
|
|
|
export const buildTable = ({
|
|
adapter,
|
|
baseColumns = {},
|
|
baseExtraConfig = {},
|
|
disableNotNull,
|
|
disableRelsTableUnique,
|
|
disableUnique = false,
|
|
fields,
|
|
locales,
|
|
rootRelationships,
|
|
rootRelationsToBuild,
|
|
rootTableIDColType,
|
|
rootTableName: incomingRootTableName,
|
|
rootUniqueRelationships,
|
|
tableName,
|
|
timestamps,
|
|
versions,
|
|
withinLocalizedArrayOrBlock,
|
|
}: Args): Result => {
|
|
const isRoot = !incomingRootTableName
|
|
const rootTableName = incomingRootTableName || tableName
|
|
const columns: Record<string, SQLiteColumnBuilder> = baseColumns
|
|
const indexes: Record<string, (cols: GenericColumns) => IndexBuilder> = {}
|
|
|
|
const localesColumns: Record<string, SQLiteColumnBuilder> = {}
|
|
const localesIndexes: Record<string, (cols: GenericColumns) => IndexBuilder> = {}
|
|
let localesTable: GenericTable | SQLiteTableWithColumns<any>
|
|
let textsTable: GenericTable | SQLiteTableWithColumns<any>
|
|
let numbersTable: GenericTable | SQLiteTableWithColumns<any>
|
|
|
|
// Relationships to the base collection
|
|
const relationships: Set<string> = rootRelationships || new Set()
|
|
const uniqueRelationships: Set<string> = rootUniqueRelationships || new Set()
|
|
|
|
let relationshipsTable: GenericTable | SQLiteTableWithColumns<any>
|
|
|
|
// Drizzle relations
|
|
const relationsToBuild: RelationMap = new Map()
|
|
|
|
const idColType: IDType = setColumnID({ columns, fields })
|
|
|
|
const {
|
|
hasLocalizedField,
|
|
hasLocalizedManyNumberField,
|
|
hasLocalizedManyTextField,
|
|
hasLocalizedRelationshipField,
|
|
hasManyNumberField,
|
|
hasManyTextField,
|
|
} = traverseFields({
|
|
adapter,
|
|
columns,
|
|
disableNotNull,
|
|
disableRelsTableUnique,
|
|
disableUnique,
|
|
fields,
|
|
indexes,
|
|
locales,
|
|
localesColumns,
|
|
localesIndexes,
|
|
newTableName: tableName,
|
|
parentTableName: tableName,
|
|
relationships,
|
|
relationsToBuild,
|
|
rootRelationsToBuild: rootRelationsToBuild || relationsToBuild,
|
|
rootTableIDColType: rootTableIDColType || idColType,
|
|
rootTableName,
|
|
uniqueRelationships,
|
|
versions,
|
|
withinLocalizedArrayOrBlock,
|
|
})
|
|
|
|
// split the relationsToBuild by localized and non-localized
|
|
const localizedRelations = new Map()
|
|
const nonLocalizedRelations = new Map()
|
|
|
|
relationsToBuild.forEach(({ type, localized, relationName, target }, key) => {
|
|
const map = localized ? localizedRelations : nonLocalizedRelations
|
|
map.set(key, { type, relationName, target })
|
|
})
|
|
|
|
if (timestamps) {
|
|
columns.createdAt = text('created_at')
|
|
.default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`)
|
|
.notNull()
|
|
columns.updatedAt = text('updated_at')
|
|
.default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`)
|
|
.notNull()
|
|
}
|
|
|
|
const table = sqliteTable(tableName, columns, (cols) => {
|
|
const extraConfig = Object.entries(baseExtraConfig).reduce((config, [key, func]) => {
|
|
config[key] = func(cols)
|
|
return config
|
|
}, {})
|
|
|
|
const result = Object.entries(indexes).reduce((acc, [colName, func]) => {
|
|
acc[colName] = func(cols)
|
|
return acc
|
|
}, extraConfig)
|
|
|
|
return result
|
|
})
|
|
|
|
adapter.tables[tableName] = table
|
|
|
|
if (hasLocalizedField || localizedRelations.size) {
|
|
const localeTableName = `${tableName}${adapter.localesSuffix}`
|
|
localesColumns.id = integer('id', { mode: 'number' }).primaryKey({ autoIncrement: true })
|
|
localesColumns._locale = text('_locale', { enum: locales }).notNull()
|
|
localesColumns._parentID = getIDColumn({
|
|
name: '_parent_id',
|
|
type: idColType,
|
|
notNull: true,
|
|
primaryKey: false,
|
|
})
|
|
|
|
localesTable = sqliteTable(localeTableName, localesColumns, (cols) => {
|
|
return Object.entries(localesIndexes).reduce(
|
|
(acc, [colName, func]) => {
|
|
acc[colName] = func(cols)
|
|
return acc
|
|
},
|
|
{
|
|
_localeParent: unique(`${localeTableName}_locale_parent_id_unique`).on(
|
|
cols._locale,
|
|
cols._parentID,
|
|
),
|
|
_parentIdFk: foreignKey({
|
|
name: `${localeTableName}_parent_id_fk`,
|
|
columns: [cols._parentID],
|
|
foreignColumns: [table.id],
|
|
}).onDelete('cascade'),
|
|
},
|
|
)
|
|
})
|
|
|
|
adapter.tables[localeTableName] = localesTable
|
|
|
|
adapter.relations[`relations_${localeTableName}`] = relations(localesTable, ({ many, one }) => {
|
|
const result: Record<string, Relation<string>> = {}
|
|
|
|
result._parentID = one(table, {
|
|
fields: [localesTable._parentID],
|
|
references: [table.id],
|
|
// name the relationship by what the many() relationName is
|
|
relationName: '_locales',
|
|
})
|
|
|
|
localizedRelations.forEach(({ type, target }, key) => {
|
|
if (type === 'one') {
|
|
result[key] = one(adapter.tables[target], {
|
|
fields: [localesTable[key]],
|
|
references: [adapter.tables[target].id],
|
|
relationName: key,
|
|
})
|
|
}
|
|
if (type === 'many') {
|
|
result[key] = many(adapter.tables[target], {
|
|
relationName: key,
|
|
})
|
|
}
|
|
})
|
|
|
|
return result
|
|
})
|
|
}
|
|
|
|
if (isRoot) {
|
|
if (hasManyTextField) {
|
|
const textsTableName = `${rootTableName}_texts`
|
|
const columns: Record<string, SQLiteColumnBuilder> = {
|
|
id: integer('id', { mode: 'number' }).primaryKey({ autoIncrement: true }),
|
|
order: integer('order').notNull(),
|
|
parent: getIDColumn({
|
|
name: 'parent_id',
|
|
type: idColType,
|
|
notNull: true,
|
|
primaryKey: false,
|
|
}),
|
|
path: text('path').notNull(),
|
|
text: text('text'),
|
|
}
|
|
|
|
if (hasLocalizedManyTextField) {
|
|
columns.locale = text('locale', { enum: locales })
|
|
}
|
|
|
|
textsTable = sqliteTable(textsTableName, columns, (cols) => {
|
|
const config: Record<string, ForeignKeyBuilder | IndexBuilder> = {
|
|
orderParentIdx: index(`${textsTableName}_order_parent_idx`).on(cols.order, cols.parent),
|
|
parentFk: foreignKey({
|
|
name: `${textsTableName}_parent_fk`,
|
|
columns: [cols.parent],
|
|
foreignColumns: [table.id],
|
|
}).onDelete('cascade'),
|
|
}
|
|
|
|
if (hasManyTextField === 'index') {
|
|
config.text_idx = index(`${textsTableName}_text_idx`).on(cols.text)
|
|
}
|
|
|
|
if (hasLocalizedManyTextField) {
|
|
config.localeParent = index(`${textsTableName}_locale_parent`).on(
|
|
cols.locale,
|
|
cols.parent,
|
|
)
|
|
}
|
|
|
|
return config
|
|
})
|
|
|
|
adapter.tables[textsTableName] = textsTable
|
|
|
|
adapter.relations[`relations_${textsTableName}`] = relations(textsTable, ({ one }) => ({
|
|
parent: one(table, {
|
|
fields: [textsTable.parent],
|
|
references: [table.id],
|
|
relationName: '_texts',
|
|
}),
|
|
}))
|
|
}
|
|
|
|
if (hasManyNumberField) {
|
|
const numbersTableName = `${rootTableName}_numbers`
|
|
const columns: Record<string, SQLiteColumnBuilder> = {
|
|
id: integer('id', { mode: 'number' }).primaryKey({ autoIncrement: true }),
|
|
number: numeric('number'),
|
|
order: integer('order').notNull(),
|
|
parent: getIDColumn({
|
|
name: 'parent_id',
|
|
type: idColType,
|
|
notNull: true,
|
|
primaryKey: false,
|
|
}),
|
|
path: text('path').notNull(),
|
|
}
|
|
|
|
if (hasLocalizedManyNumberField) {
|
|
columns.locale = text('locale', { enum: locales })
|
|
}
|
|
|
|
numbersTable = sqliteTable(numbersTableName, columns, (cols) => {
|
|
const config: Record<string, ForeignKeyBuilder | IndexBuilder> = {
|
|
orderParentIdx: index(`${numbersTableName}_order_parent_idx`).on(cols.order, cols.parent),
|
|
parentFk: foreignKey({
|
|
name: `${numbersTableName}_parent_fk`,
|
|
columns: [cols.parent],
|
|
foreignColumns: [table.id],
|
|
}).onDelete('cascade'),
|
|
}
|
|
|
|
if (hasManyNumberField === 'index') {
|
|
config.numberIdx = index(`${numbersTableName}_number_idx`).on(cols.number)
|
|
}
|
|
|
|
if (hasLocalizedManyNumberField) {
|
|
config.localeParent = index(`${numbersTableName}_locale_parent`).on(
|
|
cols.locale,
|
|
cols.parent,
|
|
)
|
|
}
|
|
|
|
return config
|
|
})
|
|
|
|
adapter.tables[numbersTableName] = numbersTable
|
|
|
|
adapter.relations[`relations_${numbersTableName}`] = relations(numbersTable, ({ one }) => ({
|
|
parent: one(table, {
|
|
fields: [numbersTable.parent],
|
|
references: [table.id],
|
|
relationName: '_numbers',
|
|
}),
|
|
}))
|
|
}
|
|
|
|
if (relationships.size) {
|
|
const relationshipColumns: Record<string, SQLiteColumnBuilder> = {
|
|
id: integer('id', { mode: 'number' }).primaryKey({ autoIncrement: true }),
|
|
order: integer('order'),
|
|
parent: getIDColumn({
|
|
name: 'parent_id',
|
|
type: idColType,
|
|
notNull: true,
|
|
primaryKey: false,
|
|
}),
|
|
path: text('path').notNull(),
|
|
}
|
|
|
|
if (hasLocalizedRelationshipField) {
|
|
relationshipColumns.locale = text('locale', { enum: locales })
|
|
}
|
|
|
|
const relationExtraConfig: BaseExtraConfig = {}
|
|
const relationshipsTableName = `${tableName}${adapter.relationshipsSuffix}`
|
|
|
|
relationships.forEach((relationTo) => {
|
|
const relationshipConfig = adapter.payload.collections[relationTo].config
|
|
const formattedRelationTo = createTableName({
|
|
adapter,
|
|
config: relationshipConfig,
|
|
})
|
|
let colType: IDType = 'integer'
|
|
const relatedCollectionCustomIDType =
|
|
adapter.payload.collections[relationshipConfig.slug]?.customIDType
|
|
|
|
if (relatedCollectionCustomIDType === 'number') {
|
|
colType = 'numeric'
|
|
}
|
|
if (relatedCollectionCustomIDType === 'text') {
|
|
colType = 'text'
|
|
}
|
|
|
|
const colName = `${relationTo}ID`
|
|
|
|
relationshipColumns[colName] = getIDColumn({
|
|
name: `${formattedRelationTo}_id`,
|
|
type: colType,
|
|
primaryKey: false,
|
|
})
|
|
|
|
relationExtraConfig[`${relationTo}IdFk`] = (cols) =>
|
|
foreignKey({
|
|
name: `${relationshipsTableName}_${toSnakeCase(relationTo)}_fk`,
|
|
columns: [cols[colName]],
|
|
foreignColumns: [adapter.tables[formattedRelationTo].id],
|
|
}).onDelete('cascade')
|
|
|
|
const indexColumns = [colName]
|
|
|
|
const unique = !disableUnique && uniqueRelationships.has(relationTo)
|
|
|
|
if (unique) {
|
|
indexColumns.push('path')
|
|
}
|
|
if (hasLocalizedRelationshipField) {
|
|
indexColumns.push('locale')
|
|
}
|
|
|
|
const indexName = buildIndexName({
|
|
name: `${relationshipsTableName}_${formattedRelationTo}_id`,
|
|
adapter: adapter as unknown as DrizzleAdapter,
|
|
})
|
|
|
|
relationExtraConfig[indexName] = createIndex({
|
|
name: indexColumns,
|
|
indexName,
|
|
unique,
|
|
})
|
|
})
|
|
|
|
relationshipsTable = sqliteTable(relationshipsTableName, relationshipColumns, (cols) => {
|
|
const result: Record<string, ForeignKeyBuilder | IndexBuilder> = Object.entries(
|
|
relationExtraConfig,
|
|
).reduce(
|
|
(config, [key, func]) => {
|
|
config[key] = func(cols)
|
|
return config
|
|
},
|
|
{
|
|
order: index(`${relationshipsTableName}_order_idx`).on(cols.order),
|
|
parentFk: foreignKey({
|
|
name: `${relationshipsTableName}_parent_fk`,
|
|
columns: [cols.parent],
|
|
foreignColumns: [table.id],
|
|
}).onDelete('cascade'),
|
|
parentIdx: index(`${relationshipsTableName}_parent_idx`).on(cols.parent),
|
|
pathIdx: index(`${relationshipsTableName}_path_idx`).on(cols.path),
|
|
},
|
|
)
|
|
|
|
if (hasLocalizedRelationshipField) {
|
|
result.localeIdx = index(`${relationshipsTableName}_locale_idx`).on(cols.locale)
|
|
}
|
|
|
|
return result
|
|
})
|
|
|
|
adapter.tables[relationshipsTableName] = relationshipsTable
|
|
|
|
adapter.relations[`relations_${relationshipsTableName}`] = relations(
|
|
relationshipsTable,
|
|
({ one }) => {
|
|
const result: Record<string, Relation<string>> = {
|
|
parent: one(table, {
|
|
fields: [relationshipsTable.parent],
|
|
references: [table.id],
|
|
relationName: '_rels',
|
|
}),
|
|
}
|
|
|
|
relationships.forEach((relationTo) => {
|
|
const relatedTableName = createTableName({
|
|
adapter,
|
|
config: adapter.payload.collections[relationTo].config,
|
|
})
|
|
const idColumnName = `${relationTo}ID`
|
|
result[idColumnName] = one(adapter.tables[relatedTableName], {
|
|
fields: [relationshipsTable[idColumnName]],
|
|
references: [adapter.tables[relatedTableName].id],
|
|
relationName: relationTo,
|
|
})
|
|
})
|
|
|
|
return result
|
|
},
|
|
)
|
|
}
|
|
}
|
|
|
|
adapter.relations[`relations_${tableName}`] = relations(table, ({ many, one }) => {
|
|
const result: Record<string, Relation<string>> = {}
|
|
|
|
nonLocalizedRelations.forEach(({ type, relationName, target }, key) => {
|
|
if (type === 'one') {
|
|
result[key] = one(adapter.tables[target], {
|
|
fields: [table[key]],
|
|
references: [adapter.tables[target].id],
|
|
relationName: key,
|
|
})
|
|
}
|
|
if (type === 'many') {
|
|
result[key] = many(adapter.tables[target], { relationName: relationName || key })
|
|
}
|
|
})
|
|
|
|
if (hasLocalizedField) {
|
|
result._locales = many(localesTable, { relationName: '_locales' })
|
|
}
|
|
|
|
if (hasManyTextField) {
|
|
result._texts = many(textsTable, { relationName: '_texts' })
|
|
}
|
|
|
|
if (hasManyNumberField) {
|
|
result._numbers = many(numbersTable, { relationName: '_numbers' })
|
|
}
|
|
|
|
if (relationships.size && relationshipsTable) {
|
|
result._rels = many(relationshipsTable, {
|
|
relationName: '_rels',
|
|
})
|
|
}
|
|
|
|
return result
|
|
})
|
|
|
|
return {
|
|
hasLocalizedManyNumberField,
|
|
hasLocalizedManyTextField,
|
|
hasLocalizedRelationshipField,
|
|
hasManyNumberField,
|
|
hasManyTextField,
|
|
relationsToBuild,
|
|
}
|
|
}
|