This PR improves speed and memory efficiency across all operations with the Mongoose adapter. ### How? - Removes Mongoose layer from all database calls, instead uses MongoDB directly. (this doesn't remove building mongoose schema since it's still needed for indexes + users in theory can use it) - Replaces deep copying of read results using `JSON.parse(JSON.stringify(data))` with the `transform` `operation: 'read'` function which converts Date's, ObjectID's in relationships / joins to strings. As before, it also handles transformations for write operations. - Faster `hasNearConstraint` for potentially large `where`'s - `traverseFields` now can accept `flattenedFields` which we use in `transform`. Less recursive calls with tabs/rows/collapsible Additional fixes - Uses current transaction for querying nested relationships properties in `buildQuery`, previously it wasn't used which could've led to wrong results - Allows to clear not required point fields with passing `null` from the Local API. Previously it didn't work in both, MongoDB and Postgres Benchmarks using this file https://github.com/payloadcms/payload/blob/chore/db-benchmark/test/_community/int.spec.ts ### Small Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 1170 | 844 | 27.86% | | `payload.db.create` (ms) | 1413 | 691 | 51.12% | | `payload.db.find` (ms) | 2856 | 2204 | 22.83% | | `payload.db.deleteMany` (ms) | 15206 | 8439 | 44.53% | | `payload.db.updateOne` (ms) | 21444 | 12162 | 43.30% | | `payload.db.findOne` (ms) | 159 | 112 | 29.56% | | `payload.db.deleteOne` (ms) | 3729 | 2578 | 30.89% | | DB small FULL (ms) | 64473 | 46451 | 27.93% | --- ### Medium Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 9407 | 6210 | 33.99% | | `payload.db.create` (ms) | 10270 | 4321 | 57.93% | | `payload.db.find` (ms) | 20814 | 16036 | 22.93% | | `payload.db.deleteMany` (ms) | 126351 | 61789 | 51.11% | | `payload.db.updateOne` (ms) | 201782 | 99943 | 50.49% | | `payload.db.findOne` (ms) | 1081 | 817 | 24.43% | | `payload.db.deleteOne` (ms) | 28534 | 23363 | 18.12% | | DB medium FULL (ms) | 519518 | 342194 | 34.13% | --- ### Large Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 26575 | 17509 | 34.14% | | `payload.db.create` (ms) | 29085 | 12196 | 58.08% | | `payload.db.find` (ms) | 58497 | 43838 | 25.04% | | `payload.db.deleteMany` (ms) | 372195 | 173218 | 53.47% | | `payload.db.updateOne` (ms) | 544089 | 288350 | 47.00% | | `payload.db.findOne` (ms) | 3058 | 2197 | 28.14% | | `payload.db.deleteOne` (ms) | 82444 | 64730 | 21.49% | | DB large FULL (ms) | 1461097 | 969714 | 33.62% |
368 lines
7.9 KiB
TypeScript
368 lines
7.9 KiB
TypeScript
import { flattenAllFields, type Field, type SanitizedConfig } from 'payload'
|
|
|
|
import { Types } from 'mongoose'
|
|
|
|
import { transform } from './transform.js'
|
|
import { MongooseAdapter } from '..'
|
|
|
|
const flattenRelationshipValues = (obj: Record<string, any>, prefix = ''): Record<string, any> => {
|
|
return Object.keys(obj).reduce(
|
|
(acc, key) => {
|
|
const fullKey = prefix ? `${prefix}.${key}` : key
|
|
const value = obj[key]
|
|
|
|
if (value && typeof value === 'object' && !(value instanceof Types.ObjectId)) {
|
|
Object.assign(acc, flattenRelationshipValues(value, fullKey))
|
|
// skip relationTo and blockType
|
|
} else if (!fullKey.endsWith('relationTo') && !fullKey.endsWith('blockType')) {
|
|
acc[fullKey] = value
|
|
}
|
|
|
|
return acc
|
|
},
|
|
{} as Record<string, any>,
|
|
)
|
|
}
|
|
|
|
const relsFields: Field[] = [
|
|
{
|
|
name: 'rel_1',
|
|
type: 'relationship',
|
|
relationTo: 'rels',
|
|
},
|
|
{
|
|
name: 'rel_1_l',
|
|
type: 'relationship',
|
|
localized: true,
|
|
relationTo: 'rels',
|
|
},
|
|
{
|
|
name: 'rel_2',
|
|
type: 'relationship',
|
|
hasMany: true,
|
|
relationTo: 'rels',
|
|
},
|
|
{
|
|
name: 'rel_2_l',
|
|
type: 'relationship',
|
|
hasMany: true,
|
|
localized: true,
|
|
relationTo: 'rels',
|
|
},
|
|
{
|
|
name: 'rel_3',
|
|
type: 'relationship',
|
|
relationTo: ['rels'],
|
|
},
|
|
{
|
|
name: 'rel_3_l',
|
|
type: 'relationship',
|
|
localized: true,
|
|
relationTo: ['rels'],
|
|
},
|
|
{
|
|
name: 'rel_4',
|
|
type: 'relationship',
|
|
hasMany: true,
|
|
relationTo: ['rels'],
|
|
},
|
|
{
|
|
name: 'rel_4_l',
|
|
type: 'relationship',
|
|
hasMany: true,
|
|
localized: true,
|
|
relationTo: ['rels'],
|
|
},
|
|
]
|
|
|
|
const config = {
|
|
collections: [
|
|
{
|
|
slug: 'docs',
|
|
fields: [
|
|
...relsFields,
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: [
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: relsFields,
|
|
},
|
|
{
|
|
name: 'blocks',
|
|
type: 'blocks',
|
|
blocks: [{ slug: 'block', fields: relsFields }],
|
|
},
|
|
...relsFields,
|
|
],
|
|
},
|
|
{
|
|
name: 'arrayLocalized',
|
|
type: 'array',
|
|
fields: [
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: relsFields,
|
|
},
|
|
{
|
|
name: 'blocks',
|
|
type: 'blocks',
|
|
blocks: [{ slug: 'block', fields: relsFields }],
|
|
},
|
|
...relsFields,
|
|
],
|
|
localized: true,
|
|
},
|
|
{
|
|
name: 'blocks',
|
|
type: 'blocks',
|
|
blocks: [
|
|
{
|
|
slug: 'block',
|
|
fields: [
|
|
...relsFields,
|
|
{
|
|
name: 'group',
|
|
type: 'group',
|
|
fields: relsFields,
|
|
},
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: relsFields,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
{
|
|
name: 'group',
|
|
type: 'group',
|
|
fields: [
|
|
...relsFields,
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: relsFields,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
name: 'groupLocalized',
|
|
type: 'group',
|
|
fields: [
|
|
...relsFields,
|
|
{
|
|
name: 'array',
|
|
type: 'array',
|
|
fields: relsFields,
|
|
},
|
|
],
|
|
localized: true,
|
|
},
|
|
{
|
|
name: 'groupAndRow',
|
|
type: 'group',
|
|
fields: [
|
|
{
|
|
type: 'row',
|
|
fields: [
|
|
...relsFields,
|
|
{
|
|
type: 'array',
|
|
name: 'array',
|
|
fields: relsFields,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
{
|
|
type: 'tabs',
|
|
tabs: [
|
|
{
|
|
name: 'tab',
|
|
fields: relsFields,
|
|
},
|
|
{
|
|
name: 'tabLocalized',
|
|
fields: relsFields,
|
|
localized: true,
|
|
},
|
|
{
|
|
label: 'another',
|
|
fields: [
|
|
{
|
|
type: 'tabs',
|
|
tabs: [
|
|
{
|
|
name: 'nestedTab',
|
|
fields: relsFields,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
{
|
|
slug: 'rels',
|
|
fields: [],
|
|
},
|
|
],
|
|
localization: {
|
|
defaultLocale: 'en',
|
|
localeCodes: ['en', 'es'],
|
|
locales: [
|
|
{ code: 'en', label: 'EN' },
|
|
{ code: 'es', label: 'ES' },
|
|
],
|
|
},
|
|
} as SanitizedConfig
|
|
|
|
const relsData = {
|
|
rel_1: new Types.ObjectId().toHexString(),
|
|
rel_1_l: {
|
|
en: new Types.ObjectId().toHexString(),
|
|
es: new Types.ObjectId().toHexString(),
|
|
},
|
|
rel_2: [new Types.ObjectId().toHexString()],
|
|
rel_2_l: {
|
|
en: [new Types.ObjectId().toHexString()],
|
|
es: [new Types.ObjectId().toHexString()],
|
|
},
|
|
rel_3: {
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
rel_3_l: {
|
|
en: {
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
es: {
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
},
|
|
rel_4: [
|
|
{
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
],
|
|
rel_4_l: {
|
|
en: [
|
|
{
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
],
|
|
es: [
|
|
{
|
|
relationTo: 'rels',
|
|
value: new Types.ObjectId().toHexString(),
|
|
},
|
|
],
|
|
},
|
|
}
|
|
|
|
describe('transform', () => {
|
|
it('should sanitize relationships with transform write', () => {
|
|
const data = {
|
|
...relsData,
|
|
array: [
|
|
{
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
blocks: [
|
|
{
|
|
blockType: 'block',
|
|
...relsData,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
arrayLocalized: {
|
|
en: [
|
|
{
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
blocks: [
|
|
{
|
|
blockType: 'block',
|
|
...relsData,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
es: [
|
|
{
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
blocks: [
|
|
{
|
|
blockType: 'block',
|
|
...relsData,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
blocks: [
|
|
{
|
|
blockType: 'block',
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
group: { ...relsData },
|
|
},
|
|
],
|
|
group: {
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
},
|
|
groupAndRow: {
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
},
|
|
groupLocalized: {
|
|
en: {
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
},
|
|
es: {
|
|
...relsData,
|
|
array: [{ ...relsData }],
|
|
},
|
|
},
|
|
tab: { ...relsData },
|
|
tabLocalized: {
|
|
en: { ...relsData },
|
|
es: { ...relsData },
|
|
},
|
|
nestedTab: { ...relsData },
|
|
}
|
|
const flattenValuesBefore = Object.values(flattenRelationshipValues(data))
|
|
|
|
const mockAdapter = { payload: { config } } as MongooseAdapter
|
|
|
|
const fields = flattenAllFields({ fields: config.collections[0].fields })
|
|
|
|
transform({ type: 'write', adapter: mockAdapter, data, fields })
|
|
|
|
const flattenValuesAfter = Object.values(flattenRelationshipValues(data))
|
|
|
|
flattenValuesAfter.forEach((value, i) => {
|
|
expect(value).toBeInstanceOf(Types.ObjectId)
|
|
expect(flattenValuesBefore[i]).toBe(value.toHexString())
|
|
})
|
|
|
|
transform({ type: 'read', adapter: mockAdapter, data, fields })
|
|
})
|
|
})
|