Files
payload/packages/db-mongodb/src/queries/sanitizeQueryValue.ts
Sasha e468292039 perf(db-mongodb): improve performance of all operations, up to 50% faster (#9594)
This PR improves speed and memory efficiency across all operations with
the Mongoose adapter.

### How?

- Removes Mongoose layer from all database calls, instead uses MongoDB
directly. (this doesn't remove building mongoose schema since it's still
needed for indexes + users in theory can use it)
- Replaces deep copying of read results using
`JSON.parse(JSON.stringify(data))` with the `transform` `operation:
'read'` function which converts Date's, ObjectID's in relationships /
joins to strings. As before, it also handles transformations for write
operations.
- Faster `hasNearConstraint` for potentially large `where`'s
- `traverseFields` now can accept `flattenedFields` which we use in
`transform`. Less recursive calls with tabs/rows/collapsible

Additional fixes
- Uses current transaction for querying nested relationships properties
in `buildQuery`, previously it wasn't used which could've led to wrong
results
- Allows to clear not required point fields with passing `null` from the
Local API. Previously it didn't work in both, MongoDB and Postgres

Benchmarks using this file
https://github.com/payloadcms/payload/blob/chore/db-benchmark/test/_community/int.spec.ts

### Small Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 1170 | 844 | 27.86% |
| `payload.db.create` (ms) | 1413 | 691 | 51.12% |
| `payload.db.find` (ms) | 2856 | 2204 | 22.83% |
| `payload.db.deleteMany` (ms) | 15206 | 8439 | 44.53% |
| `payload.db.updateOne` (ms) | 21444 | 12162 | 43.30% |
| `payload.db.findOne` (ms) | 159 | 112 | 29.56% |
| `payload.db.deleteOne` (ms) | 3729 | 2578 | 30.89% |
| DB small FULL (ms) | 64473 | 46451 | 27.93% |

---

### Medium Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 9407 | 6210 | 33.99% |
| `payload.db.create` (ms) | 10270 | 4321 | 57.93% |
| `payload.db.find` (ms) | 20814 | 16036 | 22.93% |
| `payload.db.deleteMany` (ms) | 126351 | 61789 | 51.11% |
| `payload.db.updateOne` (ms) | 201782 | 99943 | 50.49% |
| `payload.db.findOne` (ms) | 1081 | 817 | 24.43% |
| `payload.db.deleteOne` (ms) | 28534 | 23363 | 18.12% |
| DB medium FULL (ms) | 519518 | 342194 | 34.13% |

---

### Large Dataset Performance

| Metric | Before Optimization | After Optimization | Improvement (%) |

|---------------------------|---------------------|--------------------|-----------------|
| Average FULL (ms) | 26575 | 17509 | 34.14% |
| `payload.db.create` (ms) | 29085 | 12196 | 58.08% |
| `payload.db.find` (ms) | 58497 | 43838 | 25.04% |
| `payload.db.deleteMany` (ms) | 372195 | 173218 | 53.47% |
| `payload.db.updateOne` (ms) | 544089 | 288350 | 47.00% |
| `payload.db.findOne` (ms) | 3058 | 2197 | 28.14% |
| `payload.db.deleteOne` (ms) | 82444 | 64730 | 21.49% |
| DB large FULL (ms) | 1461097 | 969714 | 33.62% |
2024-12-19 13:20:39 -05:00

443 lines
12 KiB
TypeScript

import type { FlattenedBlock, FlattenedField, Payload, RelationshipField } from 'payload'
import { Types } from 'mongoose'
import { createArrayFromCommaDelineated } from 'payload'
type SanitizeQueryValueArgs = {
field: FlattenedField
hasCustomID: boolean
locale?: string
operator: string
path: string
payload: Payload
val: any
}
const buildExistsQuery = (formattedValue, path, treatEmptyString = true) => {
if (formattedValue) {
return {
rawQuery: {
$and: [
{ [path]: { $exists: true } },
{ [path]: { $ne: null } },
...(treatEmptyString ? [{ [path]: { $ne: '' } }] : []), // Treat empty string as null / undefined
],
},
}
} else {
return {
rawQuery: {
$or: [
{ [path]: { $exists: false } },
{ [path]: { $eq: null } },
...(treatEmptyString ? [{ [path]: { $eq: '' } }] : []), // Treat empty string as null / undefined
],
},
}
}
}
const sanitizeCoordinates = (coordinates: unknown[]): unknown[] => {
const result: unknown[] = []
for (const value of coordinates) {
if (typeof value === 'string') {
result.push(Number(value))
} else if (Array.isArray(value)) {
result.push(sanitizeCoordinates(value))
} else {
result.push(value)
}
}
return result
}
// returns nestedField Field object from blocks.nestedField path because getLocalizedPaths splits them only for relationships
const getFieldFromSegments = ({
field,
segments,
}: {
field: FlattenedBlock | FlattenedField
segments: string[]
}) => {
if ('blocks' in field) {
for (const block of field.blocks) {
const field = getFieldFromSegments({ field: block, segments })
if (field) {
return field
}
}
}
if ('fields' in field) {
for (let i = 0; i < segments.length; i++) {
const foundField = field.flattenedFields.find((each) => each.name === segments[i])
if (!foundField) {
break
}
if (foundField && segments.length - 1 === i) {
return foundField
}
segments.shift()
return getFieldFromSegments({ field: foundField, segments })
}
}
}
export const sanitizeQueryValue = ({
field,
hasCustomID,
locale,
operator,
path,
payload,
val,
}: SanitizeQueryValueArgs): {
operator?: string
rawQuery?: unknown
val?: unknown
} => {
let formattedValue = val
let formattedOperator = operator
if (['array', 'blocks', 'group', 'tab'].includes(field.type) && path.includes('.')) {
const segments = path.split('.')
segments.shift()
const foundField = getFieldFromSegments({ field, segments })
if (foundField) {
field = foundField
}
}
// Disregard invalid _ids
if (path === '_id') {
if (typeof val === 'string' && val.split(',').length === 1) {
if (!hasCustomID) {
const isValid = Types.ObjectId.isValid(val)
if (!isValid) {
return { operator: formattedOperator, val: undefined }
} else {
if (['in', 'not_in'].includes(operator)) {
formattedValue = createArrayFromCommaDelineated(formattedValue).map(
(id) => new Types.ObjectId(id),
)
} else {
formattedValue = new Types.ObjectId(val)
}
}
}
if (field.type === 'number') {
const parsedNumber = parseFloat(val)
if (Number.isNaN(parsedNumber)) {
return { operator: formattedOperator, val: undefined }
}
}
} else if (Array.isArray(val) || (typeof val === 'string' && val.split(',').length > 1)) {
if (typeof val === 'string') {
formattedValue = createArrayFromCommaDelineated(val)
}
formattedValue = formattedValue.reduce((formattedValues, inVal) => {
if (!hasCustomID) {
if (Types.ObjectId.isValid(inVal)) {
formattedValues.push(new Types.ObjectId(inVal))
}
}
if (field.type === 'number') {
const parsedNumber = parseFloat(inVal)
if (!Number.isNaN(parsedNumber)) {
formattedValues.push(parsedNumber)
}
} else {
formattedValues.push(inVal)
}
return formattedValues
}, [])
}
}
// Cast incoming values as proper searchable types
if (field.type === 'checkbox' && typeof val === 'string') {
if (val.toLowerCase() === 'true') {
formattedValue = true
}
if (val.toLowerCase() === 'false') {
formattedValue = false
}
}
if (['all', 'in', 'not_in'].includes(operator) && typeof formattedValue === 'string') {
formattedValue = createArrayFromCommaDelineated(formattedValue)
if (field.type === 'number') {
formattedValue = formattedValue.map((arrayVal) => parseFloat(arrayVal))
}
}
if (field.type === 'number') {
if (typeof formattedValue === 'string' && operator !== 'exists') {
formattedValue = Number(val)
}
if (operator === 'exists') {
formattedValue = val === 'true' ? true : val === 'false' ? false : Boolean(val)
return buildExistsQuery(formattedValue, path)
}
}
if (field.type === 'date' && typeof val === 'string' && operator !== 'exists') {
formattedValue = new Date(val)
if (Number.isNaN(Date.parse(formattedValue))) {
return undefined
}
}
if (['relationship', 'upload'].includes(field.type)) {
if (val === 'null') {
formattedValue = null
}
// Object equality requires the value to be the first key in the object that is being queried.
if (
operator === 'equals' &&
formattedValue &&
typeof formattedValue === 'object' &&
formattedValue.value &&
formattedValue.relationTo
) {
const { value } = formattedValue
const isValid = Types.ObjectId.isValid(value)
if (isValid) {
formattedValue.value = new Types.ObjectId(value)
}
let localizedPath = path
if (field.localized && payload.config.localization && locale) {
localizedPath = `${path}.${locale}`
}
return {
rawQuery: {
$or: [
{
[localizedPath]: {
$eq: {
// disable auto sort
/* eslint-disable */
value: formattedValue.value,
relationTo: formattedValue.relationTo,
/* eslint-enable */
},
},
},
{
[localizedPath]: {
$eq: {
relationTo: formattedValue.relationTo,
value: formattedValue.value,
},
},
},
],
},
}
}
const relationTo = (field as RelationshipField).relationTo
if (['in', 'not_in'].includes(operator) && Array.isArray(formattedValue)) {
formattedValue = formattedValue.reduce((formattedValues, inVal) => {
if (!inVal) {
return formattedValues
}
if (typeof relationTo === 'string' && payload.collections[relationTo].customIDType) {
if (payload.collections[relationTo].customIDType === 'number') {
const parsedNumber = parseFloat(inVal)
if (!Number.isNaN(parsedNumber)) {
formattedValues.push(parsedNumber)
return formattedValues
}
}
formattedValues.push(inVal)
return formattedValues
}
if (
Array.isArray(relationTo) &&
relationTo.some((relationTo) => !!payload.collections[relationTo].customIDType)
) {
if (Types.ObjectId.isValid(inVal.toString())) {
formattedValues.push(new Types.ObjectId(inVal))
} else {
formattedValues.push(inVal)
}
return formattedValues
}
if (Types.ObjectId.isValid(inVal.toString())) {
formattedValues.push(new Types.ObjectId(inVal))
}
return formattedValues
}, [])
}
if (
['contains', 'equals', 'like', 'not_equals'].includes(operator) &&
(!Array.isArray(relationTo) || !path.endsWith('.relationTo'))
) {
if (typeof relationTo === 'string') {
const customIDType = payload.collections[relationTo].customIDType
if (customIDType) {
if (customIDType === 'number') {
formattedValue = parseFloat(val)
if (Number.isNaN(formattedValue)) {
return { operator: formattedOperator, val: undefined }
}
}
} else {
if (!Types.ObjectId.isValid(formattedValue)) {
return { operator: formattedOperator, val: undefined }
}
formattedValue = new Types.ObjectId(formattedValue)
}
} else {
const hasCustomIDType = relationTo.some(
(relationTo) => !!payload.collections[relationTo].customIDType,
)
if (hasCustomIDType) {
if (typeof val === 'string') {
const formattedNumber = Number(val)
formattedValue = [Types.ObjectId.isValid(val) ? new Types.ObjectId(val) : val]
formattedOperator = operator === 'not_equals' ? 'not_in' : 'in'
if (!Number.isNaN(formattedNumber)) {
formattedValue.push(formattedNumber)
}
}
} else {
if (!Types.ObjectId.isValid(formattedValue)) {
return { operator: formattedOperator, val: undefined }
}
formattedValue = new Types.ObjectId(formattedValue)
}
}
}
}
// Set up specific formatting necessary by operators
if (operator === 'near') {
let lng
let lat
let maxDistance
let minDistance
if (Array.isArray(formattedValue)) {
;[lng, lat, maxDistance, minDistance] = formattedValue
}
if (typeof formattedValue === 'string') {
;[lng, lat, maxDistance, minDistance] = createArrayFromCommaDelineated(formattedValue)
}
if (lng == null || lat == null || (maxDistance == null && minDistance == null)) {
formattedValue = undefined
} else {
formattedValue = {
$geometry: { type: 'Point', coordinates: [parseFloat(lng), parseFloat(lat)] },
}
if (maxDistance) {
formattedValue.$maxDistance = parseFloat(maxDistance)
}
if (minDistance) {
formattedValue.$minDistance = parseFloat(minDistance)
}
}
}
if (operator === 'within' || operator === 'intersects') {
if (
formattedValue &&
typeof formattedValue === 'object' &&
Array.isArray(formattedValue.coordinates)
) {
formattedValue.coordinates = sanitizeCoordinates(formattedValue.coordinates)
}
formattedValue = {
$geometry: formattedValue,
}
}
if (path !== '_id' || (path === '_id' && hasCustomID && field.type === 'text')) {
if (operator === 'contains' && !Types.ObjectId.isValid(formattedValue)) {
formattedValue = {
$options: 'i',
$regex: formattedValue.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&'),
}
}
if (operator === 'exists') {
formattedValue = formattedValue === 'true' || formattedValue === true
// _id can't be empty string, will error Cast to ObjectId failed for value ""
return buildExistsQuery(
formattedValue,
path,
!['relationship', 'upload'].includes(field.type),
)
}
}
if (
(path === '_id' || path === 'parent') &&
operator === 'like' &&
formattedValue.length === 24 &&
!hasCustomID
) {
formattedOperator = 'equals'
}
if (operator === 'exists') {
formattedValue = formattedValue === 'true' || formattedValue === true
// Clearable fields
if (['relationship', 'select', 'upload'].includes(field.type)) {
if (formattedValue) {
return {
rawQuery: {
$and: [{ [path]: { $exists: true } }, { [path]: { $ne: null } }],
},
}
} else {
return {
rawQuery: {
$or: [{ [path]: { $exists: false } }, { [path]: { $eq: null } }],
},
}
}
}
}
return { operator: formattedOperator, val: formattedValue }
}