This PR improves speed and memory efficiency across all operations with the Mongoose adapter. ### How? - Removes Mongoose layer from all database calls, instead uses MongoDB directly. (this doesn't remove building mongoose schema since it's still needed for indexes + users in theory can use it) - Replaces deep copying of read results using `JSON.parse(JSON.stringify(data))` with the `transform` `operation: 'read'` function which converts Date's, ObjectID's in relationships / joins to strings. As before, it also handles transformations for write operations. - Faster `hasNearConstraint` for potentially large `where`'s - `traverseFields` now can accept `flattenedFields` which we use in `transform`. Less recursive calls with tabs/rows/collapsible Additional fixes - Uses current transaction for querying nested relationships properties in `buildQuery`, previously it wasn't used which could've led to wrong results - Allows to clear not required point fields with passing `null` from the Local API. Previously it didn't work in both, MongoDB and Postgres Benchmarks using this file https://github.com/payloadcms/payload/blob/chore/db-benchmark/test/_community/int.spec.ts ### Small Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 1170 | 844 | 27.86% | | `payload.db.create` (ms) | 1413 | 691 | 51.12% | | `payload.db.find` (ms) | 2856 | 2204 | 22.83% | | `payload.db.deleteMany` (ms) | 15206 | 8439 | 44.53% | | `payload.db.updateOne` (ms) | 21444 | 12162 | 43.30% | | `payload.db.findOne` (ms) | 159 | 112 | 29.56% | | `payload.db.deleteOne` (ms) | 3729 | 2578 | 30.89% | | DB small FULL (ms) | 64473 | 46451 | 27.93% | --- ### Medium Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 9407 | 6210 | 33.99% | | `payload.db.create` (ms) | 10270 | 4321 | 57.93% | | `payload.db.find` (ms) | 20814 | 16036 | 22.93% | | `payload.db.deleteMany` (ms) | 126351 | 61789 | 51.11% | | `payload.db.updateOne` (ms) | 201782 | 99943 | 50.49% | | `payload.db.findOne` (ms) | 1081 | 817 | 24.43% | | `payload.db.deleteOne` (ms) | 28534 | 23363 | 18.12% | | DB medium FULL (ms) | 519518 | 342194 | 34.13% | --- ### Large Dataset Performance | Metric | Before Optimization | After Optimization | Improvement (%) | |---------------------------|---------------------|--------------------|-----------------| | Average FULL (ms) | 26575 | 17509 | 34.14% | | `payload.db.create` (ms) | 29085 | 12196 | 58.08% | | `payload.db.find` (ms) | 58497 | 43838 | 25.04% | | `payload.db.deleteMany` (ms) | 372195 | 173218 | 53.47% | | `payload.db.updateOne` (ms) | 544089 | 288350 | 47.00% | | `payload.db.findOne` (ms) | 3058 | 2197 | 28.14% | | `payload.db.deleteOne` (ms) | 82444 | 64730 | 21.49% | | DB large FULL (ms) | 1461097 | 969714 | 33.62% |
443 lines
12 KiB
TypeScript
443 lines
12 KiB
TypeScript
import type { FlattenedBlock, FlattenedField, Payload, RelationshipField } from 'payload'
|
|
|
|
import { Types } from 'mongoose'
|
|
import { createArrayFromCommaDelineated } from 'payload'
|
|
|
|
type SanitizeQueryValueArgs = {
|
|
field: FlattenedField
|
|
hasCustomID: boolean
|
|
locale?: string
|
|
operator: string
|
|
path: string
|
|
payload: Payload
|
|
val: any
|
|
}
|
|
|
|
const buildExistsQuery = (formattedValue, path, treatEmptyString = true) => {
|
|
if (formattedValue) {
|
|
return {
|
|
rawQuery: {
|
|
$and: [
|
|
{ [path]: { $exists: true } },
|
|
{ [path]: { $ne: null } },
|
|
...(treatEmptyString ? [{ [path]: { $ne: '' } }] : []), // Treat empty string as null / undefined
|
|
],
|
|
},
|
|
}
|
|
} else {
|
|
return {
|
|
rawQuery: {
|
|
$or: [
|
|
{ [path]: { $exists: false } },
|
|
{ [path]: { $eq: null } },
|
|
...(treatEmptyString ? [{ [path]: { $eq: '' } }] : []), // Treat empty string as null / undefined
|
|
],
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
const sanitizeCoordinates = (coordinates: unknown[]): unknown[] => {
|
|
const result: unknown[] = []
|
|
|
|
for (const value of coordinates) {
|
|
if (typeof value === 'string') {
|
|
result.push(Number(value))
|
|
} else if (Array.isArray(value)) {
|
|
result.push(sanitizeCoordinates(value))
|
|
} else {
|
|
result.push(value)
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// returns nestedField Field object from blocks.nestedField path because getLocalizedPaths splits them only for relationships
|
|
const getFieldFromSegments = ({
|
|
field,
|
|
segments,
|
|
}: {
|
|
field: FlattenedBlock | FlattenedField
|
|
segments: string[]
|
|
}) => {
|
|
if ('blocks' in field) {
|
|
for (const block of field.blocks) {
|
|
const field = getFieldFromSegments({ field: block, segments })
|
|
if (field) {
|
|
return field
|
|
}
|
|
}
|
|
}
|
|
|
|
if ('fields' in field) {
|
|
for (let i = 0; i < segments.length; i++) {
|
|
const foundField = field.flattenedFields.find((each) => each.name === segments[i])
|
|
|
|
if (!foundField) {
|
|
break
|
|
}
|
|
|
|
if (foundField && segments.length - 1 === i) {
|
|
return foundField
|
|
}
|
|
|
|
segments.shift()
|
|
return getFieldFromSegments({ field: foundField, segments })
|
|
}
|
|
}
|
|
}
|
|
|
|
export const sanitizeQueryValue = ({
|
|
field,
|
|
hasCustomID,
|
|
locale,
|
|
operator,
|
|
path,
|
|
payload,
|
|
val,
|
|
}: SanitizeQueryValueArgs): {
|
|
operator?: string
|
|
rawQuery?: unknown
|
|
val?: unknown
|
|
} => {
|
|
let formattedValue = val
|
|
let formattedOperator = operator
|
|
|
|
if (['array', 'blocks', 'group', 'tab'].includes(field.type) && path.includes('.')) {
|
|
const segments = path.split('.')
|
|
segments.shift()
|
|
const foundField = getFieldFromSegments({ field, segments })
|
|
|
|
if (foundField) {
|
|
field = foundField
|
|
}
|
|
}
|
|
|
|
// Disregard invalid _ids
|
|
if (path === '_id') {
|
|
if (typeof val === 'string' && val.split(',').length === 1) {
|
|
if (!hasCustomID) {
|
|
const isValid = Types.ObjectId.isValid(val)
|
|
|
|
if (!isValid) {
|
|
return { operator: formattedOperator, val: undefined }
|
|
} else {
|
|
if (['in', 'not_in'].includes(operator)) {
|
|
formattedValue = createArrayFromCommaDelineated(formattedValue).map(
|
|
(id) => new Types.ObjectId(id),
|
|
)
|
|
} else {
|
|
formattedValue = new Types.ObjectId(val)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (field.type === 'number') {
|
|
const parsedNumber = parseFloat(val)
|
|
|
|
if (Number.isNaN(parsedNumber)) {
|
|
return { operator: formattedOperator, val: undefined }
|
|
}
|
|
}
|
|
} else if (Array.isArray(val) || (typeof val === 'string' && val.split(',').length > 1)) {
|
|
if (typeof val === 'string') {
|
|
formattedValue = createArrayFromCommaDelineated(val)
|
|
}
|
|
|
|
formattedValue = formattedValue.reduce((formattedValues, inVal) => {
|
|
if (!hasCustomID) {
|
|
if (Types.ObjectId.isValid(inVal)) {
|
|
formattedValues.push(new Types.ObjectId(inVal))
|
|
}
|
|
}
|
|
|
|
if (field.type === 'number') {
|
|
const parsedNumber = parseFloat(inVal)
|
|
if (!Number.isNaN(parsedNumber)) {
|
|
formattedValues.push(parsedNumber)
|
|
}
|
|
} else {
|
|
formattedValues.push(inVal)
|
|
}
|
|
|
|
return formattedValues
|
|
}, [])
|
|
}
|
|
}
|
|
|
|
// Cast incoming values as proper searchable types
|
|
if (field.type === 'checkbox' && typeof val === 'string') {
|
|
if (val.toLowerCase() === 'true') {
|
|
formattedValue = true
|
|
}
|
|
if (val.toLowerCase() === 'false') {
|
|
formattedValue = false
|
|
}
|
|
}
|
|
|
|
if (['all', 'in', 'not_in'].includes(operator) && typeof formattedValue === 'string') {
|
|
formattedValue = createArrayFromCommaDelineated(formattedValue)
|
|
|
|
if (field.type === 'number') {
|
|
formattedValue = formattedValue.map((arrayVal) => parseFloat(arrayVal))
|
|
}
|
|
}
|
|
|
|
if (field.type === 'number') {
|
|
if (typeof formattedValue === 'string' && operator !== 'exists') {
|
|
formattedValue = Number(val)
|
|
}
|
|
|
|
if (operator === 'exists') {
|
|
formattedValue = val === 'true' ? true : val === 'false' ? false : Boolean(val)
|
|
|
|
return buildExistsQuery(formattedValue, path)
|
|
}
|
|
}
|
|
|
|
if (field.type === 'date' && typeof val === 'string' && operator !== 'exists') {
|
|
formattedValue = new Date(val)
|
|
if (Number.isNaN(Date.parse(formattedValue))) {
|
|
return undefined
|
|
}
|
|
}
|
|
|
|
if (['relationship', 'upload'].includes(field.type)) {
|
|
if (val === 'null') {
|
|
formattedValue = null
|
|
}
|
|
|
|
// Object equality requires the value to be the first key in the object that is being queried.
|
|
if (
|
|
operator === 'equals' &&
|
|
formattedValue &&
|
|
typeof formattedValue === 'object' &&
|
|
formattedValue.value &&
|
|
formattedValue.relationTo
|
|
) {
|
|
const { value } = formattedValue
|
|
const isValid = Types.ObjectId.isValid(value)
|
|
|
|
if (isValid) {
|
|
formattedValue.value = new Types.ObjectId(value)
|
|
}
|
|
|
|
let localizedPath = path
|
|
|
|
if (field.localized && payload.config.localization && locale) {
|
|
localizedPath = `${path}.${locale}`
|
|
}
|
|
|
|
return {
|
|
rawQuery: {
|
|
$or: [
|
|
{
|
|
[localizedPath]: {
|
|
$eq: {
|
|
// disable auto sort
|
|
/* eslint-disable */
|
|
value: formattedValue.value,
|
|
relationTo: formattedValue.relationTo,
|
|
/* eslint-enable */
|
|
},
|
|
},
|
|
},
|
|
{
|
|
[localizedPath]: {
|
|
$eq: {
|
|
relationTo: formattedValue.relationTo,
|
|
value: formattedValue.value,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
}
|
|
}
|
|
|
|
const relationTo = (field as RelationshipField).relationTo
|
|
|
|
if (['in', 'not_in'].includes(operator) && Array.isArray(formattedValue)) {
|
|
formattedValue = formattedValue.reduce((formattedValues, inVal) => {
|
|
if (!inVal) {
|
|
return formattedValues
|
|
}
|
|
|
|
if (typeof relationTo === 'string' && payload.collections[relationTo].customIDType) {
|
|
if (payload.collections[relationTo].customIDType === 'number') {
|
|
const parsedNumber = parseFloat(inVal)
|
|
if (!Number.isNaN(parsedNumber)) {
|
|
formattedValues.push(parsedNumber)
|
|
return formattedValues
|
|
}
|
|
}
|
|
|
|
formattedValues.push(inVal)
|
|
return formattedValues
|
|
}
|
|
|
|
if (
|
|
Array.isArray(relationTo) &&
|
|
relationTo.some((relationTo) => !!payload.collections[relationTo].customIDType)
|
|
) {
|
|
if (Types.ObjectId.isValid(inVal.toString())) {
|
|
formattedValues.push(new Types.ObjectId(inVal))
|
|
} else {
|
|
formattedValues.push(inVal)
|
|
}
|
|
return formattedValues
|
|
}
|
|
|
|
if (Types.ObjectId.isValid(inVal.toString())) {
|
|
formattedValues.push(new Types.ObjectId(inVal))
|
|
}
|
|
|
|
return formattedValues
|
|
}, [])
|
|
}
|
|
|
|
if (
|
|
['contains', 'equals', 'like', 'not_equals'].includes(operator) &&
|
|
(!Array.isArray(relationTo) || !path.endsWith('.relationTo'))
|
|
) {
|
|
if (typeof relationTo === 'string') {
|
|
const customIDType = payload.collections[relationTo].customIDType
|
|
|
|
if (customIDType) {
|
|
if (customIDType === 'number') {
|
|
formattedValue = parseFloat(val)
|
|
|
|
if (Number.isNaN(formattedValue)) {
|
|
return { operator: formattedOperator, val: undefined }
|
|
}
|
|
}
|
|
} else {
|
|
if (!Types.ObjectId.isValid(formattedValue)) {
|
|
return { operator: formattedOperator, val: undefined }
|
|
}
|
|
formattedValue = new Types.ObjectId(formattedValue)
|
|
}
|
|
} else {
|
|
const hasCustomIDType = relationTo.some(
|
|
(relationTo) => !!payload.collections[relationTo].customIDType,
|
|
)
|
|
|
|
if (hasCustomIDType) {
|
|
if (typeof val === 'string') {
|
|
const formattedNumber = Number(val)
|
|
formattedValue = [Types.ObjectId.isValid(val) ? new Types.ObjectId(val) : val]
|
|
formattedOperator = operator === 'not_equals' ? 'not_in' : 'in'
|
|
if (!Number.isNaN(formattedNumber)) {
|
|
formattedValue.push(formattedNumber)
|
|
}
|
|
}
|
|
} else {
|
|
if (!Types.ObjectId.isValid(formattedValue)) {
|
|
return { operator: formattedOperator, val: undefined }
|
|
}
|
|
formattedValue = new Types.ObjectId(formattedValue)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set up specific formatting necessary by operators
|
|
|
|
if (operator === 'near') {
|
|
let lng
|
|
let lat
|
|
let maxDistance
|
|
let minDistance
|
|
|
|
if (Array.isArray(formattedValue)) {
|
|
;[lng, lat, maxDistance, minDistance] = formattedValue
|
|
}
|
|
|
|
if (typeof formattedValue === 'string') {
|
|
;[lng, lat, maxDistance, minDistance] = createArrayFromCommaDelineated(formattedValue)
|
|
}
|
|
|
|
if (lng == null || lat == null || (maxDistance == null && minDistance == null)) {
|
|
formattedValue = undefined
|
|
} else {
|
|
formattedValue = {
|
|
$geometry: { type: 'Point', coordinates: [parseFloat(lng), parseFloat(lat)] },
|
|
}
|
|
|
|
if (maxDistance) {
|
|
formattedValue.$maxDistance = parseFloat(maxDistance)
|
|
}
|
|
if (minDistance) {
|
|
formattedValue.$minDistance = parseFloat(minDistance)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (operator === 'within' || operator === 'intersects') {
|
|
if (
|
|
formattedValue &&
|
|
typeof formattedValue === 'object' &&
|
|
Array.isArray(formattedValue.coordinates)
|
|
) {
|
|
formattedValue.coordinates = sanitizeCoordinates(formattedValue.coordinates)
|
|
}
|
|
|
|
formattedValue = {
|
|
$geometry: formattedValue,
|
|
}
|
|
}
|
|
|
|
if (path !== '_id' || (path === '_id' && hasCustomID && field.type === 'text')) {
|
|
if (operator === 'contains' && !Types.ObjectId.isValid(formattedValue)) {
|
|
formattedValue = {
|
|
$options: 'i',
|
|
$regex: formattedValue.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&'),
|
|
}
|
|
}
|
|
|
|
if (operator === 'exists') {
|
|
formattedValue = formattedValue === 'true' || formattedValue === true
|
|
|
|
// _id can't be empty string, will error Cast to ObjectId failed for value ""
|
|
return buildExistsQuery(
|
|
formattedValue,
|
|
path,
|
|
!['relationship', 'upload'].includes(field.type),
|
|
)
|
|
}
|
|
}
|
|
|
|
if (
|
|
(path === '_id' || path === 'parent') &&
|
|
operator === 'like' &&
|
|
formattedValue.length === 24 &&
|
|
!hasCustomID
|
|
) {
|
|
formattedOperator = 'equals'
|
|
}
|
|
|
|
if (operator === 'exists') {
|
|
formattedValue = formattedValue === 'true' || formattedValue === true
|
|
|
|
// Clearable fields
|
|
if (['relationship', 'select', 'upload'].includes(field.type)) {
|
|
if (formattedValue) {
|
|
return {
|
|
rawQuery: {
|
|
$and: [{ [path]: { $exists: true } }, { [path]: { $ne: null } }],
|
|
},
|
|
}
|
|
} else {
|
|
return {
|
|
rawQuery: {
|
|
$or: [{ [path]: { $exists: false } }, { [path]: { $eq: null } }],
|
|
},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return { operator: formattedOperator, val: formattedValue }
|
|
}
|