From 4081953c187af46febc6f4646f61be1a5ef846d0 Mon Sep 17 00:00:00 2001 From: Sasha <64744993+r1tsuu@users.noreply.github.com> Date: Thu, 20 Mar 2025 22:49:21 +0200 Subject: [PATCH] feat(db-mongodb): support sorting by fields in other collections through a relationship field (#11803) This is already supported in Postgres / SQLite. For example: ``` const result = await payload.find({ collection: 'directors', depth: 0, sort: '-movies.name', // movies is a relationship field here }) ``` Removes the condition in tests: ``` // no support for sort by relation in mongodb if (isMongoose(payload)) { return } ``` --- packages/db-mongodb/src/find.ts | 11 +- packages/db-mongodb/src/findGlobalVersions.ts | 3 +- packages/db-mongodb/src/findVersions.ts | 3 +- .../db-mongodb/src/queries/buildSortParam.ts | 132 +++++++++++++++++- packages/db-mongodb/src/queryDrafts.ts | 13 +- packages/db-mongodb/src/updateMany.ts | 1 + .../src/utilities/aggregatePaginate.ts | 8 ++ .../src/utilities/buildJoinAggregation.ts | 2 + packages/payload/src/index.ts | 25 ++-- test/relationships/config.ts | 6 + test/relationships/int.spec.ts | 91 +++++++++++- test/relationships/payload-types.ts | 12 +- 12 files changed, 277 insertions(+), 30 deletions(-) diff --git a/packages/db-mongodb/src/find.ts b/packages/db-mongodb/src/find.ts index 8dc64c1993..f33df57990 100644 --- a/packages/db-mongodb/src/find.ts +++ b/packages/db-mongodb/src/find.ts @@ -1,4 +1,4 @@ -import type { PaginateOptions } from 'mongoose' +import type { PaginateOptions, PipelineStage } from 'mongoose' import type { Find } from 'payload' import { flattenWhereToOperators } from 'payload' @@ -41,13 +41,17 @@ export const find: Find = async function find( hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near')) } + const sortAggregation: PipelineStage[] = [] + let sort if (!hasNearConstraint) { sort = buildSortParam({ + adapter: this, config: this.payload.config, fields: collectionConfig.flattenedFields, locale, sort: sortArg || collectionConfig.defaultSort, + sortAggregation, timestamps: true, }) } @@ -128,8 +132,8 @@ export const find: Find = async function find( locale, query, }) - // build join aggregation - if (aggregate) { + + if (aggregate || sortAggregation.length > 0) { result = await aggregatePaginate({ adapter: this, collation: paginationOptions.collation, @@ -142,6 +146,7 @@ export const find: Find = async function find( query, session: paginationOptions.options?.session ?? undefined, sort: paginationOptions.sort as object, + sortAggregation, useEstimatedCount: paginationOptions.useEstimatedCount, }) } else { diff --git a/packages/db-mongodb/src/findGlobalVersions.ts b/packages/db-mongodb/src/findGlobalVersions.ts index 1386302f53..453cbf0e01 100644 --- a/packages/db-mongodb/src/findGlobalVersions.ts +++ b/packages/db-mongodb/src/findGlobalVersions.ts @@ -48,6 +48,7 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV let sort if (!hasNearConstraint) { sort = buildSortParam({ + adapter: this, config: this.payload.config, fields: versionFields, locale, @@ -103,7 +104,7 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV if (limit && limit >= 0) { paginationOptions.limit = limit // limit must also be set here, it's ignored when pagination is false - + paginationOptions.options!.limit = limit // Disable pagination if limit is 0 diff --git a/packages/db-mongodb/src/findVersions.ts b/packages/db-mongodb/src/findVersions.ts index 3872ce98f5..454e87f791 100644 --- a/packages/db-mongodb/src/findVersions.ts +++ b/packages/db-mongodb/src/findVersions.ts @@ -50,6 +50,7 @@ export const findVersions: FindVersions = async function findVersions( let sort if (!hasNearConstraint) { sort = buildSortParam({ + adapter: this, config: this.payload.config, fields: collectionConfig.flattenedFields, locale, @@ -111,7 +112,7 @@ export const findVersions: FindVersions = async function findVersions( if (limit && limit >= 0) { paginationOptions.limit = limit // limit must also be set here, it's ignored when pagination is false - + paginationOptions.options!.limit = limit // Disable pagination if limit is 0 diff --git a/packages/db-mongodb/src/queries/buildSortParam.ts b/packages/db-mongodb/src/queries/buildSortParam.ts index 6aa399eaaf..e629b0b42e 100644 --- a/packages/db-mongodb/src/queries/buildSortParam.ts +++ b/packages/db-mongodb/src/queries/buildSortParam.ts @@ -1,14 +1,28 @@ -import type { FlattenedField, SanitizedConfig, Sort } from 'payload' +import type { PipelineStage } from 'mongoose' +import { + APIError, + type FlattenedField, + getFieldByPath, + type SanitizedConfig, + type Sort, +} from 'payload' + +import type { MongooseAdapter } from '../index.js' + +import { getCollection } from '../utilities/getEntity.js' import { getLocalizedSortProperty } from './getLocalizedSortProperty.js' type Args = { + adapter: MongooseAdapter config: SanitizedConfig fields: FlattenedField[] locale?: string parentIsLocalized?: boolean sort: Sort + sortAggregation?: PipelineStage[] timestamps: boolean + versions?: boolean } export type SortArgs = { @@ -18,13 +32,111 @@ export type SortArgs = { export type SortDirection = 'asc' | 'desc' +const relationshipSort = ({ + adapter, + fields, + locale, + path, + sort, + sortAggregation, + sortDirection, + versions, +}: { + adapter: MongooseAdapter + fields: FlattenedField[] + locale?: string + path: string + sort: Record + sortAggregation: PipelineStage[] + sortDirection: SortDirection + versions?: boolean +}) => { + let currentFields = fields + const segments = path.split('.') + if (segments.length < 2) { + return false + } + + for (const [i, segment] of segments.entries()) { + if (versions && i === 0 && segment === 'version') { + segments.shift() + continue + } + + const field = currentFields.find((each) => each.name === segment) + + if (!field) { + return false + } + + if ('fields' in field) { + currentFields = field.flattenedFields + } else if ( + (field.type === 'relationship' || field.type === 'upload') && + i !== segments.length - 1 + ) { + const relationshipPath = segments.slice(0, i + 1).join('.') + let sortFieldPath = segments.slice(i + 1, segments.length).join('.') + if (Array.isArray(field.relationTo)) { + throw new APIError('Not supported') + } + + const foreignCollection = getCollection({ adapter, collectionSlug: field.relationTo }) + + const foreignFieldPath = getFieldByPath({ + fields: foreignCollection.collectionConfig.flattenedFields, + path: sortFieldPath, + }) + + if (!foreignFieldPath) { + return false + } + + if (foreignFieldPath.pathHasLocalized && locale) { + sortFieldPath = foreignFieldPath.localizedPath.replace('', locale) + } + + if ( + !sortAggregation.some((each) => { + return '$lookup' in each && each.$lookup.as === `__${path}` + }) + ) { + sortAggregation.push({ + $lookup: { + as: `__${path}`, + foreignField: '_id', + from: foreignCollection.Model.collection.name, + localField: relationshipPath, + pipeline: [ + { + $project: { + [sortFieldPath]: true, + }, + }, + ], + }, + }) + + sort[`__${path}.${sortFieldPath}`] = sortDirection + + return true + } + } + } + + return false +} + export const buildSortParam = ({ + adapter, config, fields, locale, parentIsLocalized = false, sort, + sortAggregation, timestamps, + versions, }: Args): Record => { if (!sort) { if (timestamps) { @@ -52,6 +164,23 @@ export const buildSortParam = ({ acc['_id'] = sortDirection return acc } + + if ( + sortAggregation && + relationshipSort({ + adapter, + fields, + locale, + path: sortProperty, + sort: acc, + sortAggregation, + sortDirection, + versions, + }) + ) { + return acc + } + const localizedProperty = getLocalizedSortProperty({ config, fields, @@ -60,6 +189,7 @@ export const buildSortParam = ({ segments: sortProperty.split('.'), }) acc[localizedProperty] = sortDirection + return acc }, {}) diff --git a/packages/db-mongodb/src/queryDrafts.ts b/packages/db-mongodb/src/queryDrafts.ts index e04c96a48f..e0e7a9e3ea 100644 --- a/packages/db-mongodb/src/queryDrafts.ts +++ b/packages/db-mongodb/src/queryDrafts.ts @@ -1,4 +1,4 @@ -import type { PaginateOptions, QueryOptions } from 'mongoose' +import type { PaginateOptions, PipelineStage, QueryOptions } from 'mongoose' import type { QueryDrafts } from 'payload' import { buildVersionCollectionFields, combineQueries, flattenWhereToOperators } from 'payload' @@ -47,19 +47,24 @@ export const queryDrafts: QueryDrafts = async function queryDrafts( hasNearConstraint = constraints.some((prop) => Object.keys(prop).some((key) => key === 'near')) } + const fields = buildVersionCollectionFields(this.payload.config, collectionConfig, true) + + const sortAggregation: PipelineStage[] = [] if (!hasNearConstraint) { sort = buildSortParam({ + adapter: this, config: this.payload.config, - fields: collectionConfig.flattenedFields, + fields, locale, sort: sortArg || collectionConfig.defaultSort, + sortAggregation, timestamps: true, + versions: true, }) } const combinedWhere = combineQueries({ latest: { equals: true } }, where) - const fields = buildVersionCollectionFields(this.payload.config, collectionConfig, true) const versionQuery = await buildQuery({ adapter: this, fields, @@ -133,7 +138,7 @@ export const queryDrafts: QueryDrafts = async function queryDrafts( }) // build join aggregation - if (aggregate) { + if (aggregate || sortAggregation.length > 0) { result = await aggregatePaginate({ adapter: this, collation: paginationOptions.collation, diff --git a/packages/db-mongodb/src/updateMany.ts b/packages/db-mongodb/src/updateMany.ts index a43ed685ef..c0ef009ba3 100644 --- a/packages/db-mongodb/src/updateMany.ts +++ b/packages/db-mongodb/src/updateMany.ts @@ -39,6 +39,7 @@ export const updateMany: UpdateMany = async function updateMany( let sort: Record | undefined if (!hasNearConstraint) { sort = buildSortParam({ + adapter: this, config: this.payload.config, fields: collectionConfig.flattenedFields, locale, diff --git a/packages/db-mongodb/src/utilities/aggregatePaginate.ts b/packages/db-mongodb/src/utilities/aggregatePaginate.ts index de6a3ed6bd..237d0a00c9 100644 --- a/packages/db-mongodb/src/utilities/aggregatePaginate.ts +++ b/packages/db-mongodb/src/utilities/aggregatePaginate.ts @@ -16,6 +16,7 @@ export const aggregatePaginate = async ({ query, session, sort, + sortAggregation, useEstimatedCount, }: { adapter: MongooseAdapter @@ -29,10 +30,17 @@ export const aggregatePaginate = async ({ query: Record session?: ClientSession sort?: object + sortAggregation?: PipelineStage[] useEstimatedCount?: boolean }): Promise> => { const aggregation: PipelineStage[] = [{ $match: query }] + if (sortAggregation && sortAggregation.length > 0) { + for (const stage of sortAggregation) { + aggregation.push(stage) + } + } + if (sort) { const $sort: Record = {} diff --git a/packages/db-mongodb/src/utilities/buildJoinAggregation.ts b/packages/db-mongodb/src/utilities/buildJoinAggregation.ts index 3dd629b004..ca01765236 100644 --- a/packages/db-mongodb/src/utilities/buildJoinAggregation.ts +++ b/packages/db-mongodb/src/utilities/buildJoinAggregation.ts @@ -87,6 +87,7 @@ export const buildJoinAggregation = async ({ } const sort = buildSortParam({ + adapter, config: adapter.payload.config, fields: aggregatedFields, locale, @@ -279,6 +280,7 @@ export const buildJoinAggregation = async ({ } const sort = buildSortParam({ + adapter, config: adapter.payload.config, fields: collectionConfig.flattenedFields, locale, diff --git a/packages/payload/src/index.ts b/packages/payload/src/index.ts index f67e1af1e2..725aae8652 100644 --- a/packages/payload/src/index.ts +++ b/packages/payload/src/index.ts @@ -27,6 +27,7 @@ import type { SelectFromCollectionSlug, TypeWithID, } from './collections/config/types.js' +export type { FieldState } from './admin/forms/Form.js' import type { Options as CountOptions } from './collections/operations/local/count.js' import type { Options as CreateOptions } from './collections/operations/local/create.js' import type { @@ -63,7 +64,7 @@ import type { TransformGlobalWithSelect, } from './types/index.js' import type { TraverseFieldsCallback } from './utilities/traverseFields.js' -export type { FieldState } from './admin/forms/Form.js' +export type * from './admin/types.js' import { Cron } from 'croner' import type { TypeWithVersion } from './versions/types.js' @@ -83,8 +84,8 @@ import { getLogger } from './utilities/logger.js' import { serverInit as serverInitTelemetry } from './utilities/telemetry/events/serverInit.js' import { traverseFields } from './utilities/traverseFields.js' -export type * from './admin/types.js' export { default as executeAccess } from './auth/executeAccess.js' +export { executeAuthStrategies } from './auth/executeAuthStrategies.js' export interface GeneratedTypes { authUntyped: { @@ -971,7 +972,6 @@ interface RequestContext { // eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface DatabaseAdapter extends BaseDatabaseAdapter {} export type { Payload, RequestContext } -export { executeAuthStrategies } from './auth/executeAuthStrategies.js' export { extractAccessFromPermission } from './auth/extractAccessFromPermission.js' export { getAccessResults } from './auth/getAccessResults.js' export { getFieldsToSign } from './auth/getFieldsToSign.js' @@ -989,7 +989,6 @@ export { resetPasswordOperation } from './auth/operations/resetPassword.js' export { unlockOperation } from './auth/operations/unlock.js' export { verifyEmailOperation } from './auth/operations/verifyEmail.js' export { JWTAuthentication } from './auth/strategies/jwt.js' - export type { AuthStrategyFunction, AuthStrategyFunctionArgs, @@ -1011,8 +1010,8 @@ export type { } from './auth/types.js' export { generateImportMap } from './bin/generateImportMap/index.js' -export type { ImportMap } from './bin/generateImportMap/index.js' +export type { ImportMap } from './bin/generateImportMap/index.js' export { genImportMapIterateFields } from './bin/generateImportMap/iterateFields.js' export { @@ -1059,6 +1058,7 @@ export type { TypeWithID, TypeWithTimestamps, } from './collections/config/types.js' + export type { CompoundIndex } from './collections/config/types.js' export type { SanitizedCompoundIndex } from './collections/config/types.js' export { createDataloaderCacheKey, getDataLoader } from './collections/dataloader.js' @@ -1075,8 +1075,8 @@ export { findVersionsOperation } from './collections/operations/findVersions.js' export { restoreVersionOperation } from './collections/operations/restoreVersion.js' export { updateOperation } from './collections/operations/update.js' export { updateByIDOperation } from './collections/operations/updateByID.js' - export { buildConfig } from './config/build.js' + export { type ClientConfig, createClientConfig, @@ -1196,19 +1196,18 @@ export { ValidationError, ValidationErrorName, } from './errors/index.js' - export type { ValidationFieldError } from './errors/index.js' + export { baseBlockFields } from './fields/baseFields/baseBlockFields.js' export { baseIDField } from './fields/baseFields/baseIDField.js' - export { createClientField, createClientFields, type ServerOnlyFieldAdminProperties, type ServerOnlyFieldProperties, } from './fields/config/client.js' -export { sanitizeFields } from './fields/config/sanitize.js' +export { sanitizeFields } from './fields/config/sanitize.js' export type { AdminClient, ArrayField, @@ -1312,16 +1311,16 @@ export type { ValidateOptions, ValueWithRelation, } from './fields/config/types.js' + export { getDefaultValue } from './fields/getDefaultValue.js' export { traverseFields as afterChangeTraverseFields } from './fields/hooks/afterChange/traverseFields.js' export { promise as afterReadPromise } from './fields/hooks/afterRead/promise.js' export { traverseFields as afterReadTraverseFields } from './fields/hooks/afterRead/traverseFields.js' export { traverseFields as beforeChangeTraverseFields } from './fields/hooks/beforeChange/traverseFields.js' export { traverseFields as beforeValidateTraverseFields } from './fields/hooks/beforeValidate/traverseFields.js' - export { default as sortableFieldTypes } from './fields/sortableFieldTypes.js' -export { validations } from './fields/validations.js' +export { validations } from './fields/validations.js' export type { ArrayFieldValidation, BlocksFieldValidation, @@ -1373,6 +1372,7 @@ export type { GlobalConfig, SanitizedGlobalConfig, } from './globals/config/types.js' + export { docAccessOperation as docAccessOperationGlobal } from './globals/operations/docAccess.js' export { findOneOperation } from './globals/operations/findOne.js' export { findVersionByIDOperation as findVersionByIDOperationGlobal } from './globals/operations/findVersionByID.js' @@ -1420,8 +1420,8 @@ export { importHandlerPath } from './queues/operations/runJobs/runJob/importHand export { getLocalI18n } from './translations/getLocalI18n.js' export * from './types/index.js' export { getFileByPath } from './uploads/getFileByPath.js' - export type * from './uploads/types.js' + export { addDataAndFileToRequest } from './utilities/addDataAndFileToRequest.js' export { addLocalesToRequestFromData, sanitizeLocales } from './utilities/addLocalesToRequest.js' export { commitTransaction } from './utilities/commitTransaction.js' @@ -1463,6 +1463,7 @@ export { formatErrors } from './utilities/formatErrors.js' export { formatLabels, formatNames, toWords } from './utilities/formatLabels.js' export { getBlockSelect } from './utilities/getBlockSelect.js' export { getCollectionIDFieldTypes } from './utilities/getCollectionIDFieldTypes.js' +export { getFieldByPath } from './utilities/getFieldByPath.js' export { getObjectDotNotation } from './utilities/getObjectDotNotation.js' export { getRequestLanguage } from './utilities/getRequestLanguage.js' export { handleEndpoints } from './utilities/handleEndpoints.js' diff --git a/test/relationships/config.ts b/test/relationships/config.ts index d6eda7fc40..4fe5c397f8 100644 --- a/test/relationships/config.ts +++ b/test/relationships/config.ts @@ -223,6 +223,7 @@ export default buildConfigWithDefaults({ }, { slug: 'movies', + versions: { drafts: true }, fields: [ { name: 'name', @@ -242,6 +243,11 @@ export default buildConfigWithDefaults({ name: 'name', type: 'text', }, + { + name: 'localized', + type: 'text', + localized: true, + }, { name: 'movies', type: 'relationship', diff --git a/test/relationships/int.spec.ts b/test/relationships/int.spec.ts index 6a56237032..cf1b11db4b 100644 --- a/test/relationships/int.spec.ts +++ b/test/relationships/int.spec.ts @@ -593,11 +593,6 @@ describe('Relationships', () => { }) it('should sort by a property of a hasMany relationship', async () => { - // no support for sort by relation in mongodb - if (isMongoose(payload)) { - return - } - const movie1 = await payload.create({ collection: 'movies', data: { @@ -638,6 +633,92 @@ describe('Relationships', () => { expect(result.docs[0].id).toStrictEqual(director1.id) }) + it('should sort by a property of a relationship', async () => { + await payload.delete({ collection: 'directors', where: {} }) + await payload.delete({ collection: 'movies', where: {} }) + + const director_1 = await payload.create({ + collection: 'directors', + data: { name: 'Dan', localized: 'Dan' }, + }) + + await payload.update({ + collection: 'directors', + id: director_1.id, + locale: 'de', + data: { localized: 'Mr. Dan' }, + }) + + const director_2 = await payload.create({ + collection: 'directors', + data: { name: 'Mr. Dan', localized: 'Mr. Dan' }, + }) + + await payload.update({ + collection: 'directors', + id: director_2.id, + locale: 'de', + data: { localized: 'Dan' }, + }) + + const movie_1 = await payload.create({ + collection: 'movies', + depth: 0, + data: { director: director_1.id, name: 'Some Movie 1' }, + }) + + const movie_2 = await payload.create({ + collection: 'movies', + depth: 0, + data: { director: director_2.id, name: 'Some Movie 2' }, + }) + + const res_1 = await payload.find({ + collection: 'movies', + sort: '-director.name', + depth: 0, + }) + const res_2 = await payload.find({ + collection: 'movies', + sort: 'director.name', + depth: 0, + }) + + expect(res_1.docs).toStrictEqual([movie_2, movie_1]) + expect(res_2.docs).toStrictEqual([movie_1, movie_2]) + + const draft_res_1 = await payload.find({ + collection: 'movies', + sort: '-director.name', + depth: 0, + draft: true, + }) + const draft_res_2 = await payload.find({ + collection: 'movies', + sort: 'director.name', + depth: 0, + draft: true, + }) + + expect(draft_res_1.docs).toStrictEqual([movie_2, movie_1]) + expect(draft_res_2.docs).toStrictEqual([movie_1, movie_2]) + + const localized_res_1 = await payload.find({ + collection: 'movies', + sort: 'director.localized', + depth: 0, + locale: 'de', + }) + const localized_res_2 = await payload.find({ + collection: 'movies', + sort: 'director.localized', + depth: 0, + }) + + expect(localized_res_1.docs).toStrictEqual([movie_2, movie_1]) + expect(localized_res_2.docs).toStrictEqual([movie_1, movie_2]) + }) + it('should query using "in" by hasMany relationship field', async () => { const tree1 = await payload.create({ collection: treeSlug, diff --git a/test/relationships/payload-types.ts b/test/relationships/payload-types.ts index 9b4d9698cc..cea19dae4e 100644 --- a/test/relationships/payload-types.ts +++ b/test/relationships/payload-types.ts @@ -54,6 +54,7 @@ export type SupportedTimezones = | 'Asia/Singapore' | 'Asia/Tokyo' | 'Asia/Seoul' + | 'Australia/Brisbane' | 'Australia/Sydney' | 'Pacific/Guam' | 'Pacific/Noumea' @@ -268,6 +269,7 @@ export interface Movie { director?: (string | null) | Director; updatedAt: string; createdAt: string; + _status?: ('draft' | 'published') | null; } /** * This interface was referenced by `Config`'s JSON-Schema @@ -276,6 +278,7 @@ export interface Movie { export interface Director { id: string; name?: string | null; + localized?: string | null; movies?: (string | Movie)[] | null; directors?: (string | Director)[] | null; updatedAt: string; @@ -460,9 +463,10 @@ export interface Item { id: string; status?: ('completed' | 'failed' | 'pending') | null; relation?: { - docs?: (string | Relation1)[] | null; - hasNextPage?: boolean | null; - } | null; + docs?: (string | Relation1)[]; + hasNextPage?: boolean; + totalDocs?: number; + }; updatedAt: string; createdAt: string; } @@ -728,6 +732,7 @@ export interface MoviesSelect { director?: T; updatedAt?: T; createdAt?: T; + _status?: T; } /** * This interface was referenced by `Config`'s JSON-Schema @@ -735,6 +740,7 @@ export interface MoviesSelect { */ export interface DirectorsSelect { name?: T; + localized?: T; movies?: T; directors?: T; updatedAt?: T;