diff --git a/.vscode/launch.json b/.vscode/launch.json index 37077c2e2f..52c2374b19 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -28,16 +28,7 @@ "type": "node", "request": "launch", "name": "Launch Program", - "env": { - "BABEL_ENV": "development" - }, "program": "${workspaceFolder}/test/dev.js", - "skipFiles": [ - "/**" - ], - "runtimeArgs": [ - "--nolazy" - ], "args": [ "fields" ] diff --git a/package.json b/package.json index b465fed993..dd29aed36a 100644 --- a/package.json +++ b/package.json @@ -134,7 +134,6 @@ "minimist": "^1.2.0", "mkdirp": "^1.0.4", "mongoose": "6.5.0", - "mongoose-aggregate-paginate-v2": "^1.0.6", "mongoose-paginate-v2": "^1.6.1", "nodemailer": "^6.4.2", "object-to-formdata": "^4.1.0", diff --git a/src/collections/buildSchema.ts b/src/collections/buildSchema.ts index b4f8a65608..6a22567751 100644 --- a/src/collections/buildSchema.ts +++ b/src/collections/buildSchema.ts @@ -1,6 +1,4 @@ import paginate from 'mongoose-paginate-v2'; -import aggregatePaginate from 'mongoose-aggregate-paginate-v2'; - import { Schema } from 'mongoose'; import { SanitizedConfig } from '../config/types'; import buildQueryPlugin from '../mongoose/buildQuery'; @@ -26,10 +24,6 @@ const buildCollectionSchema = (collection: SanitizedCollectionConfig, config: Sa schema.plugin(paginate, { useEstimatedCount: true }) .plugin(buildQueryPlugin); - if (collection.versions?.drafts) { - schema.plugin(aggregatePaginate); - } - return schema; }; diff --git a/src/collections/operations/find.ts b/src/collections/operations/find.ts index c42edaf57d..46e5677bff 100644 --- a/src/collections/operations/find.ts +++ b/src/collections/operations/find.ts @@ -9,7 +9,7 @@ import flattenWhereConstraints from '../../utilities/flattenWhereConstraints'; import { buildSortParam } from '../../mongoose/buildSortParam'; import { AccessResult } from '../../config/types'; import { afterRead } from '../../fields/hooks/afterRead'; -import { buildDraftMergeAggregate } from '../../versions/drafts/buildDraftMergeAggregate'; +import { mergeDrafts } from '../../versions/drafts/mergeDrafts'; export type Arguments = { collection: Collection @@ -51,6 +51,7 @@ async function find(incomingArgs: Arguments): Promis depth, currentDepth, draft: draftsEnabled, + collection, collection: { Model, config: collectionConfig, @@ -159,12 +160,15 @@ async function find(incomingArgs: Arguments): Promis }; if (collectionConfig.versions?.drafts && draftsEnabled) { - const aggregate = Model.aggregate(buildDraftMergeAggregate({ - config: collectionConfig, + result = await mergeDrafts({ + accessResult, + collection, + locale, + paginationOptions, + payload, query, - })); - - result = await Model.aggregatePaginate(aggregate, paginationOptions); + where, + }); } else { result = await Model.paginate(query, paginationOptions); } diff --git a/src/versions/drafts/buildDraftMergeAggregate.ts b/src/versions/drafts/buildDraftMergeAggregate.ts deleted file mode 100644 index 7e59594b88..0000000000 --- a/src/versions/drafts/buildDraftMergeAggregate.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { PipelineStage } from 'mongoose'; -import { SanitizedCollectionConfig } from '../../collections/config/types'; - -type Args = { - config: SanitizedCollectionConfig - query: Record -} - -export const buildDraftMergeAggregate = ({ config, query }: Args): PipelineStage[] => [ - // Add string-based ID to query with - { - $addFields: { id: { $toString: '$_id' } }, - }, - - // Merge in one version - // that has the same parent ID - // and is newer, sorting by updatedAt - { - $lookup: { - from: `_${config.slug}_versions`, - as: 'docs', - let: { - id: { $toString: '$_id' }, - updatedAt: '$updatedAt', - }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$parent', '$$id'] }, - { $gt: ['$updatedAt', '$$updatedAt'] }, - ], - }, - }, - }, - { $sort: { updatedAt: -1 } }, - { $limit: 1 }, - ], - }, - }, - - // Add a new field - // with the first doc returned - { - $addFields: { - doc: { - $arrayElemAt: ['$docs', 0], - }, - }, - }, - - // If newer version exists, - // merge the version into the root and - // replace the updatedAt - // Otherwise, do nothing to the root - { - $replaceRoot: { - newRoot: { - $cond: { - if: { - $ne: ['$doc', undefined], - }, - then: { - $mergeObjects: [ - '$$ROOT', - { - $mergeObjects: [ - '$doc.version', - { - updatedAt: '$doc.updatedAt', - }, - ], - }, - ], - }, - else: '$$ROOT', - }, - }, - }, - }, - - // Clear out the temporarily added `docs` - { - $project: { - doc: 0, - docs: 0, - }, - }, - - // Run the original query on the results - { - $match: query, - }, -]; diff --git a/src/versions/drafts/mergeDrafts.ts b/src/versions/drafts/mergeDrafts.ts new file mode 100644 index 0000000000..5e555f0ef7 --- /dev/null +++ b/src/versions/drafts/mergeDrafts.ts @@ -0,0 +1,222 @@ +import { AccessResult } from '../../config/types'; +import { Where } from '../../types'; +import { Payload } from '../..'; +import { PaginatedDocs } from '../../mongoose/types'; +import { Collection, CollectionModel, TypeWithID } from '../../collections/config/types'; +import { hasWhereAccessResult } from '../../auth'; +import { appendVersionToQueryKey } from './appendVersionToQueryKey'; +import sanitizeInternalFields from '../../utilities/sanitizeInternalFields'; +import replaceWithDraftIfAvailable from './replaceWithDraftIfAvailable'; + +type AggregateVersion = { + _id: string + version: T + updatedAt: string + createdAt: string +} + +type VersionCollectionMatchMap = { + [_id: string | number]: { + updatedAt: string + createdAt: string + version: T + } +} + +type Args = { + accessResult: AccessResult + collection: Collection + locale: string + paginationOptions: any + payload: Payload + query: Record + where: Where +} + +export const mergeDrafts = async ({ + accessResult, + collection, + locale, + payload, + paginationOptions, + query, + where: incomingWhere, +}: Args): Promise> => { + console.log('---------STARTING---------'); + // Query the main collection, non-paginated, for any IDs that match the query + const mainCollectionFind = await collection.Model.find(query, { updatedAt: 1 }, { limit: paginationOptions.limit }).lean(); + + // Create object "map" for performant lookup + const mainCollectionMatchMap = mainCollectionFind.reduce((map, { _id, updatedAt }) => { + const newMap = map; + newMap[_id] = updatedAt; + return newMap; + }, {}); + + console.log({ mainCollectionMatchMap }); + + // Query the versions collection with a version-specific query + const VersionModel = payload.versions[collection.config.slug] as CollectionModel; + + const where = appendVersionToQueryKey(incomingWhere || {}); + + const versionQueryToBuild: { where: Where } = { + where: { + ...where, + and: [ + ...where?.and || [], + { + 'version._status': { + equals: 'draft', + }, + }, + ], + }, + }; + + if (hasWhereAccessResult(accessResult)) { + const versionAccessResult = appendVersionToQueryKey(accessResult); + versionQueryToBuild.where.and.push(versionAccessResult); + } + + const versionQuery = await VersionModel.buildQuery(versionQueryToBuild, locale); + + const versionCollectionQuery = await VersionModel.aggregate>([ + { $sort: { updatedAt: -1 } }, + { + $group: { + _id: '$parent', + versionID: { $first: '$_id' }, + version: { $first: '$version' }, + updatedAt: { $first: '$updatedAt' }, + createdAt: { $first: '$createdAt' }, + }, + }, + { $match: versionQuery }, + { $limit: paginationOptions.limit }, + ]); + + const includedParentIDs: (string | number)[] = []; + + // Create object "map" for performant lookup + // and in the same loop, check if there are matched versions without a matched parent + // This means that the newer version's parent should appear in the main query. + // To do so, add the version's parent ID into an explicit `includedIDs` array + + const versionCollectionMatchMap = versionCollectionQuery.reduce>((map, { _id, updatedAt, createdAt, version }) => { + const newMap = map; + newMap[_id] = { version, updatedAt, createdAt }; + + const matchedParent = mainCollectionMatchMap[_id]; + if (!matchedParent) includedParentIDs.push(_id); + return newMap; + }, {}); + + console.log({ versionCollectionMatchMap }); + console.log({ includedParentIDs }); + + // Now we need to explicitly exclude any parent matches that have newer versions + // which did NOT appear in the versions query + + const parentsWithoutNewerVersions = await Promise.all(Object.entries(mainCollectionMatchMap).map(async ([parentDocID, parentDocUpdatedAt]) => { + // If there is a matched version, and it's newer, this parent should remain + if (versionCollectionMatchMap[parentDocID] && versionCollectionMatchMap[parentDocID].updatedAt > parentDocUpdatedAt) { + return null; + } + + // Otherwise, we need to check if there are newer versions present + // that did not get returned from the versions query. If there are, + // this says that the newest version does not match the incoming query, + // and the parent ID should be excluded + const versionsQuery = await VersionModel.find({ + updatedAt: { + $gt: parentDocUpdatedAt, + }, + }, {}, { limit: 1 }).lean(); + + if (versionsQuery.length > 0) { + return parentDocID; + } + + return null; + })); + + console.log({ parentsWithoutNewerVersions }); + + const excludedParentIDs = parentsWithoutNewerVersions.filter((result) => Boolean(result)); + + console.log({ excludedParentIDs }); + + // Run a final query against the main collection, + // passing in the ids to exclude and include + // so that they appear correctly + // in paginated results, properly paginated + const finalQueryToBuild: { where: Where } = { + where: { + and: [], + }, + }; + + finalQueryToBuild.where.and.push({ or: [] }); + + if (hasWhereAccessResult(accessResult)) { + finalQueryToBuild.where.and.push(accessResult); + } + + if (where) { + finalQueryToBuild.where.and[0].or.push(where); + } + + if (includedParentIDs.length > 0) { + finalQueryToBuild.where.and[0].or.push({ + id: { + in: includedParentIDs, + }, + }); + } + + if (excludedParentIDs.length > 0) { + finalQueryToBuild.where.and[0].or.push({ + id: { + not_in: excludedParentIDs, + }, + }); + } + + console.log({ finalPayloadQuery: JSON.stringify(finalQueryToBuild, null, 2) }); + + const finalQuery = await collection.Model.buildQuery(finalQueryToBuild, locale); + + let result = await collection.Model.paginate(finalQuery, paginationOptions); + + result = { + ...result, + docs: await Promise.all(result.docs.map(async (doc) => { + let sanitizedDoc = JSON.parse(JSON.stringify(doc)); + sanitizedDoc.id = sanitizedDoc._id; + sanitizedDoc = sanitizeInternalFields(sanitizedDoc); + + const matchedVersion = versionCollectionMatchMap[sanitizedDoc.id]; + + if (matchedVersion) { + return { + ...sanitizedDoc, + ...matchedVersion.version, + createdAt: matchedVersion.createdAt, + updatedAt: matchedVersion.updatedAt, + }; + } + + return replaceWithDraftIfAvailable({ + accessResult, + payload, + entity: collection.config, + entityType: 'collection', + doc: sanitizedDoc, + locale, + }); + })), + }; + + return result; +}; diff --git a/yarn.lock b/yarn.lock index 550b9c9c80..ca47c70840 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8514,11 +8514,6 @@ mongodb@^3.7.3: optionalDependencies: saslprep "^1.0.0" -mongoose-aggregate-paginate-v2@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/mongoose-aggregate-paginate-v2/-/mongoose-aggregate-paginate-v2-1.0.6.tgz#fd2f2564d1bbf52f49a196f0b7b03675913dacca" - integrity sha512-UuALu+mjhQa1K9lMQvjLL3vm3iALvNw8PQNIh2gp1b+tO5hUa0NC0Wf6/8QrT9PSJVTihXaD8hQVy3J4e0jO0Q== - mongoose-paginate-v2@*, mongoose-paginate-v2@^1.6.1: version "1.7.1" resolved "https://registry.yarnpkg.com/mongoose-paginate-v2/-/mongoose-paginate-v2-1.7.1.tgz#0b390f5eb8e5dca55ffcb1fd7b4d8078636cb8f1"