feat: ensures compatibility with azure cosmos and aws documentdb

.vscode/launch.json

@@ -28,16 +28,7 @@
       "type": "node",
       "request": "launch",
       "name": "Launch Program",
-      "env": {
-        "BABEL_ENV": "development"
-      },
       "program": "${workspaceFolder}/test/dev.js",
-      "skipFiles": [
-        "<node_internals>/**"
-      ],
-      "runtimeArgs": [
-        "--nolazy"
-      ],
       "args": [
         "fields"
       ]

package.json

@@ -134,7 +134,6 @@
     "minimist": "^1.2.0",
     "mkdirp": "^1.0.4",
     "mongoose": "6.5.0",
-    "mongoose-aggregate-paginate-v2": "^1.0.6",
     "mongoose-paginate-v2": "^1.6.1",
     "nodemailer": "^6.4.2",
     "object-to-formdata": "^4.1.0",

src/collections/buildSchema.ts

@@ -1,6 +1,4 @@
 import paginate from 'mongoose-paginate-v2';
-import aggregatePaginate from 'mongoose-aggregate-paginate-v2';
-
 import { Schema } from 'mongoose';
 import { SanitizedConfig } from '../config/types';
 import buildQueryPlugin from '../mongoose/buildQuery';
@@ -26,10 +24,6 @@ const buildCollectionSchema = (collection: SanitizedCollectionConfig, config: Sa
   schema.plugin(paginate, { useEstimatedCount: true })
     .plugin(buildQueryPlugin);
 
-  if (collection.versions?.drafts) {
-    schema.plugin(aggregatePaginate);
-  }
-
   return schema;
 };
 

src/collections/operations/find.ts

@@ -9,7 +9,7 @@ import flattenWhereConstraints from '../../utilities/flattenWhereConstraints';
 import { buildSortParam } from '../../mongoose/buildSortParam';
 import { AccessResult } from '../../config/types';
 import { afterRead } from '../../fields/hooks/afterRead';
-import { buildDraftMergeAggregate } from '../../versions/drafts/buildDraftMergeAggregate';
+import { mergeDrafts } from '../../versions/drafts/mergeDrafts';
 
 export type Arguments = {
   collection: Collection
@@ -51,6 +51,7 @@ async function find<T extends TypeWithID = any>(incomingArgs: Arguments): Promis
     depth,
     currentDepth,
     draft: draftsEnabled,
+    collection,
     collection: {
       Model,
       config: collectionConfig,
@@ -159,12 +160,15 @@ async function find<T extends TypeWithID = any>(incomingArgs: Arguments): Promis
   };
 
   if (collectionConfig.versions?.drafts && draftsEnabled) {
-    const aggregate = Model.aggregate(buildDraftMergeAggregate({
+    result = await mergeDrafts({
-      config: collectionConfig,
+      accessResult,
+      collection,
+      locale,
+      paginationOptions,
+      payload,
       query,
-    }));
+      where,
-
+    });
-    result = await Model.aggregatePaginate(aggregate, paginationOptions);
   } else {
     result = await Model.paginate(query, paginationOptions);
   }

src/versions/drafts/buildDraftMergeAggregate.ts

@@ -1,95 +0,0 @@
-import { PipelineStage } from 'mongoose';
-import { SanitizedCollectionConfig } from '../../collections/config/types';
-
-type Args = {
-  config: SanitizedCollectionConfig
-  query: Record<string, unknown>
-}
-
-export const buildDraftMergeAggregate = ({ config, query }: Args): PipelineStage[] => [
-  // Add string-based ID to query with
-  {
-    $addFields: { id: { $toString: '$_id' } },
-  },
-
-  // Merge in one version
-  // that has the same parent ID
-  // and is newer, sorting by updatedAt
-  {
-    $lookup: {
-      from: `_${config.slug}_versions`,
-      as: 'docs',
-      let: {
-        id: { $toString: '$_id' },
-        updatedAt: '$updatedAt',
-      },
-      pipeline: [
-        {
-          $match: {
-            $expr: {
-              $and: [
-                { $eq: ['$parent', '$$id'] },
-                { $gt: ['$updatedAt', '$$updatedAt'] },
-              ],
-            },
-          },
-        },
-        { $sort: { updatedAt: -1 } },
-        { $limit: 1 },
-      ],
-    },
-  },
-
-  // Add a new field
-  // with the first doc returned
-  {
-    $addFields: {
-      doc: {
-        $arrayElemAt: ['$docs', 0],
-      },
-    },
-  },
-
-  // If newer version exists,
-  // merge the version into the root and
-  // replace the updatedAt
-  // Otherwise, do nothing to the root
-  {
-    $replaceRoot: {
-      newRoot: {
-        $cond: {
-          if: {
-            $ne: ['$doc', undefined],
-          },
-          then: {
-            $mergeObjects: [
-              '$$ROOT',
-              {
-                $mergeObjects: [
-                  '$doc.version',
-                  {
-                    updatedAt: '$doc.updatedAt',
-                  },
-                ],
-              },
-            ],
-          },
-          else: '$$ROOT',
-        },
-      },
-    },
-  },
-
-  // Clear out the temporarily added `docs`
-  {
-    $project: {
-      doc: 0,
-      docs: 0,
-    },
-  },
-
-  // Run the original query on the results
-  {
-    $match: query,
-  },
-];

src/versions/drafts/mergeDrafts.ts

@@ -0,0 +1,222 @@
+import { AccessResult } from '../../config/types';
+import { Where } from '../../types';
+import { Payload } from '../..';
+import { PaginatedDocs } from '../../mongoose/types';
+import { Collection, CollectionModel, TypeWithID } from '../../collections/config/types';
+import { hasWhereAccessResult } from '../../auth';
+import { appendVersionToQueryKey } from './appendVersionToQueryKey';
+import sanitizeInternalFields from '../../utilities/sanitizeInternalFields';
+import replaceWithDraftIfAvailable from './replaceWithDraftIfAvailable';
+
+type AggregateVersion<T> = {
+  _id: string
+  version: T
+  updatedAt: string
+  createdAt: string
+}
+
+type VersionCollectionMatchMap<T> = {
+  [_id: string | number]: {
+    updatedAt: string
+    createdAt: string
+    version: T
+  }
+}
+
+type Args = {
+  accessResult: AccessResult
+  collection: Collection
+  locale: string
+  paginationOptions: any
+  payload: Payload
+  query: Record<string, unknown>
+  where: Where
+}
+
+export const mergeDrafts = async <T extends TypeWithID>({
+  accessResult,
+  collection,
+  locale,
+  payload,
+  paginationOptions,
+  query,
+  where: incomingWhere,
+}: Args): Promise<PaginatedDocs<T>> => {
+  console.log('---------STARTING---------');
+  // Query the main collection, non-paginated, for any IDs that match the query
+  const mainCollectionFind = await collection.Model.find(query, { updatedAt: 1 }, { limit: paginationOptions.limit }).lean();
+
+  // Create object "map" for performant lookup
+  const mainCollectionMatchMap = mainCollectionFind.reduce((map, { _id, updatedAt }) => {
+    const newMap = map;
+    newMap[_id] = updatedAt;
+    return newMap;
+  }, {});
+
+  console.log({ mainCollectionMatchMap });
+
+  // Query the versions collection with a version-specific query
+  const VersionModel = payload.versions[collection.config.slug] as CollectionModel;
+
+  const where = appendVersionToQueryKey(incomingWhere || {});
+
+  const versionQueryToBuild: { where: Where } = {
+    where: {
+      ...where,
+      and: [
+        ...where?.and || [],
+        {
+          'version._status': {
+            equals: 'draft',
+          },
+        },
+      ],
+    },
+  };
+
+  if (hasWhereAccessResult(accessResult)) {
+    const versionAccessResult = appendVersionToQueryKey(accessResult);
+    versionQueryToBuild.where.and.push(versionAccessResult);
+  }
+
+  const versionQuery = await VersionModel.buildQuery(versionQueryToBuild, locale);
+
+  const versionCollectionQuery = await VersionModel.aggregate<AggregateVersion<T>>([
+    { $sort: { updatedAt: -1 } },
+    {
+      $group: {
+        _id: '$parent',
+        versionID: { $first: '$_id' },
+        version: { $first: '$version' },
+        updatedAt: { $first: '$updatedAt' },
+        createdAt: { $first: '$createdAt' },
+      },
+    },
+    { $match: versionQuery },
+    { $limit: paginationOptions.limit },
+  ]);
+
+  const includedParentIDs: (string | number)[] = [];
+
+  // Create object "map" for performant lookup
+  // and in the same loop, check if there are matched versions without a matched parent
+  // This means that the newer version's parent should appear in the main query.
+  // To do so, add the version's parent ID into an explicit `includedIDs` array
+
+  const versionCollectionMatchMap = versionCollectionQuery.reduce<VersionCollectionMatchMap<T>>((map, { _id, updatedAt, createdAt, version }) => {
+    const newMap = map;
+    newMap[_id] = { version, updatedAt, createdAt };
+
+    const matchedParent = mainCollectionMatchMap[_id];
+    if (!matchedParent) includedParentIDs.push(_id);
+    return newMap;
+  }, {});
+
+  console.log({ versionCollectionMatchMap });
+  console.log({ includedParentIDs });
+
+  // Now we need to explicitly exclude any parent matches that have newer versions
+  // which did NOT appear in the versions query
+
+  const parentsWithoutNewerVersions = await Promise.all(Object.entries(mainCollectionMatchMap).map(async ([parentDocID, parentDocUpdatedAt]) => {
+    // If there is a matched version, and it's newer, this parent should remain
+    if (versionCollectionMatchMap[parentDocID] && versionCollectionMatchMap[parentDocID].updatedAt > parentDocUpdatedAt) {
+      return null;
+    }
+
+    // Otherwise, we need to check if there are newer versions present
+    // that did not get returned from the versions query. If there are,
+    // this says that the newest version does not match the incoming query,
+    // and the parent ID should be excluded
+    const versionsQuery = await VersionModel.find({
+      updatedAt: {
+        $gt: parentDocUpdatedAt,
+      },
+    }, {}, { limit: 1 }).lean();
+
+    if (versionsQuery.length > 0) {
+      return parentDocID;
+    }
+
+    return null;
+  }));
+
+  console.log({ parentsWithoutNewerVersions });
+
+  const excludedParentIDs = parentsWithoutNewerVersions.filter((result) => Boolean(result));
+
+  console.log({ excludedParentIDs });
+
+  // Run a final query against the main collection,
+  // passing in the ids to exclude and include
+  // so that they appear correctly
+  // in paginated results, properly paginated
+  const finalQueryToBuild: { where: Where } = {
+    where: {
+      and: [],
+    },
+  };
+
+  finalQueryToBuild.where.and.push({ or: [] });
+
+  if (hasWhereAccessResult(accessResult)) {
+    finalQueryToBuild.where.and.push(accessResult);
+  }
+
+  if (where) {
+    finalQueryToBuild.where.and[0].or.push(where);
+  }
+
+  if (includedParentIDs.length > 0) {
+    finalQueryToBuild.where.and[0].or.push({
+      id: {
+        in: includedParentIDs,
+      },
+    });
+  }
+
+  if (excludedParentIDs.length > 0) {
+    finalQueryToBuild.where.and[0].or.push({
+      id: {
+        not_in: excludedParentIDs,
+      },
+    });
+  }
+
+  console.log({ finalPayloadQuery: JSON.stringify(finalQueryToBuild, null, 2) });
+
+  const finalQuery = await collection.Model.buildQuery(finalQueryToBuild, locale);
+
+  let result = await collection.Model.paginate(finalQuery, paginationOptions);
+
+  result = {
+    ...result,
+    docs: await Promise.all(result.docs.map(async (doc) => {
+      let sanitizedDoc = JSON.parse(JSON.stringify(doc));
+      sanitizedDoc.id = sanitizedDoc._id;
+      sanitizedDoc = sanitizeInternalFields(sanitizedDoc);
+
+      const matchedVersion = versionCollectionMatchMap[sanitizedDoc.id];
+
+      if (matchedVersion) {
+        return {
+          ...sanitizedDoc,
+          ...matchedVersion.version,
+          createdAt: matchedVersion.createdAt,
+          updatedAt: matchedVersion.updatedAt,
+        };
+      }
+
+      return replaceWithDraftIfAvailable({
+        accessResult,
+        payload,
+        entity: collection.config,
+        entityType: 'collection',
+        doc: sanitizedDoc,
+        locale,
+      });
+    })),
+  };
+
+  return result;
+};

yarn.lock

@@ -8514,11 +8514,6 @@ mongodb@^3.7.3:
   optionalDependencies:
     saslprep "^1.0.0"
 
-mongoose-aggregate-paginate-v2@^1.0.6:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/mongoose-aggregate-paginate-v2/-/mongoose-aggregate-paginate-v2-1.0.6.tgz#fd2f2564d1bbf52f49a196f0b7b03675913dacca"
-  integrity sha512-UuALu+mjhQa1K9lMQvjLL3vm3iALvNw8PQNIh2gp1b+tO5hUa0NC0Wf6/8QrT9PSJVTihXaD8hQVy3J4e0jO0Q==
-
 mongoose-paginate-v2@*, mongoose-paginate-v2@^1.6.1:
   version "1.7.1"
   resolved "https://registry.yarnpkg.com/mongoose-paginate-v2/-/mongoose-paginate-v2-1.7.1.tgz#0b390f5eb8e5dca55ffcb1fd7b4d8078636cb8f1"