feat: allow to count related docs for join fields (#11395)

### What? For the join field query adds ability to specify `count: true`, example: ```ts const result = await payload.find({ joins: { 'group.relatedPosts': { sort: '-title', count: true, }, }, collection: "categories", }) result.group?.relatedPosts?.totalDocs // available ``` ### Why? Can be useful to implement full pagination / show total related documents count in the UI. ### How? Implements the logic in database adapters. In MongoDB it's additional `$lookup` that has `$count` in the pipeline. In SQL, it's additional subquery with `COUNT(*)`. Preserves the current behavior by default, since counting introduces overhead. Additionally, fixes a typescript generation error for join fields. Before, `docs` and `hasNextPage` were marked as nullable, which is not true, these fields cannot be `null`. Additionally, fixes threading of `joinQuery` in `transform/read/traverseFields` for group / tab fields recursive calls.
2025-02-27 18:05:48 +02:00
parent bcc68572bf
commit 3436fb16ea
9 changed files with 345 additions and 139 deletions
--- a/docs/fields/join.mdx
+++ b/docs/fields/join.mdx
@@ -158,6 +158,7 @@ object with:

 - `docs` an array of related documents or only IDs if the depth is reached
 - `hasNextPage` a boolean indicating if there are additional documents
+- `totalDocs` a total number of documents, exists only if `count: true` is passed to the join query

 ```json
 {
@@ -171,7 +172,8 @@ object with:
      }
      // { ... }
    ],
-    "hasNextPage": false
+    "hasNextPage": false,
+    "totalDocs": 10, // if count: true is passed
  }
  // other fields...
 }
@@ -184,6 +186,7 @@ object with:

 - `docs` an array of `relationTo` - the collection slug of the document and `value` - the document itself or the ID if the depth is reached
 - `hasNextPage` a boolean indicating if there are additional documents
+- `totalDocs` a total number of documents, exists only if `count: true` is passed to the join query

 ```json
 {
@@ -200,7 +203,8 @@ object with:
      }
      // { ... }
    ],
-    "hasNextPage": false
+    "hasNextPage": false,
+    "totalDocs": 10, // if count: true is passed
  }
  // other fields...
 }
@@ -215,10 +219,11 @@ returning. This is useful for performance reasons when you don't need the relate
 The following query options are supported:

 | Property    | Description                                                                                         |
-|-------------|-----------------------------------------------------------------------------------------------------|
+| ----------- | --------------------------------------------------------------------------------------------------- |
 | **`limit`** | The maximum related documents to be returned, default is 10.                                        |
 | **`where`** | An optional `Where` query to filter joined documents. Will be merged with the field `where` object. |
 | **`sort`**  | A string used to order related results                                                              |
+| **`count`** | Whether include the count of related documents or not. Not included by default                      |

 These can be applied to the local API, GraphQL, and REST API.

--- a/packages/db-mongodb/src/utilities/buildJoinAggregation.ts
+++ b/packages/db-mongodb/src/utilities/buildJoinAggregation.ts
@@ -78,6 +78,7 @@ export const buildJoinAggregation = async ({
    }

    const {
+      count = false,
      limit: limitJoin = join.field.defaultLimit ?? 10,
      page,
      sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
@@ -121,14 +122,7 @@ export const buildJoinAggregation = async ({
      const alias = `${as}.docs.${collectionSlug}`
      aliases.push(alias)

-      aggregate.push({
-        $lookup: {
-          as: alias,
-          from: adapter.collections[collectionSlug].collection.name,
-          let: {
-            root_id_: '$_id',
-          },
-          pipeline: [
+      const basePipeline = [
        {
          $addFields: {
            relationTo: {
@@ -148,6 +142,17 @@ export const buildJoinAggregation = async ({
            ],
          },
        },
+      ]
+
+      aggregate.push({
+        $lookup: {
+          as: alias,
+          from: adapter.collections[collectionSlug].collection.name,
+          let: {
+            root_id_: '$_id',
+          },
+          pipeline: [
+            ...basePipeline,
            {
              $sort: {
                [sortProperty]: sortDirection,
@@ -169,6 +174,24 @@ export const buildJoinAggregation = async ({
          ],
        },
      })
+
+      if (count) {
+        aggregate.push({
+          $lookup: {
+            as: `${as}.totalDocs.${alias}`,
+            from: adapter.collections[collectionSlug].collection.name,
+            let: {
+              root_id_: '$_id',
+            },
+            pipeline: [
+              ...basePipeline,
+              {
+                $count: 'result',
+              },
+            ],
+          },
+        })
+      }
    }

    aggregate.push({
@@ -179,6 +202,23 @@ export const buildJoinAggregation = async ({
      },
    })

+    if (count) {
+      aggregate.push({
+        $addFields: {
+          [`${as}.totalDocs`]: {
+            $add: aliases.map((alias) => ({
+              $ifNull: [
+                {
+                  $first: `$${as}.totalDocs.${alias}.result`,
+                },
+                0,
+              ],
+            })),
+          },
+        },
+      })
+    }
+
    aggregate.push({
      $set: {
        [`${as}.docs`]: {
@@ -222,6 +262,7 @@ export const buildJoinAggregation = async ({
      }

      const {
+        count,
        limit: limitJoin = join.field.defaultLimit ?? 10,
        page,
        sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
@@ -274,6 +315,31 @@ export const buildJoinAggregation = async ({
        polymorphicSuffix = '.value'
      }

+      const addTotalDocsAggregation = (as: string, foreignField: string) =>
+        aggregate.push(
+          {
+            $lookup: {
+              as: `${as}.totalDocs`,
+              foreignField,
+              from: adapter.collections[slug].collection.name,
+              localField: versions ? 'parent' : '_id',
+              pipeline: [
+                {
+                  $match,
+                },
+                {
+                  $count: 'result',
+                },
+              ],
+            },
+          },
+          {
+            $addFields: {
+              [`${as}.totalDocs`]: { $ifNull: [{ $first: `$${as}.totalDocs.result` }, 0] },
+            },
+          },
+        )
+
      if (adapter.payload.config.localization && locale === 'all') {
        adapter.payload.config.localization.localeCodes.forEach((code) => {
          const as = `${versions ? `version.${join.joinPath}` : join.joinPath}${code}`
@@ -304,6 +370,7 @@ export const buildJoinAggregation = async ({
              },
            },
          )
+
          if (limitJoin > 0) {
            aggregate.push({
              $addFields: {
@@ -313,6 +380,10 @@ export const buildJoinAggregation = async ({
              },
            })
          }
+
+          if (count) {
+            addTotalDocsAggregation(as, `${join.field.on}${code}${polymorphicSuffix}`)
+          }
        })
      } else {
        const localeSuffix =
@@ -359,6 +430,11 @@ export const buildJoinAggregation = async ({
            },
          },
        )
+
+        if (count) {
+          addTotalDocsAggregation(as, foreignField)
+        }
+
        if (limitJoin > 0) {
          aggregate.push({
            $addFields: {
--- a/packages/drizzle/src/find/traverseFields.ts
+++ b/packages/drizzle/src/find/traverseFields.ts
@@ -2,7 +2,7 @@ import type { LibSQLDatabase } from 'drizzle-orm/libsql'
 import type { SQLiteSelectBase } from 'drizzle-orm/sqlite-core'
 import type { FlattenedField, JoinQuery, SelectMode, SelectType, Where } from 'payload'

-import { and, asc, desc, eq, or, sql } from 'drizzle-orm'
+import { and, asc, count, desc, eq, or, sql } from 'drizzle-orm'
 import { fieldIsVirtual, fieldShouldBeLocalized } from 'payload/shared'
 import toSnakeCase from 'to-snake-case'

@@ -386,6 +386,7 @@ export const traverseFields = ({
        }

        const {
+          count: shouldCount = false,
          limit: limitArg = field.defaultLimit ?? 10,
          page,
          sort = field.defaultSort,
@@ -480,6 +481,13 @@ export const traverseFields = ({
            sqlWhere = and(sqlWhere, buildSQLWhere(where, subQueryAlias))
          }

+          if (shouldCount) {
+            currentArgs.extras[`${columnName}_count`] = sql`${db
+              .select({ count: count() })
+              .from(sql`${currentQuery.as(subQueryAlias)}`)
+              .where(sqlWhere)}`.as(`${columnName}_count`)
+          }
+
          currentQuery = currentQuery.orderBy(sortOrder(sql`"sortPath"`)) as SQLSelect

          if (page && limit !== 0) {
@@ -611,6 +619,20 @@ export const traverseFields = ({
              .orderBy(() => orderBy.map(({ column, order }) => order(column))),
          }).as(subQueryAlias)

+          if (shouldCount) {
+            currentArgs.extras[`${columnName}_count`] = sql`${db
+              .select({
+                count: count(),
+              })
+              .from(
+                sql`${db
+                  .select(selectFields as any)
+                  .from(newAliasTable)
+                  .where(subQueryWhere)
+                  .as(`${subQueryAlias}_count_subquery`)}`,
+              )}`.as(`${subQueryAlias}_count`)
+          }
+
          currentArgs.extras[columnName] = sql`${db
            .select({
              result: jsonAggBuildObject(adapter, {
--- a/packages/drizzle/src/transform/read/traverseFields.ts
+++ b/packages/drizzle/src/transform/read/traverseFields.ts
@@ -1,6 +1,7 @@
 import type { FlattenedBlock, FlattenedField, JoinQuery, SanitizedConfig } from 'payload'

 import { fieldIsVirtual, fieldShouldBeLocalized } from 'payload/shared'
+import toSnakeCase from 'to-snake-case'

 import type { DrizzleAdapter } from '../../types.js'
 import type { BlocksMap } from '../../utilities/createBlocksMap.js'
@@ -398,7 +399,7 @@ export const traverseFields = <T extends Record<string, unknown>>({
    }

    if (field.type === 'join') {
-      const { limit = field.defaultLimit ?? 10 } =
+      const { count, limit = field.defaultLimit ?? 10 } =
        joinQuery?.[`${fieldPrefix.replaceAll('_', '.')}${field.name}`] || {}

      // raw hasMany results from SQLite
@@ -407,8 +408,8 @@ export const traverseFields = <T extends Record<string, unknown>>({
      }

      let fieldResult:
-        | { docs: unknown[]; hasNextPage: boolean }
-        | Record<string, { docs: unknown[]; hasNextPage: boolean }>
+        | { docs: unknown[]; hasNextPage: boolean; totalDocs?: number }
+        | Record<string, { docs: unknown[]; hasNextPage: boolean; totalDocs?: number }>
      if (Array.isArray(fieldData)) {
        if (isLocalized && adapter.payload.config.localization) {
          fieldResult = fieldData.reduce(
@@ -449,6 +450,17 @@ export const traverseFields = <T extends Record<string, unknown>>({
        }
      }

+      if (count) {
+        const countPath = `${fieldName}_count`
+        if (typeof table[countPath] !== 'undefined') {
+          let value = Number(table[countPath])
+          if (Number.isNaN(value)) {
+            value = 0
+          }
+          fieldResult.totalDocs = value
+        }
+      }
+
      result[field.name] = fieldResult
      return result
    }
@@ -607,6 +619,7 @@ export const traverseFields = <T extends Record<string, unknown>>({
            deletions,
            fieldPrefix: groupFieldPrefix,
            fields: field.flattenedFields,
+            joinQuery,
            numbers,
            parentIsLocalized: parentIsLocalized || field.localized,
            path: `${sanitizedPath}${field.name}`,
--- a/packages/payload/src/types/index.ts
+++ b/packages/payload/src/types/index.ts
@@ -145,6 +145,7 @@ export type JoinQuery<TSlug extends CollectionSlug = string> =
        | Partial<{
            [K in keyof TypedCollectionJoins[TSlug]]:
              | {
+                  count?: boolean
                  limit?: number
                  page?: number
                  sort?: string
--- a/packages/payload/src/utilities/configToJSONSchema.ts
+++ b/packages/payload/src/utilities/configToJSONSchema.ts
@@ -434,14 +434,15 @@ export function fieldsToJSONSchema(

            fieldSchema = {
              ...baseFieldSchema,
-              type: withNullableJSONSchemaType('object', false),
+              type: 'object',
              additionalProperties: false,
              properties: {
                docs: {
-                  type: withNullableJSONSchemaType('array', false),
+                  type: 'array',
                  items,
                },
-                hasNextPage: { type: withNullableJSONSchemaType('boolean', false) },
+                hasNextPage: { type: 'boolean' },
+                totalDocs: { type: 'number' },
              },
            }
            break
--- a/packages/payload/src/utilities/sanitizeJoinParams.ts
+++ b/packages/payload/src/utilities/sanitizeJoinParams.ts
@@ -27,6 +27,7 @@ export const sanitizeJoinParams = (
      joinQuery[schemaPath] = false
    } else {
      joinQuery[schemaPath] = {
+        count: joins[schemaPath].count === 'true',
        limit: isNumber(joins[schemaPath]?.limit) ? Number(joins[schemaPath].limit) : undefined,
        page: isNumber(joins[schemaPath]?.page) ? Number(joins[schemaPath].page) : undefined,
        sort: joins[schemaPath]?.sort ? joins[schemaPath].sort : undefined,
--- a/test/joins/int.spec.ts
+++ b/test/joins/int.spec.ts
@@ -186,6 +186,36 @@ describe('Joins Field', () => {
    expect(categoryWithPosts.group.relatedPosts.docs[0].title).toStrictEqual('test 9')
  })

+  it('should count joins', async () => {
+    let categoryWithPosts = await payload.findByID({
+      id: category.id,
+      joins: {
+        'group.relatedPosts': {
+          sort: '-title',
+          count: true,
+        },
+      },
+      collection: categoriesSlug,
+    })
+
+    expect(categoryWithPosts.group.relatedPosts?.totalDocs).toBe(15)
+
+    // With limit 1
+    categoryWithPosts = await payload.findByID({
+      id: category.id,
+      joins: {
+        'group.relatedPosts': {
+          sort: '-title',
+          count: true,
+          limit: 1,
+        },
+      },
+      collection: categoriesSlug,
+    })
+
+    expect(categoryWithPosts.group.relatedPosts?.totalDocs).toBe(15)
+  })
+
  it('should populate relationships in joins', async () => {
    const { docs } = await payload.find({
      limit: 1,
@@ -1302,6 +1332,39 @@ describe('Joins Field', () => {

      expect(parent.children?.docs).toHaveLength(1)
      expect(parent.children.docs[0]?.value.title).toBe('doc-2')
+
+      // counting
+      parent = await payload.findByID({
+        collection: 'multiple-collections-parents',
+        id: parent.id,
+        depth: 1,
+        joins: {
+          children: {
+            count: true,
+          },
+        },
+      })
+
+      expect(parent.children?.totalDocs).toBe(2)
+
+      // counting filtered
+      parent = await payload.findByID({
+        collection: 'multiple-collections-parents',
+        id: parent.id,
+        depth: 1,
+        joins: {
+          children: {
+            count: true,
+            where: {
+              relationTo: {
+                equals: 'multiple-collections-2',
+              },
+            },
+          },
+        },
+      })
+
+      expect(parent.children?.totalDocs).toBe(1)
    })
  })
 })
--- a/test/joins/payload-types.ts
+++ b/test/joins/payload-types.ts
@@ -216,9 +216,10 @@ export interface UserAuthOperations {
 export interface User {
  id: string;
  posts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
  email: string;
@@ -324,9 +325,10 @@ export interface Post {
 export interface Upload {
  id: string;
  relatedPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
  url?: string | null;
@@ -347,74 +349,90 @@ export interface Category {
  id: string;
  name?: string | null;
  relatedPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  /**
   * Static Description
   */
  hasManyPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  hasManyPostsLocalized?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  hiddenPosts?: {
-    docs?: (string | HiddenPost)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | HiddenPost)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  group?: {
    relatedPosts?: {
-      docs?: (string | Post)[] | null;
-      hasNextPage?: boolean | null;
-    } | null;
+      docs?: (string | Post)[];
+      hasNextPage?: boolean;
+      totalDocs?: number;
+    };
    camelCasePosts?: {
-      docs?: (string | Post)[] | null;
-      hasNextPage?: boolean | null;
-    } | null;
+      docs?: (string | Post)[];
+      hasNextPage?: boolean;
+      totalDocs?: number;
+    };
  };
  arrayPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  localizedArrayPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  blocksPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  polymorphic?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  polymorphics?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  localizedPolymorphic?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  localizedPolymorphics?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  singulars?: {
-    docs?: (string | Singular)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Singular)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  filtered?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  joinWithError?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  enableErrorOnJoin?: boolean | null;
  updatedAt: string;
  createdAt: string;
@@ -460,13 +478,15 @@ export interface Version {
 export interface CategoriesVersion {
  id: string;
  relatedVersions?: {
-    docs?: (string | Version)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Version)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  relatedVersionsMany?: {
-    docs?: (string | Version)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Version)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
  _status?: ('draft' | 'published') | null;
@@ -479,9 +499,10 @@ export interface SelfJoin {
  id: string;
  rel?: (string | null) | SelfJoin;
  joins?: {
-    docs?: (string | SelfJoin)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | SelfJoin)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -504,9 +525,10 @@ export interface LocalizedCategory {
  id: string;
  name?: string | null;
  relatedPosts?: {
-    docs?: (string | LocalizedPost)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | LocalizedPost)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -518,9 +540,10 @@ export interface RestrictedCategory {
  id: string;
  name?: string | null;
  restrictedPosts?: {
-    docs?: (string | Post)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | Post)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -532,9 +555,10 @@ export interface CategoriesJoinRestricted {
  id: string;
  name?: string | null;
  collectionRestrictedJoin?: {
-    docs?: (string | CollectionRestricted)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | CollectionRestricted)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -570,9 +594,10 @@ export interface DepthJoins1 {
  id: string;
  rel?: (string | null) | DepthJoins2;
  joins?: {
-    docs?: (string | DepthJoins3)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | DepthJoins3)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -583,9 +608,10 @@ export interface DepthJoins1 {
 export interface DepthJoins2 {
  id: string;
  joins?: {
-    docs?: (string | DepthJoins1)[] | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    docs?: (string | DepthJoins1)[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -606,8 +632,7 @@ export interface DepthJoins3 {
 export interface MultipleCollectionsParent {
  id: string;
  children?: {
-    docs?:
-      | (
+    docs?: (
      | {
          relationTo?: 'multiple-collections-1';
          value: string | MultipleCollections1;
@@ -616,10 +641,10 @@ export interface MultipleCollectionsParent {
          relationTo?: 'multiple-collections-2';
          value: string | MultipleCollections2;
        }
-        )[]
-      | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    )[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }
@@ -656,8 +681,7 @@ export interface Folder {
  folder?: (string | null) | Folder;
  title?: string | null;
  children?: {
-    docs?:
-      | (
+    docs?: (
      | {
          relationTo?: 'folders';
          value: string | Folder;
@@ -670,10 +694,10 @@ export interface Folder {
          relationTo?: 'example-posts';
          value: string | ExamplePost;
        }
-        )[]
-      | null;
-    hasNextPage?: boolean | null;
-  } | null;
+    )[];
+    hasNextPage?: boolean;
+    totalDocs?: number;
+  };
  updatedAt: string;
  createdAt: string;
 }