tabshift/payloadcms
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: allow to count related docs for join fields (#11395)

Browse Source 
### What?
For the join field query adds ability to specify `count: true`, example:
```ts
const result = await payload.find({
  joins: {
    'group.relatedPosts': {
      sort: '-title',
      count: true,
    },
  },
  collection: "categories",
})

result.group?.relatedPosts?.totalDocs // available
```

### Why?
Can be useful to implement full pagination / show total related
documents count in the UI.

### How?
Implements the logic in database adapters. In MongoDB it's additional
`$lookup` that has `$count` in the pipeline. In SQL, it's additional
subquery with `COUNT(*)`. Preserves the current behavior by default,
since counting introduces overhead.


Additionally, fixes a typescript generation error for join fields.
Before, `docs` and `hasNextPage` were marked as nullable, which is not
true, these fields cannot be `null`.
Additionally, fixes threading of `joinQuery` in
`transform/read/traverseFields` for group / tab fields recursive calls.
This commit is contained in:
Sasha
2025-02-27 18:05:48 +02:00
committed by GitHub
parent bcc68572bf
commit 3436fb16ea
9 changed files with 345 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
docs/fields/join.mdx
View File

@@ -158,6 +158,7 @@ object with:
- `docs` an array of related documents or only IDs if the depth is reached
- `hasNextPage` a boolean indicating if there are additional documents
- `totalDocs` a total number of documents, exists only if `count: true` is passed to the join query
```json
{
@@ -171,7 +172,8 @@ object with:
}
// { ... }
],
"hasNextPage": false
"hasNextPage": false,
"totalDocs": 10, // if count: true is passed
}
// other fields...
}
@@ -184,6 +186,7 @@ object with:
- `docs` an array of `relationTo` - the collection slug of the document and `value` - the document itself or the ID if the depth is reached
- `hasNextPage` a boolean indicating if there are additional documents
- `totalDocs` a total number of documents, exists only if `count: true` is passed to the join query
```json
{
@@ -200,7 +203,8 @@ object with:
}
// { ... }
],
"hasNextPage": false
"hasNextPage": false,
"totalDocs": 10, // if count: true is passed
}
// other fields...
}
@@ -215,10 +219,11 @@ returning. This is useful for performance reasons when you don't need the relate
The following query options are supported:
| Property | Description |
|-------------|-----------------------------------------------------------------------------------------------------|
| ----------- | --------------------------------------------------------------------------------------------------- |
| **`limit`** | The maximum related documents to be returned, default is 10. |
| **`where`** | An optional `Where` query to filter joined documents. Will be merged with the field `where` object. |
| **`sort`** | A string used to order related results |
| **`count`** | Whether include the count of related documents or not. Not included by default |
These can be applied to the local API, GraphQL, and REST API.

114
packages/db-mongodb/src/utilities/buildJoinAggregation.ts
View File

@@ -78,6 +78,7 @@ export const buildJoinAggregation = async ({
}
const {
count = false,
limit: limitJoin = join.field.defaultLimit ?? 10,
page,
sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
@@ -121,6 +122,28 @@ export const buildJoinAggregation = async ({
const alias = `${as}.docs.${collectionSlug}`
aliases.push(alias)
const basePipeline = [
{
$addFields: {
relationTo: {
$literal: collectionSlug,
},
},
},
{
$match: {
$and: [
{
$expr: {
$eq: [`$${join.field.on}`, '$$root_id_'],
},
},
$match,
],
},
},
]
aggregate.push({
$lookup: {
as: alias,
@@ -129,25 +152,7 @@ export const buildJoinAggregation = async ({
root_id_: '$_id',
},
pipeline: [
{
$addFields: {
relationTo: {
$literal: collectionSlug,
},
},
},
{
$match: {
$and: [
{
$expr: {
$eq: [`$${join.field.on}`, '$$root_id_'],
},
},
$match,
],
},
},
...basePipeline,
{
$sort: {
[sortProperty]: sortDirection,
@@ -169,6 +174,24 @@ export const buildJoinAggregation = async ({
],
},
})
if (count) {
aggregate.push({
$lookup: {
as: `${as}.totalDocs.${alias}`,
from: adapter.collections[collectionSlug].collection.name,
let: {
root_id_: '$_id',
},
pipeline: [
...basePipeline,
{
$count: 'result',
},
],
},
})
}
}
aggregate.push({
@@ -179,6 +202,23 @@ export const buildJoinAggregation = async ({
},
})
if (count) {
aggregate.push({
$addFields: {
[`${as}.totalDocs`]: {
$add: aliases.map((alias) => ({
$ifNull: [
{
$first: `$${as}.totalDocs.${alias}.result`,
},
0,
],
})),
},
},
})
}
aggregate.push({
$set: {
[`${as}.docs`]: {
@@ -222,6 +262,7 @@ export const buildJoinAggregation = async ({
}
const {
count,
limit: limitJoin = join.field.defaultLimit ?? 10,
page,
sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
@@ -274,6 +315,31 @@ export const buildJoinAggregation = async ({
polymorphicSuffix = '.value'
}
const addTotalDocsAggregation = (as: string, foreignField: string) =>
aggregate.push(
{
$lookup: {
as: `${as}.totalDocs`,
foreignField,
from: adapter.collections[slug].collection.name,
localField: versions ? 'parent' : '_id',
pipeline: [
{
$match,
},
{
$count: 'result',
},
],
},
},
{
$addFields: {
[`${as}.totalDocs`]: { $ifNull: [{ $first: `$${as}.totalDocs.result` }, 0] },
},
},
)
if (adapter.payload.config.localization && locale === 'all') {
adapter.payload.config.localization.localeCodes.forEach((code) => {
const as = `${versions ? `version.${join.joinPath}` : join.joinPath}${code}`
@@ -304,6 +370,7 @@ export const buildJoinAggregation = async ({
},
},
)
if (limitJoin > 0) {
aggregate.push({
$addFields: {
@@ -313,6 +380,10 @@ export const buildJoinAggregation = async ({
},
})
}
if (count) {
addTotalDocsAggregation(as, `${join.field.on}${code}${polymorphicSuffix}`)
}
})
} else {
const localeSuffix =
@@ -359,6 +430,11 @@ export const buildJoinAggregation = async ({
},
},
)
if (count) {
addTotalDocsAggregation(as, foreignField)
}
if (limitJoin > 0) {
aggregate.push({
$addFields: {

24
packages/drizzle/src/find/traverseFields.ts
View File

@@ -2,7 +2,7 @@ import type { LibSQLDatabase } from 'drizzle-orm/libsql'
import type { SQLiteSelectBase } from 'drizzle-orm/sqlite-core'
import type { FlattenedField, JoinQuery, SelectMode, SelectType, Where } from 'payload'
import { and, asc, desc, eq, or, sql } from 'drizzle-orm'
import { and, asc, count, desc, eq, or, sql } from 'drizzle-orm'
import { fieldIsVirtual, fieldShouldBeLocalized } from 'payload/shared'
import toSnakeCase from 'to-snake-case'
@@ -386,6 +386,7 @@ export const traverseFields = ({
}
const {
count: shouldCount = false,
limit: limitArg = field.defaultLimit ?? 10,
page,
sort = field.defaultSort,
@@ -480,6 +481,13 @@ export const traverseFields = ({
sqlWhere = and(sqlWhere, buildSQLWhere(where, subQueryAlias))
}
if (shouldCount) {
currentArgs.extras[`${columnName}_count`] = sql`${db
.select({ count: count() })
.from(sql`${currentQuery.as(subQueryAlias)}`)
.where(sqlWhere)}`.as(`${columnName}_count`)
}
currentQuery = currentQuery.orderBy(sortOrder(sql`"sortPath"`)) as SQLSelect
if (page && limit !== 0) {
@@ -611,6 +619,20 @@ export const traverseFields = ({
.orderBy(() => orderBy.map(({ column, order }) => order(column))),
}).as(subQueryAlias)
if (shouldCount) {
currentArgs.extras[`${columnName}_count`] = sql`${db
.select({
count: count(),
})
.from(
sql`${db
.select(selectFields as any)
.from(newAliasTable)
.where(subQueryWhere)
.as(`${subQueryAlias}_count_subquery`)}`,
)}`.as(`${subQueryAlias}_count`)
}
currentArgs.extras[columnName] = sql`${db
.select({
result: jsonAggBuildObject(adapter, {

19
packages/drizzle/src/transform/read/traverseFields.ts
View File

@@ -1,6 +1,7 @@
import type { FlattenedBlock, FlattenedField, JoinQuery, SanitizedConfig } from 'payload'
import { fieldIsVirtual, fieldShouldBeLocalized } from 'payload/shared'
import toSnakeCase from 'to-snake-case'
import type { DrizzleAdapter } from '../../types.js'
import type { BlocksMap } from '../../utilities/createBlocksMap.js'
@@ -398,7 +399,7 @@ export const traverseFields = <T extends Record<string, unknown>>({
}
if (field.type === 'join') {
const { limit = field.defaultLimit ?? 10 } =
const { count, limit = field.defaultLimit ?? 10 } =
joinQuery?.[`${fieldPrefix.replaceAll('_', '.')}${field.name}`] || {}
// raw hasMany results from SQLite
@@ -407,8 +408,8 @@ export const traverseFields = <T extends Record<string, unknown>>({
}
let fieldResult:
| { docs: unknown[]; hasNextPage: boolean }
| Record<string, { docs: unknown[]; hasNextPage: boolean }>
| { docs: unknown[]; hasNextPage: boolean; totalDocs?: number }
| Record<string, { docs: unknown[]; hasNextPage: boolean; totalDocs?: number }>
if (Array.isArray(fieldData)) {
if (isLocalized && adapter.payload.config.localization) {
fieldResult = fieldData.reduce(
@@ -449,6 +450,17 @@ export const traverseFields = <T extends Record<string, unknown>>({
}
}
if (count) {
const countPath = `${fieldName}_count`
if (typeof table[countPath] !== 'undefined') {
let value = Number(table[countPath])
if (Number.isNaN(value)) {
value = 0
}
fieldResult.totalDocs = value
}
}
result[field.name] = fieldResult
return result
}
@@ -607,6 +619,7 @@ export const traverseFields = <T extends Record<string, unknown>>({
deletions,
fieldPrefix: groupFieldPrefix,
fields: field.flattenedFields,
joinQuery,
numbers,
parentIsLocalized: parentIsLocalized || field.localized,
path: `${sanitizedPath}${field.name}`,

1
packages/payload/src/types/index.ts
View File

@@ -145,6 +145,7 @@ export type JoinQuery<TSlug extends CollectionSlug = string> =
| Partial<{
[K in keyof TypedCollectionJoins[TSlug]]:
| {
count?: boolean
limit?: number
page?: number
sort?: string

7
packages/payload/src/utilities/configToJSONSchema.ts
View File

@@ -434,14 +434,15 @@ export function fieldsToJSONSchema(
fieldSchema = {
...baseFieldSchema,
type: withNullableJSONSchemaType('object', false),
type: 'object',
additionalProperties: false,
properties: {
docs: {
type: withNullableJSONSchemaType('array', false),
type: 'array',
items,
},
hasNextPage: { type: withNullableJSONSchemaType('boolean', false) },
hasNextPage: { type: 'boolean' },
totalDocs: { type: 'number' },
},
}
break

1
packages/payload/src/utilities/sanitizeJoinParams.ts
View File

@@ -27,6 +27,7 @@ export const sanitizeJoinParams = (
joinQuery[schemaPath] = false
} else {
joinQuery[schemaPath] = {
count: joins[schemaPath].count === 'true',
limit: isNumber(joins[schemaPath]?.limit) ? Number(joins[schemaPath].limit) : undefined,
page: isNumber(joins[schemaPath]?.page) ? Number(joins[schemaPath].page) : undefined,
sort: joins[schemaPath]?.sort ? joins[schemaPath].sort : undefined,

63
test/joins/int.spec.ts
View File

@@ -186,6 +186,36 @@ describe('Joins Field', () => {
expect(categoryWithPosts.group.relatedPosts.docs[0].title).toStrictEqual('test 9')
})
it('should count joins', async () => {
let categoryWithPosts = await payload.findByID({
id: category.id,
joins: {
'group.relatedPosts': {
sort: '-title',
count: true,
},
},
collection: categoriesSlug,
})
expect(categoryWithPosts.group.relatedPosts?.totalDocs).toBe(15)
// With limit 1
categoryWithPosts = await payload.findByID({
id: category.id,
joins: {
'group.relatedPosts': {
sort: '-title',
count: true,
limit: 1,
},
},
collection: categoriesSlug,
})
expect(categoryWithPosts.group.relatedPosts?.totalDocs).toBe(15)
})
it('should populate relationships in joins', async () => {
const { docs } = await payload.find({
limit: 1,
@@ -1302,6 +1332,39 @@ describe('Joins Field', () => {
expect(parent.children?.docs).toHaveLength(1)
expect(parent.children.docs[0]?.value.title).toBe('doc-2')
// counting
parent = await payload.findByID({
collection: 'multiple-collections-parents',
id: parent.id,
depth: 1,
joins: {
children: {
count: true,
},
},
})
expect(parent.children?.totalDocs).toBe(2)
// counting filtered
parent = await payload.findByID({
collection: 'multiple-collections-parents',
id: parent.id,
depth: 1,
joins: {
children: {
count: true,
where: {
relationTo: {
equals: 'multiple-collections-2',
},
},
},
},
})
expect(parent.children?.totalDocs).toBe(1)
})
})
})

244
test/joins/payload-types.ts
View File

@@ -216,9 +216,10 @@ export interface UserAuthOperations {
export interface User {
id: string;
posts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
email: string;
@@ -324,9 +325,10 @@ export interface Post {
export interface Upload {
id: string;
relatedPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
url?: string | null;
@@ -347,74 +349,90 @@ export interface Category {
id: string;
name?: string | null;
relatedPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
/**
* Static Description
*/
hasManyPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
hasManyPostsLocalized?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
hiddenPosts?: {
docs?: (string | HiddenPost)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | HiddenPost)[];
hasNextPage?: boolean;
totalDocs?: number;
};
group?: {
relatedPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
camelCasePosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
};
arrayPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
localizedArrayPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
blocksPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
polymorphic?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
polymorphics?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
localizedPolymorphic?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
localizedPolymorphics?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
singulars?: {
docs?: (string | Singular)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Singular)[];
hasNextPage?: boolean;
totalDocs?: number;
};
filtered?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
joinWithError?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
enableErrorOnJoin?: boolean | null;
updatedAt: string;
createdAt: string;
@@ -460,13 +478,15 @@ export interface Version {
export interface CategoriesVersion {
id: string;
relatedVersions?: {
docs?: (string | Version)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Version)[];
hasNextPage?: boolean;
totalDocs?: number;
};
relatedVersionsMany?: {
docs?: (string | Version)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Version)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
_status?: ('draft' | 'published') | null;
@@ -479,9 +499,10 @@ export interface SelfJoin {
id: string;
rel?: (string | null) | SelfJoin;
joins?: {
docs?: (string | SelfJoin)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | SelfJoin)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -504,9 +525,10 @@ export interface LocalizedCategory {
id: string;
name?: string | null;
relatedPosts?: {
docs?: (string | LocalizedPost)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | LocalizedPost)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -518,9 +540,10 @@ export interface RestrictedCategory {
id: string;
name?: string | null;
restrictedPosts?: {
docs?: (string | Post)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | Post)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -532,9 +555,10 @@ export interface CategoriesJoinRestricted {
id: string;
name?: string | null;
collectionRestrictedJoin?: {
docs?: (string | CollectionRestricted)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | CollectionRestricted)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -570,9 +594,10 @@ export interface DepthJoins1 {
id: string;
rel?: (string | null) | DepthJoins2;
joins?: {
docs?: (string | DepthJoins3)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | DepthJoins3)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -583,9 +608,10 @@ export interface DepthJoins1 {
export interface DepthJoins2 {
id: string;
joins?: {
docs?: (string | DepthJoins1)[] | null;
hasNextPage?: boolean | null;
} | null;
docs?: (string | DepthJoins1)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -606,20 +632,19 @@ export interface DepthJoins3 {
export interface MultipleCollectionsParent {
id: string;
children?: {
docs?:
| (
| {
relationTo?: 'multiple-collections-1';
value: string | MultipleCollections1;
}
| {
relationTo?: 'multiple-collections-2';
value: string | MultipleCollections2;
}
)[]
| null;
hasNextPage?: boolean | null;
} | null;
docs?: (
| {
relationTo?: 'multiple-collections-1';
value: string | MultipleCollections1;
}
| {
relationTo?: 'multiple-collections-2';
value: string | MultipleCollections2;
}
)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}
@@ -656,24 +681,23 @@ export interface Folder {
folder?: (string | null) | Folder;
title?: string | null;
children?: {
docs?:
| (
| {
relationTo?: 'folders';
value: string | Folder;
}
| {
relationTo?: 'example-pages';
value: string | ExamplePage;
}
| {
relationTo?: 'example-posts';
value: string | ExamplePost;
}
)[]
| null;
hasNextPage?: boolean | null;
} | null;
docs?: (
| {
relationTo?: 'folders';
value: string | Folder;
}
| {
relationTo?: 'example-pages';
value: string | ExamplePage;
}
| {
relationTo?: 'example-posts';
value: string | ExamplePost;
}
)[];
hasNextPage?: boolean;
totalDocs?: number;
};
updatedAt: string;
createdAt: string;
}