feat: join field works with hasMany relationships (#8493)

Join field works on relationships and uploads having `hasMany: true`

---------

Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>

packages/db-sqlite/src/init.ts

@@ -70,6 +70,7 @@ export const init: Init = async function init(this: SQLiteAdapter) {
       disableNotNull: !!collection?.versions?.drafts,
       disableUnique: false,
       fields: collection.fields,
+      joins: collection.joins,
       locales,
       tableName,
       timestamps: collection.timestamps,

packages/db-sqlite/src/schema/build.ts

@@ -7,7 +7,7 @@ import type {
   SQLiteTableWithColumns,
   UniqueConstraintBuilder,
 } from 'drizzle-orm/sqlite-core'
-import type { Field } from 'payload'
+import type { Field, SanitizedJoins } from 'payload'
 
 import { createTableName } from '@payloadcms/drizzle'
 import { relations, sql } from 'drizzle-orm'
@@ -58,6 +58,7 @@ type Args = {
   disableNotNull: boolean
   disableUnique: boolean
   fields: Field[]
+  joins?: SanitizedJoins
   locales?: [string, ...string[]]
   rootRelationships?: Set<string>
   rootRelationsToBuild?: RelationMap
@@ -89,6 +90,7 @@ export const buildTable = ({
   disableNotNull,
   disableUnique = false,
   fields,
+  joins,
   locales,
   rootRelationships,
   rootRelationsToBuild,
@@ -134,6 +136,7 @@ export const buildTable = ({
     disableUnique,
     fields,
     indexes,
+    joins,
     locales,
     localesColumns,
     localesIndexes,

packages/db-sqlite/src/schema/traverseFields.ts

@@ -1,6 +1,6 @@
 import type { Relation } from 'drizzle-orm'
 import type { IndexBuilder, SQLiteColumnBuilder } from 'drizzle-orm/sqlite-core'
-import type { Field, TabAsField } from 'payload'
+import type { Field, SanitizedJoins, TabAsField } from 'payload'
 
 import {
   createTableName,
@@ -41,6 +41,7 @@ type Args = {
   fields: (Field | TabAsField)[]
   forceLocalized?: boolean
   indexes: Record<string, (cols: GenericColumns) => IndexBuilder>
+  joins?: SanitizedJoins
   locales: [string, ...string[]]
   localesColumns: Record<string, SQLiteColumnBuilder>
   localesIndexes: Record<string, (cols: GenericColumns) => IndexBuilder>
@@ -78,6 +79,7 @@ export const traverseFields = ({
   fields,
   forceLocalized,
   indexes,
+  joins,
   locales,
   localesColumns,
   localesIndexes,
@@ -651,6 +653,7 @@ export const traverseFields = ({
             fields: field.fields,
             forceLocalized,
             indexes,
+            joins,
             locales,
             localesColumns,
             localesIndexes,
@@ -705,6 +708,7 @@ export const traverseFields = ({
           fields: field.fields,
           forceLocalized: field.localized,
           indexes,
+          joins,
           locales,
           localesColumns,
           localesIndexes,
@@ -760,6 +764,7 @@ export const traverseFields = ({
           fields: field.tabs.map((tab) => ({ ...tab, type: 'tab' })),
           forceLocalized,
           indexes,
+          joins,
           locales,
           localesColumns,
           localesIndexes,
@@ -815,6 +820,7 @@ export const traverseFields = ({
           fields: field.fields,
           forceLocalized,
           indexes,
+          joins,
           locales,
           localesColumns,
           localesIndexes,
@@ -905,9 +911,18 @@ export const traverseFields = ({
 
       case 'join': {
         // fieldName could be 'posts' or 'group_posts'
-        // using on as the key for the relation
+        // using `on` as the key for the relation
         const localized = adapter.payload.config.localization && field.localized
-        const target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${localized ? adapter.localesSuffix : ''}`
+        const fieldSchemaPath = `${fieldPrefix || ''}${field.name}`
+        let target: string
+        const joinConfig = joins[field.collection].find(
+          ({ schemaPath }) => fieldSchemaPath === schemaPath,
+        )
+        if (joinConfig.targetField.hasMany) {
+          target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${adapter.relationshipsSuffix}`
+        } else {
+          target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${localized ? adapter.localesSuffix : ''}`
+        }
         relationsToBuild.set(fieldName, {
           type: 'many',
           // joins are not localized on the parent table

packages/drizzle/src/find/buildFindManyArgs.ts

@@ -36,6 +36,7 @@ export const buildFindManyArgs = ({
   tableName,
 }: BuildFindQueryArgs): Record<string, unknown> => {
   const result: Result = {
+    extras: {},
     with: {},
   }
 
@@ -44,6 +45,7 @@ export const buildFindManyArgs = ({
       id: false,
       _parentID: false,
     },
+    extras: {},
     with: {},
   }
 

packages/drizzle/src/find/traverseFields.ts

@@ -1,14 +1,15 @@
-import type { DBQueryConfig } from 'drizzle-orm'
+import type { LibSQLDatabase } from 'drizzle-orm/libsql'
 import type { Field, JoinQuery } from 'payload'
 
+import { and, type DBQueryConfig, eq, sql } from 'drizzle-orm'
 import { fieldAffectsData, fieldIsVirtual, tabHasName } from 'payload/shared'
 import toSnakeCase from 'to-snake-case'
 
-import type { BuildQueryJoinAliases, DrizzleAdapter } from '../types.js'
+import type { BuildQueryJoinAliases, ChainedMethods, DrizzleAdapter } from '../types.js'
 import type { Result } from './buildFindManyArgs.js'
 
-import { buildOrderBy } from '../queries/buildOrderBy.js'
 import buildQuery from '../queries/buildQuery.js'
+import { chainMethods } from './chainMethods.js'
 
 type TraverseFieldArgs = {
   _locales: Result
@@ -241,24 +242,93 @@ export const traverseFields = ({
             // get an additional document and slice it later to determine if there is a next page
             limit += 1
           }
+
           const fields = adapter.payload.collections[field.collection].config.fields
+          const joinCollectionTableName = adapter.tableNameMap.get(toSnakeCase(field.collection))
           const joinTableName = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${
             field.localized && adapter.payload.config.localization ? adapter.localesSuffix : ''
           }`
+
+          if (!adapter.tables[joinTableName][field.on]) {
+            const db = adapter.drizzle as LibSQLDatabase
+            const joinTable = `${joinTableName}${adapter.relationshipsSuffix}`
+
+            const joins: BuildQueryJoinAliases = [
+              {
+                type: 'innerJoin',
+                condition: and(
+                  eq(adapter.tables[joinTable].parent, adapter.tables[joinTableName].id),
+                  eq(
+                    sql.raw(`"${joinTable}"."${topLevelTableName}_id"`),
+                    adapter.tables[currentTableName].id,
+                  ),
+                ),
+                table: adapter.tables[joinTable],
+              },
+            ]
+
+            const { orderBy, where: subQueryWhere } = buildQuery({
+              adapter,
+              fields,
+              joins,
+              locale,
+              sort,
+              tableName: joinCollectionTableName,
+              where: {},
+            })
+
+            const chainedMethods: ChainedMethods = []
+
+            joins.forEach(({ type, condition, table }) => {
+              chainedMethods.push({
+                args: [table, condition],
+                method: type ?? 'leftJoin',
+              })
+            })
+
+            const subQuery = chainMethods({
+              methods: chainedMethods,
+              query: db
+                .select({
+                  id: adapter.tables[joinTableName].id,
+                })
+                .from(adapter.tables[joinTableName])
+                .where(subQueryWhere)
+                .orderBy(orderBy.order(orderBy.column))
+                .limit(11),
+            })
+
+            const columnName = `${path.replaceAll('.', '_')}${field.name}`
+
+            const extras = field.localized ? _locales.extras : currentArgs.extras
+
+            if (adapter.name === 'sqlite') {
+              extras[columnName] = sql`
+              COALESCE((
+                SELECT json_group_array("id")
+                FROM (
+                  ${subQuery}
+                ) AS ${sql.raw(`${columnName}_sub`)}
+              ), '[]')
+            `.as(columnName)
+            } else {
+              extras[columnName] = sql`
+              COALESCE((
+                SELECT json_agg("id")
+                FROM (
+                  ${subQuery}
+                ) AS ${sql.raw(`${columnName}_sub`)}
+              ), '[]'::json)
+            `.as(columnName)
+            }
+
+            break
+          }
+
           const selectFields = {}
 
-          const orderBy = buildOrderBy({
-            adapter,
-            fields,
-            joins: [],
-            locale,
-            selectFields,
-            sort,
-            tableName: joinTableName,
-          })
           const withJoin: DBQueryConfig<'many', true, any, any> = {
             columns: selectFields,
-            orderBy: () => [orderBy.order(orderBy.column)],
           }
           if (limit) {
             withJoin.limit = limit
@@ -269,20 +339,21 @@ export const traverseFields = ({
             withJoin.columns._parentID = true
           } else {
             withJoin.columns.id = true
+            withJoin.columns.parent = true
           }
-
+          const { orderBy, where: joinWhere } = buildQuery({
-          if (where) {
+            adapter,
-            const { where: joinWhere } = buildQuery({
+            fields,
-              adapter,
+            joins,
-              fields,
+            locale,
-              joins,
+            sort,
-              locale,
+            tableName: joinTableName,
-              sort,
+            where,
-              tableName: joinTableName,
+          })
-              where,
+          if (joinWhere) {
-            })
             withJoin.where = () => joinWhere
           }
+          withJoin.orderBy = orderBy.order(orderBy.column)
           currentArgs.with[`${path.replaceAll('.', '_')}${field.name}`] = withJoin
           break
         }

packages/drizzle/src/postgres/init.ts

@@ -57,6 +57,7 @@ export const init: Init = async function init(this: BasePostgresAdapter) {
       disableNotNull: !!collection?.versions?.drafts,
       disableUnique: false,
       fields: collection.fields,
+      joins: collection.joins,
       tableName,
       timestamps: collection.timestamps,
       versions: false,

packages/drizzle/src/postgres/schema/build.ts

@@ -5,7 +5,7 @@ import type {
   PgColumnBuilder,
   PgTableWithColumns,
 } from 'drizzle-orm/pg-core'
-import type { Field } from 'payload'
+import type { Field, SanitizedJoins } from 'payload'
 
 import { relations } from 'drizzle-orm'
 import {
@@ -47,6 +47,7 @@ type Args = {
   disableNotNull: boolean
   disableUnique: boolean
   fields: Field[]
+  joins?: SanitizedJoins
   rootRelationships?: Set<string>
   rootRelationsToBuild?: RelationMap
   rootTableIDColType?: string
@@ -77,6 +78,7 @@ export const buildTable = ({
   disableNotNull,
   disableUnique = false,
   fields,
+  joins,
   rootRelationships,
   rootRelationsToBuild,
   rootTableIDColType,
@@ -121,6 +123,7 @@ export const buildTable = ({
     disableUnique,
     fields,
     indexes,
+    joins,
     localesColumns,
     localesIndexes,
     newTableName: tableName,

packages/drizzle/src/postgres/schema/traverseFields.ts

@@ -1,6 +1,6 @@
 import type { Relation } from 'drizzle-orm'
 import type { IndexBuilder, PgColumnBuilder } from 'drizzle-orm/pg-core'
-import type { Field, TabAsField } from 'payload'
+import type { Field, SanitizedJoins, TabAsField } from 'payload'
 
 import { relations } from 'drizzle-orm'
 import {
@@ -48,6 +48,7 @@ type Args = {
   fields: (Field | TabAsField)[]
   forceLocalized?: boolean
   indexes: Record<string, (cols: GenericColumns) => IndexBuilder>
+  joins?: SanitizedJoins
   localesColumns: Record<string, PgColumnBuilder>
   localesIndexes: Record<string, (cols: GenericColumns) => IndexBuilder>
   newTableName: string
@@ -84,6 +85,7 @@ export const traverseFields = ({
   fields,
   forceLocalized,
   indexes,
+  joins,
   localesColumns,
   localesIndexes,
   newTableName,
@@ -658,6 +660,7 @@ export const traverseFields = ({
             fields: field.fields,
             forceLocalized,
             indexes,
+            joins,
             localesColumns,
             localesIndexes,
             newTableName,
@@ -711,6 +714,7 @@ export const traverseFields = ({
           fields: field.fields,
           forceLocalized: field.localized,
           indexes,
+          joins,
           localesColumns,
           localesIndexes,
           newTableName: `${parentTableName}_${columnName}`,
@@ -765,6 +769,7 @@ export const traverseFields = ({
           fields: field.tabs.map((tab) => ({ ...tab, type: 'tab' })),
           forceLocalized,
           indexes,
+          joins,
           localesColumns,
           localesIndexes,
           newTableName,
@@ -819,6 +824,7 @@ export const traverseFields = ({
           fields: field.fields,
           forceLocalized,
           indexes,
+          joins,
           localesColumns,
           localesIndexes,
           newTableName,
@@ -908,9 +914,18 @@ export const traverseFields = ({
 
       case 'join': {
         // fieldName could be 'posts' or 'group_posts'
-        // using on as the key for the relation
+        // using `on` as the key for the relation
         const localized = adapter.payload.config.localization && field.localized
-        const target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${localized ? adapter.localesSuffix : ''}`
+        const fieldSchemaPath = `${fieldPrefix || ''}${field.name}`
+        let target: string
+        const joinConfig = joins[field.collection].find(
+          ({ schemaPath }) => fieldSchemaPath === schemaPath,
+        )
+        if (joinConfig.targetField.hasMany) {
+          target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${adapter.relationshipsSuffix}`
+        } else {
+          target = `${adapter.tableNameMap.get(toSnakeCase(field.collection))}${localized ? adapter.localesSuffix : ''}`
+        }
         relationsToBuild.set(fieldName, {
           type: 'many',
           // joins are not localized on the parent table

packages/drizzle/src/queries/buildQuery.ts

@@ -10,6 +10,7 @@ import { parseParams } from './parseParams.js'
 export type BuildQueryJoinAliases = {
   condition: SQL
   table: GenericTable | PgTableWithColumns<any>
+  type?: 'innerJoin' | 'leftJoin' | 'rightJoin'
 }[]
 
 type BuildQueryArgs = {

packages/drizzle/src/transform/read/traverseFields.ts

@@ -137,7 +137,7 @@ export const traverseFields = <T extends Record<string, unknown>>({
       }
 
       const fieldName = `${fieldPrefix || ''}${field.name}`
-      const fieldData = table[fieldName]
+      let fieldData = table[fieldName]
       const localizedFieldData = {}
       const valuesToTransform: {
         ref: Record<string, unknown>
@@ -422,6 +422,11 @@ export const traverseFields = <T extends Record<string, unknown>>({
       if (field.type === 'join') {
         const { limit = 10 } = joinQuery?.[`${fieldPrefix.replaceAll('_', '.')}${field.name}`] || {}
 
+        // raw hasMany results from SQLite
+        if (typeof fieldData === 'string') {
+          fieldData = JSON.parse(fieldData)
+        }
+
         let fieldResult:
           | { docs: unknown[]; hasNextPage: boolean }
           | Record<string, { docs: unknown[]; hasNextPage: boolean }>
@@ -447,7 +452,9 @@ export const traverseFields = <T extends Record<string, unknown>>({
           } else {
             const hasNextPage = limit !== 0 && fieldData.length > limit
             fieldResult = {
-              docs: hasNextPage ? fieldData.slice(0, limit) : fieldData,
+              docs: (hasNextPage ? fieldData.slice(0, limit) : fieldData).map((objOrID) => ({
+                id: typeof objOrID === 'object' ? objOrID.id : objOrID,
+              })),
               hasNextPage,
             }
           }

packages/payload/src/collections/config/types.ts

@@ -31,7 +31,7 @@ import type {
   StaticLabel,
 } from '../../config/types.js'
 import type { DBIdentifierName } from '../../database/types.js'
-import type { Field, JoinField } from '../../fields/config/types.js'
+import type { Field, JoinField, RelationshipField, UploadField } from '../../fields/config/types.js'
 import type {
   CollectionSlug,
   JsonObject,
@@ -485,6 +485,7 @@ export type SanitizedJoin = {
    * The schemaPath of the join field in dot notation
    */
   schemaPath: string
+  targetField: RelationshipField | UploadField
 }
 
 export type SanitizedJoins = {

packages/payload/src/fields/config/sanitizeJoinField.ts

@@ -1,4 +1,4 @@
-import type { SanitizedJoins } from '../../collections/config/types.js'
+import type { SanitizedJoin, SanitizedJoins } from '../../collections/config/types.js'
 import type { Config } from '../../config/types.js'
 import type { JoinField, RelationshipField, UploadField } from './types.js'
 
@@ -23,9 +23,10 @@ export const sanitizeJoinField = ({
   if (!field.maxDepth) {
     field.maxDepth = 1
   }
-  const join = {
+  const join: SanitizedJoin = {
     field,
     schemaPath: `${schemaPath || ''}${schemaPath ? '.' : ''}${field.name}`,
+    targetField: undefined,
   }
   const joinCollection = config.collections.find(
     (collection) => collection.slug === field.collection,
@@ -73,11 +74,12 @@ export const sanitizeJoinField = ({
   if (!joinRelationship) {
     throw new InvalidFieldJoin(join.field)
   }
-
+  if (Array.isArray(joinRelationship.relationTo)) {
-  if (joinRelationship.hasMany) {
+    throw new APIError('Join fields cannot be used with polymorphic relationships.')
-    throw new APIError('Join fields cannot be used with hasMany relationships.')
   }
 
+  join.targetField = joinRelationship
+
   // override the join field localized property to use whatever the relationship field has
   field.localized = joinRelationship.localized
 

packages/payload/src/index.ts

@@ -725,6 +725,7 @@ export type {
   RequiredDataFromCollection,
   RequiredDataFromCollectionSlug,
   SanitizedCollectionConfig,
+  SanitizedJoins,
   TypeWithID,
   TypeWithTimestamps,
 } from './collections/config/types.js'

test/joins/collections/Categories.ts

@@ -49,6 +49,12 @@ export const Categories: CollectionConfig = {
       collection: postsSlug,
       on: 'category',
     },
+    {
+      name: 'hasManyPosts',
+      type: 'join',
+      collection: postsSlug,
+      on: 'categories',
+    },
     {
       name: 'group',
       type: 'group',

test/joins/collections/Posts.ts

@@ -23,6 +23,12 @@ export const Posts: CollectionConfig = {
       type: 'relationship',
       relationTo: categoriesSlug,
     },
+    {
+      name: 'categories',
+      type: 'relationship',
+      relationTo: categoriesSlug,
+      hasMany: true,
+    },
     {
       name: 'group',
       type: 'group',

test/joins/int.spec.ts

@@ -23,6 +23,7 @@ const { email, password } = devUser
 
 describe('Joins Field', () => {
   let category: Category
+  let otherCategory: Category
   let categoryID
   // --__--__--__--__--__--__--__--__--__
   // Boilerplate test setup/teardown
@@ -49,6 +50,14 @@ describe('Joins Field', () => {
       },
     })
 
+    otherCategory = await payload.create({
+      collection: categoriesSlug,
+      data: {
+        name: 'otherCategory',
+        group: {},
+      },
+    })
+
     // create an upload
     const imageFilePath = path.resolve(dirname, './image.png')
     const imageFile = await getFileByPath(imageFilePath)
@@ -62,10 +71,15 @@ describe('Joins Field', () => {
     categoryID = idToString(category.id, payload)
 
     for (let i = 0; i < 15; i++) {
+      let categories = [category.id]
+      if (i % 2 === 0) {
+        categories = [category.id, otherCategory.id]
+      }
       await createPost({
         title: `test ${i}`,
         category: category.id,
         upload: uploadedImage,
+        categories,
         group: {
           category: category.id,
           camelCaseCategory: category.id,
@@ -90,6 +104,15 @@ describe('Joins Field', () => {
       },
       collection: 'categories',
     })
+    // const sortCategoryWithPosts = await payload.findByID({
+    //   id: category.id,
+    //   joins: {
+    //     'group.relatedPosts': {
+    //       sort: 'title',
+    //     },
+    //   },
+    //   collection: 'categories',
+    // })
 
     expect(categoryWithPosts.group.relatedPosts.docs).toHaveLength(10)
     expect(categoryWithPosts.group.relatedPosts.docs[0]).toHaveProperty('id')
@@ -164,6 +187,31 @@ describe('Joins Field', () => {
     expect(categoryWithPosts.group.relatedPosts.docs[0].title).toBe('test 14')
   })
 
+  it('should populate joins using find with hasMany relationships', async () => {
+    const result = await payload.find({
+      collection: 'categories',
+      where: {
+        id: { equals: category.id },
+      },
+    })
+    const otherResult = await payload.find({
+      collection: 'categories',
+      where: {
+        id: { equals: otherCategory.id },
+      },
+    })
+
+    const [categoryWithPosts] = result.docs
+    const [otherCategoryWithPosts] = otherResult.docs
+
+    expect(categoryWithPosts.hasManyPosts.docs).toHaveLength(10)
+    expect(categoryWithPosts.hasManyPosts.docs[0]).toHaveProperty('title')
+    expect(categoryWithPosts.hasManyPosts.docs[0].title).toBe('test 14')
+    expect(otherCategoryWithPosts.hasManyPosts.docs).toHaveLength(8)
+    expect(otherCategoryWithPosts.hasManyPosts.docs[0]).toHaveProperty('title')
+    expect(otherCategoryWithPosts.hasManyPosts.docs[0].title).toBe('test 14')
+  })
+
   it('should not error when deleting documents with joins', async () => {
     const category = await payload.create({
       collection: 'categories',

test/joins/payload-types.ts

@@ -57,6 +57,7 @@ export interface Post {
   title?: string | null;
   upload?: (string | null) | Upload;
   category?: (string | null) | Category;
+  categories?: (string | Category)[] | null;
   group?: {
     category?: (string | null) | Category;
     camelCaseCategory?: (string | null) | Category;
@@ -97,6 +98,10 @@ export interface Category {
     docs?: (string | Post)[] | null;
     hasNextPage?: boolean | null;
   } | null;
+  hasManyPosts?: {
+    docs?: (string | Post)[] | null;
+    hasNextPage?: boolean | null;
+  } | null;
   group?: {
     relatedPosts?: {
       docs?: (string | Post)[] | null;

test/relationships/int.spec.ts

@@ -17,6 +17,7 @@ import type {
 } from './payload-types.js'
 
 import { initPayloadInt } from '../helpers/initPayloadInt.js'
+import { isMongoose } from '../helpers/isMongoose.js'
 import {
   chainedRelSlug,
   customIdNumberSlug,
@@ -397,6 +398,52 @@ describe('Relationships', () => {
           expect(query2.totalDocs).toStrictEqual(2)
         })
 
+        it('should sort by a property of a hasMany relationship', async () => {
+          // no support for sort by relation in mongodb
+          if (isMongoose(payload)) {
+            return
+          }
+
+          const movie1 = await payload.create({
+            collection: 'movies',
+            data: {
+              name: 'Pulp Fiction',
+            },
+          })
+
+          const movie2 = await payload.create({
+            collection: 'movies',
+            data: {
+              name: 'Inception',
+            },
+          })
+
+          await payload.delete({ collection: 'directors', where: {} })
+
+          const director1 = await payload.create({
+            collection: 'directors',
+            data: {
+              name: 'Quentin Tarantino',
+              movies: [movie1.id],
+            },
+          })
+          const director2 = await payload.create({
+            collection: 'directors',
+            data: {
+              name: 'Christopher Nolan',
+              movies: [movie2.id],
+            },
+          })
+
+          const result = await payload.find({
+            collection: 'directors',
+            depth: 0,
+            sort: '-movies.name',
+          })
+
+          expect(result.docs[0].id).toStrictEqual(director1.id)
+        })
+
         it('should query using "in" by hasMany relationship field', async () => {
           const tree1 = await payload.create({
             collection: treeSlug,