feat: select fields (#8550)

Adds `select` which is used to specify the field projection for local
and rest API calls. This is available as an optimization to reduce the
payload's of requests and make the database queries more efficient.

Includes:
- [x] generate types for the `select` property
- [x] infer the return type by `select` with 2 modes - include (`field:
true`) and exclude (`field: false`)
- [x] lots of integration tests, including deep fields / localization
etc
- [x] implement the property in db adapters
- [x] implement the property in the local api for most operations
- [x] implement the property in the rest api 
- [x] docs

---------

Co-authored-by: Dan Ribbens <dan.ribbens@gmail.com>

packages/db-mongodb/src/deleteOne.ts

@@ -2,12 +2,13 @@ import type { DeleteOne, Document, PayloadRequest } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
 export const deleteOne: DeleteOne = async function deleteOne(
   this: MongooseAdapter,
-  { collection, req = {} as PayloadRequest, where },
+  { collection, req = {} as PayloadRequest, select, where },
 ) {
   const Model = this.collections[collection]
   const options = await withSession(this, req)
@@ -17,7 +18,14 @@ export const deleteOne: DeleteOne = async function deleteOne(
     where,
   })
 
-  const doc = await Model.findOneAndDelete(query, options).lean()
+  const doc = await Model.findOneAndDelete(query, {
+    ...options,
+    projection: buildProjectionFromSelect({
+      adapter: this,
+      fields: this.payload.collections[collection].config.fields,
+      select,
+    }),
+  }).lean()
 
   let result: Document = JSON.parse(JSON.stringify(doc))
 

packages/db-mongodb/src/find.ts

@@ -7,6 +7,7 @@ import type { MongooseAdapter } from './index.js'
 
 import { buildSortParam } from './queries/buildSortParam.js'
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
@@ -21,6 +22,7 @@ export const find: Find = async function find(
     pagination,
     projection,
     req = {} as PayloadRequest,
+    select,
     sort: sortArg,
     where,
   },
@@ -67,6 +69,14 @@ export const find: Find = async function find(
     useEstimatedCount,
   }
 
+  if (select) {
+    paginationOptions.projection = buildProjectionFromSelect({
+      adapter: this,
+      fields: collectionConfig.fields,
+      select,
+    })
+  }
+
   if (this.collation) {
     const defaultLocale = 'en'
     paginationOptions.collation = {

packages/db-mongodb/src/findGlobal.ts

@@ -4,17 +4,23 @@ import { combineQueries } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
 export const findGlobal: FindGlobal = async function findGlobal(
   this: MongooseAdapter,
-  { slug, locale, req = {} as PayloadRequest, where },
+  { slug, locale, req = {} as PayloadRequest, select, where },
 ) {
   const Model = this.globals
   const options = {
     ...(await withSession(this, req)),
     lean: true,
+    select: buildProjectionFromSelect({
+      adapter: this,
+      fields: this.payload.globals.config.find((each) => each.slug === slug).fields,
+      select,
+    }),
   }
 
   const query = await Model.buildQuery({

packages/db-mongodb/src/findGlobalVersions.ts

@@ -6,6 +6,7 @@ import { buildVersionGlobalFields, flattenWhereToOperators } from 'payload'
 import type { MongooseAdapter } from './index.js'
 
 import { buildSortParam } from './queries/buildSortParam.js'
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
@@ -18,6 +19,7 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV
     page,
     pagination,
     req = {} as PayloadRequest,
+    select,
     skip,
     sort: sortArg,
     where,
@@ -69,6 +71,7 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV
     options,
     page,
     pagination,
+    projection: buildProjectionFromSelect({ adapter: this, fields: versionFields, select }),
     sort,
     useEstimatedCount,
   }

packages/db-mongodb/src/findOne.ts

@@ -1,15 +1,16 @@
-import type { MongooseQueryOptions } from 'mongoose'
+import type { MongooseQueryOptions, QueryOptions } from 'mongoose'
 import type { Document, FindOne, PayloadRequest } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
 export const findOne: FindOne = async function findOne(
   this: MongooseAdapter,
-  { collection, joins, locale, req = {} as PayloadRequest, where },
+  { collection, joins, locale, req = {} as PayloadRequest, select, where },
 ) {
   const Model = this.collections[collection]
   const collectionConfig = this.payload.collections[collection].config
@@ -24,6 +25,12 @@ export const findOne: FindOne = async function findOne(
     where,
   })
 
+  const projection = buildProjectionFromSelect({
+    adapter: this,
+    fields: collectionConfig.fields,
+    select,
+  })
+
   const aggregate = await buildJoinAggregation({
     adapter: this,
     collection,
@@ -31,6 +38,7 @@ export const findOne: FindOne = async function findOne(
     joins,
     limit: 1,
     locale,
+    projection,
     query,
   })
 
@@ -38,6 +46,7 @@ export const findOne: FindOne = async function findOne(
   if (aggregate) {
     ;[doc] = await Model.aggregate(aggregate, options)
   } else {
+    ;(options as Record<string, unknown>).projection = projection
     doc = await Model.findOne(query, {}, options)
   }
 

packages/db-mongodb/src/findVersions.ts

@@ -1,11 +1,12 @@
 import type { PaginateOptions } from 'mongoose'
 import type { FindVersions, PayloadRequest } from 'payload'
 
-import { flattenWhereToOperators } from 'payload'
+import { buildVersionCollectionFields, flattenWhereToOperators } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
 import { buildSortParam } from './queries/buildSortParam.js'
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
@@ -18,6 +19,7 @@ export const findVersions: FindVersions = async function findVersions(
     page,
     pagination,
     req = {} as PayloadRequest,
+    select,
     skip,
     sort: sortArg,
     where,
@@ -65,6 +67,11 @@ export const findVersions: FindVersions = async function findVersions(
     options,
     page,
     pagination,
+    projection: buildProjectionFromSelect({
+      adapter: this,
+      fields: buildVersionCollectionFields(this.payload.config, collectionConfig),
+      select,
+    }),
     sort,
     useEstimatedCount,
   }

packages/db-mongodb/src/queryDrafts.ts

@@ -1,12 +1,13 @@
 import type { PaginateOptions } from 'mongoose'
 import type { PayloadRequest, QueryDrafts } from 'payload'
 
-import { combineQueries, flattenWhereToOperators } from 'payload'
+import { buildVersionCollectionFields, combineQueries, flattenWhereToOperators } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
 import { buildSortParam } from './queries/buildSortParam.js'
 import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { withSession } from './withSession.js'
 
@@ -20,6 +21,7 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
     page,
     pagination,
     req = {} as PayloadRequest,
+    select,
     sort: sortArg,
     where,
   },
@@ -54,6 +56,11 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
     where: combinedWhere,
   })
 
+  const projection = buildProjectionFromSelect({
+    adapter: this,
+    fields: buildVersionCollectionFields(this.payload.config, collectionConfig),
+    select,
+  })
   // useEstimatedCount is faster, but not accurate, as it ignores any filters. It is thus set to true if there are no filters.
   const useEstimatedCount =
     hasNearConstraint || !versionQuery || Object.keys(versionQuery).length === 0
@@ -64,6 +71,7 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
     options,
     page,
     pagination,
+    projection,
     sort,
     useEstimatedCount,
   }
@@ -109,6 +117,7 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
     joins,
     limit,
     locale,
+    projection,
     query: versionQuery,
     versions: true,
   })

packages/db-mongodb/src/updateGlobal.ts

@@ -2,19 +2,23 @@ import type { PayloadRequest, UpdateGlobal } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
 import { withSession } from './withSession.js'
 
 export const updateGlobal: UpdateGlobal = async function updateGlobal(
   this: MongooseAdapter,
-  { slug, data, req = {} as PayloadRequest },
+  { slug, data, req = {} as PayloadRequest, select },
 ) {
   const Model = this.globals
+  const fields = this.payload.config.globals.find((global) => global.slug === slug).fields
+
   const options = {
     ...(await withSession(this, req)),
     lean: true,
     new: true,
+    projection: buildProjectionFromSelect({ adapter: this, fields, select }),
   }
 
   let result
@@ -22,7 +26,7 @@ export const updateGlobal: UpdateGlobal = async function updateGlobal(
   const sanitizedData = sanitizeRelationshipIDs({
     config: this.payload.config,
     data,
-    fields: this.payload.config.globals.find((global) => global.slug === slug).fields,
+    fields,
   })
 
   result = await Model.findOneAndUpdate({ globalType: slug }, sanitizedData, options)

packages/db-mongodb/src/updateGlobalVersion.ts

@@ -7,6 +7,7 @@ import {
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
 import { withSession } from './withSession.js'
 
@@ -17,16 +18,23 @@ export async function updateGlobalVersion<T extends TypeWithID>(
     global: globalSlug,
     locale,
     req = {} as PayloadRequest,
+    select,
     versionData,
     where,
   }: UpdateGlobalVersionArgs<T>,
 ) {
   const VersionModel = this.versions[globalSlug]
   const whereToUse = where || { id: { equals: id } }
+  const fields = buildVersionGlobalFields(
+    this.payload.config,
+    this.payload.config.globals.find((global) => global.slug === globalSlug),
+  )
+
   const options = {
     ...(await withSession(this, req)),
     lean: true,
     new: true,
+    projection: buildProjectionFromSelect({ adapter: this, fields, select }),
   }
 
   const query = await VersionModel.buildQuery({
@@ -38,10 +46,7 @@ export async function updateGlobalVersion<T extends TypeWithID>(
   const sanitizedData = sanitizeRelationshipIDs({
     config: this.payload.config,
     data: versionData,
-    fields: buildVersionGlobalFields(
-      this.payload.config,
-      this.payload.config.globals.find((global) => global.slug === globalSlug),
-    ),
+    fields,
   })
 
   const doc = await VersionModel.findOneAndUpdate(query, sanitizedData, options)

packages/db-mongodb/src/updateOne.ts

@@ -3,6 +3,7 @@ import type { PayloadRequest, UpdateOne } from 'payload'
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { handleError } from './utilities/handleError.js'
 import { sanitizeInternalFields } from './utilities/sanitizeInternalFields.js'
 import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
@@ -17,16 +18,19 @@ export const updateOne: UpdateOne = async function updateOne(
     locale,
     options: optionsArgs = {},
     req = {} as PayloadRequest,
+    select,
     where: whereArg,
   },
 ) {
   const where = id ? { id: { equals: id } } : whereArg
   const Model = this.collections[collection]
+  const fields = this.payload.collections[collection].config.fields
   const options: QueryOptions = {
     ...optionsArgs,
     ...(await withSession(this, req)),
     lean: true,
     new: true,
+    projection: buildProjectionFromSelect({ adapter: this, fields, select }),
   }
 
   const query = await Model.buildQuery({
@@ -40,7 +44,7 @@ export const updateOne: UpdateOne = async function updateOne(
   const sanitizedData = sanitizeRelationshipIDs({
     config: this.payload.config,
     data,
-    fields: this.payload.collections[collection].config.fields,
+    fields,
   })
 
   try {

packages/db-mongodb/src/updateVersion.ts

@@ -2,19 +2,26 @@ import { buildVersionCollectionFields, type PayloadRequest, type UpdateVersion }
 
 import type { MongooseAdapter } from './index.js'
 
+import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { sanitizeRelationshipIDs } from './utilities/sanitizeRelationshipIDs.js'
 import { withSession } from './withSession.js'
 
 export const updateVersion: UpdateVersion = async function updateVersion(
   this: MongooseAdapter,
-  { id, collection, locale, req = {} as PayloadRequest, versionData, where },
+  { id, collection, locale, req = {} as PayloadRequest, select, versionData, where },
 ) {
   const VersionModel = this.versions[collection]
   const whereToUse = where || { id: { equals: id } }
+  const fields = buildVersionCollectionFields(
+    this.payload.config,
+    this.payload.collections[collection].config,
+  )
+
   const options = {
     ...(await withSession(this, req)),
     lean: true,
     new: true,
+    projection: buildProjectionFromSelect({ adapter: this, fields, select }),
   }
 
   const query = await VersionModel.buildQuery({
@@ -26,10 +33,7 @@ export const updateVersion: UpdateVersion = async function updateVersion(
   const sanitizedData = sanitizeRelationshipIDs({
     config: this.payload.config,
     data: versionData,
-    fields: buildVersionCollectionFields(
-      this.payload.config,
-      this.payload.collections[collection].config,
-    ),
+    fields,
   })
 
   const doc = await VersionModel.findOneAndUpdate(query, sanitizedData, options)

packages/db-mongodb/src/upsert.ts

@@ -4,7 +4,7 @@ import type { MongooseAdapter } from './index.js'
 
 export const upsert: Upsert = async function upsert(
   this: MongooseAdapter,
-  { collection, data, locale, req = {} as PayloadRequest, where },
+  { collection, data, locale, req = {} as PayloadRequest, select, where },
 ) {
-  return this.updateOne({ collection, data, locale, options: { upsert: true }, req, where })
+  return this.updateOne({ collection, data, locale, options: { upsert: true }, req, select, where })
 }

packages/db-mongodb/src/utilities/buildJoinAggregation.ts

@@ -13,6 +13,7 @@ type BuildJoinAggregationArgs = {
   // the number of docs to get at the top collection level
   limit?: number
   locale: string
+  projection?: Record<string, true>
   // the where clause for the top collection
   query?: Where
   /** whether the query is from drafts */
@@ -26,6 +27,7 @@ export const buildJoinAggregation = async ({
   joins,
   limit,
   locale,
+  projection,
   query,
   versions,
 }: BuildJoinAggregationArgs): Promise<PipelineStage[] | undefined> => {
@@ -56,6 +58,10 @@ export const buildJoinAggregation = async ({
     for (const join of joinConfig[slug]) {
       const joinModel = adapter.collections[join.field.collection]
 
+      if (projection && !projection[join.schemaPath]) {
+        continue
+      }
+
       const {
         limit: limitJoin = join.field.defaultLimit ?? 10,
         sort: sortJoin = join.field.defaultSort || collectionConfig.defaultSort,
@@ -174,5 +180,9 @@ export const buildJoinAggregation = async ({
     }
   }
 
+  if (projection) {
+    aggregate.push({ $project: projection })
+  }
+
   return aggregate
 }

packages/db-mongodb/src/utilities/buildProjectionFromSelect.ts

@@ -0,0 +1,234 @@
+import {
+  deepCopyObjectSimple,
+  type Field,
+  type FieldAffectingData,
+  type SelectMode,
+  type SelectType,
+  type TabAsField,
+} from 'payload'
+import { fieldAffectsData, getSelectMode } from 'payload/shared'
+
+import type { MongooseAdapter } from '../index.js'
+
+const addFieldToProjection = ({
+  adapter,
+  databaseSchemaPath,
+  field,
+  projection,
+  withinLocalizedField,
+}: {
+  adapter: MongooseAdapter
+  databaseSchemaPath: string
+  field: FieldAffectingData
+  projection: Record<string, true>
+  withinLocalizedField: boolean
+}) => {
+  const { config } = adapter.payload
+
+  if (withinLocalizedField && config.localization) {
+    for (const locale of config.localization.localeCodes) {
+      const localeDatabaseSchemaPath = databaseSchemaPath.replace('<locale>', locale)
+      projection[`${localeDatabaseSchemaPath}${field.name}`] = true
+    }
+  } else {
+    projection[`${databaseSchemaPath}${field.name}`] = true
+  }
+}
+
+const traverseFields = ({
+  adapter,
+  databaseSchemaPath = '',
+  fields,
+  projection,
+  select,
+  selectAllOnCurrentLevel = false,
+  selectMode,
+  withinLocalizedField = false,
+}: {
+  adapter: MongooseAdapter
+  databaseSchemaPath?: string
+  fields: (Field | TabAsField)[]
+  projection: Record<string, true>
+  select: SelectType
+  selectAllOnCurrentLevel?: boolean
+  selectMode: SelectMode
+  withinLocalizedField?: boolean
+}) => {
+  for (const field of fields) {
+    if (fieldAffectsData(field)) {
+      if (selectMode === 'include') {
+        if (select[field.name] === true || selectAllOnCurrentLevel) {
+          addFieldToProjection({
+            adapter,
+            databaseSchemaPath,
+            field,
+            projection,
+            withinLocalizedField,
+          })
+          continue
+        }
+
+        if (!select[field.name]) {
+          continue
+        }
+      }
+
+      if (selectMode === 'exclude') {
+        if (typeof select[field.name] === 'undefined') {
+          addFieldToProjection({
+            adapter,
+            databaseSchemaPath,
+            field,
+            projection,
+            withinLocalizedField,
+          })
+          continue
+        }
+
+        if (select[field.name] === false) {
+          continue
+        }
+      }
+    }
+
+    let fieldDatabaseSchemaPath = databaseSchemaPath
+    let fieldWithinLocalizedField = withinLocalizedField
+
+    if (fieldAffectsData(field)) {
+      fieldDatabaseSchemaPath = `${databaseSchemaPath}${field.name}.`
+
+      if (field.localized) {
+        fieldDatabaseSchemaPath = `${fieldDatabaseSchemaPath}<locale>.`
+        fieldWithinLocalizedField = true
+      }
+    }
+
+    switch (field.type) {
+      case 'collapsible':
+      case 'row':
+        traverseFields({
+          adapter,
+          databaseSchemaPath,
+          fields: field.fields,
+          projection,
+          select,
+          selectMode,
+          withinLocalizedField,
+        })
+        break
+
+      case 'tabs':
+        traverseFields({
+          adapter,
+          databaseSchemaPath,
+          fields: field.tabs.map((tab) => ({ ...tab, type: 'tab' })),
+          projection,
+          select,
+          selectMode,
+          withinLocalizedField,
+        })
+        break
+
+      case 'group':
+      case 'tab':
+      case 'array':
+        if (field.type === 'array' && selectMode === 'include') {
+          select[field.name]['id'] = true
+        }
+
+        traverseFields({
+          adapter,
+          databaseSchemaPath: fieldDatabaseSchemaPath,
+          fields: field.fields,
+          projection,
+          select: select[field.name] as SelectType,
+          selectMode,
+          withinLocalizedField: fieldWithinLocalizedField,
+        })
+
+        break
+
+      case 'blocks': {
+        const blocksSelect = select[field.name] as SelectType
+
+        for (const block of field.blocks) {
+          if (
+            (selectMode === 'include' && blocksSelect[block.slug] === true) ||
+            (selectMode === 'exclude' && typeof blocksSelect[block.slug] === 'undefined')
+          ) {
+            traverseFields({
+              adapter,
+              databaseSchemaPath: fieldDatabaseSchemaPath,
+              fields: block.fields,
+              projection,
+              select: {},
+              selectAllOnCurrentLevel: true,
+              selectMode: 'include',
+              withinLocalizedField: fieldWithinLocalizedField,
+            })
+            continue
+          }
+
+          let blockSelectMode = selectMode
+
+          if (selectMode === 'exclude' && blocksSelect[block.slug] === false) {
+            blockSelectMode = 'include'
+          }
+
+          if (typeof blocksSelect[block.slug] !== 'object') {
+            blocksSelect[block.slug] = {}
+          }
+
+          if (blockSelectMode === 'include') {
+            blocksSelect[block.slug]['id'] = true
+            blocksSelect[block.slug]['blockType'] = true
+          }
+
+          traverseFields({
+            adapter,
+            databaseSchemaPath: fieldDatabaseSchemaPath,
+            fields: block.fields,
+            projection,
+            select: blocksSelect[block.slug] as SelectType,
+            selectMode: blockSelectMode,
+            withinLocalizedField: fieldWithinLocalizedField,
+          })
+        }
+
+        break
+      }
+
+      default:
+        break
+    }
+  }
+}
+
+export const buildProjectionFromSelect = ({
+  adapter,
+  fields,
+  select,
+}: {
+  adapter: MongooseAdapter
+  fields: Field[]
+  select?: SelectType
+}): Record<string, true> | undefined => {
+  if (!select) {
+    return
+  }
+
+  const projection: Record<string, true> = {
+    _id: true,
+  }
+
+  traverseFields({
+    adapter,
+    fields,
+    projection,
+    // Clone to safely mutate it later
+    select: deepCopyObjectSimple(select),
+    selectMode: getSelectMode(select),
+  })
+
+  return projection
+}