feat(plugin-import-export): add custom toCSV function on fields (#12533)

This makes it possible to add custom logic into how we map the document
data into the CSV data on a field-by-field basis.

- Allow custom data transformation to be added to
`custom.['plugin-import-export'].toCSV inside the field config
- Add type declaration to FieldCustom to improve types
- Export with `depth: 1`

Example:
```ts
    {
      name: 'customRelationship',
      type: 'relationship',
      relationTo: 'users',
      custom: {
        'plugin-import-export': {
          toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
            row[`${columnName}_id`] = value.id
            row[`${columnName}_email`] = value.email
          },
        },
      },
    },
```

docs/plugins/form-builder.mdx

@@ -551,4 +551,4 @@ Below are some common troubleshooting tips. To help other developers, please con
 
 ![screenshot 5](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-5.jpg?raw=true)
 
-![screenshot 6](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-6.jpg?raw=true)
+![screenshot 6](https://github.com/payloadcms/plugin-form-builder/blob/main/images/screenshot-6.jpg?raw=true)

packages/payload/src/fields/config/types.ts

@@ -128,6 +128,7 @@ import type {
   CollectionSlug,
   DateFieldValidation,
   EmailFieldValidation,
+  FieldCustom,
   JSONFieldValidation,
   PointFieldValidation,
   RadioFieldValidation,
@@ -482,7 +483,7 @@ export interface FieldBase {
   }
   admin?: Admin
   /** Extension point to add your custom data. Server only. */
-  custom?: Record<string, any>
+  custom?: FieldCustom
   defaultValue?: DefaultValue
   hidden?: boolean
   hooks?: {

packages/payload/src/index.ts

@@ -1238,6 +1238,9 @@ export {
 } from './fields/config/client.js'
 
 export { sanitizeFields } from './fields/config/sanitize.js'
+
+export interface FieldCustom extends Record<string, any> {}
+
 export type {
   AdminClient,
   ArrayField,

packages/payload/src/utilities/traverseFields.ts

@@ -304,12 +304,12 @@ export const traverseFields = ({
       return
     }
 
-    if (field.type !== 'tab' && (fieldHasSubFields(field) || field.type === 'blocks')) {
+    if (field.type === 'tab' || fieldHasSubFields(field) || field.type === 'blocks') {
       if ('name' in field && field.name) {
         currentParentRef = currentRef
         if (!ref[field.name]) {
           if (fillEmpty) {
-            if (field.type === 'group') {
+            if (field.type === 'group' || field.type === 'tab') {
               if (fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! })) {
                 ref[field.name] = {
                   en: {},
@@ -334,7 +334,7 @@ export const traverseFields = ({
       }
 
       if (
-        field.type === 'group' &&
+        (field.type === 'tab' || field.type === 'group') &&
         fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! }) &&
         currentRef &&
         typeof currentRef === 'object'

packages/plugin-import-export/src/export/createExport.ts

@@ -6,6 +6,7 @@ import { APIError } from 'payload'
 import { Readable } from 'stream'
 
 import { flattenObject } from './flattenObject.js'
+import { getCustomFieldFunctions } from './getCustomFieldFunctions.js'
 import { getFilename } from './getFilename.js'
 import { getSelect } from './getSelect.js'
 
@@ -79,6 +80,7 @@ export const createExport = async (args: CreateExportArgs) => {
 
   const name = `${nameArg ?? `${getFilename()}-${collectionSlug}`}.${format}`
   const isCSV = format === 'csv'
+  const select = Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined
 
   if (debug) {
     req.payload.logger.info({ message: 'Export configuration:', name, isCSV, locale })
@@ -86,13 +88,13 @@ export const createExport = async (args: CreateExportArgs) => {
 
   const findArgs = {
     collection: collectionSlug,
-    depth: 0,
+    depth: 1,
     draft: drafts === 'yes',
     limit: 100,
     locale,
     overrideAccess: false,
     page: 0,
-    select: Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined,
+    select,
     sort,
     user,
     where,
@@ -104,6 +106,11 @@ export const createExport = async (args: CreateExportArgs) => {
 
   let result: PaginatedDocs = { hasNextPage: true } as PaginatedDocs
 
+  const toCSVFunctions = getCustomFieldFunctions({
+    fields: collectionConfig.flattenedFields,
+    select,
+  })
+
   if (download) {
     if (debug) {
       req.payload.logger.info('Starting download stream')
@@ -120,7 +127,7 @@ export const createExport = async (args: CreateExportArgs) => {
               `Processing batch ${findArgs.page + 1} with ${result.docs.length} documents`,
             )
           }
-          const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
+          const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
           const csvString = stringify(csvInput, { header: isFirstBatch })
           this.push(encoder.encode(csvString))
           isFirstBatch = false
@@ -164,7 +171,7 @@ export const createExport = async (args: CreateExportArgs) => {
     }
 
     if (isCSV) {
-      const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
+      const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
       outputData.push(stringify(csvInput, { header: isFirstBatch }))
       isFirstBatch = false
     } else {

packages/plugin-import-export/src/export/flattenObject.ts

@@ -1,16 +1,24 @@
 import type { Document } from 'payload'
 
+import type { ToCSVFunction } from '../types.js'
+
 type Args = {
   doc: Document
   fields?: string[]
   prefix?: string
+  toCSVFunctions: Record<string, ToCSVFunction>
 }
 
-export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unknown> => {
+export const flattenObject = ({
-  const result: Record<string, unknown> = {}
+  doc,
+  fields,
+  prefix,
+  toCSVFunctions,
+}: Args): Record<string, unknown> => {
+  const row: Record<string, unknown> = {}
 
-  const flatten = (doc: Document, prefix?: string) => {
+  const flatten = (siblingDoc: Document, prefix?: string) => {
-    Object.entries(doc).forEach(([key, value]) => {
+    Object.entries(siblingDoc).forEach(([key, value]) => {
       const newKey = prefix ? `${prefix}_${key}` : key
 
       if (Array.isArray(value)) {
@@ -18,13 +26,44 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
           if (typeof item === 'object' && item !== null) {
             flatten(item, `${newKey}_${index}`)
           } else {
-            result[`${newKey}_${index}`] = item
+            if (toCSVFunctions?.[newKey]) {
+              const columnName = `${newKey}_${index}`
+              row[columnName] = toCSVFunctions[newKey]({
+                columnName,
+                doc,
+                row,
+                siblingDoc,
+                value: item,
+              })
+            } else {
+              row[`${newKey}_${index}`] = item
+            }
           }
         })
       } else if (typeof value === 'object' && value !== null) {
-        flatten(value, newKey)
+        if (!toCSVFunctions?.[newKey]) {
+          flatten(value, newKey)
+        } else {
+          row[newKey] = toCSVFunctions[newKey]({
+            columnName: newKey,
+            doc,
+            row,
+            siblingDoc,
+            value,
+          })
+        }
       } else {
-        result[newKey] = value
+        if (toCSVFunctions?.[newKey]) {
+          row[newKey] = toCSVFunctions[newKey]({
+            columnName: newKey,
+            doc,
+            row,
+            siblingDoc,
+            value,
+          })
+        } else {
+          row[newKey] = value
+        }
       }
     })
   }
@@ -41,14 +80,14 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
     }
 
     fields.forEach((field) => {
-      if (result[field.replace(/\./g, '_')]) {
+      if (row[field.replace(/\./g, '_')]) {
         const sanitizedField = field.replace(/\./g, '_')
-        orderedResult[sanitizedField] = result[sanitizedField]
+        orderedResult[sanitizedField] = row[sanitizedField]
       } else {
         const regex = fieldToRegex(field)
-        Object.keys(result).forEach((key) => {
+        Object.keys(row).forEach((key) => {
           if (regex.test(key)) {
-            orderedResult[key] = result[key]
+            orderedResult[key] = row[key]
           }
         })
       }
@@ -57,5 +96,5 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
     return orderedResult
   }
 
-  return result
+  return row
 }

packages/plugin-import-export/src/export/getCustomFieldFunctions.ts

@@ -0,0 +1,93 @@
+import {
+  type FlattenedField,
+  type SelectIncludeType,
+  traverseFields,
+  type TraverseFieldsCallback,
+} from 'payload'
+
+import type { ToCSVFunction } from '../types.js'
+
+type Args = {
+  fields: FlattenedField[]
+  select: SelectIncludeType | undefined
+}
+
+export const getCustomFieldFunctions = ({
+  fields,
+  select,
+}: Args): Record<string, ToCSVFunction> => {
+  const result: Record<string, ToCSVFunction> = {}
+
+  const buildCustomFunctions: TraverseFieldsCallback = ({ field, parentRef, ref }) => {
+    // @ts-expect-error ref is untyped
+    ref.prefix = parentRef.prefix || ''
+    if (field.type === 'group' || field.type === 'tab') {
+      // @ts-expect-error ref is untyped
+      const parentPrefix = parentRef?.prefix ? `${parentRef.prefix}_` : ''
+      // @ts-expect-error ref is untyped
+      ref.prefix = `${parentPrefix}${field.name}_`
+    }
+
+    if (typeof field.custom?.['plugin-import-export']?.toCSV === 'function') {
+      // @ts-expect-error ref is untyped
+      result[`${ref.prefix}${field.name}`] = field.custom['plugin-import-export']?.toCSV
+    } else if (field.type === 'relationship' || field.type === 'upload') {
+      if (field.hasMany !== true) {
+        if (!Array.isArray(field.relationTo)) {
+          // monomorphic single
+          // @ts-expect-error ref is untyped
+          result[`${ref.prefix}${field.name}`] = ({ value }) =>
+            typeof value === 'object' && value && 'id' in value ? value.id : value
+        } else {
+          // polymorphic single
+          // @ts-expect-error ref is untyped
+          result[`${ref.prefix}${field.name}`] = ({ data, value }) => {
+            // @ts-expect-error ref is untyped
+            data[`${ref.prefix}${field.name}_id`] = value.id
+            // @ts-expect-error ref is untyped
+            data[`${ref.prefix}${field.name}_relationTo`] = value.relationTo
+            return undefined
+          }
+        }
+      } else {
+        if (!Array.isArray(field.relationTo)) {
+          // monomorphic many
+          // @ts-expect-error ref is untyped
+          result[`${ref.prefix}${field.name}`] = ({
+            value,
+          }: {
+            value: Record<string, unknown>[]
+          }) =>
+            value.map((val: number | Record<string, unknown> | string) =>
+              typeof val === 'object' ? val.id : val,
+            )
+        } else {
+          // polymorphic many
+          // @ts-expect-error ref is untyped
+          result[`${ref.prefix}${field.name}`] = ({
+            data,
+            value,
+          }: {
+            data: Record<string, unknown>
+            value: Record<string, unknown>[]
+          }) =>
+            value.map((val: number | Record<string, unknown> | string, i) => {
+              // @ts-expect-error ref is untyped
+              data[`${ref.prefix}${field.name}_${i}_id`] = val.id
+              // @ts-expect-error ref is untyped
+              data[`${ref.prefix}${field.name}_${i}_relationTo`] = val.relationTo
+              return undefined
+            })
+        }
+      }
+    }
+
+    // TODO: do this so we only return the functions needed based on the select used
+    ////@ts-expect-error ref is untyped
+    // ref.select = typeof select !== 'undefined' || select[field.name] ? select : {}
+  }
+
+  traverseFields({ callback: buildCustomFunctions, fields })
+
+  return result
+}

packages/plugin-import-export/src/export/getSelect.ts

@@ -1,17 +1,13 @@
-import type { SelectType } from 'payload'
+import type { SelectIncludeType } from 'payload'
 
 /**
  * Takes an input of array of string paths in dot notation and returns a select object
  * example args: ['id', 'title', 'group.value', 'createdAt', 'updatedAt']
  */
-export const getSelect = (fields: string[]): SelectType => {
+export const getSelect = (fields: string[]): SelectIncludeType => {
-  const select: SelectType = {}
+  const select: SelectIncludeType = {}
 
   fields.forEach((field) => {
-    // TODO: this can likely be removed, the form was not saving, leaving in for now
-    if (!field) {
-      return
-    }
     const segments = field.split('.')
     let selectRef = select
 
@@ -22,7 +18,7 @@ export const getSelect = (fields: string[]): SelectType => {
         if (!selectRef[segment]) {
           selectRef[segment] = {}
         }
-        selectRef = selectRef[segment] as SelectType
+        selectRef = selectRef[segment] as SelectIncludeType
       }
     })
   })

packages/plugin-import-export/src/exports/types.ts

@@ -1 +1 @@
-export type { ImportExportPluginConfig } from '../types.js'
+export type { ImportExportPluginConfig, ToCSVFunction } from '../types.js'

packages/plugin-import-export/src/getExportCollection.ts

@@ -1,6 +1,5 @@
 import type {
   CollectionAfterChangeHook,
-  CollectionBeforeChangeHook,
   CollectionBeforeOperationHook,
   CollectionConfig,
   Config,

packages/plugin-import-export/src/index.ts

@@ -3,7 +3,7 @@ import type { Config, JobsConfig } from 'payload'
 import { deepMergeSimple } from 'payload'
 
 import type { PluginDefaultTranslationsObject } from './translations/types.js'
-import type { ImportExportPluginConfig } from './types.js'
+import type { ImportExportPluginConfig, ToCSVFunction } from './types.js'
 
 import { getCreateCollectionExportTask } from './export/getCreateExportCollectionTask.js'
 import { getExportCollection } from './getExportCollection.js'
@@ -91,3 +91,11 @@ export const importExportPlugin =
 
     return config
   }
+
+declare module 'payload' {
+  export interface FieldCustom {
+    'plugin-import-export'?: {
+      toCSVFunction?: ToCSVFunction
+    }
+  }
+}

packages/plugin-import-export/src/types.ts

@@ -26,3 +26,29 @@ export type ImportExportPluginConfig = {
    */
   overrideExportCollection?: (collection: CollectionOverride) => CollectionOverride
 }
+
+/**
+ * Custom function used to modify the outgoing csv data by manipulating the data, siblingData or by returning the desired value
+ */
+export type ToCSVFunction = (args: {
+  /**
+   * The path of the column for the field, for arrays this includes the index (zero-based)
+   */
+  columnName: string
+  /**
+   * The top level document
+   */
+  doc: Document
+  /**
+   * The object data that can be manipulated to assign data to the CSV
+   */
+  row: Record<string, unknown>
+  /**
+   * The document data at the level where it belongs
+   */
+  siblingDoc: Record<string, unknown>
+  /**
+   * The data for the field.
+   */
+  value: unknown
+}) => unknown

test/plugin-import-export/collections/Pages.ts

@@ -26,6 +26,31 @@ export const Pages: CollectionConfig = {
       type: 'text',
       localized: true,
     },
+    {
+      name: 'custom',
+      type: 'text',
+      defaultValue: 'my custom csv transformer',
+      custom: {
+        'plugin-import-export': {
+          toCSV: ({ value, columnName, row, siblingDoc }) => {
+            return value + ' toCSV'
+          },
+        },
+      },
+    },
+    {
+      name: 'customRelationship',
+      type: 'relationship',
+      relationTo: 'users',
+      custom: {
+        'plugin-import-export': {
+          toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
+            row[`${columnName}_id`] = value.id
+            row[`${columnName}_email`] = value.email
+          },
+        },
+      },
+    },
     {
       name: 'group',
       type: 'group',
@@ -53,6 +78,58 @@ export const Pages: CollectionConfig = {
             },
           ],
         },
+        {
+          name: 'custom',
+          type: 'text',
+          defaultValue: 'my custom csv transformer',
+          custom: {
+            'plugin-import-export': {
+              toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
+                return value + ' toCSV'
+              },
+            },
+          },
+        },
+      ],
+    },
+    {
+      name: 'tabs',
+      type: 'tabs',
+      tabs: [
+        {
+          label: 'No Name',
+          fields: [
+            {
+              name: 'tabToCSV',
+              type: 'text',
+              defaultValue: 'my custom csv transformer',
+              custom: {
+                'plugin-import-export': {
+                  toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
+                    return value + ' toCSV'
+                  },
+                },
+              },
+            },
+          ],
+        },
+        {
+          name: 'namedTab',
+          fields: [
+            {
+              name: 'tabToCSV',
+              type: 'text',
+              defaultValue: 'my custom csv transformer',
+              custom: {
+                'plugin-import-export': {
+                  toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
+                    return value + ' toCSV'
+                  },
+                },
+              },
+            },
+          ],
+        },
       ],
     },
     {

test/plugin-import-export/int.spec.ts

@@ -368,6 +368,47 @@ describe('@payloadcms/plugin-import-export', () => {
       expect(data[0].blocks_1_blockType).toStrictEqual('content')
     })
 
+    it('should run custom toCSV function on a field', async () => {
+      const fields = [
+        'id',
+        'custom',
+        'group.custom',
+        'customRelationship',
+        'tabToCSV',
+        'namedTab.tabToCSV',
+      ]
+      const doc = await payload.create({
+        collection: 'exports',
+        user,
+        data: {
+          collectionSlug: 'pages',
+          fields,
+          format: 'csv',
+          where: {
+            title: { contains: 'Custom ' },
+          },
+        },
+      })
+
+      const exportDoc = await payload.findByID({
+        collection: 'exports',
+        id: doc.id,
+      })
+
+      expect(exportDoc.filename).toBeDefined()
+      const expectedPath = path.join(dirname, './uploads', exportDoc.filename as string)
+      const data = await readCSV(expectedPath)
+
+      // Assert that the csv file contains the expected virtual fields
+      expect(data[0].custom).toStrictEqual('my custom csv transformer toCSV')
+      expect(data[0].group_custom).toStrictEqual('my custom csv transformer toCSV')
+      expect(data[0].tabToCSV).toStrictEqual('my custom csv transformer toCSV')
+      expect(data[0].namedTab_tabToCSV).toStrictEqual('my custom csv transformer toCSV')
+      expect(data[0].customRelationship_id).toBeDefined()
+      expect(data[0].customRelationship_email).toBeDefined()
+      expect(data[0].customRelationship_createdAt).toBeUndefined()
+    })
+
     it('should create a JSON file for collection', async () => {
       let doc = await payload.create({
         collection: 'exports',

test/plugin-import-export/payload-types.ts

@@ -151,6 +151,8 @@ export interface Page {
   id: string;
   title: string;
   localized?: string | null;
+  custom?: string | null;
+  customRelationship?: (string | null) | User;
   group?: {
     value?: string | null;
     ignore?: string | null;
@@ -161,6 +163,11 @@ export interface Page {
           id?: string | null;
         }[]
       | null;
+    custom?: string | null;
+  };
+  tabToCSV?: string | null;
+  namedTab?: {
+    tabToCSV?: string | null;
   };
   array?:
     | {
@@ -465,6 +472,8 @@ export interface UsersSelect<T extends boolean = true> {
 export interface PagesSelect<T extends boolean = true> {
   title?: T;
   localized?: T;
+  custom?: T;
+  customRelationship?: T;
   group?:
     | T
     | {
@@ -477,6 +486,13 @@ export interface PagesSelect<T extends boolean = true> {
               field2?: T;
               id?: T;
             };
+        custom?: T;
+      };
+  tabToCSV?: T;
+  namedTab?:
+    | T
+    | {
+        tabToCSV?: T;
       };
   array?:
     | T

test/plugin-import-export/seed/index.ts

@@ -91,6 +91,16 @@ export const seed = async (payload: Payload): Promise<boolean> => {
       })
     }
 
+    for (let i = 0; i < 5; i++) {
+      await payload.create({
+        collection: 'pages',
+        data: {
+          customRelationship: user.id,
+          title: `Custom ${i}`,
+        },
+      })
+    }
+
     for (let i = 0; i < 5; i++) {
       await payload.create({
         collection: 'pages',

tsconfig.base.json

@@ -31,7 +31,7 @@
       }
     ],
     "paths": {
-      "@payload-config": ["./test/_community/config.ts"],
+      "@payload-config": ["./test/plugin-import-export/config.ts"],
       "@payloadcms/admin-bar": ["./packages/admin-bar/src"],
       "@payloadcms/live-preview": ["./packages/live-preview/src"],
       "@payloadcms/live-preview-react": ["./packages/live-preview-react/src/index.ts"],