tabshift/payloadcms
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat(plugin-import-export): add custom toCSV function on fields (#12533)

Browse Source 
This makes it possible to add custom logic into how we map the document
data into the CSV data on a field-by-field basis.

- Allow custom data transformation to be added to
`custom.['plugin-import-export'].toCSV inside the field config
- Add type declaration to FieldCustom to improve types
- Export with `depth: 1`

Example:
```ts
    {
      name: 'customRelationship',
      type: 'relationship',
      relationTo: 'users',
      custom: {
        'plugin-import-export': {
          toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
            row[`${columnName}_id`] = value.id
            row[`${columnName}_email`] = value.email
          },
        },
      },
    },
```
This commit is contained in:
Dan Ribbens
2025-06-09 13:53:30 -04:00
committed by GitHub
parent 773e4ad4dd
commit 8f4c4423f3
17 changed files with 349 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/payload/src/fields/config/types.ts
View File

@@ -128,6 +128,7 @@ import type {
CollectionSlug,
DateFieldValidation,
EmailFieldValidation,
FieldCustom,
JSONFieldValidation,
PointFieldValidation,
RadioFieldValidation,
@@ -482,7 +483,7 @@ export interface FieldBase {
}
admin?: Admin
/** Extension point to add your custom data. Server only. */
custom?: Record<string, any>
custom?: FieldCustom
defaultValue?: DefaultValue
hidden?: boolean
hooks?: {

3
packages/payload/src/index.ts
View File

@@ -1238,6 +1238,9 @@ export {
} from './fields/config/client.js'
export { sanitizeFields } from './fields/config/sanitize.js'
export interface FieldCustom extends Record<string, any> {}
export type {
AdminClient,
ArrayField,

6
packages/payload/src/utilities/traverseFields.ts
View File

@@ -304,12 +304,12 @@ export const traverseFields = ({
return
}
if (field.type !== 'tab' && (fieldHasSubFields(field) || field.type === 'blocks')) {
if (field.type === 'tab' || fieldHasSubFields(field) || field.type === 'blocks') {
if ('name' in field && field.name) {
currentParentRef = currentRef
if (!ref[field.name]) {
if (fillEmpty) {
if (field.type === 'group') {
if (field.type === 'group' || field.type === 'tab') {
if (fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! })) {
ref[field.name] = {
en: {},
@@ -334,7 +334,7 @@ export const traverseFields = ({
}
if (
field.type === 'group' &&
(field.type === 'tab' || field.type === 'group') &&
fieldShouldBeLocalized({ field, parentIsLocalized: parentIsLocalized! }) &&
currentRef &&
typeof currentRef === 'object'

15
packages/plugin-import-export/src/export/createExport.ts
View File

@@ -6,6 +6,7 @@ import { APIError } from 'payload'
import { Readable } from 'stream'
import { flattenObject } from './flattenObject.js'
import { getCustomFieldFunctions } from './getCustomFieldFunctions.js'
import { getFilename } from './getFilename.js'
import { getSelect } from './getSelect.js'
@@ -79,6 +80,7 @@ export const createExport = async (args: CreateExportArgs) => {
const name = `${nameArg ?? `${getFilename()}-${collectionSlug}`}.${format}`
const isCSV = format === 'csv'
const select = Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined
if (debug) {
req.payload.logger.info({ message: 'Export configuration:', name, isCSV, locale })
@@ -86,13 +88,13 @@ export const createExport = async (args: CreateExportArgs) => {
const findArgs = {
collection: collectionSlug,
depth: 0,
depth: 1,
draft: drafts === 'yes',
limit: 100,
locale,
overrideAccess: false,
page: 0,
select: Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined,
select,
sort,
user,
where,
@@ -104,6 +106,11 @@ export const createExport = async (args: CreateExportArgs) => {
let result: PaginatedDocs = { hasNextPage: true } as PaginatedDocs
const toCSVFunctions = getCustomFieldFunctions({
fields: collectionConfig.flattenedFields,
select,
})
if (download) {
if (debug) {
req.payload.logger.info('Starting download stream')
@@ -120,7 +127,7 @@ export const createExport = async (args: CreateExportArgs) => {
`Processing batch ${findArgs.page + 1} with ${result.docs.length} documents`,
)
}
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
const csvString = stringify(csvInput, { header: isFirstBatch })
this.push(encoder.encode(csvString))
isFirstBatch = false
@@ -164,7 +171,7 @@ export const createExport = async (args: CreateExportArgs) => {
}
if (isCSV) {
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields }))
const csvInput = result.docs.map((doc) => flattenObject({ doc, fields, toCSVFunctions }))
outputData.push(stringify(csvInput, { header: isFirstBatch }))
isFirstBatch = false
} else {

61
packages/plugin-import-export/src/export/flattenObject.ts
View File

@@ -1,16 +1,24 @@
import type { Document } from 'payload'
import type { ToCSVFunction } from '../types.js'
type Args = {
doc: Document
fields?: string[]
prefix?: string
toCSVFunctions: Record<string, ToCSVFunction>
}
export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unknown> => {
const result: Record<string, unknown> = {}
export const flattenObject = ({
doc,
fields,
prefix,
toCSVFunctions,
}: Args): Record<string, unknown> => {
const row: Record<string, unknown> = {}
const flatten = (doc: Document, prefix?: string) => {
Object.entries(doc).forEach(([key, value]) => {
const flatten = (siblingDoc: Document, prefix?: string) => {
Object.entries(siblingDoc).forEach(([key, value]) => {
const newKey = prefix ? `${prefix}_${key}` : key
if (Array.isArray(value)) {
@@ -18,13 +26,44 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
if (typeof item === 'object' && item !== null) {
flatten(item, `${newKey}_${index}`)
} else {
result[`${newKey}_${index}`] = item
if (toCSVFunctions?.[newKey]) {
const columnName = `${newKey}_${index}`
row[columnName] = toCSVFunctions[newKey]({
columnName,
doc,
row,
siblingDoc,
value: item,
})
} else {
row[`${newKey}_${index}`] = item
}
}
})
} else if (typeof value === 'object' && value !== null) {
if (!toCSVFunctions?.[newKey]) {
flatten(value, newKey)
} else {
result[newKey] = value
row[newKey] = toCSVFunctions[newKey]({
columnName: newKey,
doc,
row,
siblingDoc,
value,
})
}
} else {
if (toCSVFunctions?.[newKey]) {
row[newKey] = toCSVFunctions[newKey]({
columnName: newKey,
doc,
row,
siblingDoc,
value,
})
} else {
row[newKey] = value
}
}
})
}
@@ -41,14 +80,14 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
}
fields.forEach((field) => {
if (result[field.replace(/\./g, '_')]) {
if (row[field.replace(/\./g, '_')]) {
const sanitizedField = field.replace(/\./g, '_')
orderedResult[sanitizedField] = result[sanitizedField]
orderedResult[sanitizedField] = row[sanitizedField]
} else {
const regex = fieldToRegex(field)
Object.keys(result).forEach((key) => {
Object.keys(row).forEach((key) => {
if (regex.test(key)) {
orderedResult[key] = result[key]
orderedResult[key] = row[key]
}
})
}
@@ -57,5 +96,5 @@ export const flattenObject = ({ doc, fields, prefix }: Args): Record<string, unk
return orderedResult
}
return result
return row
}

93
packages/plugin-import-export/src/export/getCustomFieldFunctions.ts Normal file
View File

@@ -0,0 +1,93 @@
import {
type FlattenedField,
type SelectIncludeType,
traverseFields,
type TraverseFieldsCallback,
} from 'payload'
import type { ToCSVFunction } from '../types.js'
type Args = {
fields: FlattenedField[]
select: SelectIncludeType | undefined
}
export const getCustomFieldFunctions = ({
fields,
select,
}: Args): Record<string, ToCSVFunction> => {
const result: Record<string, ToCSVFunction> = {}
const buildCustomFunctions: TraverseFieldsCallback = ({ field, parentRef, ref }) => {
// @ts-expect-error ref is untyped
ref.prefix = parentRef.prefix || ''
if (field.type === 'group' || field.type === 'tab') {
// @ts-expect-error ref is untyped
const parentPrefix = parentRef?.prefix ? `${parentRef.prefix}_` : ''
// @ts-expect-error ref is untyped
ref.prefix = `${parentPrefix}${field.name}_`
}
if (typeof field.custom?.['plugin-import-export']?.toCSV === 'function') {
// @ts-expect-error ref is untyped
result[`${ref.prefix}${field.name}`] = field.custom['plugin-import-export']?.toCSV
} else if (field.type === 'relationship' || field.type === 'upload') {
if (field.hasMany !== true) {
if (!Array.isArray(field.relationTo)) {
// monomorphic single
// @ts-expect-error ref is untyped
result[`${ref.prefix}${field.name}`] = ({ value }) =>
typeof value === 'object' && value && 'id' in value ? value.id : value
} else {
// polymorphic single
// @ts-expect-error ref is untyped
result[`${ref.prefix}${field.name}`] = ({ data, value }) => {
// @ts-expect-error ref is untyped
data[`${ref.prefix}${field.name}_id`] = value.id
// @ts-expect-error ref is untyped
data[`${ref.prefix}${field.name}_relationTo`] = value.relationTo
return undefined
}
}
} else {
if (!Array.isArray(field.relationTo)) {
// monomorphic many
// @ts-expect-error ref is untyped
result[`${ref.prefix}${field.name}`] = ({
value,
}: {
value: Record<string, unknown>[]
}) =>
value.map((val: number | Record<string, unknown> | string) =>
typeof val === 'object' ? val.id : val,
)
} else {
// polymorphic many
// @ts-expect-error ref is untyped
result[`${ref.prefix}${field.name}`] = ({
data,
value,
}: {
data: Record<string, unknown>
value: Record<string, unknown>[]
}) =>
value.map((val: number | Record<string, unknown> | string, i) => {
// @ts-expect-error ref is untyped
data[`${ref.prefix}${field.name}_${i}_id`] = val.id
// @ts-expect-error ref is untyped
data[`${ref.prefix}${field.name}_${i}_relationTo`] = val.relationTo
return undefined
})
}
}
}
// TODO: do this so we only return the functions needed based on the select used
////@ts-expect-error ref is untyped
// ref.select = typeof select !== 'undefined' || select[field.name] ? select : {}
}
traverseFields({ callback: buildCustomFunctions, fields })
return result
}

12
packages/plugin-import-export/src/export/getSelect.ts
View File

@@ -1,17 +1,13 @@
import type { SelectType } from 'payload'
import type { SelectIncludeType } from 'payload'
/**
* Takes an input of array of string paths in dot notation and returns a select object
* example args: ['id', 'title', 'group.value', 'createdAt', 'updatedAt']
*/
export const getSelect = (fields: string[]): SelectType => {
const select: SelectType = {}
export const getSelect = (fields: string[]): SelectIncludeType => {
const select: SelectIncludeType = {}
fields.forEach((field) => {
// TODO: this can likely be removed, the form was not saving, leaving in for now
if (!field) {
return
}
const segments = field.split('.')
let selectRef = select
@@ -22,7 +18,7 @@ export const getSelect = (fields: string[]): SelectType => {
if (!selectRef[segment]) {
selectRef[segment] = {}
}
selectRef = selectRef[segment] as SelectType
selectRef = selectRef[segment] as SelectIncludeType
}
})
})

2
packages/plugin-import-export/src/exports/types.ts
View File

@@ -1 +1 @@
export type { ImportExportPluginConfig } from '../types.js'
export type { ImportExportPluginConfig, ToCSVFunction } from '../types.js'

1
packages/plugin-import-export/src/getExportCollection.ts
View File

@@ -1,6 +1,5 @@
import type {
CollectionAfterChangeHook,
CollectionBeforeChangeHook,
CollectionBeforeOperationHook,
CollectionConfig,
Config,

10
packages/plugin-import-export/src/index.ts
View File

@@ -3,7 +3,7 @@ import type { Config, JobsConfig } from 'payload'
import { deepMergeSimple } from 'payload'
import type { PluginDefaultTranslationsObject } from './translations/types.js'
import type { ImportExportPluginConfig } from './types.js'
import type { ImportExportPluginConfig, ToCSVFunction } from './types.js'
import { getCreateCollectionExportTask } from './export/getCreateExportCollectionTask.js'
import { getExportCollection } from './getExportCollection.js'
@@ -91,3 +91,11 @@ export const importExportPlugin =
return config
}
declare module 'payload' {
export interface FieldCustom {
'plugin-import-export'?: {
toCSVFunction?: ToCSVFunction
}
}
}

26
packages/plugin-import-export/src/types.ts
View File

@@ -26,3 +26,29 @@ export type ImportExportPluginConfig = {
*/
overrideExportCollection?: (collection: CollectionOverride) => CollectionOverride
}
/**
* Custom function used to modify the outgoing csv data by manipulating the data, siblingData or by returning the desired value
*/
export type ToCSVFunction = (args: {
/**
* The path of the column for the field, for arrays this includes the index (zero-based)
*/
columnName: string
/**
* The top level document
*/
doc: Document
/**
* The object data that can be manipulated to assign data to the CSV
*/
row: Record<string, unknown>
/**
* The document data at the level where it belongs
*/
siblingDoc: Record<string, unknown>
/**
* The data for the field.
*/
value: unknown
}) => unknown

77
test/plugin-import-export/collections/Pages.ts
View File

@@ -26,6 +26,31 @@ export const Pages: CollectionConfig = {
type: 'text',
localized: true,
},
{
name: 'custom',
type: 'text',
defaultValue: 'my custom csv transformer',
custom: {
'plugin-import-export': {
toCSV: ({ value, columnName, row, siblingDoc }) => {
return value + ' toCSV'
},
},
},
},
{
name: 'customRelationship',
type: 'relationship',
relationTo: 'users',
custom: {
'plugin-import-export': {
toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
row[`${columnName}_id`] = value.id
row[`${columnName}_email`] = value.email
},
},
},
},
{
name: 'group',
type: 'group',
@@ -53,6 +78,58 @@ export const Pages: CollectionConfig = {
},
],
},
{
name: 'custom',
type: 'text',
defaultValue: 'my custom csv transformer',
custom: {
'plugin-import-export': {
toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
return value + ' toCSV'
},
},
},
},
],
},
{
name: 'tabs',
type: 'tabs',
tabs: [
{
label: 'No Name',
fields: [
{
name: 'tabToCSV',
type: 'text',
defaultValue: 'my custom csv transformer',
custom: {
'plugin-import-export': {
toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
return value + ' toCSV'
},
},
},
},
],
},
{
name: 'namedTab',
fields: [
{
name: 'tabToCSV',
type: 'text',
defaultValue: 'my custom csv transformer',
custom: {
'plugin-import-export': {
toCSV: ({ value, columnName, row, siblingDoc, doc }) => {
return value + ' toCSV'
},
},
},
},
],
},
],
},
{

41
test/plugin-import-export/int.spec.ts
View File

@@ -368,6 +368,47 @@ describe('@payloadcms/plugin-import-export', () => {
expect(data[0].blocks_1_blockType).toStrictEqual('content')
})
it('should run custom toCSV function on a field', async () => {
const fields = [
'id',
'custom',
'group.custom',
'customRelationship',
'tabToCSV',
'namedTab.tabToCSV',
]
const doc = await payload.create({
collection: 'exports',
user,
data: {
collectionSlug: 'pages',
fields,
format: 'csv',
where: {
title: { contains: 'Custom ' },
},
},
})
const exportDoc = await payload.findByID({
collection: 'exports',
id: doc.id,
})
expect(exportDoc.filename).toBeDefined()
const expectedPath = path.join(dirname, './uploads', exportDoc.filename as string)
const data = await readCSV(expectedPath)
// Assert that the csv file contains the expected virtual fields
expect(data[0].custom).toStrictEqual('my custom csv transformer toCSV')
expect(data[0].group_custom).toStrictEqual('my custom csv transformer toCSV')
expect(data[0].tabToCSV).toStrictEqual('my custom csv transformer toCSV')
expect(data[0].namedTab_tabToCSV).toStrictEqual('my custom csv transformer toCSV')
expect(data[0].customRelationship_id).toBeDefined()
expect(data[0].customRelationship_email).toBeDefined()
expect(data[0].customRelationship_createdAt).toBeUndefined()
})
it('should create a JSON file for collection', async () => {
let doc = await payload.create({
collection: 'exports',

16
test/plugin-import-export/payload-types.ts
View File

@@ -151,6 +151,8 @@ export interface Page {
id: string;
title: string;
localized?: string | null;
custom?: string | null;
customRelationship?: (string | null) | User;
group?: {
value?: string | null;
ignore?: string | null;
@@ -161,6 +163,11 @@ export interface Page {
id?: string | null;
}[]
| null;
custom?: string | null;
};
tabToCSV?: string | null;
namedTab?: {
tabToCSV?: string | null;
};
array?:
| {
@@ -465,6 +472,8 @@ export interface UsersSelect<T extends boolean = true> {
export interface PagesSelect<T extends boolean = true> {
title?: T;
localized?: T;
custom?: T;
customRelationship?: T;
group?:
| T
| {
@@ -477,6 +486,13 @@ export interface PagesSelect<T extends boolean = true> {
field2?: T;
id?: T;
};
custom?: T;
};
tabToCSV?: T;
namedTab?:
| T
| {
tabToCSV?: T;
};
array?:
| T

10
test/plugin-import-export/seed/index.ts
View File

@@ -91,6 +91,16 @@ export const seed = async (payload: Payload): Promise<boolean> => {
})
}
for (let i = 0; i < 5; i++) {
await payload.create({
collection: 'pages',
data: {
customRelationship: user.id,
title: `Custom ${i}`,
},
})
}
for (let i = 0; i < 5; i++) {
await payload.create({
collection: 'pages',

2
tsconfig.base.json
View File

@@ -31,7 +31,7 @@
}
],
"paths": {
"@payload-config": ["./test/_community/config.ts"],
"@payload-config": ["./test/plugin-import-export/config.ts"],
"@payloadcms/admin-bar": ["./packages/admin-bar/src"],
"@payloadcms/live-preview": ["./packages/live-preview/src"],
"@payloadcms/live-preview-react": ["./packages/live-preview-react/src/index.ts"],