tabshift/payloadcms
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix(plugin-import-export): json preview and downloads preserve nesting and exclude disabled fields (#13210)

Browse Source 
### What?

Improves both the JSON preview and export functionality in the
import-export plugin:
- Preserves proper nesting of object and array fields (e.g., groups,
tabs, arrays)
- Excludes any fields explicitly marked as `disabled` via
`custom.plugin-import-export`
- Ensures downloaded files use proper JSON formatting when `format` is
`json` (no CSV-style flattening)

### Why?

Previously:
- The JSON preview flattened all fields to a single level and included
disabled fields.
- Exported files with `format: json` were still CSV-style data encoded
as `.json`, rather than real JSON.

### How?

- Refactored `/preview-data` JSON handling to preserve original document
shape.
- Applied `removeDisabledFields` to clean nested fields using
dot-notation paths.
- Updated `createExport` to skip `flattenObject` for JSON formats, using
a nested JSON filter instead.
- Fixed streaming and buffered export paths to output valid JSON arrays
when `format` is `json`.
This commit is contained in:
Patrik
2025-07-24 11:36:46 -04:00
committed by GitHub
parent e48427e59a
commit 8f85da8931
10 changed files with 413 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
packages/plugin-import-export/src/components/Preview/index.tsx
View File

@@ -68,6 +68,7 @@ export const Preview = () => {
collectionSlug,
draft,
fields,
format,
limit,
locale,
sort,
@@ -115,8 +116,13 @@ export const Preview = () => {
const fieldKeys =
Array.isArray(fields) && fields.length > 0
? selectedKeys // strictly only what was selected
: [...selectedKeys, ...defaultMetaFields.filter((key) => allKeys.includes(key))]
? selectedKeys // strictly use selected fields only
: [
...selectedKeys,
...defaultMetaFields.filter(
(key) => allKeys.includes(key) && !selectedKeys.includes(key),
),
]
// Build columns based on flattened keys
const newColumns: Column[] = fieldKeys.map((key) => ({
@@ -158,6 +164,7 @@ export const Preview = () => {
disabledFieldRegexes,
draft,
fields,
format,
i18n,
limit,
locale,

124
packages/plugin-import-export/src/export/createExport.ts
View File

@@ -114,7 +114,7 @@ export const createExport = async (args: CreateExportArgs) => {
const disabledRegexes: RegExp[] = disabledFields.map(buildDisabledFieldRegex)
const filterDisabled = (row: Record<string, unknown>): Record<string, unknown> => {
const filterDisabledCSV = (row: Record<string, unknown>): Record<string, unknown> => {
const filtered: Record<string, unknown> = {}
for (const [key, value] of Object.entries(row)) {
@@ -127,35 +127,62 @@ export const createExport = async (args: CreateExportArgs) => {
return filtered
}
const filterDisabledJSON = (doc: any, parentPath = ''): any => {
if (Array.isArray(doc)) {
return doc.map((item) => filterDisabledJSON(item, parentPath))
}
if (typeof doc !== 'object' || doc === null) {
return doc
}
const filtered: Record<string, any> = {}
for (const [key, value] of Object.entries(doc)) {
const currentPath = parentPath ? `${parentPath}.${key}` : key
// Only remove if this exact path is disabled
const isDisabled = disabledFields.includes(currentPath)
if (!isDisabled) {
filtered[key] = filterDisabledJSON(value, currentPath)
}
}
return filtered
}
if (download) {
if (debug) {
req.payload.logger.debug('Pre-scanning all columns before streaming')
}
const allColumnsSet = new Set<string>()
const allColumns: string[] = []
let scanPage = 1
let hasMore = true
while (hasMore) {
const result = await payload.find({ ...findArgs, page: scanPage })
if (isCSV) {
const allColumnsSet = new Set<string>()
let scanPage = 1
let hasMore = true
result.docs.forEach((doc) => {
const flat = filterDisabled(flattenObject({ doc, fields, toCSVFunctions }))
Object.keys(flat).forEach((key) => {
if (!allColumnsSet.has(key)) {
allColumnsSet.add(key)
allColumns.push(key)
}
while (hasMore) {
const result = await payload.find({ ...findArgs, page: scanPage })
result.docs.forEach((doc) => {
const flat = filterDisabledCSV(flattenObject({ doc, fields, toCSVFunctions }))
Object.keys(flat).forEach((key) => {
if (!allColumnsSet.has(key)) {
allColumnsSet.add(key)
allColumns.push(key)
}
})
})
})
hasMore = result.hasNextPage
scanPage += 1
}
hasMore = result.hasNextPage
scanPage += 1
}
if (debug) {
req.payload.logger.debug(`Discovered ${allColumns.length} columns`)
if (debug) {
req.payload.logger.debug(`Discovered ${allColumns.length} columns`)
}
}
const encoder = new TextEncoder()
@@ -171,28 +198,48 @@ export const createExport = async (args: CreateExportArgs) => {
}
if (result.docs.length === 0) {
// Close JSON array properly if JSON
if (!isCSV) {
this.push(encoder.encode(']'))
}
this.push(null)
return
}
const batchRows = result.docs.map((doc) =>
filterDisabled(flattenObject({ doc, fields, toCSVFunctions })),
)
if (isCSV) {
// --- CSV Streaming ---
const batchRows = result.docs.map((doc) =>
filterDisabledCSV(flattenObject({ doc, fields, toCSVFunctions })),
)
const paddedRows = batchRows.map((row) => {
const fullRow: Record<string, unknown> = {}
for (const col of allColumns) {
fullRow[col] = row[col] ?? ''
const paddedRows = batchRows.map((row) => {
const fullRow: Record<string, unknown> = {}
for (const col of allColumns) {
fullRow[col] = row[col] ?? ''
}
return fullRow
})
const csvString = stringify(paddedRows, {
header: isFirstBatch,
columns: allColumns,
})
this.push(encoder.encode(csvString))
} else {
// --- JSON Streaming ---
const batchRows = result.docs.map((doc) => filterDisabledJSON(doc))
// Convert each filtered/flattened row into JSON string
const batchJSON = batchRows.map((row) => JSON.stringify(row)).join(',')
if (isFirstBatch) {
this.push(encoder.encode('[' + batchJSON))
} else {
this.push(encoder.encode(',' + batchJSON))
}
return fullRow
})
}
const csvString = stringify(paddedRows, {
header: isFirstBatch,
columns: allColumns,
})
this.push(encoder.encode(csvString))
isFirstBatch = false
streamPage += 1
@@ -200,6 +247,9 @@ export const createExport = async (args: CreateExportArgs) => {
if (debug) {
req.payload.logger.debug('Stream complete - no more pages')
}
if (!isCSV) {
this.push(encoder.encode(']'))
}
this.push(null) // End the stream
}
},
@@ -239,7 +289,7 @@ export const createExport = async (args: CreateExportArgs) => {
if (isCSV) {
const batchRows = result.docs.map((doc) =>
filterDisabled(flattenObject({ doc, fields, toCSVFunctions })),
filterDisabledCSV(flattenObject({ doc, fields, toCSVFunctions })),
)
// Track discovered column keys
@@ -254,8 +304,8 @@ export const createExport = async (args: CreateExportArgs) => {
rows.push(...batchRows)
} else {
const jsonInput = result.docs.map((doc) => JSON.stringify(doc))
outputData.push(jsonInput.join(',\n'))
const batchRows = result.docs.map((doc) => filterDisabledJSON(doc))
outputData.push(batchRows.map((doc) => JSON.stringify(doc)).join(',\n'))
}
hasNextPage = result.hasNextPage

67
packages/plugin-import-export/src/index.ts
View File

@@ -13,6 +13,9 @@ import { getExportCollection } from './getExportCollection.js'
import { translations } from './translations/index.js'
import { collectDisabledFieldPaths } from './utilities/collectDisabledFieldPaths.js'
import { getFlattenedFieldKeys } from './utilities/getFlattenedFieldKeys.js'
import { getValueAtPath } from './utilities/getvalueAtPath.js'
import { removeDisabledFields } from './utilities/removeDisabledFields.js'
import { setNestedValue } from './utilities/setNestedValue.js'
export const importExportPlugin =
(pluginConfig: ImportExportPluginConfig) =>
@@ -91,6 +94,7 @@ export const importExportPlugin =
collectionSlug: string
draft?: 'no' | 'yes'
fields?: string[]
format?: 'csv' | 'json'
limit?: number
locale?: string
sort?: any
@@ -120,29 +124,58 @@ export const importExportPlugin =
where,
})
const isCSV = req?.data?.format === 'csv'
const docs = result.docs
const toCSVFunctions = getCustomFieldFunctions({
fields: collection.config.fields as FlattenedField[],
})
let transformed: Record<string, unknown>[] = []
const possibleKeys = getFlattenedFieldKeys(collection.config.fields as FlattenedField[])
const transformed = docs.map((doc) => {
const row = flattenObject({
doc,
fields,
toCSVFunctions,
if (isCSV) {
const toCSVFunctions = getCustomFieldFunctions({
fields: collection.config.fields as FlattenedField[],
})
for (const key of possibleKeys) {
if (!(key in row)) {
row[key] = null
}
}
const possibleKeys = getFlattenedFieldKeys(collection.config.fields as FlattenedField[])
return row
})
transformed = docs.map((doc) => {
const row = flattenObject({
doc,
fields,
toCSVFunctions,
})
for (const key of possibleKeys) {
if (!(key in row)) {
row[key] = null
}
}
return row
})
} else {
const disabledFields =
collection.config.admin.custom?.['plugin-import-export']?.disabledFields
transformed = docs.map((doc) => {
let output: Record<string, unknown> = { ...doc }
// Remove disabled fields first
output = removeDisabledFields(output, disabledFields)
// Then trim to selected fields only (if fields are provided)
if (Array.isArray(fields) && fields.length > 0) {
const trimmed: Record<string, unknown> = {}
for (const key of fields) {
const value = getValueAtPath(output, key)
setNestedValue(trimmed, key, value ?? null)
}
output = trimmed
}
return output
})
}
return Response.json({
docs: transformed,

42
packages/plugin-import-export/src/utilities/getFlattenedFieldKeys.ts
View File

@@ -22,21 +22,18 @@ export const getFlattenedFieldKeys = (fields: FieldWithPresentational[], prefix
'plugin-import-export' in field.custom &&
field.custom['plugin-import-export']?.toCSV
if (!('name' in field) || typeof field.name !== 'string' || fieldHasToCSVFunction) {
return
}
const name = prefix ? `${prefix}_${field.name}` : field.name
const name = 'name' in field && typeof field.name === 'string' ? field.name : undefined
const fullKey = name && prefix ? `${prefix}_${name}` : (name ?? prefix)
switch (field.type) {
case 'array': {
const subKeys = getFlattenedFieldKeys(field.fields as FlattenedField[], `${name}_0`)
const subKeys = getFlattenedFieldKeys(field.fields as FlattenedField[], `${fullKey}_0`)
keys.push(...subKeys)
break
}
case 'blocks': {
field.blocks.forEach((block) => {
const blockPrefix = `${name}_0_${block.slug}`
const blockPrefix = `${fullKey}_0_${block.slug}`
keys.push(`${blockPrefix}_blockType`)
keys.push(`${blockPrefix}_id`)
keys.push(...getFlattenedFieldKeys(block.fields as FlattenedField[], blockPrefix))
@@ -46,45 +43,42 @@ export const getFlattenedFieldKeys = (fields: FieldWithPresentational[], prefix
case 'collapsible':
case 'group':
case 'row':
keys.push(...getFlattenedFieldKeys(field.fields as FlattenedField[], name))
keys.push(...getFlattenedFieldKeys(field.fields as FlattenedField[], fullKey))
break
case 'relationship':
if (field.hasMany) {
if (Array.isArray(field.relationTo)) {
// hasMany polymorphic
keys.push(`${name}_0_relationTo`, `${name}_0_id`)
keys.push(`${fullKey}_0_relationTo`, `${fullKey}_0_id`)
} else {
// hasMany monomorphic
keys.push(`${name}_0`)
keys.push(`${fullKey}_0`)
}
} else {
if (Array.isArray(field.relationTo)) {
// hasOne polymorphic
keys.push(`${name}_relationTo`, `${name}_id`)
keys.push(`${fullKey}_relationTo`, `${fullKey}_id`)
} else {
// hasOne monomorphic
keys.push(name)
keys.push(fullKey)
}
}
break
case 'tabs':
if (field.tabs) {
field.tabs.forEach((tab) => {
if (tab.name) {
const tabPrefix = prefix ? `${prefix}_${tab.name}` : tab.name
keys.push(...getFlattenedFieldKeys(tab.fields, tabPrefix))
} else {
keys.push(...getFlattenedFieldKeys(tab.fields, prefix))
}
})
}
field.tabs?.forEach((tab) => {
const tabPrefix = tab.name ? `${fullKey}_${tab.name}` : fullKey
keys.push(...getFlattenedFieldKeys(tab.fields || [], tabPrefix))
})
break
default:
if (!name || fieldHasToCSVFunction) {
break
}
if ('hasMany' in field && field.hasMany) {
// Push placeholder for first index
keys.push(`${name}_0`)
keys.push(`${fullKey}_0`)
} else {
keys.push(name)
keys.push(fullKey)
}
break
}

59
packages/plugin-import-export/src/utilities/getvalueAtPath.ts Normal file
View File

@@ -0,0 +1,59 @@
/**
* Safely retrieves a deeply nested value from an object using a dot-notation path.
*
* Supports:
* - Indexed array access (e.g., "array.0.field1")
* - Polymorphic blocks or keyed unions (e.g., "blocks.0.hero.title"), where the block key
* (e.g., "hero") maps to a nested object inside the block item.
*
*
* @param obj - The input object to traverse.
* @param path - A dot-separated string representing the path to retrieve.
* @returns The value at the specified path, or undefined if not found.
*/
export const getValueAtPath = (obj: unknown, path: string): unknown => {
if (!obj || typeof obj !== 'object') {
return undefined
}
const parts = path.split('.')
let current: any = obj
for (const part of parts) {
if (current == null) {
return undefined
}
// If the path part is a number, treat it as an array index
if (!isNaN(Number(part))) {
current = current[Number(part)]
continue
}
// Special case: if current is an array of blocks like [{ hero: { title: '...' } }]
// and the path is "blocks.0.hero.title", then `part` would be "hero"
if (Array.isArray(current)) {
const idx = Number(parts[parts.indexOf(part) - 1])
const blockItem = current[idx]
if (typeof blockItem === 'object') {
const keys = Object.keys(blockItem)
// Find the key (e.g., "hero") that maps to an object
const matchingBlock = keys.find(
(key) => blockItem[key] && typeof blockItem[key] === 'object',
)
if (matchingBlock && part === matchingBlock) {
current = blockItem[matchingBlock]
continue
}
}
}
// Fallback to plain object key access
current = current[part]
}
return current
}

80
packages/plugin-import-export/src/utilities/removeDisabledFields.ts Normal file
View File

@@ -0,0 +1,80 @@
/**
* Recursively removes fields from a deeply nested object based on dot-notation paths.
*
* This utility supports removing:
* - Nested fields in plain objects (e.g., "group.value")
* - Fields inside arrays of objects (e.g., "group.array.field1")
*
* It safely traverses both object and array structures and avoids mutating the original input.
*
* @param obj - The original object to clean.
* @param disabled - An array of dot-separated paths indicating which fields to remove.
* @returns A deep clone of the original object with specified fields removed.
*/
export const removeDisabledFields = (
obj: Record<string, unknown>,
disabled: string[] = [],
): Record<string, unknown> => {
if (!disabled.length) {
return obj
}
const clone = structuredClone(obj)
// Process each disabled path independently
for (const path of disabled) {
const parts = path.split('.')
/**
* Recursively walks the object tree according to the dot path,
* and deletes the field once the full path is reached.
*
* @param target - The current object or array being traversed
* @param i - The index of the current path part
*/
const removeRecursively = (target: any, i = 0): void => {
if (target == null) {
return
}
const key = parts[i]
// If at the final part of the path, perform the deletion
if (i === parts.length - 1) {
// If the current level is an array, delete the key from each item
if (Array.isArray(target)) {
for (const item of target) {
if (item && typeof item === 'object' && key !== undefined) {
delete item[key as keyof typeof item]
}
}
} else if (typeof target === 'object' && key !== undefined) {
delete target[key]
}
return
}
if (key === undefined) {
return
}
// Traverse to the next level in the path
const next = target[key]
if (Array.isArray(next)) {
// If the next value is an array, recurse into each item
for (const item of next) {
removeRecursively(item, i + 1)
}
} else {
// Otherwise, continue down the object path
removeRecursively(next, i + 1)
}
}
removeRecursively(clone)
}
return clone
}

65
packages/plugin-import-export/src/utilities/setNestedValue.ts Normal file
View File

@@ -0,0 +1,65 @@
/**
* Sets a value deeply into a nested object or array, based on a dot-notation path.
*
* This function:
* - Supports array indexing (e.g., "array.0.field1")
* - Creates intermediate arrays/objects as needed
* - Mutates the target object directly
*
* @example
* const obj = {}
* setNestedValue(obj, 'group.array.0.field1', 'hello')
* // Result: { group: { array: [ { field1: 'hello' } ] } }
*
* @param obj - The target object to mutate.
* @param path - A dot-separated string path indicating where to assign the value.
* @param value - The value to set at the specified path.
*/
export const setNestedValue = (
obj: Record<string, unknown>,
path: string,
value: unknown,
): void => {
const parts = path.split('.')
let current: any = obj
for (let i = 0; i < parts.length; i++) {
const part = parts[i]
const isLast = i === parts.length - 1
const isIndex = !Number.isNaN(Number(part))
if (isIndex) {
const index = Number(part)
// Ensure the current target is an array
if (!Array.isArray(current)) {
current = []
}
// Ensure the array slot is initialized
if (!current[index]) {
current[index] = {}
}
if (isLast) {
current[index] = value
} else {
current = current[index] as Record<string, unknown>
}
} else {
// Ensure the object key exists
if (isLast) {
if (typeof part === 'string') {
current[part] = value
}
} else {
if (typeof current[part as string] !== 'object' || current[part as string] === null) {
current[part as string] = {}
}
current = current[part as string] as Record<string, unknown>
}
}
}
}

20
test/plugin-import-export/collections/Pages.ts
View File

@@ -61,6 +61,11 @@ export const Pages: CollectionConfig = {
name: 'value',
type: 'text',
defaultValue: 'group value',
// custom: {
// 'plugin-import-export': {
// disabled: true,
// },
// },
},
{
name: 'ignore',
@@ -216,5 +221,20 @@ export const Pages: CollectionConfig = {
relationTo: ['users', 'posts'],
hasMany: true,
},
{
type: 'collapsible',
label: 'Collapsible Field',
fields: [
{
name: 'textFieldInCollapsible',
type: 'text',
// custom: {
// 'plugin-import-export': {
// disabled: true,
// },
// },
},
],
},
],
}

23
test/plugin-import-export/int.spec.ts
View File

@@ -467,6 +467,29 @@ describe('@payloadcms/plugin-import-export', () => {
expect(data[0].title).toStrictEqual('JSON 0')
})
it('should download an existing export JSON file', async () => {
const response = await restClient.POST('/exports/download', {
body: JSON.stringify({
data: {
collectionSlug: 'pages',
fields: ['id', 'title'],
format: 'json',
sort: 'title',
},
}),
headers: { 'Content-Type': 'application/json' },
})
expect(response.status).toBe(200)
expect(response.headers.get('content-type')).toMatch(/application\/json/)
const data = await response.json()
expect(Array.isArray(data)).toBe(true)
expect(['string', 'number']).toContain(typeof data[0].id)
expect(typeof data[0].title).toBe('string')
})
it('should create an export with every field when no fields are defined', async () => {
let doc = await payload.create({
collection: 'exports',

2
test/plugin-import-export/payload-types.ts
View File

@@ -242,6 +242,7 @@ export interface Page {
}
)[]
| null;
textFieldInCollapsible?: string | null;
updatedAt: string;
createdAt: string;
_status?: ('draft' | 'published') | null;
@@ -579,6 +580,7 @@ export interface PagesSelect<T extends boolean = true> {
excerpt?: T;
hasOnePolymorphic?: T;
hasManyPolymorphic?: T;
textFieldInCollapsible?: T;
updatedAt?: T;
createdAt?: T;
_status?: T;