From 47a1eee765e554db29a9370927bf90f0fb4947f2 Mon Sep 17 00:00:00 2001 From: Dan Ribbens Date: Tue, 29 Apr 2025 12:28:16 -0700 Subject: [PATCH] fix(plugin-import-export): csv export column order (#12258) ### What? The order of fields, when specified for the create export function was not used for constructing the data. Now the fields order will be used. ### Why? This is important to building CSV data for consumption in other systems. ### How? Adds logic to handle ordering the field values assigned to the export data prior to building the CSV. --- .../src/export/createExport.ts | 4 +- .../src/export/flattenObject.ts | 70 ++++++++++++++----- test/plugin-import-export/int.spec.ts | 34 +++++++++ 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/packages/plugin-import-export/src/export/createExport.ts b/packages/plugin-import-export/src/export/createExport.ts index 76021900a..cee0c80f1 100644 --- a/packages/plugin-import-export/src/export/createExport.ts +++ b/packages/plugin-import-export/src/export/createExport.ts @@ -87,7 +87,7 @@ export const createExport = async (args: CreateExportArgs) => { let isFirstBatch = true while (result.docs.length > 0) { - const csvInput = result.docs.map((doc) => flattenObject(doc)) + const csvInput = result.docs.map((doc) => flattenObject({ doc, fields })) const csvString = stringify(csvInput, { header: isFirstBatch }) this.push(encoder.encode(csvString)) isFirstBatch = false @@ -119,7 +119,7 @@ export const createExport = async (args: CreateExportArgs) => { result = await payload.find(findArgs) if (isCSV) { - const csvInput = result.docs.map((doc) => flattenObject(doc)) + const csvInput = result.docs.map((doc) => flattenObject({ doc, fields })) outputData.push(stringify(csvInput, { header: isFirstBatch })) isFirstBatch = false } else { diff --git a/packages/plugin-import-export/src/export/flattenObject.ts b/packages/plugin-import-export/src/export/flattenObject.ts index 8fe2c83f2..ccc2de988 100644 --- a/packages/plugin-import-export/src/export/flattenObject.ts +++ b/packages/plugin-import-export/src/export/flattenObject.ts @@ -1,23 +1,61 @@ -export const flattenObject = (obj: any, prefix: string = ''): Record => { +import type { Document } from 'payload' + +type Args = { + doc: Document + fields?: string[] + prefix?: string +} + +export const flattenObject = ({ doc, fields, prefix }: Args): Record => { const result: Record = {} - Object.entries(obj).forEach(([key, value]) => { - const newKey = prefix ? `${prefix}_${key}` : key + const flatten = (doc: Document, prefix?: string) => { + Object.entries(doc).forEach(([key, value]) => { + const newKey = prefix ? `${prefix}_${key}` : key - if (Array.isArray(value)) { - value.forEach((item, index) => { - if (typeof item === 'object' && item !== null) { - Object.assign(result, flattenObject(item, `${newKey}_${index}`)) - } else { - result[`${newKey}_${index}`] = item - } - }) - } else if (typeof value === 'object' && value !== null) { - Object.assign(result, flattenObject(value, newKey)) - } else { - result[newKey] = value + if (Array.isArray(value)) { + value.forEach((item, index) => { + if (typeof item === 'object' && item !== null) { + flatten(item, `${newKey}_${index}`) + } else { + result[`${newKey}_${index}`] = item + } + }) + } else if (typeof value === 'object' && value !== null) { + flatten(value, newKey) + } else { + result[newKey] = value + } + }) + } + + flatten(doc, prefix) + + if (fields) { + const orderedResult: Record = {} + + const fieldToRegex = (field: string): RegExp => { + const parts = field.split('.').map((part) => `${part}(?:_\\d+)?`) + const pattern = `^${parts.join('_')}` + return new RegExp(pattern) } - }) + + fields.forEach((field) => { + if (result[field.replace(/\./g, '_')]) { + const sanitizedField = field.replace(/\./g, '_') + orderedResult[sanitizedField] = result[sanitizedField] + } else { + const regex = fieldToRegex(field) + Object.keys(result).forEach((key) => { + if (regex.test(key)) { + orderedResult[key] = result[key] + } + }) + } + }) + + return orderedResult + } return result } diff --git a/test/plugin-import-export/int.spec.ts b/test/plugin-import-export/int.spec.ts index 6bf5f6f14..b0f27f80d 100644 --- a/test/plugin-import-export/int.spec.ts +++ b/test/plugin-import-export/int.spec.ts @@ -1,5 +1,6 @@ import type { CollectionSlug, Payload } from 'payload' +import fs from 'fs' import path from 'path' import { fileURLToPath } from 'url' @@ -221,6 +222,39 @@ describe('@payloadcms/plugin-import-export', () => { expect(data[0].array_1_field2).toStrictEqual('baz') }) + it('should create a CSV file with columns matching the order of the fields array', async () => { + const fields = ['id', 'group.value', 'group.array.field1', 'title', 'createdAt', 'updatedAt'] + const doc = await payload.create({ + collection: 'exports', + user, + data: { + collectionSlug: 'pages', + fields, + format: 'csv', + where: { + title: { contains: 'Title ' }, + }, + }, + }) + + const exportDoc = await payload.findByID({ + collection: 'exports', + id: doc.id, + }) + + expect(exportDoc.filename).toBeDefined() + const expectedPath = path.join(dirname, './uploads', exportDoc.filename as string) + const buffer = fs.readFileSync(expectedPath) + const str = buffer.toString() + + // Assert that the header row matches the fields array + expect(str.indexOf('id')).toBeLessThan(str.indexOf('title')) + expect(str.indexOf('group_value')).toBeLessThan(str.indexOf('title')) + expect(str.indexOf('group_value')).toBeLessThan(str.indexOf('group_array')) + expect(str.indexOf('title')).toBeLessThan(str.indexOf('createdAt')) + expect(str.indexOf('createdAt')).toBeLessThan(str.indexOf('updatedAt')) + }) + it('should create a file for collection csv from array.subfield', async () => { let doc = await payload.create({ collection: 'exports',