### What: This PR adds `limit` and `page` fields to the export options, allowing users to control the number of documents exported and the page from which to start the export. It also enforces that limit must be a positive multiple of 100. ### Why: This feature is needed to provide pagination support for large exports, enabling users to export manageable chunks of data rather than the entire dataset at once. Enforcing multiples-of-100 for `limit` ensures consistent chunking behavior and prevents unexpected export issues. ### How: - The `limit` field determines the maximum number of documents to export and **must be a positive multiple of 100**. - The `page` field defines the starting page of the export and is displayed only when a `limit` is specified. - If `limit` is cleared, the `page` resets to 1 to maintain consistency. - Export logic was adjusted to respect the `limit` and `page` values when fetching documents. --------- Co-authored-by: Patrik Kozak <35232443+PatrikKozak@users.noreply.github.com>
228 lines
7.3 KiB
TypeScript
228 lines
7.3 KiB
TypeScript
import type { Config, FlattenedField } from 'payload'
|
|
|
|
import { addDataAndFileToRequest, deepMergeSimple } from 'payload'
|
|
|
|
import type { PluginDefaultTranslationsObject } from './translations/types.js'
|
|
import type { ImportExportPluginConfig, ToCSVFunction } from './types.js'
|
|
|
|
import { flattenObject } from './export/flattenObject.js'
|
|
import { getCreateCollectionExportTask } from './export/getCreateExportCollectionTask.js'
|
|
import { getCustomFieldFunctions } from './export/getCustomFieldFunctions.js'
|
|
import { getSelect } from './export/getSelect.js'
|
|
import { getExportCollection } from './getExportCollection.js'
|
|
import { translations } from './translations/index.js'
|
|
import { collectDisabledFieldPaths } from './utilities/collectDisabledFieldPaths.js'
|
|
import { getFlattenedFieldKeys } from './utilities/getFlattenedFieldKeys.js'
|
|
import { getValueAtPath } from './utilities/getvalueAtPath.js'
|
|
import { removeDisabledFields } from './utilities/removeDisabledFields.js'
|
|
import { setNestedValue } from './utilities/setNestedValue.js'
|
|
|
|
export const importExportPlugin =
|
|
(pluginConfig: ImportExportPluginConfig) =>
|
|
(config: Config): Config => {
|
|
const exportCollection = getExportCollection({ config, pluginConfig })
|
|
if (config.collections) {
|
|
config.collections.push(exportCollection)
|
|
} else {
|
|
config.collections = [exportCollection]
|
|
}
|
|
|
|
// inject custom import export provider
|
|
config.admin = config.admin || {}
|
|
config.admin.components = config.admin.components || {}
|
|
config.admin.components.providers = config.admin.components.providers || []
|
|
config.admin.components.providers.push(
|
|
'@payloadcms/plugin-import-export/rsc#ImportExportProvider',
|
|
)
|
|
|
|
// inject the createExport job into the config
|
|
;((config.jobs ??= {}).tasks ??= []).push(getCreateCollectionExportTask(config, pluginConfig))
|
|
|
|
let collectionsToUpdate = config.collections
|
|
|
|
const usePluginCollections = pluginConfig.collections && pluginConfig.collections?.length > 0
|
|
|
|
if (usePluginCollections) {
|
|
collectionsToUpdate = config.collections?.filter((collection) => {
|
|
return pluginConfig.collections?.includes(collection.slug)
|
|
})
|
|
}
|
|
|
|
collectionsToUpdate.forEach((collection) => {
|
|
if (!collection.admin) {
|
|
collection.admin = { components: { listMenuItems: [] } }
|
|
}
|
|
const components = collection.admin.components || {}
|
|
if (!components.listMenuItems) {
|
|
components.listMenuItems = []
|
|
}
|
|
components.listMenuItems.push({
|
|
clientProps: {
|
|
exportCollectionSlug: exportCollection.slug,
|
|
},
|
|
path: '@payloadcms/plugin-import-export/rsc#ExportListMenuItem',
|
|
})
|
|
|
|
// Find fields explicitly marked as disabled for import/export
|
|
const disabledFieldAccessors = collectDisabledFieldPaths(collection.fields)
|
|
|
|
// Store disabled field accessors in the admin config for use in the UI
|
|
collection.admin.custom = {
|
|
...(collection.admin.custom || {}),
|
|
'plugin-import-export': {
|
|
...(collection.admin.custom?.['plugin-import-export'] || {}),
|
|
disabledFields: disabledFieldAccessors,
|
|
},
|
|
}
|
|
|
|
collection.admin.components = components
|
|
})
|
|
|
|
if (!config.i18n) {
|
|
config.i18n = {}
|
|
}
|
|
|
|
// config.i18n.translations = deepMergeSimple(translations, config.i18n?.translations ?? {})
|
|
|
|
// Inject custom REST endpoints into the config
|
|
config.endpoints = config.endpoints || []
|
|
config.endpoints.push({
|
|
handler: async (req) => {
|
|
await addDataAndFileToRequest(req)
|
|
|
|
const { collectionSlug, draft, fields, limit, locale, page, sort, where } = req.data as {
|
|
collectionSlug: string
|
|
draft?: 'no' | 'yes'
|
|
fields?: string[]
|
|
format?: 'csv' | 'json'
|
|
limit?: number
|
|
locale?: string
|
|
page?: number
|
|
sort?: any
|
|
where?: any
|
|
}
|
|
|
|
const collection = req.payload.collections[collectionSlug]
|
|
if (!collection) {
|
|
return Response.json(
|
|
{ error: `Collection with slug ${collectionSlug} not found` },
|
|
{ status: 400 },
|
|
)
|
|
}
|
|
|
|
const select = Array.isArray(fields) && fields.length > 0 ? getSelect(fields) : undefined
|
|
|
|
const result = await req.payload.find({
|
|
collection: collectionSlug,
|
|
depth: 1,
|
|
draft: draft === 'yes',
|
|
limit: limit && limit > 10 ? 10 : limit,
|
|
locale,
|
|
overrideAccess: false,
|
|
page,
|
|
req,
|
|
select,
|
|
sort,
|
|
where,
|
|
})
|
|
|
|
const isCSV = req?.data?.format === 'csv'
|
|
const docs = result.docs
|
|
|
|
let transformed: Record<string, unknown>[] = []
|
|
|
|
if (isCSV) {
|
|
const toCSVFunctions = getCustomFieldFunctions({
|
|
fields: collection.config.fields as FlattenedField[],
|
|
})
|
|
|
|
const possibleKeys = getFlattenedFieldKeys(collection.config.fields as FlattenedField[])
|
|
|
|
transformed = docs.map((doc) => {
|
|
const row = flattenObject({
|
|
doc,
|
|
fields,
|
|
toCSVFunctions,
|
|
})
|
|
|
|
for (const key of possibleKeys) {
|
|
if (!(key in row)) {
|
|
row[key] = null
|
|
}
|
|
}
|
|
|
|
return row
|
|
})
|
|
} else {
|
|
const disabledFields =
|
|
collection.config.admin.custom?.['plugin-import-export']?.disabledFields
|
|
|
|
transformed = docs.map((doc) => {
|
|
let output: Record<string, unknown> = { ...doc }
|
|
|
|
// Remove disabled fields first
|
|
output = removeDisabledFields(output, disabledFields)
|
|
|
|
// Then trim to selected fields only (if fields are provided)
|
|
if (Array.isArray(fields) && fields.length > 0) {
|
|
const trimmed: Record<string, unknown> = {}
|
|
|
|
for (const key of fields) {
|
|
const value = getValueAtPath(output, key)
|
|
setNestedValue(trimmed, key, value ?? null)
|
|
}
|
|
|
|
output = trimmed
|
|
}
|
|
|
|
return output
|
|
})
|
|
}
|
|
|
|
return Response.json({
|
|
docs: transformed,
|
|
totalDocs: result.totalDocs,
|
|
})
|
|
},
|
|
method: 'post',
|
|
path: '/preview-data',
|
|
})
|
|
|
|
/**
|
|
* Merge plugin translations
|
|
*/
|
|
const simplifiedTranslations = Object.entries(translations).reduce(
|
|
(acc, [key, value]) => {
|
|
acc[key] = value.translations
|
|
return acc
|
|
},
|
|
{} as Record<string, PluginDefaultTranslationsObject>,
|
|
)
|
|
|
|
config.i18n = {
|
|
...config.i18n,
|
|
translations: deepMergeSimple(simplifiedTranslations, config.i18n?.translations ?? {}),
|
|
}
|
|
|
|
return config
|
|
}
|
|
|
|
declare module 'payload' {
|
|
export interface FieldCustom {
|
|
'plugin-import-export'?: {
|
|
/**
|
|
* When `true` the field is **completely excluded** from the import-export plugin:
|
|
* - It will not appear in the “Fields to export” selector.
|
|
* - It is hidden from the preview list when no specific fields are chosen.
|
|
* - Its data is omitted from the final CSV / JSON export.
|
|
* @default false
|
|
*/
|
|
disabled?: boolean
|
|
/**
|
|
* Custom function used to modify the outgoing csv data by manipulating the data, siblingData or by returning the desired value
|
|
*/
|
|
toCSV?: ToCSVFunction
|
|
}
|
|
}
|
|
}
|