4f822a439b
Adds new plugin-import-export initial version.
Allows for direct download and creation of downloadable collection data
stored to a json or csv uses the access control of the user creating the
request to make the file.
config options:
```ts
/**
* Collections to include the Import/Export controls in
* Defaults to all collections
*/
collections?: string[]
/**
* Enable to force the export to run synchronously
*/
disableJobsQueue?: boolean
/**
* This function takes the default export collection configured in the plugin and allows you to override it by modifying and returning it
* @param collection
* @returns collection
*/
overrideExportCollection?: (collection: CollectionOverride) => CollectionOverride
// payload.config.ts:
plugins: [
importExportPlugin({
collections: ['pages', 'users'],
overrideExportCollection: (collection) => {
collection.admin.group = 'System'
collection.upload.staticDir = path.resolve(dirname, 'uploads')
return collection
},
disableJobsQueue: true,
}),
],
```
---------
Co-authored-by: Jessica Chowdhury <jessica@trbl.design>
Co-authored-by: Kendell Joseph <kendelljoseph@gmail.com>
401 lines
11 KiB
TypeScript
401 lines
11 KiB
TypeScript
import type { CollectionSlug, Payload } from 'payload'
|
|
|
|
import path from 'path'
|
|
import { fileURLToPath } from 'url'
|
|
|
|
import { devUser } from '../credentials.js'
|
|
import { initPayloadInt } from '../helpers/initPayloadInt.js'
|
|
import { readCSV, readJSON } from './helpers.js'
|
|
import { richTextData } from './seed/richTextData.js'
|
|
|
|
let payload: Payload
|
|
let user: any
|
|
|
|
const filename = fileURLToPath(import.meta.url)
|
|
const dirname = path.dirname(filename)
|
|
|
|
describe('@payloadcms/plugin-import-export', () => {
|
|
beforeAll(async () => {
|
|
;({ payload } = (await initPayloadInt(dirname)) as { payload: Payload })
|
|
user = await payload.login({
|
|
collection: 'users',
|
|
data: {
|
|
email: devUser.email,
|
|
password: devUser.password,
|
|
},
|
|
})
|
|
})
|
|
|
|
afterAll(async () => {
|
|
if (typeof payload.db.destroy === 'function') {
|
|
await payload.db.destroy()
|
|
}
|
|
})
|
|
|
|
describe('exports', () => {
|
|
it('should create a file for collection csv from defined fields', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
sort: 'createdAt',
|
|
fields: ['id', 'title', 'group.value', 'group.array.field1', 'createdAt', 'updatedAt'],
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Title ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toContain('pages.csv')
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].id).toBeDefined()
|
|
expect(data[0].title).toStrictEqual('Title 0')
|
|
expect(data[0].group_value).toStrictEqual('group value')
|
|
expect(data[0].group_ignore).toBeUndefined()
|
|
expect(data[0].group_array_0_field1).toStrictEqual('test')
|
|
expect(data[0].createdAt).toBeDefined()
|
|
expect(data[0].updatedAt).toBeDefined()
|
|
})
|
|
|
|
it('should create a file for collection csv from one locale', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'localized'],
|
|
locale: 'en',
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Localized ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].id).toBeDefined()
|
|
expect(data[0].localized).toStrictEqual('en test')
|
|
})
|
|
|
|
it('should create a file for collection csv from multiple locales', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'localized'],
|
|
locale: 'all',
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Localized ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].id).toBeDefined()
|
|
expect(data[0].localized_en).toStrictEqual('en test')
|
|
expect(data[0].localized_es).toStrictEqual('es test')
|
|
})
|
|
|
|
it('should create a file for collection csv from array', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'array'],
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Array ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].array_0_field1).toStrictEqual('foo')
|
|
expect(data[0].array_0_field2).toStrictEqual('bar')
|
|
expect(data[0].array_1_field1).toStrictEqual('foo')
|
|
expect(data[0].array_1_field2).toStrictEqual('baz')
|
|
})
|
|
|
|
it('should create a file for collection csv from array.subfield', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'array.field1'],
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Array Subfield ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].array_0_field1).toStrictEqual('foo')
|
|
expect(data[0].array_0_field2).toBeUndefined()
|
|
expect(data[0].array_1_field1).toStrictEqual('foo')
|
|
expect(data[0].array_1_field2).toBeUndefined()
|
|
})
|
|
|
|
it('should create a file for collection csv from hasMany field', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'hasManyNumber'],
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'hasMany Number ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].hasManyNumber_0).toStrictEqual('0')
|
|
expect(data[0].hasManyNumber_1).toStrictEqual('1')
|
|
expect(data[0].hasManyNumber_2).toStrictEqual('1')
|
|
expect(data[0].hasManyNumber_3).toStrictEqual('2')
|
|
expect(data[0].hasManyNumber_4).toStrictEqual('3')
|
|
})
|
|
|
|
it('should create a file for collection csv from blocks field', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'blocks'],
|
|
format: 'csv',
|
|
where: {
|
|
title: { contains: 'Blocks ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].blocks_0_blockType).toStrictEqual('hero')
|
|
expect(data[0].blocks_1_blockType).toStrictEqual('content')
|
|
})
|
|
|
|
it('should create a JSON file for collection', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'title'],
|
|
format: 'json',
|
|
sort: 'title',
|
|
where: {
|
|
title: { contains: 'JSON ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readJSON(expectedPath)
|
|
|
|
expect(data[0].title).toStrictEqual('JSON 0')
|
|
})
|
|
|
|
it('should create an export with every field when no fields are defined', async () => {
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
format: 'json',
|
|
sort: 'title',
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readJSON(expectedPath)
|
|
|
|
expect(data[0].id).toBeDefined()
|
|
expect(data[0].title).toBeDefined()
|
|
expect(data[0].createdAt).toBeDefined()
|
|
expect(data[0].updatedAt).toBeDefined()
|
|
})
|
|
|
|
it('should create jobs task for exports', async () => {
|
|
const doc = await payload.create({
|
|
collection: 'exports-tasks' as CollectionSlug,
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'title'],
|
|
format: 'csv',
|
|
sort: 'title',
|
|
where: {
|
|
title: { contains: 'Jobs ' },
|
|
},
|
|
},
|
|
})
|
|
|
|
const { docs } = await payload.find({
|
|
collection: 'payload-jobs' as CollectionSlug,
|
|
})
|
|
const job = docs[0]
|
|
|
|
expect(job).toBeDefined()
|
|
|
|
const { input } = job
|
|
|
|
expect(input.id).toBeDefined()
|
|
expect(input.name).toBeDefined()
|
|
expect(input.format).toStrictEqual('csv')
|
|
expect(input.locale).toStrictEqual('all')
|
|
expect(input.fields).toStrictEqual(['id', 'title'])
|
|
expect(input.collectionSlug).toStrictEqual('pages')
|
|
expect(input.exportsCollection).toStrictEqual('exports-tasks')
|
|
expect(input.user).toBeDefined()
|
|
expect(input.userCollection).toBeDefined()
|
|
|
|
await payload.jobs.run()
|
|
|
|
const exportDoc = await payload.findByID({
|
|
collection: 'exports-tasks' as CollectionSlug,
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(exportDoc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', exportDoc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].title).toStrictEqual('Jobs 0')
|
|
})
|
|
|
|
// disabled so we don't always run a massive test
|
|
it.skip('should create a file from a large set of collection documents', async () => {
|
|
const allPromises = []
|
|
let promises = []
|
|
for (let i = 0; i < 100000; i++) {
|
|
promises.push(
|
|
payload.create({
|
|
collectionSlug: 'pages',
|
|
data: {
|
|
title: `Array ${i}`,
|
|
blocks: [
|
|
{
|
|
blockType: 'hero',
|
|
title: 'test',
|
|
},
|
|
{
|
|
blockType: 'content',
|
|
richText: richTextData,
|
|
},
|
|
],
|
|
},
|
|
}),
|
|
)
|
|
if (promises.length >= 500) {
|
|
await Promise.all(promises)
|
|
promises = []
|
|
}
|
|
if (i % 1000 === 0) {
|
|
console.log('created', i)
|
|
}
|
|
}
|
|
await Promise.all(promises)
|
|
|
|
console.log('seeded')
|
|
|
|
let doc = await payload.create({
|
|
collection: 'exports',
|
|
user,
|
|
data: {
|
|
collectionSlug: 'pages',
|
|
fields: ['id', 'blocks'],
|
|
format: 'csv',
|
|
},
|
|
})
|
|
|
|
doc = await payload.findByID({
|
|
collection: 'exports',
|
|
id: doc.id,
|
|
})
|
|
|
|
expect(doc.filename).toBeDefined()
|
|
const expectedPath = path.join(dirname, './uploads', doc.filename as string)
|
|
const data = await readCSV(expectedPath)
|
|
|
|
expect(data[0].blocks_0_blockType).toStrictEqual('hero')
|
|
expect(data[0].blocks_1_blockType).toStrictEqual('content')
|
|
})
|
|
})
|
|
})
|