8970c6b3a6
Adds a jobs queue to Payload.
- [x] Docs, w/ examples for Vercel Cron, additional services
- [x] Type the `job` using GeneratedTypes in `JobRunnerArgs`
(@AlessioGr)
- [x] Write the `runJobs` function
- [x] Allow for some type of `payload.runTask`
- [x] Open up a new bin script for running jobs
- [x] Determine strategy for runner endpoint to either await jobs
successfully or return early and stay open until job work completes
(serverless ramifications here)
- [x] Allow for job runner to accept how many jobs to run in one
invocation
- [x] Make a Payload local API method for creating a new job easily
(payload.createJob) or similar which is strongly typed (@AlessioGr)
- [x] Make `payload.runJobs` or similar (@AlessioGr)
- [x] Write tests for retrying up to max retries for a given step
- [x] Write tests for dynamic import of a runner
The shape of the config should permit the definition of steps separate
from the job workflows themselves.
```js
const config = {
// Not sure if we need this property anymore
queues: {
},
// A job is an instance of a workflow, stored in DB
// and triggered by something at some point
jobs: {
// Be able to override the jobs collection
collectionOverrides: () => {},
// Workflows are groups of tasks that handle
// the flow from task to task.
// When defined on the config, they are considered as predefined workflows
// BUT - in the future, we'll allow for UI-based workflow definition as well.
workflows: [
{
slug: 'job-name',
// Temporary name for this
// should be able to pass function
// or path to it for Node to dynamically import
controlFlowInJS: '/my-runner.js',
// Temporary name as well
// should be able to eventually define workflows
// in UI (meaning they need to be serialized in JSON)
// Should not be able to define both control flows
controlFlowInJSON: [
{
task: 'myTask',
next: {
// etc
}
}
],
// Workflows take input
// which are a group of fields
input: [
{
name: 'post',
type: 'relationship',
relationTo: 'posts',
maxDepth: 0,
required: true,
},
{
name: 'message',
type: 'text',
required: true,
},
],
},
],
// Tasks are defined separately as isolated functions
// that can be retried on fail
tasks: [
{
slug: 'myTask',
retries: 2,
// Each task takes input
// Used to auto-type the task func args
input: [
{
name: 'post',
type: 'relationship',
relationTo: 'posts',
maxDepth: 0,
required: true,
},
{
name: 'message',
type: 'text',
required: true,
},
],
// Each task takes output
// Used to auto-type the function signature
output: [
{
name: 'success',
type: 'checkbox',
}
],
onSuccess: () => {},
onFail: () => {},
run: myRunner,
},
]
}
}
```
### `payload.createJob`
This function should allow for the creation of jobs based on either a
workflow (group of tasks) or an individual task.
To create a job using a workflow:
```js
const job = await payload.createJob({
// Accept the `name` of a workflow so we can match to either a
// code-based workflow OR a workflow defined in the DB
// Should auto-type the input
workflowName: 'myWorkflow',
input: {
// typed to the args of the workflow by name
}
})
```
To create a job using a task:
```js
const job = await payload.createJob({
// Accept the `name` of a task
task: 'myTask',
input: {
// typed to the args of the task by name
}
})
```
---------
Co-authored-by: Alessio Gravili <alessio@gravili.de>
Co-authored-by: Dan Ribbens <dan.ribbens@gmail.com>
66 lines
2.1 KiB
TypeScript
66 lines
2.1 KiB
TypeScript
import type { Page } from '@playwright/test'
|
|
|
|
import { expect, test } from '@playwright/test'
|
|
import * as path from 'path'
|
|
import { fileURLToPath } from 'url'
|
|
|
|
import type { PayloadTestSDK } from '../helpers/sdk/index.js'
|
|
import type { Config } from './payload-types.js'
|
|
|
|
import { ensureCompilationIsDone, initPageConsoleErrorCatch } from '../helpers.js'
|
|
import { AdminUrlUtil } from '../helpers/adminUrlUtil.js'
|
|
import { initPayloadE2ENoConfig } from '../helpers/initPayloadE2ENoConfig.js'
|
|
import { reInitializeDB } from '../helpers/reInitializeDB.js'
|
|
import { RESTClient } from '../helpers/rest.js'
|
|
import { TEST_TIMEOUT } from '../playwright.config.js'
|
|
|
|
const filename = fileURLToPath(import.meta.url)
|
|
const dirname = path.dirname(filename)
|
|
let serverURL: string
|
|
let payload: PayloadTestSDK<Config>
|
|
let client: RESTClient
|
|
|
|
test.describe('Queues', () => {
|
|
let page: Page
|
|
let url: AdminUrlUtil
|
|
|
|
test.beforeAll(async ({ browser }, testInfo) => {
|
|
testInfo.setTimeout(TEST_TIMEOUT)
|
|
process.env.SEED_IN_CONFIG_ONINIT = 'false' // Makes it so the payload config onInit seed is not run. Otherwise, the seed would be run unnecessarily twice for the initial test run - once for beforeEach and once for onInit
|
|
;({ payload, serverURL } = await initPayloadE2ENoConfig({ dirname }))
|
|
url = new AdminUrlUtil(serverURL, 'payload-jobs')
|
|
|
|
const context = await browser.newContext()
|
|
page = await context.newPage()
|
|
initPageConsoleErrorCatch(page)
|
|
await reInitializeDB({
|
|
serverURL,
|
|
snapshotKey: 'queuesTest',
|
|
})
|
|
await ensureCompilationIsDone({ page, serverURL })
|
|
})
|
|
|
|
test.beforeEach(async () => {
|
|
await reInitializeDB({
|
|
serverURL,
|
|
snapshotKey: 'fieldsTest',
|
|
uploadsDir: path.resolve(dirname, './collections/Upload/uploads'),
|
|
})
|
|
|
|
if (client) {
|
|
await client.logout()
|
|
}
|
|
client = new RESTClient(null, { defaultSlug: 'users', serverURL })
|
|
await client.login()
|
|
|
|
await ensureCompilationIsDone({ page, serverURL })
|
|
})
|
|
|
|
test('example test', async () => {
|
|
await page.goto(url.list)
|
|
|
|
const textCell = page.locator('.row-1 .cell-text')
|
|
await expect(textCell).toHaveText('example post')
|
|
})
|
|
})
|