tabshift/payload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: use direct db calls in job-queue system (#11489)

Browse Source 
Previously, our job queue system relied on `payload.*` operations, which
ran very frequently:
- whenever job execution starts, as all jobs need to be set to
`processing: true`
- every single time a task completes or fails, as the job log needs to
be updated
- whenever job execution stops, to mark it as completed and to delete it
(if `deleteJobOnComplete` is set)

This PR replaces these with direct `payload.db.*` calls, which are
significantly faster than payload operations. Given how often the job
queue system communicates with the database, this should be a massive
performance improvement.

## How it affects running hooks

To generate the task status, we previously used an `afterRead` hook.
Since direct db adapter calls no longer execute hooks, this PR
introduces new `updateJob` and `updateJobs` helpers to handle task
status generation outside the normal payload hook lifecycle.

Additionally, a new `runHooks` property has been added to the global job
configuration. While setting this to `true` can be useful if custom
hooks were added to the `payload-jobs` collection config, this will
revert the job system to use normal payload operations.
This should be avoided as it degrades performance. In most cases, the
`onSuccess` or `onFail` properties in the job config will be sufficient
and much faster.

Furthermore, if the `depth` property is set in the global job
configuration, the job queue system will also fall back to the slower,
normal payload operations.

---------

Co-authored-by: Dan Ribbens <DanRibbens@users.noreply.github.com>
This commit is contained in:
Alessio Gravili
2025-03-20 11:31:14 -06:00
committed by GitHub
parent 43cdccdef0
commit 032c424244
20 changed files with 320 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/queues/workflows/noRetriesSet.ts
View File

@@ -10,7 +10,7 @@ export const noRetriesSetWorkflow: WorkflowConfig<'workflowNoRetriesSet'> = {
},
],
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -23,6 +23,8 @@ export const noRetriesSetWorkflow: WorkflowConfig<'workflowNoRetriesSet'> = {
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimple('1', {
input: {
message: job.input.message,

3
test/queues/workflows/retries0.ts
View File

@@ -11,7 +11,7 @@ export const retries0Workflow: WorkflowConfig<'workflowRetries0'> = {
],
retries: 0,
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -23,6 +23,7 @@ export const retries0Workflow: WorkflowConfig<'workflowRetries0'> = {
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimple('1', {
input: {

3
test/queues/workflows/retriesBackoffTest.ts
View File

@@ -45,13 +45,14 @@ export const retriesBackoffTestWorkflow: WorkflowConfig<'retriesBackoffTest'> =
// @ts-expect-error timeTried is new arbitrary data and not in the type
job.input.timeTried[totalTried] = new Date().toISOString()
await req.payload.update({
const updated = await req.payload.update({
collection: 'payload-jobs',
data: {
input: job.input,
},
id: job.id,
})
job.input = updated.input as any
if (totalTried < 4) {
// Cleanup the post

3
test/queues/workflows/retriesTest.ts
View File

@@ -10,7 +10,7 @@ export const retriesTestWorkflow: WorkflowConfig<'retriesTest'> = {
},
],
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -22,6 +22,7 @@ export const retriesTestWorkflow: WorkflowConfig<'retriesTest'> = {
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimple('1', {
input: {

3
test/queues/workflows/retriesWorkflowLevelTest.ts
View File

@@ -11,7 +11,7 @@ export const retriesWorkflowLevelTestWorkflow: WorkflowConfig<'retriesWorkflowLe
],
retries: 2, // Even though CreateSimple has 3 retries, this workflow only has 2. Thus, it will only retry once
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -23,6 +23,7 @@ export const retriesWorkflowLevelTestWorkflow: WorkflowConfig<'retriesWorkflowLe
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimple('1', {
input: {

8
test/queues/workflows/subTaskFails.ts
View File

@@ -23,7 +23,7 @@ export const subTaskFailsWorkflow: WorkflowConfig<'subTaskFails'> = {
},
})
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -38,6 +38,8 @@ export const subTaskFailsWorkflow: WorkflowConfig<'subTaskFails'> = {
},
id: job.id,
})
job.input = updatedJob.input as any
return {
output: {
newSimple,
@@ -48,7 +50,7 @@ export const subTaskFailsWorkflow: WorkflowConfig<'subTaskFails'> = {
await inlineTask('create doc 2 - fails', {
task: async ({ req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -63,6 +65,8 @@ export const subTaskFailsWorkflow: WorkflowConfig<'subTaskFails'> = {
},
id: job.id,
})
job.input = updatedJob.input as any
throw new Error('Failed on purpose')
},
})

3
test/queues/workflows/workflowAndTasksRetriesUndefined.ts
View File

@@ -11,7 +11,7 @@ export const workflowAndTasksRetriesUndefinedWorkflow: WorkflowConfig<'workflowA
},
],
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -23,6 +23,7 @@ export const workflowAndTasksRetriesUndefinedWorkflow: WorkflowConfig<'workflowA
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimpleRetriesUndefined('1', {
input: {

3
test/queues/workflows/workflowRetries2TasksRetries0.ts
View File

@@ -12,7 +12,7 @@ export const workflowRetries2TasksRetries0Workflow: WorkflowConfig<'workflowRetr
},
],
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -24,6 +24,7 @@ export const workflowRetries2TasksRetries0Workflow: WorkflowConfig<'workflowRetr
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimpleRetries0('1', {
input: {

3
test/queues/workflows/workflowRetries2TasksRetriesUndefined.ts
View File

@@ -12,7 +12,7 @@ export const workflowRetries2TasksRetriesUndefinedWorkflow: WorkflowConfig<'work
},
],
handler: async ({ job, tasks, req }) => {
await req.payload.update({
const updatedJob = await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
@@ -24,6 +24,7 @@ export const workflowRetries2TasksRetriesUndefinedWorkflow: WorkflowConfig<'work
},
id: job.id,
})
job.input = updatedJob.input as any
await tasks.CreateSimpleRetriesUndefined('1', {
input: {