tabshift/payload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: allow running sub-tasks from tasks (#10373)

Browse Source 
Task handlers now receive `inlineTask` as an arg, which can be used to
run inline sub-tasks. In the task log, those inline tasks will have a
`parent` property that points to the parent task.

Example:

```ts
{
        slug: 'subTask',
        inputSchema: [
          {
            name: 'message',
            type: 'text',
            required: true,
          },
        ],
        handler: async ({ job, inlineTask }) => {
          await inlineTask('create two docs', {
            task: async ({ input, inlineTask }) => {
            
              const { newSimple } = await inlineTask('create doc 1', {
                task: async ({ req }) => {
                  const newSimple = await req.payload.create({
                    collection: 'simple',
                    req,
                    data: {
                      title: input.message,
                    },
                  })
                  return {
                    output: {
                      newSimple,
                    },
                  }
                },
              })

              const { newSimple2 } = await inlineTask('create doc 2', {
                task: async ({ req }) => {
                  const newSimple2 = await req.payload.create({
                    collection: 'simple',
                    req,
                    data: {
                      title: input.message,
                    },
                  })
                  return {
                    output: {
                      newSimple2,
                    },
                  }
                },
              })
              return {
                output: {
                  simpleID1: newSimple.id,
                  simpleID2: newSimple2.id,
                },
              }
            },
            input: {
              message: job.input.message,
            },
          })
        },
      } as WorkflowConfig<'subTask'>
```

Job log example:

```ts
[
  {
    executedAt: '2025-01-06T03:55:44.682Z',
    completedAt: '2025-01-06T03:55:44.684Z',
    taskSlug: 'inline',
    taskID: 'create doc 1',
    output: { newSimple: [Object] },
    parent: { taskSlug: 'inline', taskID: 'create two docs' }, // <= New
    state: 'succeeded',
    id: '677b5440ba35d345d1214d1b'
  },
  {
    executedAt: '2025-01-06T03:55:44.690Z',
    completedAt: '2025-01-06T03:55:44.692Z',
    taskSlug: 'inline',
    taskID: 'create doc 2',
    output: { newSimple2: [Object] },
    parent: { taskSlug: 'inline', taskID: 'create two docs' }, // <= New
    state: 'succeeded',
    id: '677b5440ba35d345d1214d1c'
  },
  {
    executedAt: '2025-01-06T03:55:44.681Z',
    completedAt: '2025-01-06T03:55:44.697Z',
    taskSlug: 'inline',
    taskID: 'create two docs',
    input: { message: 'hello!' },
    output: {
      simpleID1: '677b54401e34772cc63c8693',
      simpleID2: '677b54401e34772cc63c8697'
    },
    parent: {},
    state: 'succeeded',
    id: '677b5440ba35d345d1214d1d'
  }
]
```
This commit is contained in:
Alessio Gravili
2025-01-07 10:24:00 -07:00
committed by GitHub
parent ab53ababc8
commit 08fb159943
10 changed files with 416 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
test/queues/config.ts
View File

@@ -846,6 +846,142 @@ export default buildConfigWithDefaults({
})
},
} as WorkflowConfig<'retriesBackoffTest'>,
{
slug: 'subTask',
inputSchema: [
{
name: 'message',
type: 'text',
required: true,
},
],
handler: async ({ job, inlineTask }) => {
await inlineTask('create two docs', {
task: async ({ input, inlineTask }) => {
const { newSimple } = await inlineTask('create doc 1', {
task: async ({ req }) => {
const newSimple = await req.payload.create({
collection: 'simple',
req,
data: {
title: input.message,
},
})
return {
output: {
newSimple,
},
}
},
})
const { newSimple2 } = await inlineTask('create doc 2', {
task: async ({ req }) => {
const newSimple2 = await req.payload.create({
collection: 'simple',
req,
data: {
title: input.message,
},
})
return {
output: {
newSimple2,
},
}
},
})
return {
output: {
simpleID1: newSimple.id,
simpleID2: newSimple2.id,
},
}
},
input: {
message: job.input.message,
},
})
},
} as WorkflowConfig<'subTask'>,
{
slug: 'subTaskFails',
inputSchema: [
{
name: 'message',
type: 'text',
required: true,
},
],
retries: 3,
handler: async ({ job, inlineTask }) => {
await inlineTask('create two docs', {
task: async ({ input, inlineTask }) => {
const { newSimple } = await inlineTask('create doc 1 - succeeds', {
task: async ({ req }) => {
const newSimple = await req.payload.create({
collection: 'simple',
req,
data: {
title: input.message,
},
})
await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
...job.input,
amountTask1Retried:
// @ts-expect-error amountRetried is new arbitrary data and not in the type
job.input.amountTask1Retried !== undefined
? // @ts-expect-error
job.input.amountTask1Retried + 1
: 0,
},
},
id: job.id,
})
return {
output: {
newSimple,
},
}
},
})
await inlineTask('create doc 2 - fails', {
task: async ({ req }) => {
await req.payload.update({
collection: 'payload-jobs',
data: {
input: {
...job.input,
amountTask2Retried:
// @ts-expect-error amountRetried is new arbitrary data and not in the type
job.input.amountTask2Retried !== undefined
? // @ts-expect-error
job.input.amountTask2Retried + 1
: 0,
},
},
id: job.id,
})
throw new Error('Failed on purpose')
},
})
return {
output: {
simpleID1: newSimple.id,
},
}
},
input: {
message: job.input.message,
},
})
},
} as WorkflowConfig<'subTaskFails'>,
],
},
editor: lexicalEditor(),

76
test/queues/int.spec.ts
View File

@@ -1051,4 +1051,80 @@ describe('Queues', () => {
expect(allCompletedJobs.totalDocs).toBe(1)
expect((allCompletedJobs.docs[0].input as any).message).toBe('from single task 2')
})
it('can run sub-tasks', async () => {
payload.config.jobs.deleteJobOnComplete = false
const job = await payload.jobs.queue({
workflow: 'subTask',
input: {
message: 'hello!',
},
})
await payload.jobs.run()
const allSimples = await payload.find({
collection: 'simple',
limit: 100,
})
expect(allSimples.totalDocs).toBe(2)
expect(allSimples.docs[0].title).toBe('hello!')
expect(allSimples.docs[1].title).toBe('hello!')
const jobAfterRun = await payload.findByID({
collection: 'payload-jobs',
id: job.id,
})
expect(jobAfterRun.log[0].taskID).toBe('create doc 1')
//expect(jobAfterRun.log[0].parent.taskID).toBe('create two docs')
// jobAfterRun.log[0].parent should not exist
expect(jobAfterRun.log[0].parent).toBeUndefined()
expect(jobAfterRun.log[1].taskID).toBe('create doc 2')
//expect(jobAfterRun.log[1].parent.taskID).toBe('create two docs')
expect(jobAfterRun.log[1].parent).toBeUndefined()
expect(jobAfterRun.log[2].taskID).toBe('create two docs')
})
it('ensure successful sub-tasks are not retried', async () => {
payload.config.jobs.deleteJobOnComplete = false
const job = await payload.jobs.queue({
workflow: 'subTaskFails',
input: {
message: 'hello!',
},
})
let hasJobsRemaining = true
while (hasJobsRemaining) {
const response = await payload.jobs.run()
if (response.noJobsRemaining) {
hasJobsRemaining = false
}
}
const allSimples = await payload.find({
collection: 'simple',
limit: 100,
})
expect(allSimples.totalDocs).toBe(1)
expect(allSimples.docs[0].title).toBe('hello!')
const jobAfterRun = await payload.findByID({
collection: 'payload-jobs',
id: job.id,
})
// @ts-expect-error
expect(jobAfterRun.input.amountTask2Retried).toBe(3)
// @ts-expect-error
expect(jobAfterRun.input.amountTask1Retried).toBe(0)
})
})

43
test/queues/payload-types.ts
View File

@@ -66,6 +66,8 @@ export interface Config {
inlineTaskTest: WorkflowInlineTaskTest;
externalWorkflow: WorkflowExternalWorkflow;
retriesBackoffTest: WorkflowRetriesBackoffTest;
subTask: WorkflowSubTask;
subTaskFails: WorkflowSubTaskFails;
};
};
}
@@ -221,6 +223,21 @@ export interface PayloadJob {
| number
| boolean
| null;
parent?: {
taskSlug?:
| (
| 'inline'
| 'UpdatePost'
| 'UpdatePostStep2'
| 'CreateSimple'
| 'CreateSimpleRetriesUndefined'
| 'CreateSimpleRetries0'
| 'CreateSimpleWithDuplicateMessage'
| 'ExternalTask'
)
| null;
taskID?: string | null;
};
state: 'failed' | 'succeeded';
error?:
| {
@@ -249,6 +266,8 @@ export interface PayloadJob {
| 'inlineTaskTest'
| 'externalWorkflow'
| 'retriesBackoffTest'
| 'subTask'
| 'subTaskFails'
)
| null;
taskSlug?:
@@ -390,6 +409,12 @@ export interface PayloadJobsSelect<T extends boolean = true> {
taskID?: T;
input?: T;
output?: T;
parent?:
| T
| {
taskSlug?: T;
taskID?: T;
};
state?: T;
error?: T;
id?: T;
@@ -641,6 +666,24 @@ export interface WorkflowRetriesBackoffTest {
message: string;
};
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "WorkflowSubTask".
*/
export interface WorkflowSubTask {
input: {
message: string;
};
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "WorkflowSubTaskFails".
*/
export interface WorkflowSubTaskFails {
input: {
message: string;
};
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "auth".