perf: 23% faster job queue system on postgres/sqlite (#13187)
Previously, a single run of the simplest job queue workflow (1 single task, no db calls by user code in the task - we're just testing db system overhead) would result in **22 db roundtrips** on drizzle. This PR reduces it to **17 db roundtrips** by doing the following: - Modifies db.updateJobs to use the new optimized upsertRow function if the update is simple - Do not unnecessarily pass the job log to the final job update when the workflow completes => allows using the optimized upsertRow function, as only the main table is involved --- - To see the specific tasks where the Asana app for GitHub is being used, see below: - https://app.asana.com/0/0/1210888186878606
This commit is contained in:
59
test/queues/postgres-logs.int.spec.ts
Normal file
59
test/queues/postgres-logs.int.spec.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import type { Payload } from 'payload'
|
||||
|
||||
/* eslint-disable jest/require-top-level-describe */
|
||||
import assert from 'assert'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
import { initPayloadInt } from '../helpers/initPayloadInt.js'
|
||||
import { withoutAutoRun } from './utilities.js'
|
||||
|
||||
const filename = fileURLToPath(import.meta.url)
|
||||
const dirname = path.dirname(filename)
|
||||
|
||||
const describePostgres = process.env.PAYLOAD_DATABASE?.startsWith('postgres')
|
||||
? describe
|
||||
: describe.skip
|
||||
|
||||
let payload: Payload
|
||||
|
||||
describePostgres('queues - postgres logs', () => {
|
||||
beforeAll(async () => {
|
||||
const initialized = await initPayloadInt(
|
||||
dirname,
|
||||
undefined,
|
||||
undefined,
|
||||
'config.postgreslogs.ts',
|
||||
)
|
||||
assert(initialized.payload)
|
||||
assert(initialized.restClient)
|
||||
;({ payload } = initialized)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await payload.destroy()
|
||||
})
|
||||
|
||||
it('ensure running jobs uses minimal db calls', async () => {
|
||||
await withoutAutoRun(async () => {
|
||||
await payload.jobs.queue({
|
||||
task: 'DoNothingTask',
|
||||
input: {
|
||||
message: 'test',
|
||||
},
|
||||
})
|
||||
|
||||
// Count every console log (= db call)
|
||||
const consoleCount = jest.spyOn(console, 'log').mockImplementation(() => {})
|
||||
|
||||
const res = await payload.jobs.run({})
|
||||
|
||||
expect(res).toEqual({
|
||||
jobStatus: { '1': { status: 'success' } },
|
||||
remainingJobsFromQueried: 0,
|
||||
})
|
||||
expect(consoleCount).toHaveBeenCalledTimes(17) // Should be 17 sql calls if the optimizations are used. If not, this would be 22 calls
|
||||
consoleCount.mockRestore()
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user