Files
payload/test/query-presets/payload-types.ts
Alessio Gravili 9a1c3cf4cc fix: support parallel job queue tasks (#11917)
This adds support for running multiple job queue tasks in parallel
within the same workflow while preventing conflicts. Previously, this
would have caused the following issues:
- Job log entries get lost - the final job log is incomplete, despite
all tasks having been executed
- Write conflicts in postgres, leading to unique constraint violation
errors

The solution involves handling job log data updates in a way that avoids
overwriting, and ensuring the final update reflects the latest job log
data. Each job log entry now initializes its own ID, so a given job log
entry’s ID remains the same across multiple, parallel task executions.

## Postgres

In Postgres, we need to enable transactions for the
`payload.db.updateJobs` operation; otherwise, two tasks updating the
same job in parallel can conflict. This happens because Postgres handles
array rows by deleting them all, then re-inserting (rather than
upserting). The rows are stored in a separate table, and the following
scenario can occur:

Op 1: deletes all job log rows
Op 2: deletes all job log rows
Op 1: inserts 200 job log rows
Op 2: insert the same 200 job log rows again => `error: “duplicate key
value violates unique constraint "payload_jobs_log_pkey”`

Because transactions were not used, the rows inserted by Op 1
immediately became visible to Op 2, causing the conflict. Enabling
transactions fixes this. In theory, it can still happen if Op 1 commits
before Op 2 starts inserting (due to the read committed isolation
level), but it should occur far less frequently.

Alongside this change, we should consider inserting the rows using an
upsert (update on conflict), which will get rid of this error
completely. That way, if the insertion of Op 1 is visible to Op 2, Op 2
will simply overwrite it, rather than erroring. Individual job entries
are immutable and job entries cannot be deleted, thus this shouldn't
corrupt any data.

## Mongo

In Mongo, the issue is addressed by ensuring that log row deletions
caused due to different log states in concurrent operations are not
merged back to the client job log, and by making sure the final update
includes all job logs.

There is no duplicate key error in Mongo because the array log resides
in the same document and duplicates are simply upserted. We cannot use
transactions in Mongo, as it appears to lock the document in a way that
prevents reliable parallel updates, leading to:

`MongoServerError: WriteConflict error: this operation conflicted with
another operation. Please retry your operation or multi-document
transaction`
2025-03-31 13:06:05 -06:00

386 lines
8.6 KiB
TypeScript

/* tslint:disable */
/* eslint-disable */
/**
* This file was automatically generated by Payload.
* DO NOT MODIFY IT BY HAND. Instead, modify your source Payload config,
* and re-run `payload generate:types` to regenerate this file.
*/
/**
* Supported timezones in IANA format.
*
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "supportedTimezones".
*/
export type SupportedTimezones =
| 'Pacific/Midway'
| 'Pacific/Niue'
| 'Pacific/Honolulu'
| 'Pacific/Rarotonga'
| 'America/Anchorage'
| 'Pacific/Gambier'
| 'America/Los_Angeles'
| 'America/Tijuana'
| 'America/Denver'
| 'America/Phoenix'
| 'America/Chicago'
| 'America/Guatemala'
| 'America/New_York'
| 'America/Bogota'
| 'America/Caracas'
| 'America/Santiago'
| 'America/Buenos_Aires'
| 'America/Sao_Paulo'
| 'Atlantic/South_Georgia'
| 'Atlantic/Azores'
| 'Atlantic/Cape_Verde'
| 'Europe/London'
| 'Europe/Berlin'
| 'Africa/Lagos'
| 'Europe/Athens'
| 'Africa/Cairo'
| 'Europe/Moscow'
| 'Asia/Riyadh'
| 'Asia/Dubai'
| 'Asia/Baku'
| 'Asia/Karachi'
| 'Asia/Tashkent'
| 'Asia/Calcutta'
| 'Asia/Dhaka'
| 'Asia/Almaty'
| 'Asia/Jakarta'
| 'Asia/Bangkok'
| 'Asia/Shanghai'
| 'Asia/Singapore'
| 'Asia/Tokyo'
| 'Asia/Seoul'
| 'Australia/Brisbane'
| 'Australia/Sydney'
| 'Pacific/Guam'
| 'Pacific/Noumea'
| 'Pacific/Auckland'
| 'Pacific/Fiji';
export interface Config {
auth: {
users: UserAuthOperations;
};
blocks: {};
collections: {
pages: Page;
users: User;
posts: Post;
'payload-locked-documents': PayloadLockedDocument;
'payload-preferences': PayloadPreference;
'payload-migrations': PayloadMigration;
'payload-query-presets': PayloadQueryPreset;
};
collectionsJoins: {};
collectionsSelect: {
pages: PagesSelect<false> | PagesSelect<true>;
users: UsersSelect<false> | UsersSelect<true>;
posts: PostsSelect<false> | PostsSelect<true>;
'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
'payload-preferences': PayloadPreferencesSelect<false> | PayloadPreferencesSelect<true>;
'payload-migrations': PayloadMigrationsSelect<false> | PayloadMigrationsSelect<true>;
'payload-query-presets': PayloadQueryPresetsSelect<false> | PayloadQueryPresetsSelect<true>;
};
db: {
defaultIDType: number;
};
globals: {};
globalsSelect: {};
locale: null;
user: User & {
collection: 'users';
};
jobs: {
tasks: unknown;
workflows: unknown;
};
}
export interface UserAuthOperations {
forgotPassword: {
email: string;
password: string;
};
login: {
email: string;
password: string;
};
registerFirstUser: {
email: string;
password: string;
};
unlock: {
email: string;
password: string;
};
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "pages".
*/
export interface Page {
id: number;
text?: string | null;
updatedAt: string;
createdAt: string;
_status?: ('draft' | 'published') | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "users".
*/
export interface User {
id: number;
name?: string | null;
roles?: ('admin' | 'user' | 'anonymous')[] | null;
updatedAt: string;
createdAt: string;
email: string;
resetPasswordToken?: string | null;
resetPasswordExpiration?: string | null;
salt?: string | null;
hash?: string | null;
loginAttempts?: number | null;
lockUntil?: string | null;
password?: string | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "posts".
*/
export interface Post {
id: number;
text?: string | null;
updatedAt: string;
createdAt: string;
_status?: ('draft' | 'published') | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-locked-documents".
*/
export interface PayloadLockedDocument {
id: number;
document?:
| ({
relationTo: 'pages';
value: number | Page;
} | null)
| ({
relationTo: 'users';
value: number | User;
} | null)
| ({
relationTo: 'posts';
value: number | Post;
} | null);
globalSlug?: string | null;
user: {
relationTo: 'users';
value: number | User;
};
updatedAt: string;
createdAt: string;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-preferences".
*/
export interface PayloadPreference {
id: number;
user: {
relationTo: 'users';
value: number | User;
};
key?: string | null;
value?:
| {
[k: string]: unknown;
}
| unknown[]
| string
| number
| boolean
| null;
updatedAt: string;
createdAt: string;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-migrations".
*/
export interface PayloadMigration {
id: number;
name?: string | null;
batch?: number | null;
updatedAt: string;
createdAt: string;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-query-presets".
*/
export interface PayloadQueryPreset {
id: number;
title: string;
isShared?: boolean | null;
access?: {
read?: {
constraint?: ('everyone' | 'onlyMe' | 'specificUsers' | 'specificRoles') | null;
users?: (number | User)[] | null;
roles?: ('admin' | 'user' | 'anonymous')[] | null;
};
update?: {
constraint?: ('everyone' | 'onlyMe' | 'specificUsers' | 'specificRoles') | null;
users?: (number | User)[] | null;
roles?: ('admin' | 'user' | 'anonymous')[] | null;
};
delete?: {
constraint?: ('everyone' | 'onlyMe' | 'specificUsers') | null;
users?: (number | User)[] | null;
};
};
where?:
| {
[k: string]: unknown;
}
| unknown[]
| string
| number
| boolean
| null;
columns?:
| {
[k: string]: unknown;
}
| unknown[]
| string
| number
| boolean
| null;
relatedCollection: 'pages' | 'posts';
updatedAt: string;
createdAt: string;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "pages_select".
*/
export interface PagesSelect<T extends boolean = true> {
text?: T;
updatedAt?: T;
createdAt?: T;
_status?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "users_select".
*/
export interface UsersSelect<T extends boolean = true> {
name?: T;
roles?: T;
updatedAt?: T;
createdAt?: T;
email?: T;
resetPasswordToken?: T;
resetPasswordExpiration?: T;
salt?: T;
hash?: T;
loginAttempts?: T;
lockUntil?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "posts_select".
*/
export interface PostsSelect<T extends boolean = true> {
text?: T;
updatedAt?: T;
createdAt?: T;
_status?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-locked-documents_select".
*/
export interface PayloadLockedDocumentsSelect<T extends boolean = true> {
document?: T;
globalSlug?: T;
user?: T;
updatedAt?: T;
createdAt?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-preferences_select".
*/
export interface PayloadPreferencesSelect<T extends boolean = true> {
user?: T;
key?: T;
value?: T;
updatedAt?: T;
createdAt?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-migrations_select".
*/
export interface PayloadMigrationsSelect<T extends boolean = true> {
name?: T;
batch?: T;
updatedAt?: T;
createdAt?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "payload-query-presets_select".
*/
export interface PayloadQueryPresetsSelect<T extends boolean = true> {
title?: T;
isShared?: T;
access?:
| T
| {
read?:
| T
| {
constraint?: T;
users?: T;
roles?: T;
};
update?:
| T
| {
constraint?: T;
users?: T;
roles?: T;
};
delete?:
| T
| {
constraint?: T;
users?: T;
};
};
where?: T;
columns?: T;
relatedCollection?: T;
updatedAt?: T;
createdAt?: T;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "auth".
*/
export interface Auth {
[k: string]: unknown;
}
declare module 'payload' {
// @ts-ignore
export interface GeneratedTypes extends Config {}
}