diff --git a/packages/db-mongodb/src/connect.ts b/packages/db-mongodb/src/connect.ts index ba2c9c4db..ffe5e902f 100644 --- a/packages/db-mongodb/src/connect.ts +++ b/packages/db-mongodb/src/connect.ts @@ -35,7 +35,12 @@ export const connect: Connect = async function connect( } try { - this.connection = (await mongoose.connect(urlToConnect, connectionOptions)).connection + if (!this.connection) { + this.connection = await mongoose.createConnection(urlToConnect, connectionOptions).asPromise() + } + + await this.connection.openUri(urlToConnect, connectionOptions) + if (this.useAlternativeDropDatabase) { if (this.connection.db) { // Firestore doesn't support dropDatabase, so we monkey patch @@ -75,7 +80,8 @@ export const connect: Connect = async function connect( if (!hotReload) { if (process.env.PAYLOAD_DROP_DATABASE === 'true') { this.payload.logger.info('---- DROPPING DATABASE ----') - await mongoose.connection.dropDatabase() + await this.connection.dropDatabase() + this.payload.logger.info('---- DROPPED DATABASE ----') } } diff --git a/packages/db-mongodb/src/destroy.ts b/packages/db-mongodb/src/destroy.ts index f363d4549..8d53e839a 100644 --- a/packages/db-mongodb/src/destroy.ts +++ b/packages/db-mongodb/src/destroy.ts @@ -1,11 +1,11 @@ import type { Destroy } from 'payload' -import mongoose from 'mongoose' - import type { MongooseAdapter } from './index.js' export const destroy: Destroy = async function destroy(this: MongooseAdapter) { - await mongoose.disconnect() + await this.connection.close() - Object.keys(mongoose.models).map((model) => mongoose.deleteModel(model)) + for (const name of Object.keys(this.connection.models)) { + this.connection.deleteModel(name) + } } diff --git a/packages/db-mongodb/src/init.ts b/packages/db-mongodb/src/init.ts index 25325c6c7..f048ca438 100644 --- a/packages/db-mongodb/src/init.ts +++ b/packages/db-mongodb/src/init.ts @@ -19,11 +19,14 @@ import { getBuildQueryPlugin } from './queries/getBuildQueryPlugin.js' import { getDBName } from './utilities/getDBName.js' export const init: Init = function init(this: MongooseAdapter) { + // Always create a scoped, **unopened** connection object + // (no URI here; models compile per-connection and do not require an open socket) + this.connection ??= mongoose.createConnection() + this.payload.config.collections.forEach((collection: SanitizedCollectionConfig) => { const schemaOptions = this.collectionsSchemaOptions?.[collection.slug] const schema = buildCollectionSchema(collection, this.payload, schemaOptions) - if (collection.versions) { const versionModelName = getDBName({ config: collection, versions: true }) @@ -55,7 +58,7 @@ export const init: Init = function init(this: MongooseAdapter) { const versionCollectionName = this.autoPluralization === true && !collection.dbName ? undefined : versionModelName - this.versions[collection.slug] = mongoose.model( + this.versions[collection.slug] = this.connection.model( versionModelName, versionSchema, versionCollectionName, @@ -66,14 +69,14 @@ export const init: Init = function init(this: MongooseAdapter) { const collectionName = this.autoPluralization === true && !collection.dbName ? undefined : modelName - this.collections[collection.slug] = mongoose.model( + this.collections[collection.slug] = this.connection.model( modelName, schema, collectionName, ) as CollectionModel }) - this.globals = buildGlobalModel(this.payload) as GlobalModel + this.globals = buildGlobalModel(this) as GlobalModel this.payload.config.globals.forEach((global) => { if (global.versions) { @@ -101,7 +104,7 @@ export const init: Init = function init(this: MongooseAdapter) { }), ) - this.versions[global.slug] = mongoose.model( + this.versions[global.slug] = this.connection.model( versionModelName, versionSchema, versionModelName, diff --git a/packages/db-mongodb/src/models/buildGlobalModel.ts b/packages/db-mongodb/src/models/buildGlobalModel.ts index e184c6c11..5f1ae4219 100644 --- a/packages/db-mongodb/src/models/buildGlobalModel.ts +++ b/packages/db-mongodb/src/models/buildGlobalModel.ts @@ -1,14 +1,13 @@ -import type { Payload } from 'payload' - import mongoose from 'mongoose' +import type { MongooseAdapter } from '../index.js' import type { GlobalModel } from '../types.js' import { getBuildQueryPlugin } from '../queries/getBuildQueryPlugin.js' import { buildSchema } from './buildSchema.js' -export const buildGlobalModel = (payload: Payload): GlobalModel | null => { - if (payload.config.globals && payload.config.globals.length > 0) { +export const buildGlobalModel = (adapter: MongooseAdapter): GlobalModel | null => { + if (adapter.payload.config.globals && adapter.payload.config.globals.length > 0) { const globalsSchema = new mongoose.Schema( {}, { discriminatorKey: 'globalType', minimize: false, timestamps: true }, @@ -16,9 +15,13 @@ export const buildGlobalModel = (payload: Payload): GlobalModel | null => { globalsSchema.plugin(getBuildQueryPlugin()) - const Globals = mongoose.model('globals', globalsSchema, 'globals') as unknown as GlobalModel + const Globals = adapter.connection.model( + 'globals', + globalsSchema, + 'globals', + ) as unknown as GlobalModel - Object.values(payload.config.globals).forEach((globalConfig) => { + Object.values(adapter.payload.config.globals).forEach((globalConfig) => { const globalSchema = buildSchema({ buildSchemaOptions: { options: { @@ -26,7 +29,7 @@ export const buildGlobalModel = (payload: Payload): GlobalModel | null => { }, }, configFields: globalConfig.fields, - payload, + payload: adapter.payload, }) Globals.discriminator(globalConfig.slug, globalSchema) }) diff --git a/packages/payload/src/index.ts b/packages/payload/src/index.ts index d23a502f4..38e62b0ef 100644 --- a/packages/payload/src/index.ts +++ b/packages/payload/src/index.ts @@ -625,6 +625,57 @@ export class BasePayload { [slug: string]: any // TODO: Type this } = {} + async _initializeCrons() { + if (this.config.jobs.enabled && this.config.jobs.autoRun && !isNextBuild()) { + const DEFAULT_CRON = '* * * * *' + const DEFAULT_LIMIT = 10 + + const cronJobs = + typeof this.config.jobs.autoRun === 'function' + ? await this.config.jobs.autoRun(this) + : this.config.jobs.autoRun + + await Promise.all( + cronJobs.map((cronConfig) => { + const jobAutorunCron = new Cron(cronConfig.cron ?? DEFAULT_CRON, async () => { + if ( + _internal_jobSystemGlobals.shouldAutoSchedule && + !cronConfig.disableScheduling && + this.config.jobs.scheduling + ) { + await this.jobs.handleSchedules({ + allQueues: cronConfig.allQueues, + queue: cronConfig.queue, + }) + } + + if (!_internal_jobSystemGlobals.shouldAutoRun) { + return + } + + if (typeof this.config.jobs.shouldAutoRun === 'function') { + const shouldAutoRun = await this.config.jobs.shouldAutoRun(this) + + if (!shouldAutoRun) { + jobAutorunCron.stop() + return + } + } + + await this.jobs.run({ + allQueues: cronConfig.allQueues, + limit: cronConfig.limit ?? DEFAULT_LIMIT, + queue: cronConfig.queue, + silent: cronConfig.silent, + }) + }) + + this.crons.push(jobAutorunCron) + }), + ) + } + } + async bin({ args, cwd, @@ -855,53 +906,8 @@ export class BasePayload { throw error } - if (this.config.jobs.enabled && this.config.jobs.autoRun && !isNextBuild() && options.cron) { - const DEFAULT_CRON = '* * * * *' - const DEFAULT_LIMIT = 10 - - const cronJobs = - typeof this.config.jobs.autoRun === 'function' - ? await this.config.jobs.autoRun(this) - : this.config.jobs.autoRun - - await Promise.all( - cronJobs.map((cronConfig) => { - const jobAutorunCron = new Cron(cronConfig.cron ?? DEFAULT_CRON, async () => { - if ( - _internal_jobSystemGlobals.shouldAutoSchedule && - !cronConfig.disableScheduling && - this.config.jobs.scheduling - ) { - await this.jobs.handleSchedules({ - allQueues: cronConfig.allQueues, - queue: cronConfig.queue, - }) - } - - if (!_internal_jobSystemGlobals.shouldAutoRun) { - return - } - - if (typeof this.config.jobs.shouldAutoRun === 'function') { - const shouldAutoRun = await this.config.jobs.shouldAutoRun(this) - - if (!shouldAutoRun) { - jobAutorunCron.stop() - return - } - } - - await this.jobs.run({ - allQueues: cronConfig.allQueues, - limit: cronConfig.limit ?? DEFAULT_LIMIT, - queue: cronConfig.queue, - silent: cronConfig.silent, - }) - }) - - this.crons.push(jobAutorunCron) - }), - ) + if (options.cron) { + await this._initializeCrons() } return this @@ -932,17 +938,6 @@ const initialized = new BasePayload() // eslint-disable-next-line no-restricted-exports export default initialized -let cached: { - payload: null | Payload - promise: null | Promise - reload: boolean | Promise - ws: null | WebSocket -} = (global as any)._payload - -if (!cached) { - cached = (global as any)._payload = { payload: null, promise: null, reload: false, ws: null } -} - export const reload = async ( config: SanitizedConfig, payload: Payload, @@ -1010,14 +1005,73 @@ export const reload = async ( ;(global as any)._payload_doNotCacheClientSchemaMap = true } +let _cached: Map< + string, + { + initializedCrons: boolean + payload: null | Payload + promise: null | Promise + reload: boolean | Promise + ws: null | WebSocket + } +> = (global as any)._payload + +if (!_cached) { + _cached = (global as any)._payload = new Map() +} + +/** + * Get a payload instance. + * This function is a wrapper around new BasePayload().init() that adds the following functionality on top of that: + * + * - smartly caches Payload instance on the module scope. That way, we prevent unnecessarily initializing Payload over and over again + * when calling getPayload multiple times or from multiple locations. + * - adds HMR support and reloads the payload instance when the config changes. + */ export const getPayload = async ( - options: Pick, + options: { + /** + * A unique key to identify the payload instance. You can pass your own key if you want to cache this payload instance separately. + * This is useful if you pass a different payload config for each instance. + * + * @default 'default' + */ + key?: string + } & Pick, ): Promise => { if (!options?.config) { throw new Error('Error: the payload config is required for getPayload to work.') } + let alreadyCachedSameConfig = false + + let cached = _cached.get(options.key ?? 'default') + if (!cached) { + cached = { + initializedCrons: Boolean(options.cron), + payload: null, + promise: null, + reload: false, + ws: null, + } + _cached.set(options.key ?? 'default', cached) + } else { + alreadyCachedSameConfig = true + } + + if (alreadyCachedSameConfig) { + // alreadyCachedSameConfig => already called onInit once, but same config => no need to call onInit again. + // calling onInit again would only make sense if a different config was passed. + options.disableOnInit = true + } + if (cached.payload) { + if (options.cron && !cached.initializedCrons) { + // getPayload called with crons enabled, but existing cached version does not have crons initialized. => Initialize crons in existing cached version + cached.initializedCrons = true + await cached.payload._initializeCrons() + } + if (cached.reload === true) { let resolve!: () => void diff --git a/test/config/int.spec.ts b/test/config/int.spec.ts index 26b5127a7..398adbbd0 100644 --- a/test/config/int.spec.ts +++ b/test/config/int.spec.ts @@ -1,12 +1,12 @@ -import type { BlockField, Payload } from 'payload' - import { execSync } from 'child_process' import { existsSync, readFileSync, rmSync } from 'fs' import path from 'path' +import { type BlocksField, getPayload, type Payload } from 'payload' import { fileURLToPath } from 'url' import type { NextRESTClient } from '../helpers/NextRESTClient.js' +import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js' import { initPayloadInt } from '../helpers/initPayloadInt.js' import { testFilePath } from './testFilePath.js' @@ -40,6 +40,56 @@ describe('Config', () => { description: 'Get the sanitized payload config', }) }) + + it('should allow multiple getPayload calls using different configs in same process', async () => { + const payload2 = await getPayload({ + key: 'payload2', + config: await buildConfigWithDefaults({ + collections: [ + { + slug: 'payload2', + fields: [{ name: 'title2', type: 'text' }], + }, + ], + }), + }) + + // Use payload2 instance before creating payload3 instance, as we share the same db connection => each instance + // creation will reset the db schema. + const result2: any = await payload2.create({ + collection: 'payload2', + data: { + title2: 'Payload 2', + }, + } as any) + + expect(result2.title2).toBe('Payload 2') + + const payload3 = await getPayload({ + key: 'payload3', + config: await buildConfigWithDefaults({ + collections: [ + { + slug: 'payload3', + fields: [{ name: 'title3', type: 'text' }], + }, + ], + }), + }) + + // If payload was still incorrectly cached, this would fail, as the old payload config would still be used + const result3: any = await payload3.create({ + collection: 'payload3', + data: { + title3: 'Payload 3', + }, + } as any) + + expect(result3.title3).toBe('Payload 3') + + await payload2.destroy() + await payload3.destroy() + }) }) describe('collection config', () => { @@ -72,7 +122,7 @@ describe('Config', () => { const [collection] = payload.config.collections const [, blocksField] = collection.fields - expect((blocksField as BlockField).blocks[0].custom).toEqual({ + expect((blocksField as BlocksField).blocks[0].custom).toEqual({ description: 'The blockOne of this page', }) }) diff --git a/test/queues/cli.int.spec.ts b/test/queues/cli.int.spec.ts new file mode 100644 index 000000000..6b04bf430 --- /dev/null +++ b/test/queues/cli.int.spec.ts @@ -0,0 +1,85 @@ +import path from 'path' +import { + _internal_jobSystemGlobals, + _internal_resetJobSystemGlobals, + getPayload, + migrateCLI, + type SanitizedConfig, +} from 'payload' +import { wait } from 'payload/shared' +import { fileURLToPath } from 'url' + +import { initPayloadInt } from '../helpers/initPayloadInt.js' +import { waitUntilAutorunIsDone } from './utilities.js' + +const filename = fileURLToPath(import.meta.url) +const dirname = path.dirname(filename) + +describe('Queues - CLI', () => { + let config: SanitizedConfig + beforeAll(async () => { + ;({ config } = await initPayloadInt(dirname, undefined, false)) + }) + + it('ensure consecutive getPayload call with cron: true will autorun jobs', async () => { + const payload = await getPayload({ + config, + }) + + await payload.jobs.queue({ + workflow: 'inlineTaskTest', + queue: 'autorunSecond', + input: { + message: 'hello!', + }, + }) + + process.env.PAYLOAD_DROP_DATABASE = 'false' + + // Second instance of payload with the only purpose of running cron jobs + const _payload2 = await getPayload({ + config, + cron: true, + }) + + await waitUntilAutorunIsDone({ + payload, + queue: 'autorunSecond', + }) + + const allSimples = await payload.find({ + collection: 'simple', + limit: 100, + }) + + expect(allSimples.totalDocs).toBe(1) + expect(allSimples?.docs?.[0]?.title).toBe('hello!') + + // Shut down safely: + // Ensure no new crons are scheduled + _internal_jobSystemGlobals.shouldAutoRun = false + _internal_jobSystemGlobals.shouldAutoSchedule = false + // Wait 3 seconds to ensure all currently-running crons are done. If we shut down the db while a function is running, it can cause issues + // Cron function runs may persist after a test has finished + await wait(3000) + // Now we can destroy the payload instance + await _payload2.destroy() + await payload.destroy() + _internal_resetJobSystemGlobals() + }) + + it('can run migrate CLI without jobs attempting to run', async () => { + await migrateCLI({ + config, + parsedArgs: { + _: ['migrate'], + }, + }) + + // Wait 3 seconds to let potential autorun crons trigger + await new Promise((resolve) => setTimeout(resolve, 3000)) + + // Expect no errors. Previously, this would throw an "error: relation "payload_jobs" does not exist" error + expect(true).toBe(true) + }) +}) diff --git a/test/queues/int.spec.ts b/test/queues/int.spec.ts index e5b55963b..f35071046 100644 --- a/test/queues/int.spec.ts +++ b/test/queues/int.spec.ts @@ -4,9 +4,7 @@ import { _internal_resetJobSystemGlobals, type JobTaskStatus, type Payload, - type SanitizedConfig, } from 'payload' -import { migrateCLI } from 'payload' import { wait } from 'payload/shared' import { fileURLToPath } from 'url' @@ -17,15 +15,15 @@ import { initPayloadInt } from '../helpers/initPayloadInt.js' import { clearAndSeedEverything } from './seed.js' import { waitUntilAutorunIsDone } from './utilities.js' -let payload: Payload -let restClient: NextRESTClient -let token: string - const { email, password } = devUser const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) -describe('Queues', () => { +describe('Queues - Payload', () => { + let payload: Payload + let restClient: NextRESTClient + let token: string + beforeAll(async () => { process.env.SEED_IN_CONFIG_ONINIT = 'false' // Makes it so the payload config onInit seed is not run. Otherwise, the seed would be run unnecessarily twice for the initial test run - once for beforeEach and once for onInit ;({ payload, restClient } = await initPayloadInt(dirname)) @@ -1056,7 +1054,7 @@ describe('Queues', () => { it('ensure payload.jobs.runByID works and only runs the specified job', async () => { payload.config.jobs.deleteJobOnComplete = false - let lastJobID: null | string = null + let lastJobID: null | number | string = null for (let i = 0; i < 3; i++) { const job = await payload.jobs.queue({ task: 'CreateSimple', @@ -1100,7 +1098,7 @@ describe('Queues', () => { it('ensure where query for id in payload.jobs.run works and only runs the specified job', async () => { payload.config.jobs.deleteJobOnComplete = false - let lastJobID: null | string = null + let lastJobID: null | number | string = null for (let i = 0; i < 3; i++) { const job = await payload.jobs.queue({ task: 'CreateSimple', @@ -1467,24 +1465,3 @@ describe('Queues', () => { expect(allSimples?.docs?.[0]?.title).toBe('hello!') }) }) - -describe('Queues - CLI', () => { - let config: SanitizedConfig - beforeAll(async () => { - ;({ config } = await initPayloadInt(dirname, undefined, false)) - }) - it('can run migrate CLI without jobs attempting to run', async () => { - await migrateCLI({ - config, - parsedArgs: { - _: ['migrate'], - }, - }) - - // Wait 3 seconds to let potential autorun crons trigger - await new Promise((resolve) => setTimeout(resolve, 3000)) - - // Expect no errors. Previously, this would throw an "error: relation "payload_jobs" does not exist" error - expect(true).toBe(true) - }) -})