fix(db-mongodb): improve compatibility with Firestore database (#12763)

### What?

Adds four more arguments to the `mongooseAdapter`:

```typescript
  useJoinAggregations?: boolean  /* The big one */
  useAlternativeDropDatabase?: boolean
  useBigIntForNumberIDs?: boolean
  usePipelineInSortLookup?: boolean
```

Also export a new `compatabilityOptions` object from
`@payloadcms/db-mongodb` where each key is a mongo-compatible database
and the value is the recommended `mongooseAdapter` settings for
compatability.

### Why?

When using firestore and visiting
`/admin/collections/media/payload-folders`, we get:

```
MongoServerError: invalid field(s) in lookup: [let, pipeline], only lookup(from, localField, foreignField, as) is supported
```

Firestore doesn't support the full MongoDB aggregation API used by
Payload which gets used when building aggregations for populating join
fields.

There are several other compatability issues with Firestore:
- The invalid `pipeline` property is used in the `$lookup` aggregation
in `buildSortParams`
- Firestore only supports number IDs of type `Long`, but Mongoose
converts custom ID fields of type number to `Double`
- Firestore does not support the `dropDatabase` command
- Firestore does not support the `createIndex` command (not addressed in
this PR)

### How?

 ```typescript
useJoinAggregations?: boolean  /* The big one */
```
When this is `false` we skip the `buildJoinAggregation()` pipeline and resolve the join fields through multiple queries. This can potentially be used with AWS DocumentDB and Azure Cosmos DB to support join fields, but I have not tested with either of these databases.

 ```typescript
useAlternativeDropDatabase?: boolean
```
When `true`, monkey-patch (replace) the `dropDatabase` function so that
it calls `collection.deleteMany({})` on every collection instead of
sending a single `dropDatabase` command to the database

 ```typescript
useBigIntForNumberIDs?: boolean
```
When `true`, use `mongoose.Schema.Types.BigInt` for custom ID fields of type `number` which converts to a firestore `Long` behind the scenes

```typescript
  usePipelineInSortLookup?: boolean
```
When `false`, modify the sortAggregation pipeline in `buildSortParams()` so that we don't use the `pipeline` property in the `$lookup` aggregation. Results in slightly worse performance when sorting by relationship properties.

### Limitations

This PR does not add support for transactions or creating indexes in firestore.

### Fixes

Fixed a bug (and added a test) where you weren't able to sort by multiple properties on a relationship field.

### Future work

1. Firestore supports simple `$lookup` aggregations but other databases might not. Could add a `useSortAggregations` property which can be used to disable aggregations in sorting.

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>
This commit is contained in:
Elliott W
2025-07-17 01:02:43 +05:45
committed by GitHub
parent e6da384a43
commit 41cff6d436
20 changed files with 938 additions and 40 deletions

View File

@@ -21,6 +21,25 @@ export const allDatabaseAdapters = {
strength: 1,
},
})`,
firestore: `
import { mongooseAdapter, compatabilityOptions } from '@payloadcms/db-mongodb'
export const databaseAdapter = mongooseAdapter({
...compatabilityOptions.firestore,
url:
process.env.DATABASE_URI ||
process.env.MONGODB_MEMORY_SERVER_URI ||
'mongodb://127.0.0.1/payloadtests',
collation: {
strength: 1,
},
// The following options prevent some tests from failing.
// More work needed to get tests succeeding without these options.
ensureIndexes: true,
transactionOptions: {},
disableIndexHints: false,
useAlternativeDropDatabase: false,
})`,
postgres: `
import { postgresAdapter } from '@payloadcms/db-postgres'

View File

@@ -13,7 +13,7 @@ const dirname = path.dirname(filename)
const writeDBAdapter = process.env.WRITE_DB_ADAPTER !== 'false'
process.env.PAYLOAD_DROP_DATABASE = process.env.PAYLOAD_DROP_DATABASE || 'true'
if (process.env.PAYLOAD_DATABASE === 'mongodb') {
if (process.env.PAYLOAD_DATABASE === 'mongodb' || process.env.PAYLOAD_DATABASE === 'firestore') {
throw new Error('Not supported')
}

View File

@@ -1,5 +1,8 @@
import type { Payload } from 'payload'
export function isMongoose(_payload?: Payload) {
return _payload?.db?.name === 'mongoose' || ['mongodb'].includes(process.env.PAYLOAD_DATABASE)
return (
_payload?.db?.name === 'mongoose' ||
['firestore', 'mongodb'].includes(process.env.PAYLOAD_DATABASE)
)
}

View File

@@ -14,13 +14,17 @@ declare global {
*/
// eslint-disable-next-line no-restricted-exports
export default async () => {
if (process.env.DATABASE_URI) {
return
}
process.env.NODE_ENV = 'test'
process.env.PAYLOAD_DROP_DATABASE = 'true'
process.env.NODE_OPTIONS = '--no-deprecation'
process.env.DISABLE_PAYLOAD_HMR = 'true'
if (
(!process.env.PAYLOAD_DATABASE || process.env.PAYLOAD_DATABASE === 'mongodb') &&
(!process.env.PAYLOAD_DATABASE ||
['firestore', 'mongodb'].includes(process.env.PAYLOAD_DATABASE)) &&
!global._mongoMemoryServer
) {
console.log('Starting memory db...')

View File

@@ -38,7 +38,7 @@ const dirname = path.dirname(filename)
type EasierChained = { id: string; relation: EasierChained }
const mongoIt = process.env.PAYLOAD_DATABASE === 'mongodb' ? it : it.skip
const mongoIt = ['firestore', 'mongodb'].includes(process.env.PAYLOAD_DATABASE || '') ? it : it.skip
describe('Relationships', () => {
beforeAll(async () => {
@@ -791,6 +791,47 @@ describe('Relationships', () => {
expect(localized_res_2.docs).toStrictEqual([movie_1, movie_2])
})
it('should sort by multiple properties of a relationship', async () => {
await payload.delete({ collection: 'directors', where: {} })
await payload.delete({ collection: 'movies', where: {} })
const createDirector = {
collection: 'directors',
data: {
name: 'Dan',
},
} as const
const director_1 = await payload.create(createDirector)
const director_2 = await payload.create(createDirector)
const movie_1 = await payload.create({
collection: 'movies',
depth: 0,
data: { director: director_1.id, name: 'Some Movie 1' },
})
const movie_2 = await payload.create({
collection: 'movies',
depth: 0,
data: { director: director_2.id, name: 'Some Movie 2' },
})
const res_1 = await payload.find({
collection: 'movies',
sort: ['director.name', 'director.createdAt'],
depth: 0,
})
const res_2 = await payload.find({
collection: 'movies',
sort: ['director.name', '-director.createdAt'],
depth: 0,
})
expect(res_1.docs).toStrictEqual([movie_1, movie_2])
expect(res_2.docs).toStrictEqual([movie_2, movie_1])
})
it('should sort by a property of a hasMany relationship', async () => {
const movie1 = await payload.create({
collection: 'movies',