feat(db-postgres, db-sqlite): drizzle schema generation (#9953)

This PR allows to have full type safety on `payload.drizzle` with a
single command
```sh
pnpm payload generate:db-schema
```
Which generates TypeScript code with Drizzle declarations based on the
current database schema.

Example of generated file with the website template: 
https://gist.github.com/r1tsuu/b8687f211b51d9a3a7e78ba41e8fbf03

Video that shows the power:


https://github.com/user-attachments/assets/3ced958b-ec1d-49f5-9f51-d859d5fae236


We also now proxy drizzle package the same way we do for Lexical so you
don't have to install it (and you shouldn't because you may have version
mismatch).
Instead, you can import from Drizzle like this:
```ts
import {
  pgTable,
  index,
  foreignKey,
  integer,
  text,
  varchar,
  jsonb,
  boolean,
  numeric,
  serial,
  timestamp,
  uniqueIndex,
  pgEnum,
} from '@payloadcms/db-postgres/drizzle/pg-core'
import { sql } from '@payloadcms/db-postgres/drizzle'
import { relations } from '@payloadcms/db-postgres/drizzle/relations'
```


Fixes https://github.com/payloadcms/payload/discussions/4318

In the future we can also support types generation for mongoose / raw
mongodb results.
This commit is contained in:
Sasha
2024-12-19 18:08:17 +02:00
committed by GitHub
parent ba0e7aeee5
commit 23f1ed4a48
39 changed files with 4519 additions and 32 deletions

View File

@@ -71,15 +71,27 @@ export default buildConfig({
| `versionsSuffix` | A string appended to the end of table names for storing versions. Defaults to '_v'. |
| `beforeSchemaInit` | Drizzle schema hook. Runs before the schema is built. [More Details](#beforeschemainit) |
| `afterSchemaInit` | Drizzle schema hook. Runs after the schema is built. [More Details](#afterschemainit) |
| `generateSchemaOutputFile` | Override generated schema from `payload generate:db-schema` file path. Defaults to `{CWD}/src/payload-generated.schema.ts` |
## Access to Drizzle
After Payload is initialized, this adapter will expose the full power of Drizzle to you for use if you need it.
You can access Drizzle as follows:
To ensure type-safety, you need to generate Drizzle schema first with:
```sh
npx payload generate:db-schema
```
```text
payload.db.drizzle
Then, you can access Drizzle as follows:
```ts
import { posts } from './payload-generated-schema'
// To avoid installing Drizzle, you can import everything that drizzle has from our re-export path.
import { eq, sql, and } from '@payloadcms/db-postgres/drizzle'
// Drizzle's Querying API: https://orm.drizzle.team/docs/rqb
const posts = await payload.db.drizzle.query.posts.findMany()
// Drizzle's Select API https://orm.drizzle.team/docs/select
const result = await payload.db.drizzle.select().from(posts).where(and(eq(posts.id, 50), sql`lower(${posts.title}) = 'example post title'`))
```
## Tables, relations, and enums
@@ -114,7 +126,7 @@ Runs before the schema is built. You can use this hook to extend your database s
```ts
import { postgresAdapter } from '@payloadcms/db-postgres'
import { integer, pgTable, serial } from 'drizzle-orm/pg-core'
import { integer, pgTable, serial } from '@payloadcms/db-postgres/drizzle/pg-core'
postgresAdapter({
beforeSchemaInit: [
@@ -194,7 +206,7 @@ The following example adds the `extra_integer_column` column and a composite ind
```ts
import { postgresAdapter } from '@payloadcms/db-postgres'
import { index, integer } from 'drizzle-orm/pg-core'
import { index, integer } from '@payloadcms/db-postgres/drizzle/pg-core'
import { buildConfig } from 'payload'
export default buildConfig({
@@ -236,3 +248,43 @@ export default buildConfig({
})
```
### Note for generated schema:
Columns and tables, added in schema hooks won't be added to the generated via `payload generate:db-schema` Drizzle schema.
If you want them to be there, you either have to edit this file manually or mutate the internal Payload "raw" SQL schema in the `beforeSchemaInit`:
```ts
import { postgresAdapter } from '@payloadcms/db-postgres'
postgresAdapter({
beforeSchemaInit: [
({ schema, adapter }) => {
// Add a new table
schema.rawTables.myTable = {
name: 'my_table',
columns: [{
name: 'my_id',
type: 'serial',
primaryKey: true
}],
}
// Add a new column to generated by Payload table:
schema.rawTables.posts.columns.customColumn = {
name: 'custom_column',
// Note that Payload SQL doesn't support everything that Drizzle does.
type: 'integer',
notNull: true
}
// Add a new index to generated by Payload table:
schema.rawTables.posts.indexes.customColumnIdx = {
name: 'custom_column_idx',
unique: true,
on: ['custom_column']
}
return schema
},
],
})
```

View File

@@ -35,7 +35,7 @@ export default buildConfig({
## Options
| Option | Description |
| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `client` \* | [Client connection options](https://orm.drizzle.team/docs/get-started-sqlite#turso) that will be passed to `createClient` from `@libsql/client`. |
| `push` | Disable Drizzle's [`db push`](https://orm.drizzle.team/kit-docs/overview#prototyping-with-db-push) in development mode. By default, `push` is enabled for development mode only. |
| `migrationDir` | Customize the directory that migrations are stored. |
@@ -47,15 +47,28 @@ export default buildConfig({
| `versionsSuffix` | A string appended to the end of table names for storing versions. Defaults to '_v'. |
| `beforeSchemaInit` | Drizzle schema hook. Runs before the schema is built. [More Details](#beforeschemainit) |
| `afterSchemaInit` | Drizzle schema hook. Runs after the schema is built. [More Details](#afterschemainit) |
| `generateSchemaOutputFile` | Override generated schema from `payload generate:db-schema` file path. Defaults to `{CWD}/src/payload-generated.schema.ts` |
## Access to Drizzle
After Payload is initialized, this adapter will expose the full power of Drizzle to you for use if you need it.
You can access Drizzle as follows:
To ensure type-safety, you need to generate Drizzle schema first with:
```sh
npx payload generate:db-schema
```
```text
payload.db.drizzle
Then, you can access Drizzle as follows:
```ts
// Import table from the generated file
import { posts } from './payload-generated-schema'
// To avoid installing Drizzle, you can import everything that drizzle has from our re-export path.
import { eq, sql, and } from '@payloadcms/db-sqlite/drizzle'
// Drizzle's Querying API: https://orm.drizzle.team/docs/rqb
const posts = await payload.db.drizzle.query.posts.findMany()
// Drizzle's Select API https://orm.drizzle.team/docs/select
const result = await payload.db.drizzle.select().from(posts).where(and(eq(posts.id, 50), sql`lower(${posts.title}) = 'example post title'`))
```
## Tables and relations
@@ -89,7 +102,7 @@ Runs before the schema is built. You can use this hook to extend your database s
```ts
import { sqliteAdapter } from '@payloadcms/db-sqlite'
import { integer, sqliteTable } from 'drizzle-orm/sqlite-core'
import { integer, sqliteTable } from '@payloadcms/db-sqlite/drizzle/sqlite-core'
sqliteAdapter({
beforeSchemaInit: [
@@ -169,7 +182,7 @@ The following example adds the `extra_integer_column` column and a composite ind
```ts
import { sqliteAdapter } from '@payloadcms/db-sqlite'
import { index, integer } from 'drizzle-orm/sqlite-core'
import { index, integer } from '@payloadcms/db-sqlite/drizzle/sqlite-core'
import { buildConfig } from 'payload'
export default buildConfig({
@@ -211,3 +224,43 @@ export default buildConfig({
})
```
### Note for generated schema:
Columns and tables, added in schema hooks won't be added to the generated via `payload generate:db-schema` Drizzle schema.
If you want them to be there, you either have to edit this file manually or mutate the internal Payload "raw" SQL schema in the `beforeSchemaInit`:
```ts
import { sqliteAdapter } from '@payloadcms/db-sqlite'
sqliteAdapter({
beforeSchemaInit: [
({ schema, adapter }) => {
// Add a new table
schema.rawTables.myTable = {
name: 'my_table',
columns: [{
name: 'my_id',
type: 'integer',
primaryKey: true
}],
}
// Add a new column to generated by Payload table:
schema.rawTables.posts.columns.customColumn = {
name: 'custom_column',
// Note that Payload SQL doesn't support everything that Drizzle does.
type: 'integer',
notNull: true
}
// Add a new index to generated by Payload table:
schema.rawTables.posts.indexes.customColumnIdx = {
name: 'custom_column_idx',
unique: true,
on: ['custom_column']
}
return schema
},
],
})
```

View File

@@ -54,6 +54,7 @@
"clean:build": "node ./scripts/delete-recursively.js 'media/' '**/dist/' '**/.cache/' '**/.next/' '**/.turbo/' '**/tsconfig.tsbuildinfo' '**/payload*.tgz' '**/meta_*.json'",
"clean:cache": "node ./scripts/delete-recursively.js node_modules/.cache! packages/payload/node_modules/.cache! .next/*",
"dev": "cross-env NODE_OPTIONS=--no-deprecation tsx ./test/dev.ts",
"dev:generate-db-schema": "pnpm runts ./test/generateDatabaseSchema.ts",
"dev:generate-graphql-schema": "pnpm runts ./test/generateGraphQLSchema.ts",
"dev:generate-importmap": "pnpm runts ./test/generateImportMap.ts",
"dev:generate-types": "pnpm runts ./test/generateTypes.ts",

View File

@@ -33,6 +33,26 @@
"import": "./src/exports/migration-utils.ts",
"types": "./src/exports/migration-utils.ts",
"default": "./src/exports/migration-utils.ts"
},
"./drizzle": {
"import": "./src/drizzle-proxy/index.ts",
"types": "./src/drizzle-proxy/index.ts",
"default": "./src/drizzle-proxy/index.ts"
},
"./drizzle/pg-core": {
"import": "./src/drizzle-proxy/pg-core.ts",
"types": "./src/drizzle-proxy/pg-core.ts",
"default": "./src/drizzle-proxy/pg-core.ts"
},
"./drizzle/node-postgres": {
"import": "./src/drizzle-proxy/node-postgres.ts",
"types": "./src/drizzle-proxy/node-postgres.ts",
"default": "./src/drizzle-proxy/node-postgres.ts"
},
"./drizzle/relations": {
"import": "./src/drizzle-proxy/relations.ts",
"types": "./src/drizzle-proxy/relations.ts",
"default": "./src/drizzle-proxy/relations.ts"
}
},
"main": "./src/index.ts",
@@ -90,6 +110,26 @@
"import": "./dist/exports/migration-utils.js",
"types": "./dist/exports/migration-utils.d.ts",
"default": "./dist/exports/migration-utils.js"
},
"./drizzle": {
"import": "./dist/drizzle-proxy/index.js",
"types": "./dist/drizzle-proxy/index.d.ts",
"default": "./dist/drizzle-proxy/index.js"
},
"./drizzle/pg-core": {
"import": "./dist/drizzle-proxy/pg-core.js",
"types": "./dist/drizzle-proxy/pg-core.d.ts",
"default": "./dist/drizzle-proxy/pg-core.js"
},
"./drizzle/node-postgres": {
"import": "./dist/drizzle-proxy/node-postgres.js",
"types": "./dist/drizzle-proxy/node-postgres.d.ts",
"default": "./dist/drizzle-proxy/node-postgres.js"
},
"./drizzle/relations": {
"import": "./dist/drizzle-proxy/relations.js",
"types": "./dist/drizzle-proxy/relations.d.ts",
"default": "./dist/drizzle-proxy/relations.js"
}
},
"main": "./dist/index.js",

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/node-postgres'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/pg-core'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/relations'

View File

@@ -10,6 +10,7 @@ import {
create,
createGlobal,
createGlobalVersion,
createSchemaGenerator,
createVersion,
deleteMany,
deleteOne,
@@ -36,6 +37,7 @@ import {
updateVersion,
} from '@payloadcms/drizzle'
import {
columnToCodeConverter,
countDistinct,
createDatabase,
createExtensions,
@@ -106,6 +108,14 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj<PostgresAdapter>
json: true,
},
fieldConstraints: {},
generateSchema: createSchemaGenerator({
columnToCodeConverter,
corePackageSuffix: 'pg-core',
defaultOutputFile: args.generateSchemaOutputFile,
enumImport: 'pgEnum',
schemaImport: 'pgSchema',
tableImport: 'pgTable',
}),
idType: postgresIDType,
initializing,
localesSuffix: args.localesSuffix || '_locales',
@@ -188,4 +198,5 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj<PostgresAdapter>
}
export type { MigrateDownArgs, MigrateUpArgs } from '@payloadcms/drizzle/postgres'
export { geometryColumn } from '@payloadcms/drizzle/postgres'
export { sql } from 'drizzle-orm'

View File

@@ -8,6 +8,7 @@ import type {
} from '@payloadcms/drizzle/postgres'
import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
import type { DrizzleConfig } from 'drizzle-orm'
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
import type { PgSchema, PgTableFn, PgTransactionConfig } from 'drizzle-orm/pg-core'
import type { Pool, PoolConfig } from 'pg'
@@ -30,6 +31,8 @@ export type Args = {
*/
disableCreateDatabase?: boolean
extensions?: string[]
/** Generated schema from payload generate:db-schema file path */
generateSchemaOutputFile?: string
idType?: 'serial' | 'uuid'
localesSuffix?: string
logger?: DrizzleConfig['logger']
@@ -52,7 +55,17 @@ export type Args = {
versionsSuffix?: string
}
export interface GeneratedDatabaseSchema {
schemaUntyped: Record<string, unknown>
}
type ResolveSchemaType<T> = 'schema' extends keyof T
? T['schema']
: GeneratedDatabaseSchema['schemaUntyped']
type Drizzle = NodePgDatabase<ResolveSchemaType<GeneratedDatabaseSchema>>
export type PostgresAdapter = {
drizzle: Drizzle
pool: Pool
poolOptions: PoolConfig
} & BasePostgresAdapter
@@ -65,7 +78,7 @@ declare module 'payload' {
beforeSchemaInit: PostgresSchemaHook[]
beginTransaction: (options?: PgTransactionConfig) => Promise<null | number | string>
drizzle: PostgresDB
drizzle: Drizzle
enums: Record<string, GenericEnum>
extensions: Record<string, boolean>
/**

View File

@@ -33,6 +33,26 @@
"import": "./src/exports/migration-utils.ts",
"require": "./src/exports/migration-utils.ts",
"types": "./src/exports/migration-utils.ts"
},
"./drizzle": {
"import": "./src/drizzle-proxy/index.ts",
"types": "./src/drizzle-proxy/index.ts",
"default": "./src/drizzle-proxy/index.ts"
},
"./drizzle/sqlite-core": {
"import": "./src/drizzle-proxy/sqlite-core.ts",
"types": "./src/drizzle-proxy/sqlite-core.ts",
"default": "./src/drizzle-proxy/sqlite-core.ts"
},
"./drizzle/libsql": {
"import": "./src/drizzle-proxy/libsql.ts",
"types": "./src/drizzle-proxy/libsql.ts",
"default": "./src/drizzle-proxy/libsql.ts"
},
"./drizzle/relations": {
"import": "./src/drizzle-proxy/relations.ts",
"types": "./src/drizzle-proxy/relations.ts",
"default": "./src/drizzle-proxy/relations.ts"
}
},
"main": "./src/index.ts",
@@ -86,6 +106,26 @@
"import": "./dist/exports/migration-utils.js",
"require": "./dist/exports/migration-utils.js",
"types": "./dist/exports/migration-utils.d.ts"
},
"./drizzle": {
"import": "./dist/drizzle-proxy/index.js",
"types": "./dist/drizzle-proxy/index.d.ts",
"default": "./dist/drizzle-proxy/index.js"
},
"./drizzle/sqlite-core": {
"import": "./dist/drizzle-proxy/sqlite-core.js",
"types": "./dist/drizzle-proxy/sqlite-core.d.ts",
"default": "./dist/drizzle-proxy/sqlite-core.js"
},
"./drizzle/libsql": {
"import": "./dist/drizzle-proxy/libsql.js",
"types": "./dist/drizzle-proxy/libsql.d.ts",
"default": "./dist/drizzle-proxy/libsql.js"
},
"./drizzle/relations": {
"import": "./dist/drizzle-proxy/relations.js",
"types": "./dist/drizzle-proxy/relations.d.ts",
"default": "./dist/drizzle-proxy/relations.js"
}
},
"main": "./dist/index.js",

View File

@@ -0,0 +1,126 @@
import type { ColumnToCodeConverter } from '@payloadcms/drizzle/types'
export const columnToCodeConverter: ColumnToCodeConverter = ({
adapter,
addImport,
column,
locales,
tableKey,
}) => {
let columnBuilderFn: string = column.type
const columnBuilderArgsArray: string[] = []
let defaultStatement: null | string = null
switch (column.type) {
case 'boolean': {
columnBuilderFn = 'integer'
columnBuilderArgsArray.push("mode: 'boolean'")
break
}
case 'enum': {
let options: string[]
if ('locale' in column) {
options = locales
} else {
options = column.options
}
columnBuilderFn = 'text'
columnBuilderArgsArray.push(`enum: [${options.map((locale) => `'${locale}'`).join(', ')}]`)
break
}
case 'geometry':
case 'jsonb': {
columnBuilderFn = 'text'
columnBuilderArgsArray.push("mode: 'json'")
break
}
case 'serial': {
columnBuilderFn = 'integer'
break
}
case 'timestamp': {
columnBuilderFn = 'text'
defaultStatement = `default(sql\`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))\`)`
break
}
case 'uuid': {
columnBuilderFn = 'text'
if (column.defaultRandom) {
addImport('crypto', 'randomUUID')
defaultStatement = `$defaultFn(() => randomUUID())`
}
break
}
case 'varchar': {
columnBuilderFn = 'text'
break
}
default: {
columnBuilderFn = column.type
}
}
addImport(`${adapter.packageName}/drizzle/sqlite-core`, columnBuilderFn)
let columnBuilderArgs = ''
if (columnBuilderArgsArray.length) {
columnBuilderArgs = `, {${columnBuilderArgsArray.join(',')}}`
}
let code = `${columnBuilderFn}('${column.name}'${columnBuilderArgs})`
if (column.notNull) {
code = `${code}.notNull()`
}
if (column.primaryKey) {
code = `${code}.primaryKey()`
}
if (defaultStatement) {
code = `${code}.${defaultStatement}`
} else if (typeof column.default !== 'undefined') {
let sanitizedDefault = column.default
if (column.type === 'jsonb' || column.type === 'geometry') {
sanitizedDefault = `'${JSON.stringify(column.default)}'`
} else if (typeof column.default === 'string') {
sanitizedDefault = JSON.stringify(column.default)
} else if (column.type === 'numeric') {
sanitizedDefault = `'${column.default}'`
}
code = `${code}.default(${sanitizedDefault})`
}
if (column.reference) {
let callback = `()`
if (column.reference.table === tableKey) {
addImport(`${adapter.packageName}/drizzle/sqlite-core`, 'type AnySQLiteColumn')
callback = `${callback}: AnySQLiteColumn`
}
callback = `${callback} => ${column.reference.table}.${column.reference.name}`
code = `${code}.references(${callback}, {
${column.reference.onDelete ? `onDelete: '${column.reference.onDelete}'` : ''}
})`
}
return code
}

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/libsql'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/relations'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/sqlite-core'

View File

@@ -11,6 +11,7 @@ import {
create,
createGlobal,
createGlobalVersion,
createSchemaGenerator,
createVersion,
deleteMany,
deleteOne,
@@ -42,6 +43,7 @@ import { fileURLToPath } from 'url'
import type { Args, SQLiteAdapter } from './types.js'
import { columnToCodeConverter } from './columnToCodeConverter.js'
import { connect } from './connect.js'
import { countDistinct } from './countDistinct.js'
import { convertPathToJSONTraversal } from './createJSONQuery/convertPathToJSONTraversal.js'
@@ -93,6 +95,12 @@ export function sqliteAdapter(args: Args): DatabaseAdapterObj<SQLiteAdapter> {
json: true,
},
fieldConstraints: {},
generateSchema: createSchemaGenerator({
columnToCodeConverter,
corePackageSuffix: 'sqlite-core',
defaultOutputFile: args.generateSchemaOutputFile,
tableImport: 'sqliteTable',
}),
idType: sqliteIDType,
initializing,
localesSuffix: args.localesSuffix || '_locales',

View File

@@ -33,6 +33,26 @@
"import": "./src/exports/migration-utils.ts",
"types": "./src/exports/migration-utils.ts",
"default": "./src/exports/migration-utils.ts"
},
"./drizzle": {
"import": "./src/drizzle-proxy/index.ts",
"types": "./src/drizzle-proxy/index.ts",
"default": "./src/drizzle-proxy/index.ts"
},
"./drizzle/pg-core": {
"import": "./src/drizzle-proxy/pg-core.ts",
"types": "./src/drizzle-proxy/pg-core.ts",
"default": "./src/drizzle-proxy/pg-core.ts"
},
"./drizzle/node-postgres": {
"import": "./src/drizzle-proxy/node-postgres.ts",
"types": "./src/drizzle-proxy/node-postgres.ts",
"default": "./src/drizzle-proxy/node-postgres.ts"
},
"./drizzle/relations": {
"import": "./src/drizzle-proxy/relations.ts",
"types": "./src/drizzle-proxy/relations.ts",
"default": "./src/drizzle-proxy/relations.ts"
}
},
"main": "./src/index.ts",
@@ -91,6 +111,26 @@
"import": "./dist/exports/migration-utils.js",
"types": "./dist/exports/migration-utils.d.ts",
"default": "./dist/exports/migration-utils.js"
},
"./drizzle": {
"import": "./dist/drizzle-proxy/index.js",
"types": "./dist/drizzle-proxy/index.d.ts",
"default": "./dist/drizzle-proxy/index.js"
},
"./drizzle/pg-core": {
"import": "./dist/drizzle-proxy/pg-core.js",
"types": "./dist/drizzle-proxy/pg-core.d.ts",
"default": "./dist/drizzle-proxy/pg-core.js"
},
"./drizzle/node-postgres": {
"import": "./dist/drizzle-proxy/node-postgres.js",
"types": "./dist/drizzle-proxy/node-postgres.d.ts",
"default": "./dist/drizzle-proxy/node-postgres.js"
},
"./drizzle/relations": {
"import": "./dist/drizzle-proxy/relations.js",
"types": "./dist/drizzle-proxy/relations.d.ts",
"default": "./dist/drizzle-proxy/relations.js"
}
},
"main": "./dist/index.js",

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/node-postgres'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/pg-core'

View File

@@ -0,0 +1 @@
export * from 'drizzle-orm/relations'

View File

@@ -10,6 +10,7 @@ import {
create,
createGlobal,
createGlobalVersion,
createSchemaGenerator,
createVersion,
deleteMany,
deleteOne,
@@ -36,6 +37,7 @@ import {
updateVersion,
} from '@payloadcms/drizzle'
import {
columnToCodeConverter,
countDistinct,
createDatabase,
createExtensions,
@@ -100,6 +102,14 @@ export function vercelPostgresAdapter(args: Args = {}): DatabaseAdapterObj<Verce
},
fieldConstraints: {},
forceUseVercelPostgres: args.forceUseVercelPostgres ?? false,
generateSchema: createSchemaGenerator({
columnToCodeConverter,
corePackageSuffix: 'pg-core',
defaultOutputFile: args.generateSchemaOutputFile,
enumImport: 'pgEnum',
schemaImport: 'pgSchema',
tableImport: 'pgTable',
}),
idType: postgresIDType,
indexes: new Set<string>(),
initializing,
@@ -189,4 +199,5 @@ export function vercelPostgresAdapter(args: Args = {}): DatabaseAdapterObj<Verce
}
export type { MigrateDownArgs, MigrateUpArgs } from '@payloadcms/drizzle/postgres'
export { geometryColumn } from '@payloadcms/drizzle/postgres'
export { sql } from 'drizzle-orm'

View File

@@ -9,6 +9,7 @@ import type {
import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
import type { VercelPool, VercelPostgresPoolConfig } from '@vercel/postgres'
import type { DrizzleConfig } from 'drizzle-orm'
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
import type { PgSchema, PgTableFn, PgTransactionConfig } from 'drizzle-orm/pg-core'
export type Args = {
@@ -38,6 +39,8 @@ export type Args = {
* and you'd to spin up the database with a special Neon's Docker Compose setup - https://vercel.com/docs/storage/vercel-postgres/local-development#option-2:-local-postgres-instance-with-docker
*/
forceUseVercelPostgres?: boolean
/** Generated schema from payload generate:db-schema file path */
generateSchemaOutputFile?: string
idType?: 'serial' | 'uuid'
localesSuffix?: string
logger?: DrizzleConfig['logger']
@@ -64,7 +67,18 @@ export type Args = {
versionsSuffix?: string
}
export interface GeneratedDatabaseSchema {
schemaUntyped: Record<string, unknown>
}
type ResolveSchemaType<T> = 'schema' extends keyof T
? T['schema']
: GeneratedDatabaseSchema['schemaUntyped']
type Drizzle = NodePgDatabase<ResolveSchemaType<GeneratedDatabaseSchema>>
export type VercelPostgresAdapter = {
drizzle: Drizzle
forceUseVercelPostgres?: boolean
pool?: VercelPool
poolOptions?: Args['pool']
@@ -77,7 +91,7 @@ declare module 'payload' {
afterSchemaInit: PostgresSchemaHook[]
beforeSchemaInit: PostgresSchemaHook[]
beginTransaction: (options?: PgTransactionConfig) => Promise<null | number | string>
drizzle: PostgresDB
drizzle: Drizzle
enums: Record<string, GenericEnum>
extensions: Record<string, boolean>
extensionsFilter: Set<string>

View File

@@ -1,3 +1,4 @@
export { columnToCodeConverter } from '../postgres/columnToCodeConverter.js'
export { countDistinct } from '../postgres/countDistinct.js'
export { createDatabase } from '../postgres/createDatabase.js'
export { createExtensions } from '../postgres/createExtensions.js'
@@ -10,4 +11,5 @@ export { init } from '../postgres/init.js'
export { insert } from '../postgres/insert.js'
export { migratePostgresV2toV3 } from '../postgres/predefinedMigrations/v2-v3/index.js'
export { requireDrizzleKit } from '../postgres/requireDrizzleKit.js'
export { geometryColumn } from '../postgres/schema/geometryColumn.js'
export * from '../postgres/types.js'

View File

@@ -38,6 +38,7 @@ export { updateVersion } from './updateVersion.js'
export { upsertRow } from './upsertRow/index.js'
export { buildCreateMigration } from './utilities/buildCreateMigration.js'
export { buildIndexName } from './utilities/buildIndexName.js'
export { createSchemaGenerator } from './utilities/createSchemaGenerator.js'
export { executeSchemaHooks } from './utilities/executeSchemaHooks.js'
export { extendDrizzleTable } from './utilities/extendDrizzleTable.js'
export { hasLocalesTable } from './utilities/hasLocalesTable.js'

View File

@@ -0,0 +1,94 @@
import type { ColumnToCodeConverter } from '../types.js'
export const columnToCodeConverter: ColumnToCodeConverter = ({
adapter,
addEnum,
addImport,
column,
tableKey,
}) => {
let columnBuilderFn: string = column.type
if (column.type === 'geometry') {
columnBuilderFn = 'geometryColumn'
addImport(adapter.packageName, columnBuilderFn)
} else if (column.type === 'enum') {
if ('locale' in column) {
columnBuilderFn = `enum__locales`
} else {
addEnum(column.enumName, column.options)
columnBuilderFn = column.enumName
}
} else {
addImport(`${adapter.packageName}/drizzle/pg-core`, columnBuilderFn)
}
const columnBuilderArgsArray: string[] = []
if (column.type === 'timestamp') {
columnBuilderArgsArray.push(`mode: '${column.mode}'`)
if (column.withTimezone) {
columnBuilderArgsArray.push('withTimezone: true')
}
if (typeof column.precision === 'number') {
columnBuilderArgsArray.push(`precision: ${column.precision}`)
}
}
let columnBuilderArgs = ''
if (columnBuilderArgsArray.length) {
columnBuilderArgs = `, {${columnBuilderArgsArray.join(',')}}`
}
let code = `${columnBuilderFn}('${column.name}'${columnBuilderArgs})`
if (column.type === 'timestamp' && column.defaultNow) {
code = `${code}.defaultNow()`
}
if (column.type === 'uuid' && column.defaultRandom) {
code = `${code}.defaultRandom()`
}
if (column.notNull) {
code = `${code}.notNull()`
}
if (column.primaryKey) {
code = `${code}.primaryKey()`
}
if (typeof column.default !== 'undefined') {
let sanitizedDefault = column.default
if (column.type === 'geometry') {
sanitizedDefault = `sql\`${column.default}\``
} else if (column.type === 'jsonb') {
sanitizedDefault = `sql\`'${JSON.stringify(column.default)}'::jsonb\``
} else if (column.type === 'numeric') {
sanitizedDefault = `'${column.default}'`
} else if (typeof column.default === 'string') {
sanitizedDefault = `${JSON.stringify(column.default)}`
}
code = `${code}.default(${sanitizedDefault})`
}
if (column.reference) {
let callback = `()`
if (column.reference.table === tableKey) {
addImport(`${adapter.packageName}/drizzle/pg-core`, 'type AnyPgColumn')
callback = `${callback}: AnyPgColumn`
}
callback = `${callback} => ${column.reference.table}.${column.reference.name}`
code = `${code}.references(${callback}, {
${column.reference.onDelete ? `onDelete: '${column.reference.onDelete}'` : ''}
})`
}
return code
}

View File

@@ -286,6 +286,15 @@ export type SetColumnID = (args: {
fields: FlattenedField[]
}) => IDType
export type ColumnToCodeConverter = (args: {
adapter: DrizzleAdapter
addEnum: (name: string, options: string[]) => void
addImport: (from: string, name: string) => void
column: RawColumn
locales?: string[]
tableKey: string
}) => string
export type BuildDrizzleTable<T extends DrizzleAdapter = DrizzleAdapter> = (args: {
adapter: T
locales: string[]

View File

@@ -0,0 +1,311 @@
import type { GenerateSchema } from 'payload'
import { exec } from 'child_process'
import { existsSync } from 'fs'
import { writeFile } from 'fs/promises'
import path from 'path'
import { promisify } from 'util'
import type { ColumnToCodeConverter, DrizzleAdapter } from '../types.js'
const execAsync = promisify(exec)
/**
* @example
* console.log(sanitizeObjectKey("oneTwo")); // oneTwo
* console.log(sanitizeObjectKey("one-two")); // 'one-two'
* console.log(sanitizeObjectKey("_one$Two3")); // _one$Two3
* console.log(sanitizeObjectKey("3invalid")); // '3invalid'
*/
const sanitizeObjectKey = (key: string) => {
// Regular expression for a valid identifier
const identifierRegex = /^[a-z_$][\w$]*$/i
if (identifierRegex.test(key)) {
return key
}
return `'${key}'`
}
/**
* @example
* (columns default-valuesID) -> columns['default-valuesID']
* (columns defaultValues) -> columns.defaultValues
*/
const accessProperty = (objName: string, key: string) => {
const sanitized = sanitizeObjectKey(key)
if (sanitized.startsWith("'")) {
return `${objName}[${sanitized}]`
}
return `${objName}.${key}`
}
export const createSchemaGenerator = ({
columnToCodeConverter,
corePackageSuffix,
defaultOutputFile,
enumImport,
schemaImport,
tableImport,
}: {
columnToCodeConverter: ColumnToCodeConverter
corePackageSuffix: string
defaultOutputFile?: string
enumImport?: string
schemaImport?: string
tableImport: string
}): GenerateSchema => {
return async function generateSchema(
this: DrizzleAdapter,
{ log = true, outputFile = defaultOutputFile, prettify = true } = {},
) {
const importDeclarations: Record<string, Set<string>> = {}
const tableDeclarations: string[] = []
const enumDeclarations: string[] = []
const relationsDeclarations: string[] = []
const addImport = (from: string, name: string) => {
if (!importDeclarations[from]) {
importDeclarations[from] = new Set()
}
importDeclarations[from].add(name)
}
const corePackage = `${this.packageName}/drizzle/${corePackageSuffix}`
let schemaDeclaration: null | string = null
if (this.schemaName) {
addImport(corePackage, schemaImport)
schemaDeclaration = `export const db_schema = ${schemaImport}('${this.schemaName}')`
}
const enumFn = this.schemaName ? `db_schema.enum` : enumImport
const enumsList: string[] = []
const addEnum = (name: string, options: string[]) => {
if (enumsList.some((each) => each === name)) {
return
}
enumsList.push(name)
enumDeclarations.push(
`export const ${name} = ${enumFn}('${name}', [${options.map((option) => `'${option}'`).join(', ')}])`,
)
}
if (this.payload.config.localization && enumImport) {
addEnum('enum__locales', this.payload.config.localization.localeCodes)
}
const tableFn = this.schemaName ? `db_schema.table` : tableImport
if (!this.schemaName) {
addImport(corePackage, tableImport)
}
addImport(corePackage, 'index')
addImport(corePackage, 'uniqueIndex')
addImport(corePackage, 'foreignKey')
addImport(`${this.packageName}/drizzle`, 'sql')
addImport(`${this.packageName}/drizzle`, 'relations')
for (const tableName in this.rawTables) {
const table = this.rawTables[tableName]
const extrasDeclarations: string[] = []
if (table.indexes) {
for (const key in table.indexes) {
const index = table.indexes[key]
let indexDeclaration = `${sanitizeObjectKey(key)}: ${index.unique ? 'uniqueIndex' : 'index'}('${index.name}')`
indexDeclaration += `.on(${typeof index.on === 'string' ? `${accessProperty('columns', index.on)}` : `${index.on.map((on) => `${accessProperty('columns', on)}`).join(', ')}`}),`
extrasDeclarations.push(indexDeclaration)
}
}
if (table.foreignKeys) {
for (const key in table.foreignKeys) {
const foreignKey = table.foreignKeys[key]
let foreignKeyDeclaration = `${sanitizeObjectKey(key)}: foreignKey({
columns: [${foreignKey.columns.map((col) => `columns['${col}']`).join(', ')}],
foreignColumns: [${foreignKey.foreignColumns.map((col) => `${accessProperty(col.table, col.name)}`).join(', ')}],
name: '${foreignKey.name}'
})`
if (foreignKey.onDelete) {
foreignKeyDeclaration += `.onDelete('${foreignKey.onDelete}')`
}
if (foreignKey.onUpdate) {
foreignKeyDeclaration += `.onUpdate('${foreignKey.onDelete}')`
}
foreignKeyDeclaration += ','
extrasDeclarations.push(foreignKeyDeclaration)
}
}
const tableCode = `
export const ${tableName} = ${tableFn}('${tableName}', {
${Object.entries(table.columns)
.map(
([key, column]) =>
` ${sanitizeObjectKey(key)}: ${columnToCodeConverter({
adapter: this,
addEnum,
addImport,
column,
locales: this.payload.config.localization
? this.payload.config.localization.localeCodes
: undefined,
tableKey: tableName,
})},`,
)
.join('\n')}
}${
extrasDeclarations.length
? `, (columns) => ({
${extrasDeclarations.join('\n ')}
})`
: ''
}
)
`
tableDeclarations.push(tableCode)
}
for (const tableName in this.rawRelations) {
const relations = this.rawRelations[tableName]
const properties: string[] = []
for (const key in relations) {
const relation = relations[key]
let declaration: string
if (relation.type === 'one') {
declaration = `${sanitizeObjectKey(key)}: one(${relation.to}, {
${relation.fields.some((field) => field.table !== tableName) ? '// @ts-expect-error Drizzle TypeScript bug for ONE relationships with a field in different table' : ''}
fields: [${relation.fields.map((field) => `${accessProperty(field.table, field.name)}`).join(', ')}],
references: [${relation.references.map((col) => `${accessProperty(relation.to, col)}`).join(', ')}],
${relation.relationName ? `relationName: '${relation.relationName}',` : ''}
}),`
} else {
declaration = `${sanitizeObjectKey(key)}: many(${relation.to}, {
${relation.relationName ? `relationName: '${relation.relationName}',` : ''}
}),`
}
properties.push(declaration)
}
// beautify / lintify relations callback output, when no many for example, don't add it
const args = []
if (Object.values(relations).some((rel) => rel.type === 'one')) {
args.push('one')
}
if (Object.values(relations).some((rel) => rel.type === 'many')) {
args.push('many')
}
const arg = args.length ? `{ ${args.join(', ')} }` : ''
const declaration = `export const relations_${tableName} = relations(${tableName}, (${arg}) => ({
${properties.join('\n ')}
}))`
relationsDeclarations.push(declaration)
}
if (enumDeclarations.length && !this.schemaName) {
addImport(corePackage, enumImport)
}
const importDeclarationsSanitized: string[] = []
for (const moduleName in importDeclarations) {
const moduleImports = importDeclarations[moduleName]
importDeclarationsSanitized.push(
`import { ${Array.from(moduleImports).join(', ')} } from '${moduleName}'`,
)
}
const schemaType = `
type DatabaseSchema = {
${[
this.schemaName ? 'db_schema' : null,
...enumsList,
...Object.keys(this.rawTables),
...Object.keys(this.rawRelations).map((table) => `relations_${table}`),
]
.filter(Boolean)
.map((name) => `${name}: typeof ${name}`)
.join('\n ')}
}
`
const finalDeclaration = `
declare module '${this.packageName}/types' {
export interface GeneratedDatabaseSchema {
schema: DatabaseSchema
}
}
`
const warning = `
/* tslint:disable */
/* eslint-disable */
/**
* This file was automatically generated by Payload.
* DO NOT MODIFY IT BY HAND. Instead, modify your source Payload config,
* and re-run \`payload generate:db-schema\` to regenerate this file.
*/
`
const code = [
warning,
...importDeclarationsSanitized,
schemaDeclaration,
...enumDeclarations,
...tableDeclarations,
...relationsDeclarations,
schemaType,
finalDeclaration,
]
.filter(Boolean)
.join('\n')
if (!outputFile) {
const cwd = process.cwd()
const srcDir = path.resolve(cwd, 'src')
if (existsSync(srcDir)) {
outputFile = path.resolve(srcDir, 'payload-generated-schema.ts')
} else {
outputFile = path.resolve(cwd, 'payload-generated-schema.ts')
}
}
await writeFile(outputFile, code, 'utf-8')
if (prettify) {
try {
await execAsync(`npx prettier ${outputFile} --write`)
// eslint-disable-next-line no-empty
} catch {}
}
if (log) {
this.payload.logger.info(`Written ${outputFile}`)
}
}
}

View File

@@ -109,6 +109,23 @@ export const bin = async () => {
}
}
if (script === 'generate:db-schema') {
const payload = await getPayload({ config })
if (typeof payload.db.generateSchema !== 'function') {
payload.logger.error({
msg: `${payload.db.packageName} does not support database schema generation`,
})
process.exit(1)
}
await payload.db.generateSchema({
log: args.log === 'false' ? false : true,
prettify: args.prettify === 'false' ? false : true,
})
}
console.error(`Unknown script: "${script}".`)
process.exit(1)
}

View File

@@ -70,6 +70,8 @@ export interface BaseDatabaseAdapter {
findVersions: FindVersions
generateSchema?: GenerateSchema
/**
* Perform startup tasks required to interact with the database such as building Schema and models
*/
@@ -89,16 +91,15 @@ export interface BaseDatabaseAdapter {
* Drop the current database and run all migrate up functions
*/
migrateFresh: (args: { forceAcceptWarning?: boolean }) => Promise<void>
/**
* Run all migration down functions before running up
*/
migrateRefresh: () => Promise<void>
/**
* Run all migrate down functions
*/
migrateReset: () => Promise<void>
/**
* Read the current state of migrations and output the result to show which have been run
*/
@@ -111,6 +112,7 @@ export interface BaseDatabaseAdapter {
* The name of the database adapter
*/
name: string
/**
* Full package name of the database adapter
*
@@ -122,12 +124,12 @@ export interface BaseDatabaseAdapter {
* reference to the instance of payload
*/
payload: Payload
queryDrafts: QueryDrafts
/**
* Abort any changes since the start of the transaction.
*/
rollbackTransaction: RollbackTransaction
/**
* A key-value store of all sessions open (used for transactions)
*/
@@ -521,3 +523,11 @@ export type MigrationTemplateArgs = {
packageName?: string
upSQL?: string
}
export type GenerateSchemaArgs = {
log?: boolean
outputFile?: string
prettify?: boolean
}
export type GenerateSchema = (args?: GenerateSchemaArgs) => Promise<void>

View File

@@ -1050,6 +1050,7 @@ export type {
FindOneArgs,
FindVersions,
FindVersionsArgs,
GenerateSchema,
Init,
Migration,
MigrationData,

View File

@@ -1 +1,2 @@
migrations
*.generated-schema.ts

View File

@@ -1,6 +1,5 @@
import type { MongooseAdapter } from '@payloadcms/db-mongodb'
import type { PostgresAdapter } from '@payloadcms/db-postgres/types'
import type { Table } from 'drizzle-orm'
import type { NextRESTClient } from 'helpers/NextRESTClient.js'
import type { Payload, PayloadRequest, TypeWithID } from 'payload'
@@ -8,6 +7,7 @@ import {
migrateRelationshipsV2_V3,
migrateVersionsV1_V2,
} from '@payloadcms/db-mongodb/migration-utils'
import { desc, type Table } from 'drizzle-orm'
import * as drizzlePg from 'drizzle-orm/pg-core'
import * as drizzleSqlite from 'drizzle-orm/sqlite-core'
import fs from 'fs'
@@ -788,6 +788,61 @@ describe('database', () => {
})
})
describe('Schema generation', () => {
if (process.env.PAYLOAD_DATABASE.includes('postgres')) {
it('should generate Drizzle Postgres schema', async () => {
const generatedAdapterName = process.env.PAYLOAD_DATABASE
const outputFile = path.resolve(dirname, `${generatedAdapterName}.generated-schema.ts`)
await payload.db.generateSchema({
outputFile,
})
const module = await import(outputFile)
// Confirm that the generated module exports every relation
for (const relation in payload.db.relations) {
expect(module).toHaveProperty(relation)
}
// Confirm that module exports every table
for (const table in payload.db.tables) {
expect(module).toHaveProperty(table)
}
// Confirm that module exports every enum
for (const enumName in payload.db.enums) {
expect(module).toHaveProperty(enumName)
}
})
}
if (process.env.PAYLOAD_DATABASE.includes('sqlite')) {
it('should generate Drizzle SQLite schema', async () => {
const generatedAdapterName = process.env.PAYLOAD_DATABASE
const outputFile = path.resolve(dirname, `${generatedAdapterName}.generated-schema.ts`)
await payload.db.generateSchema({
outputFile,
})
const module = await import(outputFile)
// Confirm that the generated module exports every relation
for (const relation in payload.db.relations) {
expect(module).toHaveProperty(relation)
}
// Confirm that module exports every table
for (const table in payload.db.tables) {
expect(module).toHaveProperty(table)
}
})
}
})
describe('drizzle: schema hooks', () => {
it('should add tables with hooks', async () => {
// eslint-disable-next-line jest/no-conditional-in-test

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,48 @@
import path from 'path'
import { getPayload, type SanitizedConfig } from 'payload'
import { fileURLToPath } from 'url'
import { generateDatabaseAdapter } from './generateDatabaseAdapter.js'
import { setTestEnvPaths } from './helpers/setTestEnvPaths.js'
const [testConfigDir] = process.argv.slice(2)
const filename = fileURLToPath(import.meta.url)
const dirname = path.dirname(filename)
const writeDBAdapter = process.env.WRITE_DB_ADAPTER !== 'false'
process.env.PAYLOAD_DROP_DATABASE = process.env.PAYLOAD_DROP_DATABASE || 'true'
if (process.env.PAYLOAD_DATABASE === 'mongodb') {
throw new Error('Not supported')
}
if (writeDBAdapter) {
generateDatabaseAdapter(process.env.PAYLOAD_DATABASE || 'postgres')
process.env.WRITE_DB_ADAPTER = 'false'
}
const loadConfig = async (configPath: string): Promise<SanitizedConfig> => {
return await (
await import(configPath)
).default
}
if (!testConfigDir) {
throw new Error('Yo must Specify testConfigDir')
}
const testDir = path.resolve(dirname, testConfigDir)
const config = await loadConfig(path.resolve(testDir, 'config.ts'))
setTestEnvPaths(testDir)
const payload = await getPayload({ config })
// await payload.db.dropDatabase({ adapter: payload.db })
await payload.db.generateSchema({
outputFile: path.resolve(testDir, 'payload-generated-schema.ts'),
})
process.exit(0)

0
test/helpers/database.ts Normal file
View File

View File

@@ -25,6 +25,9 @@ jest.spyOn(nodemailer, 'createTestAccount').mockImplementation(() => {
})
})
const dbAdapter = process.env.PAYLOAD_DATABASE || 'mongodb'
if (!process.env.PAYLOAD_DATABASE) {
// Mutate env so we can use conditions by DB adapter in tests properly without ignoring // eslint no-jest-conditions.
process.env.PAYLOAD_DATABASE = 'sqlite-uuid'
}
generateDatabaseAdapter(dbAdapter)
generateDatabaseAdapter(process.env.PAYLOAD_DATABASE)

File diff suppressed because it is too large Load Diff