feat: dev push migrations handling and prompt

.vscode/launch.json

@@ -19,7 +19,7 @@
       "cwd": "${workspaceFolder}"
     },
     {
-      "command": "yarn run dev:postgres postgres",
+      "command": "yarn run dev:postgres postgres -- -I", // Allow input
       "name": "Run Dev Postgres",
       "request": "launch",
       "type": "node-terminal",

packages/db-postgres/package.json

@@ -13,9 +13,11 @@
     "payload": "^1.11.8"
   },
   "dependencies": {
+    "@libsql/client": "^0.3.1",
     "drizzle-kit": "^0.19.13-a511135",
     "drizzle-orm": "^0.27.2",
     "pg": "^8.11.1",
+    "prompts": "^2.4.2",
     "to-snake-case": "^1.0.0"
   },
   "devDependencies": {

packages/db-postgres/src/init.ts

@@ -1,12 +1,26 @@
 /* eslint-disable no-param-reassign */
-import { generateDrizzleJson, generateMigration, pushSchema } from 'drizzle-kit/utils';
+import { eq, sql } from 'drizzle-orm';
-import { buildVersionCollectionFields } from 'payload/dist/versions/buildCollectionFields';
+import {
+  numeric,
+  pgEnum,
+  pgTable,
+  varchar,
+  jsonb,
+} from 'drizzle-orm/pg-core';
+import { pushSchema } from 'drizzle-kit/utils';
 import { SanitizedCollectionConfig } from 'payload/dist/collections/config/types';
-import { getVersionsModelName } from 'payload/dist/versions/getVersionsModelName';
+import { configToJSONSchema } from 'payload/dist/utilities/configToJSONSchema';
 import type { Init } from 'payload/dist/database/types';
-import { pgEnum } from 'drizzle-orm/pg-core';
+import prompts from 'prompts';
-import type { GenericEnum, GenericRelation, GenericTable, PostgresAdapter } from './types';
 import { buildTable } from './schema/build';
+import type { GenericEnum, GenericRelation, GenericTable, PostgresAdapter } from './types';
+
+// Migration table def in order to use query using drizzle
+const migrationsSchema = pgTable('payload_migrations', {
+  name: varchar('name'),
+  batch: numeric('batch'),
+  schema: jsonb('schema'),
+});
 
 export const init: Init = async function init(
   this: PostgresAdapter,
@@ -50,6 +64,7 @@ export const init: Init = async function init(
       schema[`enum_${key}`] = val;
     });
 
+    // This will prompt if clarifications are needed for Drizzle to push new schema
     const {
       hasDataLoss,
       warnings,
@@ -57,19 +72,71 @@ export const init: Init = async function init(
       apply,
     } = await pushSchema(schema, this.db);
 
-    // TODO:
+    this.payload.logger.info({
-    // if there are warnings, make the user accept them via CLI
+      msg: 'Schema push results',
-    // Log the warnings and the statements, etc.
+      hasDataLoss,
-    // Only apply if user accepts warnings
+      warnings,
+      statementsToExecute,
+    });
 
-    // TODO:
+    if (warnings.length) {
-    // PUSH MIGRATION RECORD to db with shape of JSON schema
+      this.payload.logger.warn({
-    // this migration needs to have some "flag" that says "pushed"
+        msg: `Warnings detected during schema push: ${warnings.join('\n')}`,
-    // we don't want 1000 pushes in dev mode, just update the most recently pushed one
+        warnings,
-    // to do this, we will say "give me the most recent migration in the DB"
+      });
-    // if pushed: true, update that one with the new schema
+
-    // if pushed is false or does not exist, create a new migration
+      if (hasDataLoss) {
-    // with pushed: true and the shape of the schema (generated via generateDrizzleJSON)
+        this.payload.logger.warn({
+          msg: 'DATA LOSS WARNING: Possible data loss detected if schema is pushed.',
+        });
+      }
+
+      const { confirm: acceptWarnings } = await prompts(
+        {
+          type: 'confirm',
+          name: 'confirm',
+          message: 'Accept warnings and push schema to database?',
+          initial: false,
+        },
+        {
+          onCancel: () => {
+            process.exit(0);
+          },
+        },
+      );
+
+      // Exit if user does not accept warnings.
+      // Q: Is this the right type of exit for this interaction?
+      if (!acceptWarnings) {
+        process.exit(0);
+      }
+    }
+
+    const jsonSchema = configToJSONSchema(this.payload.config);
+
+    // This should mirror the generated table definition from schema/build.ts
+    await this.db.execute(sql`CREATE TABLE IF NOT EXISTS "payload_migrations" (
+        id SERIAL PRIMARY KEY,
+        name character varying,
+        batch numeric,
+        schema jsonb,
+        created_at timestamp without time zone DEFAULT now() NOT NULL,
+        updated_at timestamp without time zone DEFAULT now() NOT NULL
+      );`);
+
+    const devPush = await this.db.select().from(migrationsSchema).where(eq(migrationsSchema.batch, '-1'));
+
+    if (!devPush.length) {
+      await this.db.insert(migrationsSchema).values({
+        name: 'dev',
+        batch: '-1',
+        schema: JSON.stringify(jsonSchema),
+      });
+    } else {
+      await this.db.update(migrationsSchema).set({
+        schema: JSON.stringify(jsonSchema),
+      }).where(eq(migrationsSchema.batch, '-1'));
+    }
 
     await apply();
   }

packages/db-postgres/src/schema/traverseFields.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-param-reassign */
-import { AnyPgColumnBuilder, integer, pgEnum, pgTable, serial, uniqueIndex, text, varchar, PgColumn, PgTableExtraConfig, index, numeric, PgColumnHKT, IndexBuilder, PgNumericBuilder, PgVarcharBuilder } from 'drizzle-orm/pg-core';
+import { AnyPgColumnBuilder, integer, pgEnum, pgTable, serial, uniqueIndex, text, varchar, PgColumn, PgTableExtraConfig, index, numeric, PgColumnHKT, IndexBuilder, PgNumericBuilder, PgVarcharBuilder, jsonb } from 'drizzle-orm/pg-core';
 import { Field } from 'payload/types';
 import toSnakeCase from 'to-snake-case';
 import { fieldAffectsData } from 'payload/dist/fields/config/types';
@@ -93,6 +93,7 @@ export const traverseFields = ({
 
       case 'richText':
       case 'json': {
+        targetTable[`${fieldPrefix || ''}${field.name}`] = jsonb(columnName);
         break;
       }
 

src/database/migrations/migrationsCollection.ts

@@ -14,16 +14,13 @@ export const migrationsCollection: CollectionConfig = {
     {
       name: 'batch',
       type: 'number',
+      // NOTE: This value is -1 if it is a "dev push"
     },
     // TODO: determine how schema will impact migration workflow
     {
       name: 'schema',
       type: 'json',
     },
-    {
-      name: 'pushed',
-      type: 'checkbox',
-    },
     // TODO: do we need to persist the indexes separate from the schema?
     // {
     //   name: 'indexes',

test/dev.ts

@@ -36,7 +36,7 @@ const startDev = async () => {
     secret: uuid(),
     express: expressApp,
     email: {
-      logMockCredentials: true,
+      logMockCredentials: false,
       fromName: 'Payload',
       fromAddress: 'hello@payloadcms.com',
     },