perf: faster page navigation by speeding up createClientConfig, speed up version fetching, speed up lexical init. Up to 100x faster (#9457)

If you had a lot of fields and collections, createClientConfig would be
extremely slow, as it was copying a lot of memory. In my test config
with a lot of fields and collections, it took 4 seconds(!!).

And not only that, it also ran between every single page navigation.

This PR significantly speeds up the createClientConfig function. In my
test config, its execution speed went from 4 seconds to 50 ms.
Additionally, createClientConfig is now properly cached in both dev &
prod. It no longer runs between every single page navigation. Even if
you trigger a full page reload, createClientConfig will be cached and
not run again. Despite that, HMR remains fully-functional.

This will make payload feel noticeably faster for large configs -
especially if it contains a lot of richtext fields, as it was previously
deep-copying the relatively large richText editor configs over and over
again.

## Before - 40 sec navigation speed

https://github.com/user-attachments/assets/fe6b707a-459b-44c6-982a-b277f6cbb73f

## After - 1 sec navigation speed

https://github.com/user-attachments/assets/384fba63-dc32-4396-b3c2-0353fcac6639

## Todo

- [x] Implement ClientSchemaMap and cache it, to remove
createClientField call in our form state endpoint
- [x] Enable schemaMap caching for dev
- [x] Cache lexical clientField generation, or add it to the parent
clientConfig

## Lexical changes

Red: old / removed
Green: new

![CleanShot 2024-11-22 at 21 07
41@2x](https://github.com/user-attachments/assets/f8321218-763c-4120-9353-076c381f33fb)

### Speed up version queries

This PR comes with performance optimizations for fetching versions
before a document is loaded. Not only does it use the new select API to
limit the fields it queries, it also completely skips a database query
if the current document is published.

### Speed up lexical init

Removes a bunch of unnecessary deep copying of lexical objects which
caused higher memory usage and slower load times. Additionally, the
lexical default config sanitization now happens less often.

test/auth-basic/config.ts

@@ -0,0 +1,19 @@
+import { fileURLToPath } from 'node:url'
+import path from 'path'
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js'
+
+// eslint-disable-next-line no-restricted-exports
+export default buildConfigWithDefaults({
+  admin: {
+    autoLogin: false,
+    importMap: {
+      baseDir: path.resolve(dirname),
+    },
+  },
+  typescript: {
+    outputFile: path.resolve(dirname, 'payload-types.ts'),
+  },
+})

test/auth-basic/e2e.spec.ts

@@ -0,0 +1,152 @@
+import type { Page } from '@playwright/test'
+import type { SanitizedConfig } from 'payload'
+
+import { expect, test } from '@playwright/test'
+import { devUser } from 'credentials.js'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+import type { PayloadTestSDK } from '../helpers/sdk/index.js'
+import type { Config } from './payload-types.js'
+
+import { ensureCompilationIsDone, getRoutes, initPageConsoleErrorCatch } from '../helpers.js'
+import { AdminUrlUtil } from '../helpers/adminUrlUtil.js'
+import { initPayloadE2ENoConfig } from '../helpers/initPayloadE2ENoConfig.js'
+import { reInitializeDB } from '../helpers/reInitializeDB.js'
+import { POLL_TOPASS_TIMEOUT, TEST_TIMEOUT_LONG } from '../playwright.config.js'
+
+const filename = fileURLToPath(import.meta.url)
+const dirname = path.dirname(filename)
+
+let payload: PayloadTestSDK<Config>
+
+const { beforeAll, beforeEach, describe } = test
+
+const createFirstUser = async ({
+  page,
+  serverURL,
+}: {
+  customAdminRoutes?: SanitizedConfig['admin']['routes']
+  customRoutes?: SanitizedConfig['routes']
+  page: Page
+  serverURL: string
+}) => {
+  const {
+    admin: {
+      routes: { createFirstUser: createFirstUserRoute },
+    },
+    routes: { admin: adminRoute },
+  } = getRoutes({})
+
+  // wait for create first user route
+  await page.goto(serverURL + `${adminRoute}${createFirstUserRoute}`)
+
+  // forget to fill out confirm password
+  await page.locator('#field-email').fill(devUser.email)
+  await page.locator('#field-password').fill(devUser.password)
+  await page.locator('.form-submit > button').click()
+  await expect(page.locator('.field-type.confirm-password .field-error')).toHaveText(
+    'This field is required.',
+  )
+
+  // make them match, but does not pass password validation
+  await page.locator('#field-email').fill(devUser.email)
+  await page.locator('#field-password').fill('12')
+  await page.locator('#field-confirm-password').fill('12')
+  await page.locator('.form-submit > button').click()
+  await expect(page.locator('.field-type.password .field-error')).toHaveText(
+    'This value must be longer than the minimum length of 3 characters.',
+  )
+
+  await page.locator('#field-email').fill(devUser.email)
+  await page.locator('#field-password').fill(devUser.password)
+  await page.locator('#field-confirm-password').fill(devUser.password)
+  await page.locator('.form-submit > button').click()
+
+  await expect
+    .poll(() => page.url(), { timeout: POLL_TOPASS_TIMEOUT })
+    .not.toContain('create-first-user')
+}
+
+describe('auth-basic', () => {
+  let page: Page
+  let url: AdminUrlUtil
+  let serverURL: string
+  let apiURL: string
+
+  beforeAll(async ({ browser }, testInfo) => {
+    testInfo.setTimeout(TEST_TIMEOUT_LONG)
+    ;({ payload, serverURL } = await initPayloadE2ENoConfig<Config>({ dirname }))
+    apiURL = `${serverURL}/api`
+    url = new AdminUrlUtil(serverURL, 'users')
+
+    const context = await browser.newContext()
+    page = await context.newPage()
+    initPageConsoleErrorCatch(page)
+
+    await ensureCompilationIsDone({
+      page,
+      serverURL,
+      readyURL: `${serverURL}/admin/**`,
+      noAutoLogin: true,
+    })
+
+    // Undo onInit seeding, as we need to test this without having a user created, or testing create-first-user
+    await reInitializeDB({
+      serverURL,
+      snapshotKey: 'auth-basic',
+      deleteOnly: true,
+    })
+
+    await payload.delete({
+      collection: 'users',
+      where: {
+        id: {
+          exists: true,
+        },
+      },
+    })
+
+    await ensureCompilationIsDone({
+      page,
+      serverURL,
+      readyURL: `${serverURL}/admin/create-first-user`,
+    })
+  })
+
+  beforeEach(async () => {
+    await payload.delete({
+      collection: 'users',
+      where: {
+        id: {
+          exists: true,
+        },
+      },
+    })
+  })
+
+  describe('unauthenticated users', () => {
+    test('ensure create first user page only has 3 fields', async () => {
+      await page.goto(url.admin + '/create-first-user')
+
+      // Ensure there are only 2 elements with class field-type
+      await expect(page.locator('.field-type')).toHaveCount(3) // Email, Password, Confirm Password
+    })
+
+    test('ensure first user can be created', async () => {
+      await createFirstUser({ page, serverURL })
+
+      // use the api key in a fetch to assert that it is disabled
+      await expect(async () => {
+        const users = await payload.find({
+          collection: 'users',
+        })
+
+        expect(users.totalDocs).toBe(1)
+        expect(users.docs[0].email).toBe(devUser.email)
+      }).toPass({
+        timeout: POLL_TOPASS_TIMEOUT,
+      })
+    })
+  })
+})

test/auth-basic/payload-types.ts

@@ -0,0 +1,186 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * This file was automatically generated by Payload.
+ * DO NOT MODIFY IT BY HAND. Instead, modify your source Payload config,
+ * and re-run `payload generate:types` to regenerate this file.
+ */
+
+export interface Config {
+  auth: {
+    users: UserAuthOperations;
+  };
+  collections: {
+    users: User;
+    'payload-locked-documents': PayloadLockedDocument;
+    'payload-preferences': PayloadPreference;
+    'payload-migrations': PayloadMigration;
+  };
+  collectionsJoins: {};
+  collectionsSelect: {
+    users: UsersSelect<false> | UsersSelect<true>;
+    'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
+    'payload-preferences': PayloadPreferencesSelect<false> | PayloadPreferencesSelect<true>;
+    'payload-migrations': PayloadMigrationsSelect<false> | PayloadMigrationsSelect<true>;
+  };
+  db: {
+    defaultIDType: string;
+  };
+  globals: {};
+  globalsSelect: {};
+  locale: null;
+  user: User & {
+    collection: 'users';
+  };
+  jobs: {
+    tasks: unknown;
+    workflows: unknown;
+  };
+}
+export interface UserAuthOperations {
+  forgotPassword: {
+    email: string;
+    password: string;
+  };
+  login: {
+    email: string;
+    password: string;
+  };
+  registerFirstUser: {
+    email: string;
+    password: string;
+  };
+  unlock: {
+    email: string;
+    password: string;
+  };
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "users".
+ */
+export interface User {
+  id: string;
+  updatedAt: string;
+  createdAt: string;
+  email: string;
+  resetPasswordToken?: string | null;
+  resetPasswordExpiration?: string | null;
+  salt?: string | null;
+  hash?: string | null;
+  loginAttempts?: number | null;
+  lockUntil?: string | null;
+  password?: string | null;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-locked-documents".
+ */
+export interface PayloadLockedDocument {
+  id: string;
+  document?: {
+    relationTo: 'users';
+    value: string | User;
+  } | null;
+  globalSlug?: string | null;
+  user: {
+    relationTo: 'users';
+    value: string | User;
+  };
+  updatedAt: string;
+  createdAt: string;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-preferences".
+ */
+export interface PayloadPreference {
+  id: string;
+  user: {
+    relationTo: 'users';
+    value: string | User;
+  };
+  key?: string | null;
+  value?:
+    | {
+        [k: string]: unknown;
+      }
+    | unknown[]
+    | string
+    | number
+    | boolean
+    | null;
+  updatedAt: string;
+  createdAt: string;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-migrations".
+ */
+export interface PayloadMigration {
+  id: string;
+  name?: string | null;
+  batch?: number | null;
+  updatedAt: string;
+  createdAt: string;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "users_select".
+ */
+export interface UsersSelect<T extends boolean = true> {
+  updatedAt?: T;
+  createdAt?: T;
+  email?: T;
+  resetPasswordToken?: T;
+  resetPasswordExpiration?: T;
+  salt?: T;
+  hash?: T;
+  loginAttempts?: T;
+  lockUntil?: T;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-locked-documents_select".
+ */
+export interface PayloadLockedDocumentsSelect<T extends boolean = true> {
+  document?: T;
+  globalSlug?: T;
+  user?: T;
+  updatedAt?: T;
+  createdAt?: T;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-preferences_select".
+ */
+export interface PayloadPreferencesSelect<T extends boolean = true> {
+  user?: T;
+  key?: T;
+  value?: T;
+  updatedAt?: T;
+  createdAt?: T;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "payload-migrations_select".
+ */
+export interface PayloadMigrationsSelect<T extends boolean = true> {
+  name?: T;
+  batch?: T;
+  updatedAt?: T;
+  createdAt?: T;
+}
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "auth".
+ */
+export interface Auth {
+  [k: string]: unknown;
+}
+
+
+declare module 'payload' {
+  // @ts-ignore 
+  export interface GeneratedTypes extends Config {}
+}

test/auth-basic/tsconfig.eslint.json

@@ -0,0 +1,13 @@
+{
+  // extend your base config to share compilerOptions, etc
+  //"extends": "./tsconfig.json",
+  "compilerOptions": {
+    // ensure that nobody can accidentally use this config for a build
+    "noEmit": true
+  },
+  "include": [
+    // whatever paths you intend to lint
+    "./**/*.ts",
+    "./**/*.tsx"
+  ]
+}

test/auth-basic/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../tsconfig.json"
+}