chore: unflake auth e2e tests

packages/eslint-plugin-payload/customRules/no-non-retryable-assertions.js

@@ -51,7 +51,8 @@ module.exports = {
         ) {
           context.report({
             node: node.callee.property,
-            message: 'Non-retryable assertion used in Playwright test: "{{ assertion }}"',
+            message:
+              'Non-retryable, flaky assertion used in Playwright test: "{{ assertion }}". Those need to be wrapped in expect.poll or expect.toPass.',
             data: {
               assertion: node.callee.property.name,
             },

test/.eslintrc.cjs

@@ -63,7 +63,8 @@ module.exports = {
         'jest/require-top-level-describe': 'off',
         'jest-dom/prefer-to-have-attribute': 'off',
         'playwright/prefer-web-first-assertions': 'error',
-        'payload/no-non-retryable-assertions': 'error',
+        // Enable the no-non-retryable-assertions rule ONLY for hunting for flakes
+        // 'payload/no-non-retryable-assertions': 'error',
       },
     },
     {

test/auth/e2e.spec.ts

@@ -103,12 +103,19 @@ describe('auth', () => {
       await page.locator('#field-enableAPIKey').click()
 
       // assert that the value is set
-      const apiKey = await page.locator('#apiKey').inputValue()
-      expect(apiKey).toBeDefined()
+      const apiKeyLocator = page.locator('#apiKey')
+      await expect
+        .poll(async () => await apiKeyLocator.inputValue(), { timeout: 45000 })
+        .toBeDefined()
 
       await saveDocAndAssert(page)
 
-      expect(await page.locator('#apiKey').inputValue()).toStrictEqual(apiKey)
+      await expect(async () => {
+        const apiKey = await apiKeyLocator.inputValue()
+        expect(await page.locator('#apiKey').inputValue()).toStrictEqual(apiKey)
+      }).toPass({
+        timeout: 45000,
+      })
     })
 
     test('should disable api key', async () => {
@@ -123,14 +130,18 @@ describe('auth', () => {
       await saveDocAndAssert(page)
 
       // use the api key in a fetch to assert that it is disabled
-      const response = await fetch(`${apiURL}/${apiKeysSlug}/me`, {
-        headers: {
-          ...headers,
-          Authorization: `${slug} API-Key ${user.apiKey}`,
-        },
-      }).then((res) => res.json())
+      await expect(async () => {
+        const response = await fetch(`${apiURL}/${apiKeysSlug}/me`, {
+          headers: {
+            ...headers,
+            Authorization: `${slug} API-Key ${user.apiKey}`,
+          },
+        }).then((res) => res.json())
 
-      expect(response.user).toBeNull()
+        expect(response.user).toBeNull()
+      }).toPass({
+        timeout: 45000,
+      })
     })
   })
 })

test/helpers.ts

@@ -42,7 +42,7 @@ export async function delayNetwork({
   context: BrowserContext
   delay: 'Fast 3G' | 'Slow 3G' | 'Slow 4G'
   page: Page
-}): Promise<void> {
+}) {
   const cdpSession = await context.newCDPSession(page)
 
   await cdpSession.send('Network.emulateNetworkConditions', {
@@ -91,8 +91,7 @@ export async function saveDocHotkeyAndAssert(page: Page): Promise<void> {
 export async function saveDocAndAssert(page: Page, selector = '#action-save'): Promise<void> {
   await page.click(selector, { delay: 100 })
   await expect(page.locator('.Toastify')).toContainText('successfully')
-  await wait(500)
-  expect(page.url()).not.toContain('create')
+  await expect.poll(() => page.url(), { timeout: 45000 }).not.toContain('create')
 }
 
 export async function openNav(page: Page): Promise<void> {
@@ -135,13 +134,24 @@ export function exactText(text: string) {
   return new RegExp(`^${text}$`)
 }
 
-export const checkPageTitle = async (page: Page, title: string) =>
-  expect(await page.locator('.doc-header__title.render-title')?.first()?.innerText()).toBe(title)
+export const checkPageTitle = async (page: Page, title: string) => {
+  await expect
+    .poll(async () => await page.locator('.doc-header__title.render-title')?.first()?.innerText(), {
+      timeout: 45000,
+    })
+    .toBe(title)
+}
 
-export const checkBreadcrumb = async (page: Page, text: string) =>
-  expect(await page.locator('.step-nav.app-header__step-nav .step-nav__last')?.innerText()).toBe(
-    text,
-  )
+export const checkBreadcrumb = async (page: Page, text: string) => {
+  await expect
+    .poll(
+      async () => await page.locator('.step-nav.app-header__step-nav .step-nav__last')?.innerText(),
+      {
+        timeout: 45000,
+      },
+    )
+    .toBe(text)
+}
 
 export const selectTableRow = async (page: Page, title: string): Promise<void> => {
   const selector = `tbody tr:has-text("${title}") .select-row__checkbox input[type=checkbox]`

test/playwright.config.ts

@@ -4,14 +4,14 @@ export default defineConfig({
   // Look for test files in the "test" directory, relative to this configuration file
   testDir: '',
   testMatch: '*e2e.spec.ts',
-  timeout: 180000, // 3 minutes
+  timeout: 240000, // 4 minutes
   use: {
     screenshot: 'only-on-failure',
     trace: 'retain-on-failure',
     video: 'retain-on-failure',
   },
   expect: {
-    timeout: 45000,
+    timeout: 60000,
   },
   workers: 16,
 })