From 5f572dbd0475ee8dc43cb516e9d127fa38cb7c42 Mon Sep 17 00:00:00 2001 From: Alessio Gravili Date: Wed, 20 Mar 2024 23:28:56 -0400 Subject: [PATCH] chore: unflake auth e2e tests --- .../no-non-retryable-assertions.js | 3 +- test/.eslintrc.cjs | 3 +- test/auth/e2e.spec.ts | 31 +++++++++++++------ test/helpers.ts | 28 +++++++++++------ test/playwright.config.ts | 4 +-- 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/packages/eslint-plugin-payload/customRules/no-non-retryable-assertions.js b/packages/eslint-plugin-payload/customRules/no-non-retryable-assertions.js index dcb7c88916..f64de02996 100644 --- a/packages/eslint-plugin-payload/customRules/no-non-retryable-assertions.js +++ b/packages/eslint-plugin-payload/customRules/no-non-retryable-assertions.js @@ -51,7 +51,8 @@ module.exports = { ) { context.report({ node: node.callee.property, - message: 'Non-retryable assertion used in Playwright test: "{{ assertion }}"', + message: + 'Non-retryable, flaky assertion used in Playwright test: "{{ assertion }}". Those need to be wrapped in expect.poll or expect.toPass.', data: { assertion: node.callee.property.name, }, diff --git a/test/.eslintrc.cjs b/test/.eslintrc.cjs index 4e6eb6a2cb..81e4b0b717 100644 --- a/test/.eslintrc.cjs +++ b/test/.eslintrc.cjs @@ -63,7 +63,8 @@ module.exports = { 'jest/require-top-level-describe': 'off', 'jest-dom/prefer-to-have-attribute': 'off', 'playwright/prefer-web-first-assertions': 'error', - 'payload/no-non-retryable-assertions': 'error', + // Enable the no-non-retryable-assertions rule ONLY for hunting for flakes + // 'payload/no-non-retryable-assertions': 'error', }, }, { diff --git a/test/auth/e2e.spec.ts b/test/auth/e2e.spec.ts index 2da87a5b4b..f84333a2ba 100644 --- a/test/auth/e2e.spec.ts +++ b/test/auth/e2e.spec.ts @@ -103,12 +103,19 @@ describe('auth', () => { await page.locator('#field-enableAPIKey').click() // assert that the value is set - const apiKey = await page.locator('#apiKey').inputValue() - expect(apiKey).toBeDefined() + const apiKeyLocator = page.locator('#apiKey') + await expect + .poll(async () => await apiKeyLocator.inputValue(), { timeout: 45000 }) + .toBeDefined() await saveDocAndAssert(page) - expect(await page.locator('#apiKey').inputValue()).toStrictEqual(apiKey) + await expect(async () => { + const apiKey = await apiKeyLocator.inputValue() + expect(await page.locator('#apiKey').inputValue()).toStrictEqual(apiKey) + }).toPass({ + timeout: 45000, + }) }) test('should disable api key', async () => { @@ -123,14 +130,18 @@ describe('auth', () => { await saveDocAndAssert(page) // use the api key in a fetch to assert that it is disabled - const response = await fetch(`${apiURL}/${apiKeysSlug}/me`, { - headers: { - ...headers, - Authorization: `${slug} API-Key ${user.apiKey}`, - }, - }).then((res) => res.json()) + await expect(async () => { + const response = await fetch(`${apiURL}/${apiKeysSlug}/me`, { + headers: { + ...headers, + Authorization: `${slug} API-Key ${user.apiKey}`, + }, + }).then((res) => res.json()) - expect(response.user).toBeNull() + expect(response.user).toBeNull() + }).toPass({ + timeout: 45000, + }) }) }) }) diff --git a/test/helpers.ts b/test/helpers.ts index 10cb70565a..73026b5517 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -42,7 +42,7 @@ export async function delayNetwork({ context: BrowserContext delay: 'Fast 3G' | 'Slow 3G' | 'Slow 4G' page: Page -}): Promise { +}) { const cdpSession = await context.newCDPSession(page) await cdpSession.send('Network.emulateNetworkConditions', { @@ -91,8 +91,7 @@ export async function saveDocHotkeyAndAssert(page: Page): Promise { export async function saveDocAndAssert(page: Page, selector = '#action-save'): Promise { await page.click(selector, { delay: 100 }) await expect(page.locator('.Toastify')).toContainText('successfully') - await wait(500) - expect(page.url()).not.toContain('create') + await expect.poll(() => page.url(), { timeout: 45000 }).not.toContain('create') } export async function openNav(page: Page): Promise { @@ -135,13 +134,24 @@ export function exactText(text: string) { return new RegExp(`^${text}$`) } -export const checkPageTitle = async (page: Page, title: string) => - expect(await page.locator('.doc-header__title.render-title')?.first()?.innerText()).toBe(title) +export const checkPageTitle = async (page: Page, title: string) => { + await expect + .poll(async () => await page.locator('.doc-header__title.render-title')?.first()?.innerText(), { + timeout: 45000, + }) + .toBe(title) +} -export const checkBreadcrumb = async (page: Page, text: string) => - expect(await page.locator('.step-nav.app-header__step-nav .step-nav__last')?.innerText()).toBe( - text, - ) +export const checkBreadcrumb = async (page: Page, text: string) => { + await expect + .poll( + async () => await page.locator('.step-nav.app-header__step-nav .step-nav__last')?.innerText(), + { + timeout: 45000, + }, + ) + .toBe(text) +} export const selectTableRow = async (page: Page, title: string): Promise => { const selector = `tbody tr:has-text("${title}") .select-row__checkbox input[type=checkbox]` diff --git a/test/playwright.config.ts b/test/playwright.config.ts index 2ea0683f60..83ead85c1a 100644 --- a/test/playwright.config.ts +++ b/test/playwright.config.ts @@ -4,14 +4,14 @@ export default defineConfig({ // Look for test files in the "test" directory, relative to this configuration file testDir: '', testMatch: '*e2e.spec.ts', - timeout: 180000, // 3 minutes + timeout: 240000, // 4 minutes use: { screenshot: 'only-on-failure', trace: 'retain-on-failure', video: 'retain-on-failure', }, expect: { - timeout: 45000, + timeout: 60000, }, workers: 16, })