chore: new payload/no-flaky-assertions and payload/no-wait-function eslint rules (#5425)

packages/eslint-plugin-payload/customRules/no-flaky-assertions.js

@@ -0,0 +1,88 @@
+/** @type {import('eslint').Rule.RuleModule} */
+module.exports = {
+  meta: {
+    type: 'problem',
+    docs: {
+      description:
+        'Disallow non-retryable assertions in Playwright E2E tests unless they are wrapped in an expect.poll() or expect().toPass()',
+      category: 'Best Practices',
+      recommended: true,
+    },
+    schema: [],
+  },
+  create: function (context) {
+    const nonRetryableAssertions = [
+      'toBe',
+      'toBeCloseTo',
+      'toBeDefined',
+      'toBeFalsy',
+      'toBeGreaterThan',
+      'toBeGreaterThanOrEqual',
+      'toBeInstanceOf',
+      'toBeLessThan',
+      'toBeLessThanOrEqual',
+      'toBeNaN',
+      'toBeNull',
+      'toBeTruthy',
+      'toBeUndefined',
+      'toContain',
+      'toContainEqual',
+      'toEqual',
+      'toHaveLength',
+      'toHaveProperty',
+      'toMatch',
+      'toMatchObject',
+      'toStrictEqual',
+      'toThrow',
+      'any',
+      'anything',
+      'arrayContaining',
+      'closeTo',
+      'objectContaining',
+      'stringContaining',
+      'stringMatching',
+    ]
+
+    return {
+      CallExpression(node) {
+        if (
+          node.callee.type === 'MemberExpression' &&
+          //node.callee.object.name === 'expect' &&
+          node.callee.property.type === 'Identifier' &&
+          nonRetryableAssertions.includes(node.callee.property.name)
+        ) {
+          let ancestor = node
+          let hasExpectPollOrToPass = false
+
+          while (ancestor) {
+            if (
+              ancestor.type === 'CallExpression' &&
+              ancestor.callee.type === 'MemberExpression' &&
+              ((ancestor.callee.object.type === 'CallExpression' &&
+                ancestor.callee.object.callee.type === 'MemberExpression' &&
+                ancestor.callee.object.callee.property.name === 'poll') ||
+                ancestor.callee.property.name === 'toPass')
+            ) {
+              hasExpectPollOrToPass = true
+              break
+            }
+            ancestor = ancestor.parent
+          }
+
+          if (hasExpectPollOrToPass) {
+            return
+          }
+
+          context.report({
+            node: node.callee.property,
+            message:
+              'Non-retryable, flaky assertion used in Playwright test: "{{ assertion }}". Those need to be wrapped in expect.poll() or expect().toPass.',
+            data: {
+              assertion: node.callee.property.name,
+            },
+          })
+        }
+      },
+    }
+  },
+}

packages/eslint-plugin-payload/index.js

@@ -4,5 +4,22 @@ module.exports = {
     'no-jsx-import-statements': require('./customRules/no-jsx-import-statements'),
     'no-non-retryable-assertions': require('./customRules/no-non-retryable-assertions'),
     'no-relative-monorepo-imports': require('./customRules/no-relative-monorepo-imports'),
+    'no-flaky-assertions': require('./customRules/no-flaky-assertions'),
+    'no-wait-function': {
+      create: function (context) {
+        return {
+          CallExpression(node) {
+            // Check if the function being called is named "wait"
+            if (node.callee.name === 'wait') {
+              context.report({
+                node,
+                message:
+                  'Usage of "wait" function is discouraged as it\'s flaky. Proper assertions should be used instead.',
+              })
+            }
+          },
+        }
+      },
+    },
   },
 }

packages/payload/src/exports/utilities.ts

@@ -42,5 +42,5 @@ export { mapAsync } from '../utilities/mapAsync.js'
 export { mergeListSearchAndWhere } from '../utilities/mergeListSearchAndWhere.js'
 export { setsAreEqual } from '../utilities/setsAreEqual.js'
 export { default as toKebabCase } from '../utilities/toKebabCase.js'
-export { default as wait } from '../utilities/wait.js'
+export { wait } from '../utilities/wait.js'
 export { default as wordBoundariesRegex } from '../utilities/wordBoundariesRegex.js'

packages/payload/src/utilities/wait.ts

@@ -1,7 +1,5 @@
-async function wait(ms) {
+export async function wait(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms)
   })
 }
-
-export default wait

test/.eslintrc.cjs

@@ -63,6 +63,8 @@ module.exports = {
         'jest/require-top-level-describe': 'off',
         'jest-dom/prefer-to-have-attribute': 'off',
         'playwright/prefer-web-first-assertions': 'error',
+        'payload/no-flaky-assertions': 'warn',
+        'payload/no-wait-function': 'warn',
         // Enable the no-non-retryable-assertions rule ONLY for hunting for flakes
         // 'payload/no-non-retryable-assertions': 'error',
       },