chore: hide test flakes, improve playwright CI logs, significantly reduce playwright timeouts, add back test retries, cache playwright browsers in CI, disable CI telemetry, improve test throttle utility (#6155)

test/playwright.config.ts

@@ -8,14 +8,16 @@ const dirname = path.dirname(filename)
 
 dotenv.config({ path: path.resolve(dirname, 'test.env') })
 
-export const EXPECT_TIMEOUT = 45000
+export const TEST_TIMEOUT_LONG = 480000 // 8 minutes - used as timeOut for the beforeAll
+export const TEST_TIMEOUT = 60000
+export const EXPECT_TIMEOUT = 8000
 export const POLL_TOPASS_TIMEOUT = EXPECT_TIMEOUT * 4 // That way expect.poll() or expect().toPass can retry 4 times. 4x higher than default expect timeout => can retry 4 times if retryable expects are used inside
 
 export default defineConfig({
   // Look for test files in the "test" directory, relative to this configuration file
   testDir: '',
   testMatch: '*e2e.spec.ts',
-  timeout: 120000,
+  timeout: TEST_TIMEOUT, // 1 minute
   use: {
     screenshot: 'only-on-failure',
     trace: 'retain-on-failure',
@@ -25,5 +27,9 @@ export default defineConfig({
     timeout: EXPECT_TIMEOUT,
   },
   workers: 16,
-  maxFailures: process.env.CI ? 1 : undefined,
+  maxFailures: process.env.CI ? undefined : undefined,
+  retries: process.env.CI ? 5 : undefined,
+  reporter: process.env.CI
+    ? [['list', { printSteps: true }], ['json']]
+    : [['list', { printSteps: true }]],
 })