chore: and e2e benchmarks to BinaryCoStream

e2e/BinaryCoStream/package.json

@@ -27,6 +27,7 @@
     "@types/node": "^22.5.1",
     "@types/react": "^18.2.19",
     "@types/react-dom": "^18.2.7",
+    "jstat": "^1.9.6",
     "@vitejs/plugin-react-swc": "^3.3.2",
     "typescript": "^5.3.3",
     "vite": "^5.0.10"

e2e/BinaryCoStream/tests/bench.test.ts

@@ -0,0 +1,35 @@
+import { test, expect } from "@playwright/test";
+import { runBenchmarks } from "./bench/runner";
+
+test.describe("BinaryCoStream - Benchmarks", () => {
+  test("should sync a 1MB file between the two peers", async ({ browser, baseURL }) => {
+    const result = await runBenchmarks(browser, {
+        runs: 3,
+        benchmark: async (page, { isBaseline }) => {
+            if (isBaseline) {
+                await page.goto("https://binary-test-sage.vercel.app/?auto&fileSize=1000000&localSync");
+            } else {
+                const url = new URL(baseURL!);
+                url.searchParams.set("localSync", "true");
+                url.searchParams.set("auto", "true");
+                url.searchParams.set("fileSize", "1000000");
+
+                await page.goto(url.toString());
+            }
+
+            await page.getByRole("button", { name: "Upload Test File" }).click();
+
+            await page.getByTestId("sync-duration").waitFor();
+
+            return page.evaluate(() => {
+                return performance.getEntriesByName('sync')[0].duration;
+            });
+        }
+    });
+
+    console.log(result);
+
+    expect(result).toBeTruthy();
+  });
+
+});

e2e/BinaryCoStream/tests/bench/format.ts

@@ -0,0 +1,63 @@
+import { ConfidenceInterval, Difference } from './stats';
+
+export function formatConfidenceInterval(
+  ci: ConfidenceInterval,
+  format: (n: number) => string,
+) {
+  if (ci.low == null || isNaN(ci.low)) return `∞ - ∞`;
+  return `${format(ci.low)} - ${format(ci.high)}`;
+}
+
+const colorizeSign = (n: number, format: (n: number) => string) => {
+  if (n > 0) {
+    return `+${format(n)}`;
+  } else if (n < 0) {
+    // Negate the value so that we don't get a double negative sign.
+    return `-${format(-n)}`;
+  } else {
+    return format(n);
+  }
+};
+
+export function percent(n: number) {
+  return (n * 100).toFixed(0) + '%';
+}
+
+export function formatTime(n: number) {
+  if (n > 1000) return (n / 1000).toFixed(2) + 's';
+
+  return n.toFixed(0) + 'ms';
+}
+
+function negate(ci: ConfidenceInterval): ConfidenceInterval {
+  return {
+    low: -ci.high,
+    high: -ci.low,
+  };
+}
+
+export function formatDifference({ absolute, relative }: Difference) {
+  let word: string, rel: string, abs: string;
+
+  if (absolute.low > 0 && relative.low > 0) {
+    word = `worse ❌`;
+    rel = formatConfidenceInterval(relative, percent);
+    abs = formatConfidenceInterval(absolute, formatTime);
+  } else if (absolute.high < 0 && relative.high < 0) {
+    word = `better ✅`;
+    rel = formatConfidenceInterval(negate(relative), percent);
+    abs = formatConfidenceInterval(negate(absolute), formatTime);
+  } else {
+    word = `unsure 🔍`;
+    rel = formatConfidenceInterval(relative, (n) => colorizeSign(n, percent));
+    abs = formatConfidenceInterval(absolute, (n) =>
+      colorizeSign(n, formatTime),
+    );
+  }
+
+  return {
+    label: word,
+    relative: rel,
+    absolute: abs,
+  };
+}

e2e/BinaryCoStream/tests/bench/runner.ts

@@ -0,0 +1,37 @@
+import { Browser, Page } from "@playwright/test";
+import { summaryStats, computeDifference } from "./stats";
+import { formatDifference } from "./format";
+
+export type BenchmarkConfig = {
+    runs: number;
+    benchmark: (page: Page, props: { isBaseline: boolean }) => Promise<number>;
+}
+
+export async function runBenchmarks(browser: Browser, config: BenchmarkConfig) {
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    const baselineResults: number[] = [];
+    const curentTargetResults: number[] = [];
+
+    for (let i = 0; i < config.runs; i++) {
+        const baselinePage = await context.newPage();  
+       baselineResults.push(await config.benchmark(baselinePage, { isBaseline: true }));
+       baselinePage.close();
+       const currentTargetPage = await context.newPage();
+       curentTargetResults.push(await config.benchmark(currentTargetPage, { isBaseline: false }));
+       currentTargetPage.close();
+    }
+
+    const baselineStats = summaryStats(baselineResults);
+    const currentTargetStats = summaryStats(curentTargetResults);
+
+    const diff = computeDifference(currentTargetStats, baselineStats);
+
+    return {
+        baselineStats,
+        currentTargetStats,
+        result: formatDifference(diff),
+    }
+    
+}

e2e/BinaryCoStream/tests/bench/stats.ts

@@ -0,0 +1,161 @@
+// @ts-expect-error No types
+import jstat from 'jstat';
+
+// Most of these helpers are coming from tachometer source code
+// https://github.com/google/tachometer/blob/9c3ad697b27a85935c6c1bce987eedc51f507d35/src/stats.ts
+
+/**
+ * @license
+ * Copyright 2019 Google LLC
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+export type ConfidenceInterval = {
+    low: number;
+    high: number;
+  };
+  
+  export type SummaryStats = {
+    size: number;
+    median: number;
+    mean: number;
+    meanCI: ConfidenceInterval;
+    variance: number;
+    standardDeviation: number;
+    relativeStandardDeviation: number;
+  };
+
+function median(data: number[]): number {
+  // https://jstat.github.io/all.html#percentile
+  return jstat.percentile(data, 0.5);
+}
+function studenttInv(p: number, dof: number): number {
+  // https://jstat.github.io/all.html#jStat.studentt.inv
+  return jstat.studentt.inv(p, dof);
+}
+
+export function summaryStats(data: number[]): SummaryStats {
+  const size = data.length;
+  const sum = sumOf(data);
+  const mean = sum / size;
+  const squareResiduals = data.map((val) => (val - mean) ** 2);
+
+  // n - 1 due to https://en.wikipedia.org/wiki/Bessel%27s_correction
+  const variance = sumOf(squareResiduals) / (size - 1);
+  const stdDev = Math.sqrt(variance);
+
+  return {
+    size,
+    median: median(data),
+    mean,
+    meanCI: confidenceInterval95(
+      samplingDistributionOfTheMean({ mean, variance }, size),
+      size,
+    ),
+    variance,
+    standardDeviation: stdDev,
+    // aka coefficient of variation
+    relativeStandardDeviation: stdDev / mean,
+  };
+}
+
+type MeanAndVariance = { mean: number; variance: number };
+
+/**
+ * Compute a 95% confidence interval for the given distribution.
+ */
+function confidenceInterval95(
+  { mean, variance }: MeanAndVariance,
+  size: number,
+) {
+  // http://www.stat.yale.edu/Courses/1997-98/101/confint.htm
+  const t = studenttInv(1 - 0.05 / 2, size - 1);
+  const stdDev = Math.sqrt(variance);
+  const margin = t * stdDev;
+  return {
+    low: mean - margin,
+    high: mean + margin,
+  };
+}
+
+function sumOf(data: number[]) {
+  return data.reduce((acc, cur) => acc + cur);
+}
+
+export function computeDifference(a: SummaryStats, b: SummaryStats) {
+  const meanA = samplingDistributionOfTheMean(a, a.size);
+  const meanB = samplingDistributionOfTheMean(b, b.size);
+  const diffAbs = samplingDistributionOfAbsoluteDifferenceOfMeans(meanA, meanB);
+  const diffRel = samplingDistributionOfRelativeDifferenceOfMeans(meanA, meanB);
+
+  // We're assuming sample sizes are equal. If they're not for some reason, be
+  // conservative and use the smaller one for the t-distribution's degrees of
+  // freedom (since that will lead to a wider confidence interval).
+  const minSize = Math.min(a.size, b.size);
+  return {
+    absolute: confidenceInterval95(diffAbs, minSize),
+    relative: confidenceInterval95(diffRel, minSize),
+  };
+}
+
+export type Difference = {
+    absolute: ConfidenceInterval;
+    relative: ConfidenceInterval;
+}
+
+/**
+ * Estimates the sampling distribution of the mean. This models the distribution
+ * of the means that we would compute under repeated samples of the given size.
+ */
+function samplingDistributionOfTheMean(
+  dist: MeanAndVariance,
+  sampleSize: number,
+) {
+  // http://onlinestatbook.com/2/sampling_distributions/samp_dist_mean.html
+  // http://www.stat.yale.edu/Courses/1997-98/101/sampmn.htm
+  return {
+    mean: dist.mean,
+    // Error shrinks as sample size grows.
+    variance: dist.variance / sampleSize,
+  };
+}
+
+/**
+ * Estimates the sampling distribution of the difference of means (b-a). This
+ * models the distribution of the difference between two means that we would
+ * compute under repeated samples under the given two sampling distributions of
+ * means.
+ */
+function samplingDistributionOfAbsoluteDifferenceOfMeans(
+  a: MeanAndVariance,
+  b: MeanAndVariance,
+) {
+  // http://onlinestatbook.com/2/sampling_distributions/samplingdist_diff_means.html
+  // http://www.stat.yale.edu/Courses/1997-98/101/meancomp.htm
+  return {
+    mean: b.mean - a.mean,
+    // The error from both input sampling distributions of means accumulate.
+    variance: a.variance + b.variance,
+  };
+}
+
+/**
+ * Estimates the sampling distribution of the relative difference of means
+ * ((b-a)/a). This models the distribution of the relative difference between
+ * two means that we would compute under repeated samples under the given two
+ * sampling distributions of means.
+ */
+function samplingDistributionOfRelativeDifferenceOfMeans(
+  a: MeanAndVariance,
+  b: MeanAndVariance,
+) {
+  // http://blog.analytics-toolkit.com/2018/confidence-intervals-p-values-percent-change-relative-difference/
+  // Note that the above article also prevents an alternative calculation for a
+  // confidence interval for relative differences, but the one chosen here is
+  // is much simpler and passes our stochastic tests, so it seems sufficient.
+  return {
+    mean: (b.mean - a.mean) / a.mean,
+    variance:
+      (a.variance * b.mean ** 2 + b.variance * a.mean ** 2) / a.mean ** 4,
+  };
+}

pnpm-lock.yaml

@@ -76,6 +76,9 @@ importers:
       '@vitejs/plugin-react-swc':
         specifier: ^3.3.2
         version: 3.5.0(@swc/helpers@0.5.5)(vite@5.0.10(@types/node@22.5.1)(terser@5.33.0))
+      jstat:
+        specifier: ^1.9.6
+        version: 1.9.6
       typescript:
         specifier: ^5.3.3
         version: 5.3.3
@@ -3294,7 +3297,7 @@ packages:
   '@radix-ui/react-compose-refs@1.1.0':
     resolution: {integrity: sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==}
     peerDependencies:
-      '@types/react': '*'
+      '@types/react': ^18.2.32
       react: 18.3.1
     peerDependenciesMeta:
       '@types/react':
@@ -3378,8 +3381,8 @@ packages:
   '@radix-ui/react-focus-scope@1.1.0':
     resolution: {integrity: sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==}
     peerDependencies:
-      '@types/react': '*'
-      '@types/react-dom': '*'
+      '@types/react': ^18.2.32
+      '@types/react-dom': ^18.2.14
       react: 18.3.1
       react-dom: 18.3.1
     peerDependenciesMeta:
@@ -3391,7 +3394,7 @@ packages:
   '@radix-ui/react-id@1.1.0':
     resolution: {integrity: sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==}
     peerDependencies:
-      '@types/react': '*'
+      '@types/react': ^18.2.32
       react: 18.3.1
     peerDependenciesMeta:
       '@types/react':
@@ -3413,8 +3416,8 @@ packages:
   '@radix-ui/react-popper@1.2.0':
     resolution: {integrity: sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==}
     peerDependencies:
-      '@types/react': '*'
-      '@types/react-dom': '*'
+      '@types/react': ^18.2.32
+      '@types/react-dom': ^18.2.14
       react: 18.3.1
       react-dom: 18.3.1
     peerDependenciesMeta:
@@ -3491,8 +3494,8 @@ packages:
   '@radix-ui/react-primitive@2.0.0':
     resolution: {integrity: sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==}
     peerDependencies:
-      '@types/react': '*'
-      '@types/react-dom': '*'
+      '@types/react': ^18.2.32
+      '@types/react-dom': ^18.2.14
       react: 18.3.1
       react-dom: 18.3.1
     peerDependenciesMeta: