perf: skip verify step when creating a new local transaction

feat: add a stress-test app to benchmark Jazz with huge loads
perf: Optimize RawCoMap and RawCoList init and add methods to create bulk transactions
2024-12-23 11:06:33 +01:00 · 2024-12-21 01:11:36 +01:00 · 2024-12-21 00:57:44 +01:00
21 changed files with 511 additions and 17 deletions
--- a/.changeset/fast-chicken-build.md
+++ b/.changeset/fast-chicken-build.md
@@ -0,0 +1,5 @@
+---
+"cojson": patch
+---
+
+skip verify step when creating a new local transaction
--- a/.changeset/tiny-trains-vanish.md
+++ b/.changeset/tiny-trains-vanish.md
@@ -0,0 +1,5 @@
+---
+"cojson": patch
+---
+
+Optimize RawCoMap and RawCoList init and add the assign and appendItems methods to create bulk transactions
--- a/examples/stress-test/.gitignore
+++ b/examples/stress-test/.gitignore
@@ -0,0 +1,30 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+sync-db/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
--- a/examples/stress-test/index.html
+++ b/examples/stress-test/index.html
@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/png" href="/jazz-logo.png" />
+    <link rel="stylesheet" href="/src/index.css" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Jazz CoValues tests</title>
+  </head>
+  <body style="margin: 0; padding: 0;">
+    <div id="root"></div>
+    <script type="module" src="/src/app.tsx"></script>
+  </body>
+</html>
--- a/examples/stress-test/package.json
+++ b/examples/stress-test/package.json
@@ -0,0 +1,35 @@
+{
+  "name": "@jazz/stress-test-example",
+  "private": true,
+  "version": "0.0.110",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "dev:local": "VITE_WS_PEER=ws://localhost:4200/ vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview",
+    "format-and-lint": "biome check .",
+    "format-and-lint:fix": "biome check . --write",
+    "sync": "jazz-run sync"
+  },
+  "dependencies": {
+    "@faker-js/faker": "^9.3.0",
+    "@tanstack/react-virtual": "^3.11.2",
+    "cojson": "workspace:*",
+    "jazz-react": "workspace:*",
+    "jazz-tools": "workspace:*",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-router": "^6.16.0",
+    "react-router-dom": "^6.16.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.5.1",
+    "@types/react": "^18.2.19",
+    "@types/react-dom": "^18.2.7",
+    "@vitejs/plugin-react-swc": "^3.3.2",
+    "jazz-run": "workspace:*",
+    "typescript": "^5.3.3",
+    "vite": "^5.0.10"
+  }
+}
--- a/examples/stress-test/src/app.tsx
+++ b/examples/stress-test/src/app.tsx
@@ -0,0 +1,25 @@
+import React from "react";
+import ReactDOM from "react-dom/client";
+import { RouterProvider, createBrowserRouter } from "react-router-dom";
+import { AuthAndJazz } from "./jazz";
+import { HomePage } from "./pages/Home";
+import { TaskPage } from "./pages/Task";
+
+const router = createBrowserRouter([
+  {
+    path: "/task/:taskID",
+    element: <TaskPage />,
+  },
+  {
+    path: "/",
+    element: <HomePage />,
+  },
+]);
+
+ReactDOM.createRoot(document.getElementById("root")!).render(
+  <React.StrictMode>
+    <AuthAndJazz>
+      <RouterProvider router={router} />
+    </AuthAndJazz>
+  </React.StrictMode>,
+);
--- a/examples/stress-test/src/jazz.tsx
+++ b/examples/stress-test/src/jazz.tsx
@@ -0,0 +1,55 @@
+import { createJazzReactApp, useDemoAuth } from "jazz-react";
+import { useEffect, useRef } from "react";
+import { MillionTasksAccount } from "./schema";
+
+const url = new URL(window.location.href);
+
+const key = `${getUserInfo()}@jazz.tools`;
+
+let peer =
+  (url.searchParams.get("peer") as `ws://${string}`) ??
+  `wss://cloud.jazz.tools/?key=${key}`;
+
+if (url.searchParams.has("local")) {
+  peer = `ws://localhost:4200/?key=${key}`;
+}
+
+if (import.meta.env.VITE_WS_PEER) {
+  peer = import.meta.env.VITE_WS_PEER;
+}
+
+const Jazz = createJazzReactApp({ AccountSchema: MillionTasksAccount });
+
+export const { useAccount, useCoState, useAcceptInvite } = Jazz;
+
+function getUserInfo() {
+  return url.searchParams.get("userName") ?? "Mister X";
+}
+
+export function AuthAndJazz({ children }: { children: React.ReactNode }) {
+  const [auth, state] = useDemoAuth();
+
+  const signedUp = useRef(false);
+
+  useEffect(() => {
+    if (state.state === "ready" && !signedUp.current) {
+      const userName = getUserInfo();
+
+      if (state.existingUsers.includes(userName)) {
+        state.logInAs(userName);
+      } else {
+        state.signUp(userName);
+      }
+
+      signedUp.current = true;
+    }
+  }, [state.state]);
+
+  console.log(state.state, signedUp);
+
+  return (
+    <Jazz.Provider auth={auth} peer={`${peer}?key=${key}`}>
+      {children}
+    </Jazz.Provider>
+  );
+}
--- a/examples/stress-test/src/pages/Home.tsx
+++ b/examples/stress-test/src/pages/Home.tsx
@@ -0,0 +1,62 @@
+import { useVirtualizer } from "@tanstack/react-virtual";
+import { useRef } from "react";
+import { useAccount } from "../jazz";
+import { addTasks } from "../schema";
+
+export function HomePage() {
+  const { me } = useAccount({
+    root: {
+      tasks: [],
+    },
+  });
+
+  const tasks = me?.root.tasks ?? [];
+
+  const parentRef = useRef<HTMLDivElement>(null);
+  const rowVirtualizer = useVirtualizer({
+    count: tasks.length,
+    getScrollElement: () => parentRef.current,
+    estimateSize: () => 35,
+  });
+
+  return (
+    <>
+      <div>
+        <button onClick={() => me?.root.tasks && addTasks(me.root.tasks)}>
+          Add Tasks
+        </button>
+      </div>
+      <div
+        ref={parentRef}
+        style={{
+          height: `100vh`,
+          overflow: "auto", // Make it scroll!
+        }}
+      >
+        <div
+          style={{
+            height: `${rowVirtualizer.getTotalSize()}px`,
+            width: "100%",
+            position: "relative",
+          }}
+        >
+          {rowVirtualizer.getVirtualItems().map((virtualItem) => (
+            <div
+              key={virtualItem.key}
+              style={{
+                position: "absolute",
+                top: 0,
+                left: 0,
+                width: "100%",
+                height: `${virtualItem.size}px`,
+                transform: `translateY(${virtualItem.start}px)`,
+              }}
+            >
+              {tasks[virtualItem.index]?.title}
+            </div>
+          ))}
+        </div>
+      </div>
+    </>
+  );
+}
--- a/examples/stress-test/src/pages/Task.tsx
+++ b/examples/stress-test/src/pages/Task.tsx
@@ -0,0 +1,3 @@
+export function TaskPage() {
+  return <div>Task</div>;
+}
--- a/examples/stress-test/src/schema.ts
+++ b/examples/stress-test/src/schema.ts
@@ -0,0 +1,64 @@
+import { faker } from "@faker-js/faker";
+import { Account, CoList, CoMap, Group, ID, co } from "jazz-tools";
+
+export class Comment extends CoMap {
+  text = co.string;
+}
+
+export class CommentList extends CoList.Of(co.ref(Comment)) {}
+
+export class Task extends CoMap {
+  title = co.string;
+  comments = co.ref(CommentList);
+}
+
+export class TaskList extends CoList.Of(co.ref(Task)) {}
+
+export class AccountRoot extends CoMap {
+  tasks = co.ref(TaskList);
+}
+
+export class MillionTasksAccount extends Account {
+  root = co.ref(AccountRoot);
+
+  async migrate(creationProps: { name: string }) {
+    super.migrate(creationProps);
+
+    if (!this._refs.root) {
+      const tasks = await TaskList.load(
+        "co_zZsMvX5ZqKt4164YkLLq9iTd7LY" as ID<TaskList>,
+        this,
+        [],
+      );
+      this.root = AccountRoot.create(
+        {
+          tasks: tasks!,
+        },
+        { owner: this },
+      );
+    }
+  }
+}
+
+export function addTasks(taskList: TaskList) {
+  const arr = [];
+  for (let i = 0; i < 10_000; i++) {
+    arr.push(createTask(taskList._owner as Group).id);
+  }
+  taskList._raw.appendItems(arr, undefined, "private");
+}
+
+export function createTask(group: Group) {
+  return Task.create(
+    {
+      title: faker.word.words({ count: { min: 5, max: 10 } }),
+      comments: CommentList.create(
+        Array.from({ length: 10 }, () =>
+          Comment.create({ text: faker.lorem.paragraphs(5) }, { owner: group }),
+        ),
+        { owner: group },
+      ),
+    },
+    { owner: group },
+  );
+}
--- a/examples/stress-test/src/vite-env.d.ts
+++ b/examples/stress-test/src/vite-env.d.ts
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
--- a/examples/stress-test/tsconfig.json
+++ b/examples/stress-test/tsconfig.json
@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2023", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "baseUrl": "."
+  },
+  "include": ["src"]
+}
--- a/examples/stress-test/vite.config.ts
+++ b/examples/stress-test/vite.config.ts
@@ -0,0 +1,10 @@
+import react from "@vitejs/plugin-react-swc";
+import { defineConfig } from "vite";
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+  build: {
+    minify: false,
+  },
+});
--- a/packages/cojson/src/coValueCore.ts
+++ b/packages/cojson/src/coValueCore.ts
@@ -202,6 +202,7 @@ export class CoValueCore {
    newTransactions: Transaction[],
    givenExpectedNewHash: Hash | undefined,
    newSignature: Signature,
+    skipVerify: boolean = false,
  ): Result<true, TryAddTransactionsError> {
    return this.node
      .resolveAccountAgent(
@@ -231,8 +232,10 @@ export class CoValueCore {
          } satisfies InvalidHashError);
        }

-        // const beforeVerify = performance.now();
-        if (!this.crypto.verify(newSignature, expectedNewHash, signerID)) {
+        if (
+          skipVerify !== true &&
+          !this.crypto.verify(newSignature, expectedNewHash, signerID)
+        ) {
          return err({
            type: "InvalidSignature",
            id: this.id,
@@ -600,6 +603,7 @@ export class CoValueCore {
      [transaction],
      expectedNewHash,
      signature,
+      true,
    )._unsafeUnwrap({ withStackTrace: true });

    if (success) {
--- a/packages/cojson/src/coValues/coList.ts
+++ b/packages/cojson/src/coValues/coList.ts
@@ -412,6 +412,14 @@ export class RawCoList<
    item: Item,
    after?: number,
    privacy: "private" | "trusting" = "private",
+  ) {
+    this.appendItems([item], after, privacy);
+  }
+
+  appendItems(
+    items: Item[],
+    after?: number,
+    privacy: "private" | "trusting" = "private",
  ) {
    const entries = this.entries();
    after =
@@ -420,7 +428,7 @@ export class RawCoList<
          ? entries.length - 1
          : 0
        : after;
-    let opIDBefore;
+    let opIDBefore: OpID | "start";
    if (entries.length > 0) {
      const entryBefore = entries[after];
      if (!entryBefore) {
@@ -433,14 +441,17 @@ export class RawCoList<
      }
      opIDBefore = "start";
    }
+
    this.core.makeTransaction(
-      [
-        {
+      // Since the operation is "append" we need to reverse
+      // the items to keep the same insertion order
+      items
+        .map((item) => ({
          op: "app",
          value: isCoValue(item) ? item.id : item,
          after: opIDBefore,
-        },
-      ],
+        }))
+        .reverse(),
      privacy,
    );

--- a/packages/cojson/src/coValues/coMap.ts
+++ b/packages/cojson/src/coValues/coMap.ts
@@ -383,6 +383,26 @@ export class RawCoMap<
    this.processNewTransactions();
  }

+  assign(
+    entries: Partial<Shape>,
+    privacy: "private" | "trusting" = "private",
+  ): void {
+    if (this.isTimeTravelEntity()) {
+      throw new Error("Cannot set value on a time travel entity");
+    }
+
+    this.core.makeTransaction(
+      Object.entries(entries).map(([key, value]) => ({
+        op: "set",
+        key,
+        value: isCoValue(value) ? value.id : value,
+      })),
+      privacy,
+    );
+
+    this.processNewTransactions();
+  }
+
  /** Delete the given key (setting it to undefined).
   *
   * If `privacy` is `"private"` **(default)**, `key` is encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
--- a/packages/cojson/src/coValues/group.ts
+++ b/packages/cojson/src/coValues/group.ts
@@ -623,9 +623,7 @@ export class RawGroup<
      .getCurrentContent() as M;

    if (init) {
-      for (const [key, value] of Object.entries(init)) {
-        map.set(key, value, initPrivacy);
-      }
+      map.assign(init, initPrivacy);
    }

    return map;
@@ -655,10 +653,8 @@ export class RawGroup<
      })
      .getCurrentContent() as L;

-    if (init) {
-      for (const item of init) {
-        list.append(item, undefined, initPrivacy);
-      }
+    if (init?.length) {
+      list.appendItems(init, undefined, initPrivacy);
    }

    return list;
--- a/packages/cojson/src/tests/coList.test.ts
+++ b/packages/cojson/src/tests/coList.test.ts
@@ -75,6 +75,26 @@ test("Push is equivalent to append after last item", () => {
  expect(content.toJSON()).toEqual(["hello", "world", "hooray"]);
 });

+test("appendItems add an array of items at the end of the list", () => {
+  const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
+
+  const coValue = node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const content = expectList(coValue.getCurrentContent());
+
+  expect(content.type).toEqual("colist");
+
+  content.append("hello", 0, "trusting");
+  expect(content.toJSON()).toEqual(["hello"]);
+  content.appendItems(["world", "hooray", "universe"], undefined, "trusting");
+  expect(content.toJSON()).toEqual(["hello", "world", "hooray", "universe"]);
+});
+
 test("Can push into empty list", () => {
  const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);

--- a/packages/cojson/src/tests/coMap.test.ts
+++ b/packages/cojson/src/tests/coMap.test.ts
@@ -175,3 +175,35 @@ test("Can get last tx ID for a key in CoMap", () => {
  content.set("hello", "C", "trusting");
  expect(content.lastEditAt("hello")?.tx.txIndex).toEqual(2);
 });
+
+test("Can set items in bulk with assign", () => {
+  const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
+
+  const coValue = node.createCoValue({
+    type: "comap",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const content = expectMap(coValue.getCurrentContent());
+
+  expect(content.type).toEqual("comap");
+
+  content.set("key1", "set1", "trusting");
+
+  content.assign(
+    {
+      key1: "assign1",
+      key2: "assign2",
+      key3: "assign3",
+    },
+    "trusting",
+  );
+
+  expect(content.toJSON()).toEqual({
+    key1: "assign1",
+    key2: "assign2",
+    key3: "assign3",
+  });
+});
--- a/packages/jazz-tools/src/coValues/coList.ts
+++ b/packages/jazz-tools/src/coValues/coList.ts
@@ -239,9 +239,11 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
  }

  push(...items: Item[]): number {
-    for (const item of toRawItems(items as Item[], this._schema[ItemsSym])) {
-      this._raw.append(item);
-    }
+    this._raw.appendItems(
+      toRawItems(items, this._schema[ItemsSym]),
+      undefined,
+      "private",
+    );

    return this._raw.entries().length;
  }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1229,6 +1229,58 @@ importers:

  examples/richtext: {}

+  examples/stress-test:
+    dependencies:
+      '@faker-js/faker':
+        specifier: ^9.3.0
+        version: 9.3.0
+      '@tanstack/react-virtual':
+        specifier: ^3.11.2
+        version: 3.11.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+      cojson:
+        specifier: workspace:*
+        version: link:../../packages/cojson
+      jazz-react:
+        specifier: workspace:*
+        version: link:../../packages/jazz-react
+      jazz-tools:
+        specifier: workspace:*
+        version: link:../../packages/jazz-tools
+      react:
+        specifier: 18.3.1
+        version: 18.3.1
+      react-dom:
+        specifier: 18.3.1
+        version: 18.3.1(react@18.3.1)
+      react-router:
+        specifier: ^6.16.0
+        version: 6.21.0(react@18.3.1)
+      react-router-dom:
+        specifier: ^6.16.0
+        version: 6.21.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+    devDependencies:
+      '@types/node':
+        specifier: ^22.5.1
+        version: 22.5.1
+      '@types/react':
+        specifier: ^18.2.19
+        version: 18.3.12
+      '@types/react-dom':
+        specifier: ^18.2.7
+        version: 18.3.1
+      '@vitejs/plugin-react-swc':
+        specifier: ^3.3.2
+        version: 3.5.0(@swc/helpers@0.5.5)(vite@5.4.11(@types/node@22.5.1)(lightningcss@1.27.0)(terser@5.33.0))
+      jazz-run:
+        specifier: workspace:*
+        version: link:../../packages/jazz-run
+      typescript:
+        specifier: ^5.3.3
+        version: 5.6.3
+      vite:
+        specifier: ^5.0.10
+        version: 5.4.11(@types/node@22.5.1)(lightningcss@1.27.0)(terser@5.33.0)
+
  examples/todo:
    dependencies:
      '@radix-ui/react-checkbox':
@@ -3522,6 +3574,10 @@ packages:
    resolution: {integrity: sha512-sqXgo1SCv+j4VtYEwl/bukuOIBrVgx6euIoCat3Iyx5oeoXwEA2USCoeL0IPubflMxncA2INkqJ/Wr3NGrSgzw==}
    hasBin: true

+  '@faker-js/faker@9.3.0':
+    resolution: {integrity: sha512-r0tJ3ZOkMd9xsu3VRfqlFR6cz0V/jFYRswAIpC+m/DIfAUXq7g8N7wTAlhSANySXYGKzGryfDXwtwsY8TxEIDw==}
+    engines: {node: '>=18.0.0', npm: '>=9.0.0'}
+
  '@floating-ui/core@1.6.8':
    resolution: {integrity: sha512-7XJ9cPU+yI2QeLS+FCSlqNFZJq8arvswefkZrYI1yQBbftw6FyrZOxYSh+9S7z7TpeWlRt9zJ5IhM1WIL334jA==}

@@ -4960,6 +5016,15 @@ packages:
    peerDependencies:
      tailwindcss: '>=3.0.0 || insiders || >=4.0.0-alpha.20'

+  '@tanstack/react-virtual@3.11.2':
+    resolution: {integrity: sha512-OuFzMXPF4+xZgx8UzJha0AieuMihhhaWG0tCqpp6tDzlFwOmNBPYMuLOtMJ1Tr4pXLHmgjcWhG6RlknY2oNTdQ==}
+    peerDependencies:
+      react: 18.3.1
+      react-dom: 18.3.1
+
+  '@tanstack/virtual-core@3.11.2':
+    resolution: {integrity: sha512-vTtpNt7mKCiZ1pwU9hfKPhpdVO2sVzFQsxoVBGtOSHxlrRRzYr8iQ2TlwbAcRYCcEiZ9ECAM8kBzH0v2+VzfKw==}
+
  '@testing-library/dom@10.4.0':
    resolution: {integrity: sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==}
    engines: {node: '>=18'}
@@ -13520,6 +13585,8 @@ snapshots:
      find-up: 5.0.0
      js-yaml: 4.1.0

+  '@faker-js/faker@9.3.0': {}
+
  '@floating-ui/core@1.6.8':
    dependencies:
      '@floating-ui/utils': 0.2.8
@@ -15123,6 +15190,14 @@ snapshots:
      postcss-selector-parser: 6.0.10
      tailwindcss: 3.4.15(ts-node@10.9.2(@swc/core@1.7.22(@swc/helpers@0.5.5))(@types/node@22.5.1)(typescript@5.6.3))

+  '@tanstack/react-virtual@3.11.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
+    dependencies:
+      '@tanstack/virtual-core': 3.11.2
+      react: 18.3.1
+      react-dom: 18.3.1(react@18.3.1)
+
+  '@tanstack/virtual-core@3.11.2': {}
+
  '@testing-library/dom@10.4.0':
    dependencies:
      '@babel/code-frame': 7.24.7
Author	SHA1	Message	Date
Guido D'Orsi	58cfade490	perf: skip verify step when creating a new local transaction	2024-12-23 11:06:33 +01:00
Guido D'Orsi	e549256771	feat: add a stress-test app to benchmark Jazz with huge loads	2024-12-21 01:11:36 +01:00
Guido D'Orsi	4a4d7f2d15	perf: Optimize RawCoMap and RawCoList init and add methods to create bulk transactions	2024-12-21 00:57:44 +01:00