Release

Effect streams for peers

.changeset/config.json

@@ -12,7 +12,7 @@
             "jazz-react",
             "jazz-nodejs",
             "jazz-run",
-            "cojson-transport-nodejs-ws",
+            "cojson-transport-ws",
             "cojson-storage-indexeddb",
             "cojson-storage-sqlite"
         ]

examples/chat/CHANGELOG.md

@@ -1,5 +1,94 @@
 # jazz-example-chat
 
+## 0.0.61
+
+### Patch Changes
+
+- Updated dependencies
+  - cojson@0.7.14
+  - jazz-tools@0.7.14
+  - jazz-react@0.7.14
+
+## 0.0.60
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-tools@0.7.13
+  - jazz-react@0.7.13
+
+## 0.0.59
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-tools@0.7.12
+  - jazz-react@0.7.12
+
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies
+  - cojson@0.7.11
+  - jazz-react@0.7.11
+  - jazz-tools@0.7.11
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies
+  - cojson@0.7.10
+  - jazz-react@0.7.10
+  - jazz-tools@0.7.10
+
+## 0.0.56
+
+### Patch Changes
+
+- Updated dependencies
+  - cojson@0.7.9
+  - jazz-react@0.7.9
+  - jazz-tools@0.7.9
+
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-tools@0.7.8
+  - jazz-react@0.7.8
+
+## 0.0.54
+
+### Patch Changes
+
+- Updated dependencies [9fdc91c]
+  - jazz-react@0.7.7
+
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-tools@0.7.6
+  - jazz-react@0.7.6
+
+## 0.0.52
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-react@0.7.5
+
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies
+  - jazz-react@0.7.4
+
 ## 0.0.50
 
 ### Patch Changes

examples/chat/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-chat",
   "private": true,
-  "version": "0.0.50",
+  "version": "0.0.61",
   "type": "module",
   "scripts": {
     "dev": "vite",

examples/pets/CHANGELOG.md

@@ -1,5 +1,96 @@
 # jazz-example-pets
 
+## 0.0.79
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.14
+    -   jazz-react@0.7.14
+    -   jazz-browser-media-images@0.7.14
+
+## 0.0.78
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+    -   jazz-browser-media-images@0.7.13
+    -   jazz-react@0.7.13
+
+## 0.0.77
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+    -   jazz-browser-media-images@0.7.12
+    -   jazz-react@0.7.12
+
+## 0.0.76
+
+### Patch Changes
+
+-   jazz-react@0.7.11
+-   jazz-tools@0.7.11
+-   jazz-browser-media-images@0.7.11
+
+## 0.0.75
+
+### Patch Changes
+
+-   jazz-react@0.7.10
+-   jazz-tools@0.7.10
+-   jazz-browser-media-images@0.7.10
+
+## 0.0.74
+
+### Patch Changes
+
+-   jazz-react@0.7.9
+-   jazz-tools@0.7.9
+-   jazz-browser-media-images@0.7.9
+
+## 0.0.73
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+    -   jazz-browser-media-images@0.7.8
+    -   jazz-react@0.7.8
+
+## 0.0.72
+
+### Patch Changes
+
+-   Updated dependencies [9fdc91c]
+    -   jazz-react@0.7.7
+
+## 0.0.71
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+    -   jazz-browser-media-images@0.7.6
+    -   jazz-react@0.7.6
+
+## 0.0.70
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-react@0.7.5
+    -   jazz-browser-media-images@0.7.5
+
+## 0.0.69
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-react@0.7.4
+
 ## 0.0.68
 
 ### Patch Changes

examples/pets/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-pets",
   "private": true,
-  "version": "0.0.68",
+  "version": "0.0.79",
   "type": "module",
   "scripts": {
     "dev": "vite",

examples/pets/src/2_main.tsx

@@ -42,7 +42,7 @@ ReactDOM.createRoot(document.getElementById("root")!).render(
         <ThemeProvider>
             <TitleAndLogo name={appName} />
             <div className="flex flex-col h-full items-center justify-start gap-10 pt-10 pb-10 px-5">
-                <Jazz.Provider>
+                <Jazz.Provider loading={<div>Loading</div>}>
                     <App />
                 </Jazz.Provider>
             </div>

examples/todo/CHANGELOG.md

@@ -1,5 +1,87 @@
 # jazz-example-todo
 
+## 0.0.78
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.14
+    -   jazz-react@0.7.14
+
+## 0.0.77
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+    -   jazz-react@0.7.13
+
+## 0.0.76
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+    -   jazz-react@0.7.12
+
+## 0.0.75
+
+### Patch Changes
+
+-   jazz-react@0.7.11
+-   jazz-tools@0.7.11
+
+## 0.0.74
+
+### Patch Changes
+
+-   jazz-react@0.7.10
+-   jazz-tools@0.7.10
+
+## 0.0.73
+
+### Patch Changes
+
+-   jazz-react@0.7.9
+-   jazz-tools@0.7.9
+
+## 0.0.72
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+    -   jazz-react@0.7.8
+
+## 0.0.71
+
+### Patch Changes
+
+-   Updated dependencies [9fdc91c]
+    -   jazz-react@0.7.7
+
+## 0.0.70
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+    -   jazz-react@0.7.6
+
+## 0.0.69
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-react@0.7.5
+
+## 0.0.68
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-react@0.7.4
+
 ## 0.0.67
 
 ### Patch Changes

examples/todo/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-todo",
   "private": true,
-  "version": "0.0.67",
+  "version": "0.0.78",
   "type": "module",
   "scripts": {
     "dev": "vite",

homepage/homepage/app/docs/guide.mdx

@@ -1,17 +1,18 @@
 import { Slogan } from "@/components/forMdx";
 import { JazzLogo } from "@/components/logos";
 
-<h1 id="guide">Learn some Jazz.</h1>
+<h1 id="guide">Learn some <JazzLogo className="h-[1.3em] relative -top-0.5 inline-block -ml-[0.1em] -mr-[0.1em]"/></h1>
-<Slogan>Build an issue tracking app with distributed state.</Slogan>
+<Slogan>Build an issue tracker with distributed state.</Slogan>
 
 Our issues app will be quite simple, but it will have team collaboration. <span className="text-nowrap">**Let's call it... &ldquo;Circular.&rdquo;**</span>
 
 We'll build everything **step-by-step,** in typical, immediately usable stages. We'll explore many important things Jazz does &mdash; so **follow along** or **just pick things out.**
 
-<h2 id="setup">Project Setup</h2>
+<h2 id="guide-setup">Project Setup</h2>
 
-1. Create a project from a generic Vite starter template:
+1. Create a project called "circular" from a generic Vite starter template:
 
+    {/* prettier-ignore */}
     ```bash
     npx degit gardencmp/vite-ts-react-tailwind circular
     cd circular
@@ -29,33 +30,36 @@ We'll build everything **step-by-step,** in typical, immediately usable stages.
 
     <small>(in a new terminal window):</small>
 
+    {/* prettier-ignore */}
     ```bash
     cd circular
     npm install jazz-tools jazz-react
     ```
 
-3. Set up a Jazz context, by modifying `src/main.tsx`:
+3. Modify `src/main.tsx` to set up a Jazz context:
-
-    ```tsx subtle=1,2,3,4,13,15,16,17,19
-    import React from "react";
-    import ReactDOM from "react-dom/client";
-    import App from "./App.tsx";
-    import "./index.css";
-    import { JazzReact } from "jazz-react";
 
+    {/* prettier-ignore */}
+    ```tsx
+    import React from "react"; // old
+    import ReactDOM from "react-dom/client"; // old
+    import App from "./App.tsx"; // old
+    import "./index.css"; // old
+    import { createJazzReactContext, DemoAuth } from "jazz-react";
+     // old
     const Jazz = createJazzReactContext({
         auth: DemoAuth({ appName: "Circular" }),
-        peer: "wss://mesh.jazz.tools/?key=you@example.com", // <- put your email here to receive a proper API key for later
+        peer: "wss://mesh.jazz.tools/?key=you@example.com", // <- put your email here to get a proper API key later
     });
     export const { useAccount, useCoState } = Jazz;
-
+    // old
-    ReactDOM.createRoot(document.getElementById("root")!).render(
+    ReactDOM.createRoot(document.getElementById("root")!).render( // old
         <Jazz.Provider>
-            <React.StrictMode>
+            <React.StrictMode> // old
-                <App />
+                {" "}// old
-            </React.StrictMode>
+                <App /> // old
+            </React.StrictMode>{" "}// old
         </Jazz.Provider>,
-    );
+    ); // old
     ```
 
     This sets Jazz up, extracts app-specific hooks for later, and wraps our app in the provider.
@@ -75,7 +79,7 @@ We can
 -   **edit** CoValues, from anywhere, by mutating them like local state
 -   **subscribe to edits** in CoValues, whether they're local or remote
 
-<h3 id="first-schema">Declaring our own CoValues</h3>
+<h3 id="declaring-covalues">Declaring our own CoValues</h3>
 
 To make our own CoValues, we first need to declare a schema for them. Think of a schema as a combination of TypeScript types and runtime type information.
 
@@ -96,16 +100,17 @@ export class Issue extends CoMap {
 
 TODO: explain what's happening
 
-<h3>Reading from CoValues</h3>
+<h3 id="reading-covalues">Reading from CoValues</h3>
 
 CoValues are designed to be read like simple local JSON state. Let's see how we can read from an Issue by building a component to render one.
 
 Create a new file `src/components/Issue.tsx` and add the following:
 
+{/* prettier-ignore */}
 ```tsx
 import { Issue } from "../schema";
 
-export function IssueComponent({ issue }, { issue: Issue }) {
+export function IssueComponent({ issue }: { issue: Issue }) {
     return (
         <div className="grid grid-cols-6 text-sm border-r border-b [&>*]:p-2 [&>*]:border-l [&>*]:border-t">
             <h2>{issue.title}</h2>
@@ -119,42 +124,44 @@ export function IssueComponent({ issue }, { issue: Issue }) {
 
 Simple enough!
 
-<h3>Creating CoValues</h3>
+<h3 id="creating-covalues">Creating CoValues</h3>
 
 To actually see an Issue, we have to create one. This is where things start to get interesting...
 
 Let's modify `src/App.tsx` to prepare for creating an Issue and then rendering it:
 
-```tsx subtle=5,13,14,15
+{/* prettier-ignore */}
-import { useState, useCallback } from "react";
+```tsx
+import { useState } from "react";
 import { Issue } from "./schema";
-import { IssueComponent } from "./components/Issue";
+import { IssueComponent } from "./components/Issue.tsx";
-
+// old
-function App() {
+function App() {// old
     const [issue, setIssue] = useState<Issue>();
-
+    // old
     if (issue) {
         return <IssueComponent issue={issue} />;
     } else {
         return <button>Create Issue</button>;
     }
-}
+} // old
-
+// old
-export default App;
+export default App; // old
 ```
 
 Now, finally, let's implement creating an issue:
 
-```tsx subtle=1,2,3,5,6,8,23,24,25,27,28,29,30
+{/* prettier-ignore */}
-import { useState } from "react";
+```tsx
-import { Issue } from "./schema";
+import { useState } from "react"; // old
-import { IssueComponent } from "./components/Issue";
+import { Issue } from "./schema"; // old
+import { IssueComponent } from "./components/Issue.tsx"; // old
 import { useAccount } from "./main";
-
+// old
-function App() {
+function App() {// old
     const { me } = useAccount();
-    const [issue, setIssue] = useState<Issue>();
+    const [issue, setIssue] = useState<Issue>(); // old
-
+    // old
     const createIssue = () => {
         const newIssue = Issue.create(
             {
@@ -167,20 +174,20 @@ function App() {
         );
         setIssue(newIssue);
     };
-
+    // old
-    if (issue) {
+    if (issue) {// old
-        return <IssueComponent issue={issue} />;
+        return <IssueComponent issue={issue} />; // old
-    } else {
+    } else { // old
         return <button onClick={createIssue}>Create Issue</button>;
-    }
+    } // old
-}
+} // old
-
+// old
-export default App;
+export default App; // old
 ```
 
-Now you should be able to create a new issue by clicking the button and then see it rendered.
+🏁 Now you should be able to create a new issue by clicking the button and then see it rendered!
 
-<div className="text-xs uppercase text-stone-400 dark:text-stone-600 tracking-wider">
+<div className="text-xs uppercase text-stone-400 dark:text-stone-600 tracking-wider -mb-3">
     Preview
 </div>
 <div className="p-3 md:-mx-3 rounded border border-stone-100 dark:border-stone-900 bg-white dark:bg-black not-prose">
@@ -202,7 +209,7 @@ We'll already notice one interesting thing here:
 
 We'll make use of both of these facts in a bit, but for now let's start with local editing and subscribing.
 
-<h3>Editing CoValues and subscribing to edits</h3>
+<h3 id="editing-and-subscription">Editing CoValues and subscribing to edits</h3>
 
 Since we're the owner of the CoValue, we should be able to edit it, right?
 
@@ -216,49 +223,52 @@ This is exactly what the `useCoState` hook is for!
 
 Let's modify `src/App.tsx`:
 
-```tsx subtle=1,2,3,4,5,6,7,12,13,14,15,16,17,18,19,20,21,23,25,26,27,28,29,30,32
+{/* prettier-ignore */}
-import { useState } from "react";
+```tsx
-import { Issue } from "./schema";
+import { useState } from "react"; // old
-import { IssueComponent } from "./components/Issue";
+import { Issue } from "./schema"; // old
-import { useAccount } from "./main";
+import { IssueComponent } from "./components/Issue.tsx"; // old
-
+import { useAccount, useCoState } from "./main";
-function App() {
+import { ID } from "jazz-tools"
-    const { me } = useAccount();
+// old
+function App() { // old
+    const { me } = useAccount(); // old
     const [issueID, setIssueID] = useState<ID<Issue>>();
-
+    // old
     const issue = useCoState(Issue, issueID);
-
+    // old
-    const createIssue = () => {
+    const createIssue = () => {// old
-        const newIssue = Issue.create(
+        const newIssue = Issue.create(// old
-            {
+            { // old
-                title: "Buy terrarium",
+                title: "Buy terrarium", // old
-                description: "Make sure it's big enough for 10 snails.",
+                description: "Make sure it's big enough for 10 snails.", // old
-                estimate: 5,
+                estimate: 5, // old
-                status: "backlog",
+                status: "backlog", // old
-            },
+            }, // old
-            { owner: me },
+            { owner: me }, // old
-        );
+        ); // old
         setIssueID(newIssue.id);
-    };
+    }; // old
-
+    // old
-    if (issue) {
+    if (issue) { // old
-        return <IssueComponent issue={issue} />;
+        return <IssueComponent issue={issue} />; // old
-    } else {
+    } else { // old
-        return <button onClick={createIssue}>Create Issue</button>;
+        return <button onClick={createIssue}>Create Issue</button>; // old
-    }
+    } // old
-}
+} // old
-
+// old
-export default App;
+export default App; // old
 ```
 
 And now for the exciting part! Let's make `src/components/Issue.tsx` an editing component.
 
-```tsx subtle=1,3,4,5,28,29,30,31
+{/* prettier-ignore */}
-import { Issue } from "../schema";
+```tsx
-
+import { Issue } from "../schema"; // old
-export function IssueComponent({ issue }, { issue: Issue }) {
+// old
-    return (
+export function IssueComponent({ issue }: { issue: Issue }) { // old
-        <div className="grid grid-cols-6 text-sm border-r border-b [&>*]:p-2 [&>*]:border-l [&>*]:border-t">
+    return ( // old
+        <div className="grid grid-cols-6 text-sm border-r border-b [&>*]:p-2 [&>*]:border-l [&>*]:border-t"> // old
             <input type="text"
                 value={issue.title}
                 onChange={(event) => { issue.title = event.target.value }}/>
@@ -279,14 +289,14 @@ export function IssueComponent({ issue }, { issue: Issue }) {
             >
                 <option value="backlog">Backlog</option>
                 <option value="in progress">In Progress</option>
-                <option value="done">Done</options>
+                <option value="done">Done</option>
             </select>
-        </div>
+        </div> // old
-    );
+    ); // old
-}
+} // old
 ```
 
-<div className="text-xs uppercase text-stone-400 dark:text-stone-600 tracking-wider">
+<div className="text-xs uppercase text-stone-400 dark:text-stone-600 tracking-wider -mb-3">
     Preview
 </div>
 <div className="p-3 md:-mx-3 rounded border border-stone-100 dark:border-stone-900 bg-white dark:bg-black not-prose">
@@ -309,9 +319,15 @@ export function IssueComponent({ issue }, { issue: Issue }) {
     </div>
 </div>
 
+🏁 Now you should be able to edit the issue after creating it!
+
 You'll immediately notice that we're doing something non-idiomatic for React: we mutate the issue directly, by assigning to its properties.
 
-This works because CoValues intercept these edits, update their local view accordingly (React doesn't really care after rendering) and then notify subscribers of the change, who will receive a fresh, updated view of the CoValue.
+This works because CoValues
+
+-   intercept these edits
+-   update their local view accordingly (React doesn't really care after rendering)
+-   notify subscribers of the change (who will receive a fresh, updated view of the CoValue)
 
 <aside className="text-sm border border-stone-300 dark:border-stone-700 rounded px-4 my-4 max-w-3xl [&_pre]:mx-0">
     <h4 className="not-prose text-base py-2 mb-3 -mx-4 px-4 border-b border-stone-300 dark:border-stone-700">💡 A Quick Overview of Subscribing to CoValues</h4>
@@ -321,13 +337,13 @@ This works because CoValues intercept these edits, update their local view accor
     1.  Directly on an instance:
 
         ```ts
-        const unsub = issue.subscribe((updatedIssue) => console.log(updatedIssue));
+        const unsub = issue.subscribe([], (updatedIssue) => console.log(updatedIssue));
         ```
 
     2.  If you only have an ID (this will load the issue if needed):
 
         ```ts
-        const unsub = Issue.subscribe(issueID, { as: me }, (updatedIssue) => {
+        const unsub = Issue.subscribe(issueID, me, [], (updatedIssue) => {
             console.log(updatedIssue);
         });
         ```
@@ -344,7 +360,7 @@ This works because CoValues intercept these edits, update their local view accor
                 const { me } = useAccount();
                 const [value, setValue] = useState<V>();
 
-                useEffect(() => Schema.subscribe(id, { as: me }, setValue), [id]);
+                useEffect(() => Schema.subscribe(id, me, [], setValue), [id]);
 
                 return value;
             }
@@ -354,62 +370,290 @@ This works because CoValues intercept these edits, update their local view accor
 
 We have one subscriber on our Issue, with `useCoState` in `src/App.tsx`, which will cause the `App` component and its children **to** re-render whenever the Issue changes.
 
-<h3>Automatic local & cloud persistence</h3>
+<h3 id="persistence">Automatic local & cloud persistence</h3>
 
 So far our Issue CoValues just looked like ephemeral local state. We'll now start exploring the first main feature that makes CoValues special: **automatic persistence.**
 
 Actually, all the Issue CoValues we've created so far **have already been automatically persisted** to the cloud and locally - but we loose track of their ID after a reload.
 
-So let's store the ID in URL state and make sure our useState is in sync with that.
+So let's store the ID in window URL state and make sure our useState is in sync with that.
 
-```tsx subtle=1,2,3,4,5,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,28,29,30,31,32,33,34,35
+{/* prettier-ignore */}
-import { useState } from "react";
+```tsx
-import { Issue } from "./schema";
+import { useState } from "react"; // old
-import { IssueComponent } from "./components/Issue";
+import { Issue } from "./schema"; // old
-import { useAccount } from "./main";
+import { IssueComponent } from "./components/Issue.tsx"; // old
-
+import { useAccount, useCoState } from "./main"; // old
-function App() {
+import { ID } from "jazz-tools" // old
-    const { me } = useAccount();
+// old
-    const [issueID, setIssueID] = useState<ID<Issue>>(
+function App() { // old
-        window.location.search?.replace("?issue=", "") || undefined
+    const { me } = useAccount(); // old
+    const [issueID, setIssueID] = useState<ID<Issue> | undefined>(
+        (window.location.search?.replace("?issue=", "") || undefined) as ID<Issue> | undefined,
     );
-
+    // old
-    const issue = useCoState(Issue, issueID);
+    const issue = useCoState(Issue, issueID); // old
-
+    // old
-    const createIssue = () => {
+    const createIssue = () => {// old
-        const newIssue = Issue.create(
+        const newIssue = Issue.create(// old
-            {
+            { // old
-                title: "Buy terrarium",
+                title: "Buy terrarium", // old
-                description: "Make sure it's big enough for 10 snails.",
+                description: "Make sure it's big enough for 10 snails.", // old
-                estimate: 5,
+                estimate: 5, // old
-                status: "backlog",
+                status: "backlog", // old
-            },
+            }, // old
-            { owner: me },
+            { owner: me }, // old
-        );
+        ); // old
-        setIssueID(newIssue.id);
+        setIssueID(newIssue.id); // old
         window.history.pushState({}, "", `?issue=${newIssue.id}`);
-    };
+    }; // old
-
+    // old
-    if (issue) {
+    if (issue) { // old
-        return <IssueComponent issue={issue} />;
+        return <IssueComponent issue={issue} />; // old
-    } else {
+    } else { // old
-        return <button onClick={createIssue}>Create Issue</button>;
+        return <button onClick={createIssue}>Create Issue</button>; // old
-    }
+    } // old
-}
+} // old
-
+// old
-export default App;
+export default App; // old
 ```
 
-Now you should be able to create an issue, reload the page, and still see the same issue.
+🏁 Now you should be able to create an issue, edit it, reload the page, and still see the same issue.
 
+<h3 id="remote-sync">Remote sync</h3>
 
-<h3>Remote sync</h3>
+To see that sync is also already working, try the following:
 
-But even better, you should be able to:
+-   copy the URL to a new tab in the same browser window and see the same issue
+-   edit the issue and see the changes reflected in the other tab!
 
-- copy the URL to a new tab and see the same issue
+This works because we load the issue as the same account that created it and owns it (remember setting `{ owner: me }`?).
-- edit the issue and see the changes reflected in the other tab
 
+We'll learn more about access control in "Groups & Permissions", but for now let's build a super simple way of sharing an Issue by just making it publicly readable & writable.
+
+All we have to do is create a new group to own each new issue and add "everyone" as a "writer":
+
+{/* prettier-ignore */}
+```tsx
+import { useState } from "react"; // old
+import { Issue } from "./schema"; // old
+import { IssueComponent } from "./components/Issue.tsx"; // old
+import { useAccount, useCoState } from "./main"; // old
+import { ID, Group } from "jazz-tools"
+// old
+function App() { // old
+    const { me } = useAccount(); // old
+    const [issueID, setIssueID] = useState<ID<Issue> | undefined>(// old
+        (window.location.search?.replace("?issue=", "") || undefined) as ID<Issue> | undefined,// old
+    ); // old
+    // old
+    const issue = useCoState(Issue, issueID); // old
+    // old
+    const createIssue = () => { // old
+        const group = Group.create({ owner: me });
+        group.addMember("everyone", "writer");
+        // old
+        const newIssue = Issue.create( // old
+            { // old
+                title: "Buy terrarium", // old
+                description: "Make sure it's big enough for 10 snails.", // old
+                estimate: 5, // old
+                status: "backlog", // old
+            }, // old
+            { owner: group },
+        ); // old
+        setIssueID(newIssue.id); // old
+        window.history.pushState({}, "", `?issue=${newIssue.id}`); // old
+    }; // old
+    // old
+    if (issue) { // old
+        return <IssueComponent issue={issue} />; // old
+    } else { // old
+        return <button onClick={createIssue}>Create Issue</button>; // old
+    } // old
+} // old
+// old
+export default App; // old
+```
+
+🏁 Now you should be able to open the Issue (with its unique URL) on another device or browser, or send it to a friend and you should be able to **edit it together in realtime!**
+
+This concludes our intro to the essence of CoValues. Hopefully you're starting to have a feeling for how CoValues behave and how they're magically available everywhere.
+
+<h2 id="refs-and-on-demand-subscribe">Refs & Auto-Subscribe</h2>
+
+Now let's have a look at how to compose CoValues into more complex structures and build a whole app around them.
+
+Let's extend our two data model to include "Projects" which have a list of tasks and some properties of their own.
+
+Using plain objects, you would probably type a Project like this:
+
+```ts
+type Project = {
+    name: string;
+    issues: Issue[];
+};
+```
+
+In order to create this more complex structure in a fully collaborative way, we're going to need _references_ that allow us to nest or link CoValues.
+
+Add the following to `src/schema.ts`:
+
+```ts
+import { CoMap, CoList, co } from "jazz-tools";
+// old
+export class Issue extends CoMap { // old
+    title = co.string; // old
+    description = co.string; // old
+    estimate = co.number; // old
+    status? = co.literal("backlog", "in progress", "done"); // old
+} // old
+// old
+export class ListOfIssues extends CoList.Of(co.ref(Issue)) {}
+
+export class Project extends CoMap {
+    name = co.string;
+    issues = co.ref(ListOfIssues);
+}
+```
+
+Now let's change things up a bit in terms of components as well.
+
+First, we'll change `App.tsx` to create and render `Project`s instead of `Issue`s. (We'll move the `useCoState` into the `ProjectComponent` we'll create in a second).
+
+{/* prettier-ignore */}
+```tsx
+import { useState } from "react"; // old
+import { Project, ListOfIssues } from "./schema";
+import { ProjectComponent } from "./components/Project.tsx";
+import { useAccount } from "./main";
+import { ID, Group } from "jazz-tools"
+// old
+function App() { // old
+    const { me } = useAccount(); // old
+    const [projectID, setProjectID] = useState<ID<Project> | undefined>(
+        (window.location.search?.replace("?project=", "") || undefined) as ID<Project> | undefined,// old
+    );
+    // old
+    const issue = useCoState(Issue, issueID); // *bin*
+    // old
+    const createProject = () => {
+        const group = Group.create({ owner: me });
+        group.addMember("everyone", "writer");
+
+        const newProject = Project.create(
+            {
+                name: "New Project",
+                issues: ListOfIssues.create([], { owner: group })
+            },
+            { owner: group },
+        );
+        setProjectID(newProject.id);
+        window.history.pushState({}, "", `?project=${newProject.id}`);
+    };
+    // old
+    if (projectID) {
+        return <ProjectComponent projectID={projectID} />;
+    } else {
+        return <button onClick={createProject}>Create Project</button>;
+    }
+} // old
+// old
+export default App; // old
+```
+
+Now we'll actually create the `ProjectComponent` that renders a `Project` and its `Issue`s.
+
+Create a new file `src/components/Project.tsx` and add the following:
+
+{/* prettier-ignore */}
+```tsx
+import { ID } from "jazz-tools";
+import { Project, Issue } from "../schema";
+import { IssueComponent } from "./Issue.tsx";
+import { useCoState } from "../main";
+
+export function ProjectComponent({ projectID }: { projectID: ID<Project> }) {
+    const project = useCoState(Project, projectID);
+
+    const createAndAddIssue = () => {
+        project?.issues?.push(Issue.create({
+            title: "",
+            description: "",
+            estimate: 0,
+            status: "backlog",
+        }, { owner: project._owner }));
+    };
+
+    return project ? (
+        <div>
+            <h1>{project.name}</h1>
+            <div className="border-r border-b">
+                {project.issues?.map((issue) => (
+                    issue && <IssueComponent key={issue.id} issue={issue} />
+                ))}
+                <button onClick={createAndAddIssue}>Create Issue</button>
+            </div>
+        </div>
+    ) : (
+        <div>Loading project...</div>
+    );
+}
+```
+
+🏁 Now you should be able to create a project, add issues to it, share it, and edit it collaboratively!
+
+Two things to note here:
+
+-   We create a new Issue like before, and then push it into the `issues` list of the Project. By setting the `owner` to the Project's owner, we ensure that the Issue has the same access rights as the project itself.
+-   We only need to use `useCoState` on the Project, and the nested `ListOfIssues` and each `Issue` will be **automatically loaded and subscribed to when we access them.**
+-   However, because either the `Project`, `ListOfIssues`, or each `Issue` might not be loaded yet, we have to check for them being defined.
+
+The load-and-subscribe-on-access is a convenient way to have your rendering drive data loading (including in nested components!) and lets you quickly chuck UIs together without worrying too much about the shape of all data you'll need.
+
+But you can also take more precise control over loading by defining a minimum-depth to load in `useCoState`:
+
+{/* prettier-ignore */}
+```tsx
+import { ID } from "jazz-tools";// old
+import { Project, Issue } from "../schema"; // old
+import { IssueComponent } from "./Issue.tsx"; // old
+import { useCoState } from "../main"; // old
+// old
+export function ProjectComponent({ projectID }: { projectID: ID<Project> }) {// old
+    const project = useCoState(Project, projectID, { issues: [{}] });
+
+    const createAndAddIssue = () => {// old
+        project?.issues.push(Issue.create({
+            title: "",// old
+            description: "",// old
+            estimate: 0,// old
+            status: "backlog",// old
+        }, { owner: project._owner }));// old
+    };// old
+// old
+    return project ? (// old
+        <div>// old
+            <h1>{project.name}</h1>// old
+            <div className="border-r border-b">// old
+                {project.issues.map((issue) => (
+                    <IssueComponent key={issue.id} issue={issue} />
+                ))}// old
+                <button onClick={createAndAddIssue}>Create Issue</button>// old
+            </div>// old
+        </div>// old
+    ) : (// old
+        <div>Loading project...</div>// old
+    );// old
+}// old
+```
+
+The loading-depth spec `{ issues: [{}] }` means "in `Project`, load `issues` and load each item in `issues` shallowly". (Since an `Issue` doesn't have any further references, "shallowly" actually means all its properties will be available).
+
+-   Now, we can get rid of a lot of coniditional accesses because we know that once `project` is loaded, `project.issues` and each `Issue` in it will be loaded as well.
+-   This also results in only one rerender and visual update when everything is loaded, which is faster (especially for long lists) and gives you more control over the loading UX.
+
+TODO: explainer about not loaded vs not set/defined and `_refs` basics
 
 <div className="text-amber-500 mt-52">
     🚧 OH NO - This is as far as we've written the Guide. 🚧
@@ -417,10 +661,10 @@ But even better, you should be able to:
 {" -> "}
 <a href="https://github.com/gardencmp/jazz/issues/186">Complain on GitHub</a>
 
-<h2 id="refs-and-on-demand-subscribe">Refs & Auto-Subscribe</h2>
-
 <h2 id="groups-and-permissions">Groups & Permissions</h2>
 
-<h2 id="accounts-and-migrations">Accounts & Migrations</h2>
+<h2 id="auth-accounts-and-migrations">Auth, Accounts & Migrations</h2>
 
-<h2 id="backend-workers">Backend Workers</h2>
+<h2 id="edits-and-time-travel">Edit Metadata & Time Travel</h2>
+
+<h2 id="backend-workers">Backend Workers</h2>

homepage/homepage/app/docs/page.tsx

@@ -1,6 +1,7 @@
 import { DocNav } from "@/components/docs/nav";
 import { PackageDocs } from "@/components/docs/packageDocs";
 import Guide from "./guide.mdx";
+import { Prose } from "@/components/forMdx";
 
 export const metadata = {
     title: "jazz - Docs",
@@ -10,44 +11,50 @@ export const metadata = {
 export default function Page() {
     return (
         <>
-            <div className="hidden md:block bg-stone-100 dark:bg-stone-900 p-4 rounded-xl sticky overflow-y-scroll overscroll-contain w-[16rem] h-[calc(100dvh-8rem)] -mb-[calc(100dvh-8rem)] top-[6rem] mr-10 prose-sm prose-ul:pl-1 prose-ul:ml-1 prose-li:my-2 prose-li:leading-tight prose-ul:list-['-']">
+            <div className="hidden md:block bg-stone-100 dark:bg-stone-900 text-stone-700 dark:text-stone-300 p-4 rounded-xl sticky overflow-y-scroll overscroll-contain w-[16rem] h-[calc(100dvh-8rem)] -mb-[calc(100dvh-8rem)] top-[6rem] mr-10 prose-sm prose-ul:pl-1 prose-ul:ml-1 prose-li:my-2 prose-li:leading-tight prose-ul:list-['-']">
                 <DocNav />
             </div>
 
-            <div className="md:ml-[20rem]">
+            <div className="md:ml-[20rem] text-base">
-                <Guide />
+                <Prose className="prose">
+                    <Guide />
 
-                <h1 id="faq">FAQ</h1>
+                    <h1 id="faq">FAQ</h1>
-
-                <p>
-                    <span className="text-amber-500">
-                        🚧 OH NO - We don&apos;t have any FAQ yet. 🚧
-                    </span>{" "}
-                    {"->"}{" "}
-                    <a href="https://github.com/gardencmp/jazz/issues/187">
-                        Complain on GitHub
-                    </a>
-                </p>
-
-                <div className="xl:-mr-[calc(50vw-40rem)]">
-                    <h1>API Reference</h1>
 
                     <p>
                         <span className="text-amber-500">
-                            🚧 OH NO - These docs are still highly
+                            🚧 OH NO - We don&apos;t have any FAQ yet. 🚧
-                            work-in-progress. 🚧
                         </span>{" "}
                         {"->"}{" "}
-                        <a href="https://github.com/gardencmp/jazz/issues/188">
+                        <a href="https://github.com/gardencmp/jazz/issues/187">
                             Complain on GitHub
                         </a>
                     </p>
+                </Prose>
 
-                    <PackageDocs package="jazz-tools" />
+                <div className="xl:-mr-[calc(50vw-40rem)]">
-                    <PackageDocs package="jazz-react" />
+                    <Prose>
-                    <PackageDocs package="jazz-browser" />
+                        <h1>API Reference</h1>
-                    <PackageDocs package="jazz-browser-media-images" />
+
-                    <PackageDocs package="jazz-nodejs" />
+                        <p>
+                            <span className="text-amber-500">
+                                🚧 OH NO - These docs are still highly
+                                work-in-progress. 🚧
+                            </span>{" "}
+                            {"->"}{" "}
+                            <a href="https://github.com/gardencmp/jazz/issues/188">
+                                Complain on GitHub
+                            </a>
+                        </p>
+                    </Prose>
+
+                    <div className="text-stone-800 dark:text-stone-200">
+                        <PackageDocs package="jazz-tools" />
+                        <PackageDocs package="jazz-react" />
+                        <PackageDocs package="jazz-browser" />
+                        <PackageDocs package="jazz-browser-media-images" />
+                        <PackageDocs package="jazz-nodejs" />
+                    </div>
                 </div>
             </div>
         </>

homepage/homepage/app/globals.css

@@ -394,12 +394,12 @@ body {
     --shiki-color-text: #606060;
     --shiki-color-background: transparent;
     --shiki-token-constant: #00a5a5;
-    --shiki-token-string: #1aa245;
+    --shiki-token-string: #4e3a2c;
     --shiki-token-comment: #aaa;
     --shiki-token-keyword: #7b8bff;
     --shiki-token-parameter: #ff9800;
     --shiki-token-function: #445dd7;
-    --shiki-token-string-expression: #1aa245;
+    --shiki-token-string-expression: #38a35f;
     --shiki-token-punctuation: #969696;
     --shiki-token-link: #1aa245;
 }
@@ -407,7 +407,7 @@ body {
 .dark body {
     --shiki-color-text: #d1d1d1;
     --shiki-token-constant: #2dc9c9;
-    --shiki-token-string: #ffab70;
+    --shiki-token-string: #feb179;
     --shiki-token-comment: #6b737c;
     --shiki-token-keyword: #7b8bff;
     --shiki-token-parameter: #ff9800;

homepage/homepage/app/layout.tsx

@@ -101,22 +101,7 @@ export default function RootLayout({
                         docNav={<DocNav />}
                     />
                     <main className="flex min-h-screen flex-col p-8 max-w-[80rem] w-full [&_*]:scroll-mt-[6rem]">
-                        <article
+                        {children}
-                            className={[
-                                "prose lg:prose-lg max-w-none prose-stone dark:prose-invert",
-                                "prose-headings:font-display",
-                                "prose-h1:text-5xl lg:prose-h1:text-6xl prose-h1:font-medium prose-h1:tracking-tighter",
-                                "prose-h2:text-2xl lg:prose-h2:text-3xl prose-h2:font-medium prose-h2:tracking-tight",
-                                "prose-p:max-w-3xl prose-p:leading-snug",
-                                "prose-strong:font-medium",
-                                "prose-code:font-normal prose-code:leading-tight prose-code:before:content-none prose-code:after:content-none prose-code:bg-stone-100 prose-code:dark:bg-stone-900 prose-code:p-1 prose-code:-my-1 prose-code:rounded",
-                                "prose-pre:max-w-3xl prose-pre:text-[0.8em] prose-pre:leading-[1.3] prose-pre:-mt-4 prose-pre:my-4 prose-pre:px-3 prose-pre:py-2 md:prose-pre:-mx-3 prose-pre:bg-stone-100 dark:prose-pre:bg-stone-900",
-
-                                "prose-inner-code:font-normal prose-inner-code:text-[1em]",
-                            ].join(" ")}
-                        >
-                            {children}
-                        </article>
                     </main>
                     <footer className="flex z-10 mt-10 min-h-[15rem] -mb-20 bg-stone-100 dark:bg-stone-900 text-stone-600 dark:text-stone-400 w-full justify-center">
                         <div className="p-8 max-w-[80rem] w-full grid grid-cols-3 md:grid-cols-4 lg:grid-cols-7 gap-8 max-sm:mb-12">

homepage/homepage/app/mesh/page.mdx

@@ -4,6 +4,7 @@ import {
     GridCard,
     GridItem,
     ComingSoonBadge,
+    Prose,
 } from "@/components/forMdx";
 
 export const metadata = {
@@ -13,6 +14,8 @@ export const metadata = {
 
 <div className="md:pt-20" />
 
+<Prose>
+
 # Sync & Storage Mesh
 
 <Slogan>The first Collaboration Delivery Network.</Slogan>
@@ -251,3 +254,5 @@ Costs:
 </div>
 </GridCard>
 </Grid>
+
+</Prose>

homepage/homepage/app/page.mdx

@@ -7,6 +7,7 @@ import {
     MultiplayerIcon,
     ResponsiveIframe,
     ComingSoonBadge,
+    Prose
 } from "@/components/forMdx";
 import { JazzLogo, LocalFirstConfLogo } from "@/components/logos";
 import {
@@ -25,10 +26,7 @@ import Link from "next/link";
 
 <div className="md:pt-20" />
 
-<a href="https://app.localfirstconf.com/schedule/conference/every-app-secretly-wants-to-be-local-first" className="-mt-8 md:-mt-20 float-right top-[5rem] right-4 border border-stone-700 dark:border-stone-300 rounded flex gap-3 items-center px-4 py-2 mb-4 rotate-2 md:rotate-6 no-underline hover:scale-105 transition-transform">
+<Prose>
-<div className="text-sm font-bold uppercase">See you in Berlin<br/>May 30-31!</div>
-<LocalFirstConfLogo className="w-24"/>
-</a>
 
 # Instant sync.
 
@@ -276,3 +274,5 @@ Jazz Mesh is currently free &mdash; and it's set up as the default sync & storag
 -   <Link href="https://discord.gg/utDMjHYg42" target="_blank">
         Join our Discord
     </Link>
+
+</Prose>

homepage/homepage/components/docs/nav.tsx

@@ -6,7 +6,7 @@ import { PackageIcon } from "lucide-react";
 export function DocNav() {
     return (
         <>
-            <p className="mt-0 not-prose font-medium">
+            <p className="mt-0 font-medium">
                 <DocNavLink href="#guide">Guide</DocNavLink>
             </p>
 
@@ -20,38 +20,38 @@ export function DocNav() {
                     </DocNavLink>
                     <ul>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#declaring-covalues">
                                 Declaration
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#reading-covalues">
                                 Reading
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#creating-covalues">
                                 Creation
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#editing-and-subscription">
                                 Editing & Subscription
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#persistence">
                                 Persistence
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#remote-sync">
                                 Remote Sync
                             </DocNavLink>
                         </li>
                         <li>
-                            <DocNavLink href="#intro-to-covalues">
+                            <DocNavLink href="#simple-public-sharing">
-                                Public Sharing
+                                Simple Public Sharing
                             </DocNavLink>
                         </li>
                     </ul>
@@ -67,8 +67,13 @@ export function DocNav() {
                     </DocNavLink>
                 </li>
                 <li>
-                    <DocNavLink href="#accounts-and-migrations">
+                    <DocNavLink href="#auth-accounts-and-migrations">
-                        Accounts & Migrations
+                        Auth, Accounts & Migrations
+                    </DocNavLink>
+                </li>
+                <li>
+                    <DocNavLink href="#edits-and-time-travel">
+                        Edit Metadata & Time Travel
                     </DocNavLink>
                 </li>
                 <li>
@@ -100,7 +105,7 @@ export async function NavPackage({
 
     return (
         <>
-            <h2 className="text-sm not-prose mt-4 flex gap-1 items-center -mx-4 px-4 pt-4 border-t border-stone-200 dark:border-stone-800 ">
+            <h2 className="text-sm mt-4 flex gap-1 items-center -mx-4 px-4 pt-4 border-t border-stone-200 dark:border-stone-800 ">
                 <code className="font-bold">{packageName}</code>{" "}
                 <PackageIcon size={15} strokeWidth={1.5} />
             </h2>
@@ -123,7 +128,7 @@ export async function NavPackage({
                                         <>
                                             <a
                                                 key={child.id}
-                                                className="inline-block not-prose px-1 m-0.5 bg-stone-200 dark:bg-stone-800 rounded opacity-70 hover:opacity-100 cursor-pointer"
+                                                className="text-sm inline-block px-2 m-0.5 text-stone-800 dark:text-stone-200 bg-stone-200 dark:bg-stone-800 rounded opacity-70 hover:opacity-100 cursor-pointer"
                                                 href={`#${packageName}/${child.name}`}
                                             >
                                                 <code>{child.name}</code>
@@ -150,7 +155,7 @@ export function DocNavLink({
     return (
         <a
             href={href}
-            className="not-prose hover:text-black dark:hover:text-white"
+            className="hover:text-black dark:hover:text-white py-1 hover:transition-colors"
         >
             {children}
         </a>

homepage/homepage/components/docs/packageDocs.tsx

@@ -213,7 +213,14 @@ function RenderClassOrInterface({
 function renderSummary(commentSummary: CommentDisplayPart[] | undefined) {
     return commentSummary?.map((part, idx) =>
         part.kind === "text" ? (
-            <span key={idx}>{part.text}</span>
+            <span key={idx}>
+                {part.text.split("\n").map((line, i, lines) => (
+                    <>
+                        {line}
+                        {i !== lines.length - 1 && <br />}
+                    </>
+                ))}
+            </span>
         ) : part.kind === "inline-tag" ? (
             <code key={idx}>
                 {part.tag} {part.text}

homepage/homepage/components/forMdx.tsx

@@ -1,3 +1,24 @@
+export function Prose(props: { children: ReactNode; className?: string }) {
+    return (
+        <div
+            className={[
+                "max-w-none prose-stone dark:prose-invert",
+                "prose-headings:font-display",
+                "prose-h1:text-5xl lg:prose-h1:text-6xl prose-h1:font-medium prose-h1:tracking-tighter",
+                "prose-h2:text-2xl lg:prose-h2:text-3xl prose-h2:font-medium prose-h2:tracking-tight",
+                "prose-p:max-w-3xl prose-p:leading-snug",
+                "prose-strong:font-medium",
+                "prose-code:font-normal prose-code:leading-tight prose-code:before:content-none prose-code:after:content-none prose-code:bg-stone-100 prose-code:dark:bg-stone-900 prose-code:p-1 prose-code:rounded",
+                "prose-pre:text-black dark:prose-pre:text-white prose-pre:max-w-3xl prose-pre:text-[0.8em] prose-pre:leading-[1.3] prose-pre:-mt-2 prose-pre:my-4 prose-pre:px-10 prose-pre:py-2 prose-pre:-mx-10 prose-pre:bg-transparent",
+                "[&_pre_.line]:relative [&_pre_.line]:min-h-[1.3em] [&_pre_.lineNo]:text-[0.75em] [&_pre_.lineNo]:text-stone-300 [&_pre_.lineNo]:dark:text-stone-700 [&_pre_.lineNo]:absolute [&_pre_.lineNo]:text-right [&_pre_.lineNo]:w-8 [&_pre_.lineNo]:-left-10 [&_pre_.lineNo]:top-[0.3em] [&_pre_.lineNo]:select-none",
+                props.className || "prose lg:prose-lg",
+            ].join(" ")}
+        >
+            {props.children}
+        </div>
+    );
+}
+
 export function Slogan(props: { children: ReactNode; small?: boolean }) {
     return (
         <div

homepage/homepage/next.config.mjs

@@ -41,21 +41,31 @@ function highlightPlugin() {
                 "css-variables",
             );
 
-            // match a meta tag like `subtle=0,1,2,3` and parse out the line numbers
+            let lineNo = -1;
-            const subtleTag = node.meta && node.meta.match(/subtle=\S+/);
-            const subtle =
-                subtleTag && subtleTag[0].split("=")[1].split(",").map(Number);
 
             node.type = "html";
             node.value = `<pre><code class="not-prose">${lines
-                .map((line, lineI) =>
+                .map((line) => {
-                    line
+                    const isSubduedLine = line.some((token) =>
-                        .map(
+                        token.content.includes("// old"),
-                            (token) =>
+                    );
-                                `<span style="color: ${token.color};${subtle?.includes(lineI + 1) ? "opacity: 0.3;" : ""}">${escape(token.content)}</span>`,
+                    const isBinnedLine = line.some((token) =>
-                        )
+                        token.content.includes("// *bin*"),
-                        .join(""),
+                    );
-                )
+                    if (!isBinnedLine) {
+                        lineNo++;
+                    }
+                    return (
+                        `<span class="line" style="${isBinnedLine ? "opacity: 0.3; text-decoration: line-through; user-select: none" : ""}"><div class="lineNo" style="${isSubduedLine ? "opacity: 0.3;" : ""}${isBinnedLine ? "color: red;" : ""}">${node.lang === "bash" ? ">" : isBinnedLine ? "✕" : (lineNo + 1)}</div>` +
+                        line
+                            .map(
+                                (token) =>
+                                    `<span style="color: ${isBinnedLine ? "red" : token.color};${isSubduedLine ? "opacity: 0.3;" : ""}">${escape(token.content.replace("// old", "").replace("// *bin*", ""))}</span>`,
+                            )
+                            .join("") +
+                        "</span>"
+                    );
+                })
                 .join("\n")}</code></pre>`;
             node.children = [];
             return SKIP;

homepage/homepage/tailwind.config.ts

@@ -92,12 +92,6 @@ const config: Config = {
             },
         },
     },
-    plugins: [
+    plugins: [tailwindCSSAnimate, typography()],
-        tailwindCSSAnimate,
-        typography(),
-        typography({
-            className: "prose-inner",
-        }),
-    ],
 };
 export default config;

packages/cojson-storage-indexeddb/CHANGELOG.md

@@ -1,198 +1,226 @@
 # cojson-storage-indexeddb
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+
 ## 0.7.0
 
 ### Patch Changes
 
-- c4151fc: Support stricter TS lint rules
+-   c4151fc: Support stricter TS lint rules
-- 952982e: Consistent proxy based API
+-   952982e: Consistent proxy based API
-- 21771c4: Reintroduce changes from main
+-   21771c4: Reintroduce changes from main
-- 69ac514: Use effect schema much less
+-   69ac514: Use effect schema much less
-- f0f6f1b: Clean up API more & re-add jazz-nodejs
+-   f0f6f1b: Clean up API more & re-add jazz-nodejs
-- 1a44f87: Refactoring
+-   1a44f87: Refactoring
-- 627d895: Get rid of Co namespace
+-   627d895: Get rid of Co namespace
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [96c494f]
+-   Updated dependencies [96c494f]
-- Updated dependencies [19f52b7]
+-   Updated dependencies [19f52b7]
-- Updated dependencies [d8fe2b1]
+-   Updated dependencies [d8fe2b1]
-- Updated dependencies [1200aae]
+-   Updated dependencies [1200aae]
-- Updated dependencies [52675c9]
+-   Updated dependencies [52675c9]
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [e299c3e]
+-   Updated dependencies [e299c3e]
-- Updated dependencies [bf0f8ec]
+-   Updated dependencies [bf0f8ec]
-- Updated dependencies [c4151fc]
+-   Updated dependencies [c4151fc]
-- Updated dependencies [8636319]
+-   Updated dependencies [8636319]
-- Updated dependencies [952982e]
+-   Updated dependencies [952982e]
-- Updated dependencies [21771c4]
+-   Updated dependencies [21771c4]
-- Updated dependencies [69ac514]
+-   Updated dependencies [69ac514]
-- Updated dependencies [f0f6f1b]
+-   Updated dependencies [f0f6f1b]
-- Updated dependencies [1a44f87]
+-   Updated dependencies [1a44f87]
-- Updated dependencies [63374cc]
+-   Updated dependencies [63374cc]
-  - cojson@0.7.0
+    -   cojson@0.7.0
 
 ## 0.7.0-alpha.42
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.42
+    -   cojson@0.7.0-alpha.42
 
 ## 0.7.0-alpha.39
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.39
+    -   cojson@0.7.0-alpha.39
 
 ## 0.7.0-alpha.38
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.38
+    -   cojson@0.7.0-alpha.38
 
 ## 0.7.0-alpha.37
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.37
+    -   cojson@0.7.0-alpha.37
 
 ## 0.7.0-alpha.36
 
 ### Patch Changes
 
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-  - cojson@0.7.0-alpha.36
+    -   cojson@0.7.0-alpha.36
 
 ## 0.7.0-alpha.35
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.35
+    -   cojson@0.7.0-alpha.35
 
 ## 0.7.0-alpha.29
 
 ### Patch Changes
 
-- Reintroduce changes from main
+-   Reintroduce changes from main
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.29
+    -   cojson@0.7.0-alpha.29
 
 ## 0.7.0-alpha.28
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.28
+    -   cojson@0.7.0-alpha.28
 
 ## 0.7.0-alpha.27
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.27
+    -   cojson@0.7.0-alpha.27
 
 ## 0.7.0-alpha.24
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.24
+    -   cojson@0.7.0-alpha.24
 
 ## 0.7.0-alpha.11
 
 ### Patch Changes
 
-- Support stricter TS lint rules
+-   Support stricter TS lint rules
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.11
+    -   cojson@0.7.0-alpha.11
 
 ## 0.7.0-alpha.10
 
 ### Patch Changes
 
-- Clean up API more & re-add jazz-nodejs
+-   Clean up API more & re-add jazz-nodejs
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.10
+    -   cojson@0.7.0-alpha.10
 
 ## 0.6.4-alpha.4
 
 ### Patch Changes
 
-- Consistent proxy based API
+-   Consistent proxy based API
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.7
+    -   cojson@0.7.0-alpha.7
 
 ## 0.6.4-alpha.3
 
 ### Patch Changes
 
-- Refactoring
+-   Refactoring
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.5
+    -   cojson@0.7.0-alpha.5
 
 ## 0.6.4-alpha.2
 
 ### Patch Changes
 
-- Get rid of Co namespace
+-   Get rid of Co namespace
 
 ## 0.6.4-alpha.1
 
 ### Patch Changes
 
-- Use effect schema much less
+-   Use effect schema much less
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.1
+    -   cojson@0.7.0-alpha.1
 
 ## 0.6.4-alpha.0
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.0
+    -   cojson@0.7.0-alpha.0
 
 ## 0.6.3
 
 ### Patch Changes
 
-- Fix ordering bugs with indexeddb
+-   Fix ordering bugs with indexeddb
 
 ## 0.6.2
 
 ### Patch Changes
 
-- Fix TypeScript lint
+-   Fix TypeScript lint
 
 ## 0.6.1
 
 ### Patch Changes
 
-- IndexedDB & timer perf improvements
+-   IndexedDB & timer perf improvements
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.6.4
+    -   cojson@0.6.4
 
 ## 0.6.0
 
 ### Minor Changes
 
-- Make addMember and removeMember take loaded Accounts instead of just IDs
+-   Make addMember and removeMember take loaded Accounts instead of just IDs
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.6.0
+    -   cojson@0.6.0
 
 ## 0.5.0
 
 ### Minor Changes
 
-- Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
+-   Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.5.0
+    -   cojson@0.5.0

packages/cojson-storage-indexeddb/package.json

@@ -1,14 +1,14 @@
 {
   "name": "cojson-storage-indexeddb",
-  "version": "0.7.0",
+  "version": "0.7.14",
   "main": "dist/index.js",
   "type": "module",
   "types": "src/index.ts",
   "license": "MIT",
   "dependencies": {
     "cojson": "workspace:*",
-    "typescript": "^5.1.6",
+    "effect": "^3.1.5",
-    "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae"
+    "typescript": "^5.1.6"
   },
   "devDependencies": {
     "@vitest/browser": "^0.34.1",

packages/cojson-storage-indexeddb/src/index.ts

@@ -6,14 +6,11 @@ import {
     CojsonInternalTypes,
     MAX_RECOMMENDED_TX_SIZE,
     AccountID,
+    IncomingSyncStream,
+    OutgoingSyncQueue,
 } from "cojson";
-import {
-    ReadableStream,
-    WritableStream,
-    ReadableStreamDefaultReader,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
 import { SyncPromise } from "./syncPromises.js";
+import { Effect, Queue, Stream } from "effect";
 
 type CoValueRow = {
     id: CojsonInternalTypes.RawCoID;
@@ -46,39 +43,35 @@ type SignatureAfterRow = {
 
 export class IDBStorage {
     db: IDBDatabase;
-    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: OutgoingSyncQueue;
-    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
 
     constructor(
         db: IDBDatabase,
-        fromLocalNode: ReadableStream<SyncMessage>,
+        fromLocalNode: IncomingSyncStream,
-        toLocalNode: WritableStream<SyncMessage>,
+        toLocalNode: OutgoingSyncQueue,
     ) {
         this.db = db;
-        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode;
-        this.toLocalNode = toLocalNode.getWriter();
 
-        void (async () => {
+        void fromLocalNode.pipe(
-            let done = false;
+            Stream.runForEach((msg) =>
-            while (!done) {
+                Effect.tryPromise({
-                const result = await this.fromLocalNode.read();
+                    try: () => this.handleSyncMessage(msg),
-                done = result.done;
+                    catch: (e) =>
-
+                        new Error(
-                if (result.value) {
+                            `Error reading from localNode, handling msg\n\n${JSON.stringify(
-                    // console.log(
+                                msg,
-                    //     "IDB: handling msg",
+                                (k, v) =>
-                    //     result.value.id,
+                                    k === "changes" || k === "encryptedChanges"
-                    //     result.value.action
+                                        ? v.slice(0, 20) + "..."
-                    // );
+                                        : v,
-                    await this.handleSyncMessage(result.value);
+                            )}`,
-                    // console.log(
+                            { cause: e },
-                    //     "IDB: handled msg",
+                        ),
-                    //     result.value.id,
+                }),
-                    //     result.value.action
+            ),
-                    // );
+            Effect.runPromise,
-                }
+        );
-            }
-        })();
     }
 
     static async asPeer(
@@ -89,23 +82,30 @@ export class IDBStorage {
             localNodeName: "local",
         },
     ): Promise<Peer> {
-        const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
+        return Effect.runPromise(
-            localNodeName,
+            Effect.gen(function* () {
-            "storage",
+                const [localNodeAsPeer, storageAsPeer] =
-            { peer1role: "client", peer2role: "server", trace },
+                    yield* cojsonInternals.connectedPeers(
-        );
+                        localNodeName,
+                        "storage",
+                        { peer1role: "client", peer2role: "server", trace },
+                    );
 
-        await IDBStorage.open(
+                yield* Effect.promise(() =>
-            localNodeAsPeer.incoming,
+                    IDBStorage.open(
-            localNodeAsPeer.outgoing,
+                        localNodeAsPeer.incoming,
-        );
+                        localNodeAsPeer.outgoing,
+                    ),
+                );
 
-        return { ...storageAsPeer, priority: 100 };
+                return { ...storageAsPeer, priority: 100 };
+            }),
+        );
     }
 
     static async open(
-        fromLocalNode: ReadableStream<SyncMessage>,
+        fromLocalNode: IncomingSyncStream,
-        toLocalNode: WritableStream<SyncMessage>,
+        toLocalNode: OutgoingSyncQueue,
     ) {
         const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
             const request = indexedDB.open("jazz-storage", 4);
@@ -150,23 +150,6 @@ export class IDBStorage {
                         keyPath: ["ses", "idx"],
                     });
                 }
-                // if (ev.oldVersion !== 0 && ev.oldVersion <= 3) {
-                //     // fix embarrassing off-by-one error for transaction indices
-                //     console.log("Migration: fixing off-by-one error");
-                //     const transaction = (
-                //         ev.target as unknown as { transaction: IDBTransaction }
-                //     ).transaction;
-
-                //     const txsStore = transaction.objectStore("transactions");
-                //     const txs = await promised(txsStore.getAll());
-
-                //     for (const tx of txs) {
-                //         await promised(txsStore.delete([tx.ses, tx.idx]));
-                //         tx.idx -= 1;
-                //         await promised(txsStore.add(tx));
-                //     }
-                //     console.log("Migration: fixing off-by-one error - done");
-                // }
             };
         });
 
@@ -409,29 +392,35 @@ export class IDBStorage {
                             ),
                         ).then(() => {
                             // we're done with IndexedDB stuff here so can use native Promises again
-                            setTimeout(async () => {
+                            setTimeout(() =>
-                                await this.toLocalNode.write({
+                                Effect.runPromise(
-                                    action: "known",
+                                    Effect.gen(this, function* () {
-                                    ...ourKnown,
+                                        yield* Queue.offer(this.toLocalNode, {
-                                    asDependencyOf,
+                                            action: "known",
-                                });
+                                            ...ourKnown,
+                                            asDependencyOf,
+                                        });
 
-                                const nonEmptyNewContentPieces =
+                                        const nonEmptyNewContentPieces =
-                                    newContentPieces.filter(
+                                            newContentPieces.filter(
-                                        (piece) =>
+                                                (piece) =>
-                                            piece.header ||
+                                                    piece.header ||
-                                            Object.keys(piece.new).length > 0,
+                                                    Object.keys(piece.new)
-                                    );
+                                                        .length > 0,
+                                            );
 
-                                // console.log(theirKnown.id, nonEmptyNewContentPieces);
+                                        // console.log(theirKnown.id, nonEmptyNewContentPieces);
 
-                                for (const piece of nonEmptyNewContentPieces) {
+                                        for (const piece of nonEmptyNewContentPieces) {
-                                    await this.toLocalNode.write(piece);
+                                            yield* Queue.offer(
-                                    await new Promise((resolve) =>
+                                                this.toLocalNode,
-                                        setTimeout(resolve, 0),
+                                                piece,
-                                    );
+                                            );
-                                }
+                                            yield* Effect.yieldNow();
-                            }, 0);
+                                        }
+                                    }),
+                                ),
+                            );
 
                             return Promise.resolve();
                         });
@@ -456,13 +445,15 @@ export class IDBStorage {
                     const header = msg.header;
                     if (!header) {
                         console.error("Expected to be sent header first");
-                        void this.toLocalNode.write({
+                        void Effect.runPromise(
-                            action: "known",
+                            Queue.offer(this.toLocalNode, {
-                            id: msg.id,
+                                action: "known",
-                            header: false,
+                                id: msg.id,
-                            sessions: {},
+                                header: false,
-                            isCorrection: true,
+                                sessions: {},
-                        });
+                                isCorrection: true,
+                            }),
+                        );
                         throw new Error("Expected to be sent header first");
                     }
 
@@ -524,11 +515,13 @@ export class IDBStorage {
                         ),
                     ).then(() => {
                         if (invalidAssumptions) {
-                            void this.toLocalNode.write({
+                            void Effect.runPromise(
-                                action: "known",
+                                Queue.offer(this.toLocalNode, {
-                                ...ourKnown,
+                                    action: "known",
-                                isCorrection: invalidAssumptions,
+                                    ...ourKnown,
-                            });
+                                    isCorrection: invalidAssumptions,
+                                }),
+                            );
                         }
                     });
                 });

packages/cojson-storage-sqlite/CHANGELOG.md

@@ -1,155 +1,183 @@
 # cojson-storage-sqlite
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+
 ## 0.7.0
 
 ### Patch Changes
 
-- c4151fc: Support stricter TS lint rules
+-   c4151fc: Support stricter TS lint rules
-- 21771c4: Reintroduce changes from main
+-   21771c4: Reintroduce changes from main
-- 69ac514: Use effect schema much less
+-   69ac514: Use effect schema much less
-- f0f6f1b: Clean up API more & re-add jazz-nodejs
+-   f0f6f1b: Clean up API more & re-add jazz-nodejs
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [96c494f]
+-   Updated dependencies [96c494f]
-- Updated dependencies [19f52b7]
+-   Updated dependencies [19f52b7]
-- Updated dependencies [d8fe2b1]
+-   Updated dependencies [d8fe2b1]
-- Updated dependencies [1200aae]
+-   Updated dependencies [1200aae]
-- Updated dependencies [52675c9]
+-   Updated dependencies [52675c9]
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [e299c3e]
+-   Updated dependencies [e299c3e]
-- Updated dependencies [bf0f8ec]
+-   Updated dependencies [bf0f8ec]
-- Updated dependencies [c4151fc]
+-   Updated dependencies [c4151fc]
-- Updated dependencies [8636319]
+-   Updated dependencies [8636319]
-- Updated dependencies [952982e]
+-   Updated dependencies [952982e]
-- Updated dependencies [21771c4]
+-   Updated dependencies [21771c4]
-- Updated dependencies [69ac514]
+-   Updated dependencies [69ac514]
-- Updated dependencies [f0f6f1b]
+-   Updated dependencies [f0f6f1b]
-- Updated dependencies [1a44f87]
+-   Updated dependencies [1a44f87]
-- Updated dependencies [63374cc]
+-   Updated dependencies [63374cc]
-  - cojson@0.7.0
+    -   cojson@0.7.0
 
 ## 0.7.0-alpha.42
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.42
+    -   cojson@0.7.0-alpha.42
 
 ## 0.7.0-alpha.39
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.39
+    -   cojson@0.7.0-alpha.39
 
 ## 0.7.0-alpha.38
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.38
+    -   cojson@0.7.0-alpha.38
 
 ## 0.7.0-alpha.37
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.37
+    -   cojson@0.7.0-alpha.37
 
 ## 0.7.0-alpha.36
 
 ### Patch Changes
 
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-- Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
-  - cojson@0.7.0-alpha.36
+    -   cojson@0.7.0-alpha.36
 
 ## 0.7.0-alpha.35
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.35
+    -   cojson@0.7.0-alpha.35
 
 ## 0.7.0-alpha.29
 
 ### Patch Changes
 
-- Reintroduce changes from main
+-   Reintroduce changes from main
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.29
+    -   cojson@0.7.0-alpha.29
 
 ## 0.7.0-alpha.28
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.28
+    -   cojson@0.7.0-alpha.28
 
 ## 0.7.0-alpha.27
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.27
+    -   cojson@0.7.0-alpha.27
 
 ## 0.7.0-alpha.24
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.24
+    -   cojson@0.7.0-alpha.24
 
 ## 0.7.0-alpha.11
 
 ### Patch Changes
 
-- Support stricter TS lint rules
+-   Support stricter TS lint rules
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.11
+    -   cojson@0.7.0-alpha.11
 
 ## 0.7.0-alpha.10
 
 ### Patch Changes
 
-- Clean up API more & re-add jazz-nodejs
+-   Clean up API more & re-add jazz-nodejs
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.10
+    -   cojson@0.7.0-alpha.10
 
 ## 0.5.3-alpha.1
 
 ### Patch Changes
 
-- Use effect schema much less
+-   Use effect schema much less
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.1
+    -   cojson@0.7.0-alpha.1
 
 ## 0.5.3-alpha.0
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.7.0-alpha.0
+    -   cojson@0.7.0-alpha.0
 
 ## 0.5.2
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.6.0
+    -   cojson@0.6.0
 
 ## 0.5.1
 
 ### Patch Changes
 
-- Make typedefs for better-sqlite3 a normal dependency
+-   Make typedefs for better-sqlite3 a normal dependency
 
 ## 0.5.0
 
 ### Minor Changes
 
-- Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
+-   Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
 
 ### Patch Changes
 
-- Updated dependencies
+-   Updated dependencies
-  - cojson@0.5.0
+    -   cojson@0.5.0

packages/cojson-storage-sqlite/package.json

@@ -1,15 +1,15 @@
 {
   "name": "cojson-storage-sqlite",
   "type": "module",
-  "version": "0.7.0",
+  "version": "0.7.14",
   "main": "dist/index.js",
   "types": "src/index.ts",
   "license": "MIT",
   "dependencies": {
     "better-sqlite3": "^8.5.2",
     "cojson": "workspace:*",
-    "typescript": "^5.1.6",
+    "effect": "^3.1.5",
-    "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae"
+    "typescript": "^5.1.6"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.4"

packages/cojson-storage-sqlite/src/index.ts

@@ -6,15 +6,12 @@ import {
     SessionID,
     MAX_RECOMMENDED_TX_SIZE,
     AccountID,
+    IncomingSyncStream,
+    OutgoingSyncQueue,
 } from "cojson";
-import {
-    ReadableStream,
-    WritableStream,
-    ReadableStreamDefaultReader,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
 
 import Database, { Database as DatabaseT } from "better-sqlite3";
+import { Effect, Queue, Stream } from "effect";
 
 type CoValueRow = {
     id: CojsonInternalTypes.RawCoID;
@@ -46,30 +43,36 @@ type SignatureAfterRow = {
 };
 
 export class SQLiteStorage {
-    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: OutgoingSyncQueue;
-    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
     db: DatabaseT;
 
     constructor(
         db: DatabaseT,
-        fromLocalNode: ReadableStream<SyncMessage>,
+        fromLocalNode: IncomingSyncStream,
-        toLocalNode: WritableStream<SyncMessage>,
+        toLocalNode: OutgoingSyncQueue,
     ) {
         this.db = db;
-        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode;
-        this.toLocalNode = toLocalNode.getWriter();
 
-        void (async () => {
+        void fromLocalNode.pipe(
-            let done = false;
+            Stream.runForEach((msg) =>
-            while (!done) {
+                Effect.tryPromise({
-                const result = await this.fromLocalNode.read();
+                    try: () => this.handleSyncMessage(msg),
-                done = result.done;
+                    catch: (e) =>
-
+                        new Error(
-                if (result.value) {
+                            `Error reading from localNode, handling msg\n\n${JSON.stringify(
-                    await this.handleSyncMessage(result.value);
+                                msg,
-                }
+                                (k, v) =>
-            }
+                                    k === "changes" || k === "encryptedChanges"
-        })();
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                            )}`,
+                            { cause: e },
+                        ),
+                }),
+            ),
+            Effect.runPromise,
+        );
     }
 
     static async asPeer({
@@ -81,25 +84,32 @@ export class SQLiteStorage {
         trace?: boolean;
         localNodeName?: string;
     }): Promise<Peer> {
-        const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
+        return Effect.runPromise(
-            localNodeName,
+            Effect.gen(function* () {
-            "storage",
+                const [localNodeAsPeer, storageAsPeer] =
-            { peer1role: "client", peer2role: "server", trace },
+                    yield* cojsonInternals.connectedPeers(
-        );
+                        localNodeName,
+                        "storage",
+                        { peer1role: "client", peer2role: "server", trace },
+                    );
 
-        await SQLiteStorage.open(
+                yield* Effect.promise(() =>
-            filename,
+                    SQLiteStorage.open(
-            localNodeAsPeer.incoming,
+                        filename,
-            localNodeAsPeer.outgoing,
+                        localNodeAsPeer.incoming,
-        );
+                        localNodeAsPeer.outgoing,
+                    ),
+                );
 
-        return { ...storageAsPeer, priority: 100 };
+                return { ...storageAsPeer, priority: 100 };
+            }),
+        );
     }
 
     static async open(
         filename: string,
-        fromLocalNode: ReadableStream<SyncMessage>,
+        fromLocalNode: IncomingSyncStream,
-        toLocalNode: WritableStream<SyncMessage>,
+        toLocalNode: OutgoingSyncQueue,
     ) {
         const db = Database(filename);
         db.pragma("journal_mode = WAL");
@@ -431,11 +441,13 @@ export class SQLiteStorage {
             );
         }
 
-        await this.toLocalNode.write({
+        await Effect.runPromise(
-            action: "known",
+            Queue.offer(this.toLocalNode, {
-            ...ourKnown,
+                action: "known",
-            asDependencyOf,
+                ...ourKnown,
-        });
+                asDependencyOf,
+            }),
+        );
 
         const nonEmptyNewContentPieces = newContentPieces.filter(
             (piece) => piece.header || Object.keys(piece.new).length > 0,
@@ -444,7 +456,7 @@ export class SQLiteStorage {
         // console.log(theirKnown.id, nonEmptyNewContentPieces);
 
         for (const piece of nonEmptyNewContentPieces) {
-            await this.toLocalNode.write(piece);
+            await Effect.runPromise(Queue.offer(this.toLocalNode, piece));
             await new Promise((resolve) => setTimeout(resolve, 0));
         }
     }
@@ -466,13 +478,15 @@ export class SQLiteStorage {
             const header = msg.header;
             if (!header) {
                 console.error("Expected to be sent header first");
-                await this.toLocalNode.write({
+                await Effect.runPromise(
-                    action: "known",
+                    Queue.offer(this.toLocalNode, {
-                    id: msg.id,
+                        action: "known",
-                    header: false,
+                        id: msg.id,
-                    sessions: {},
+                        header: false,
-                    isCorrection: true,
+                        sessions: {},
-                });
+                        isCorrection: true,
+                    }),
+                );
                 return;
             }
 
@@ -604,11 +618,13 @@ export class SQLiteStorage {
         })();
 
         if (invalidAssumptions) {
-            await this.toLocalNode.write({
+            await Effect.runPromise(
-                action: "known",
+                Queue.offer(this.toLocalNode, {
-                ...ourKnown,
+                    action: "known",
-                isCorrection: invalidAssumptions,
+                    ...ourKnown,
-            });
+                    isCorrection: invalidAssumptions,
+                }),
+            );
         }
     }
 

packages/cojson-transport-nodejs-ws/CHANGELOG.md

@@ -1,161 +0,0 @@
-# cojson-transport-nodejs-ws
-
-## 0.7.0
-
-### Patch Changes
-
-- c4151fc: Support stricter TS lint rules
-- 21771c4: Reintroduce changes from main
-- f0f6f1b: Clean up API more & re-add jazz-nodejs
-- 627d895: Get rid of Co namespace
-- a423eee: ignore error on ws close, fixing "Invalid state: Controller is already closed"
-- Updated dependencies [1a35307]
-- Updated dependencies [96c494f]
-- Updated dependencies [19f52b7]
-- Updated dependencies [d8fe2b1]
-- Updated dependencies [1200aae]
-- Updated dependencies [52675c9]
-- Updated dependencies [1a35307]
-- Updated dependencies [e299c3e]
-- Updated dependencies [bf0f8ec]
-- Updated dependencies [c4151fc]
-- Updated dependencies [8636319]
-- Updated dependencies [952982e]
-- Updated dependencies [21771c4]
-- Updated dependencies [69ac514]
-- Updated dependencies [f0f6f1b]
-- Updated dependencies [1a44f87]
-- Updated dependencies [63374cc]
-  - cojson@0.7.0
-
-## 0.7.0-alpha.42
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.42
-
-## 0.7.0-alpha.41
-
-### Patch Changes
-
-- ignore error on ws close, fixing "Invalid state: Controller is already closed"
-
-## 0.7.0-alpha.39
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.39
-
-## 0.7.0-alpha.38
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.38
-
-## 0.7.0-alpha.37
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.37
-
-## 0.7.0-alpha.36
-
-### Patch Changes
-
-- Updated dependencies [1a35307]
-- Updated dependencies [1a35307]
-  - cojson@0.7.0-alpha.36
-
-## 0.7.0-alpha.35
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.35
-
-## 0.7.0-alpha.29
-
-### Patch Changes
-
-- Reintroduce changes from main
-- Updated dependencies
-  - cojson@0.7.0-alpha.29
-
-## 0.7.0-alpha.28
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.28
-
-## 0.7.0-alpha.27
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.27
-
-## 0.7.0-alpha.24
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.24
-
-## 0.7.0-alpha.11
-
-### Patch Changes
-
-- Support stricter TS lint rules
-- Updated dependencies
-  - cojson@0.7.0-alpha.11
-
-## 0.7.0-alpha.10
-
-### Patch Changes
-
-- Clean up API more & re-add jazz-nodejs
-- Updated dependencies
-  - cojson@0.7.0-alpha.10
-
-## 0.5.2-alpha.2
-
-### Patch Changes
-
-- Get rid of Co namespace
-
-## 0.5.2-alpha.1
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.1
-
-## 0.5.2-alpha.0
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.7.0-alpha.0
-
-## 0.5.1
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.6.0
-
-## 0.5.0
-
-### Minor Changes
-
-- Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
-
-### Patch Changes
-
-- Updated dependencies
-  - cojson@0.5.0

packages/cojson-transport-nodejs-ws/src/index.ts

@@ -1,94 +0,0 @@
-import { WebSocket } from "ws";
-import { WritableStream, ReadableStream } from "isomorphic-streams";
-
-export function websocketReadableStream<T>(ws: WebSocket) {
-    ws.binaryType = "arraybuffer";
-
-    return new ReadableStream<T>({
-        start(controller) {
-            ws.addEventListener("message", (event) => {
-                if (typeof event.data !== "string")
-                    return console.warn(
-                        "Got non-string message from client",
-                        event.data,
-                    );
-                const msg = JSON.parse(event.data);
-                if (msg.type === "ping") {
-                    // console.debug(
-                    //     "Got ping from",
-                    //     msg.dc,
-                    //     "latency",
-                    //     Date.now() - msg.time,
-                    //     "ms"
-                    // );
-                    return;
-                }
-                controller.enqueue(msg);
-            });
-            ws.addEventListener("close", () => {
-                try {
-                    controller.close();
-                } catch (ignore) {
-                    // will throw if already closed, with no way to check before-hand
-                }
-            });
-            ws.addEventListener("error", () =>
-                controller.error(new Error("The WebSocket errored!")),
-            );
-        },
-
-        cancel() {
-            ws.close();
-        },
-    });
-}
-
-export function websocketWritableStream<T>(ws: WebSocket) {
-    return new WritableStream<T>({
-        start(controller) {
-            ws.addEventListener("close", () =>
-                controller.error(
-                    new Error("The WebSocket closed unexpectedly!"),
-                ),
-            );
-            ws.addEventListener("error", () =>
-                controller.error(new Error("The WebSocket errored!")),
-            );
-
-            if (ws.readyState === WebSocket.OPEN) {
-                return;
-            }
-
-            return new Promise((resolve) =>
-                ws.addEventListener("open", resolve, { once: true }),
-            );
-        },
-
-        write(chunk) {
-            ws.send(JSON.stringify(chunk));
-            // Return immediately, since the web socket gives us no easy way to tell
-            // when the write completes.
-        },
-
-        close() {
-            return closeWS(1000);
-        },
-
-        abort(reason) {
-            return closeWS(4000, reason && reason.message);
-        },
-    });
-
-    function closeWS(code: number, reasonString?: string) {
-        return new Promise<void>((resolve, reject) => {
-            ws.onclose = (e) => {
-                if (e.wasClean) {
-                    resolve();
-                } else {
-                    reject(new Error("The connection was not closed cleanly"));
-                }
-            };
-            ws.close(code, reasonString);
-        });
-    }
-}

packages/cojson-transport-ws/CHANGELOG.md

@@ -0,0 +1,189 @@
+# cojson-transport-nodejs-ws
+
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+
+## 0.7.0
+
+### Patch Changes
+
+-   c4151fc: Support stricter TS lint rules
+-   21771c4: Reintroduce changes from main
+-   f0f6f1b: Clean up API more & re-add jazz-nodejs
+-   627d895: Get rid of Co namespace
+-   a423eee: ignore error on ws close, fixing "Invalid state: Controller is already closed"
+-   Updated dependencies [1a35307]
+-   Updated dependencies [96c494f]
+-   Updated dependencies [19f52b7]
+-   Updated dependencies [d8fe2b1]
+-   Updated dependencies [1200aae]
+-   Updated dependencies [52675c9]
+-   Updated dependencies [1a35307]
+-   Updated dependencies [e299c3e]
+-   Updated dependencies [bf0f8ec]
+-   Updated dependencies [c4151fc]
+-   Updated dependencies [8636319]
+-   Updated dependencies [952982e]
+-   Updated dependencies [21771c4]
+-   Updated dependencies [69ac514]
+-   Updated dependencies [f0f6f1b]
+-   Updated dependencies [1a44f87]
+-   Updated dependencies [63374cc]
+    -   cojson@0.7.0
+
+## 0.7.0-alpha.42
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.42
+
+## 0.7.0-alpha.41
+
+### Patch Changes
+
+-   ignore error on ws close, fixing "Invalid state: Controller is already closed"
+
+## 0.7.0-alpha.39
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.39
+
+## 0.7.0-alpha.38
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.38
+
+## 0.7.0-alpha.37
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.37
+
+## 0.7.0-alpha.36
+
+### Patch Changes
+
+-   Updated dependencies [1a35307]
+-   Updated dependencies [1a35307]
+    -   cojson@0.7.0-alpha.36
+
+## 0.7.0-alpha.35
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.35
+
+## 0.7.0-alpha.29
+
+### Patch Changes
+
+-   Reintroduce changes from main
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.29
+
+## 0.7.0-alpha.28
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.28
+
+## 0.7.0-alpha.27
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.27
+
+## 0.7.0-alpha.24
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.24
+
+## 0.7.0-alpha.11
+
+### Patch Changes
+
+-   Support stricter TS lint rules
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.11
+
+## 0.7.0-alpha.10
+
+### Patch Changes
+
+-   Clean up API more & re-add jazz-nodejs
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.10
+
+## 0.5.2-alpha.2
+
+### Patch Changes
+
+-   Get rid of Co namespace
+
+## 0.5.2-alpha.1
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.1
+
+## 0.5.2-alpha.0
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.0-alpha.0
+
+## 0.5.1
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.6.0
+
+## 0.5.0
+
+### Minor Changes
+
+-   Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.5.0

packages/cojson-transport-ws/package.json

@@ -1,15 +1,14 @@
 {
-  "name": "cojson-transport-nodejs-ws",
+  "name": "cojson-transport-ws",
   "type": "module",
-  "version": "0.7.0",
+  "version": "0.7.14",
   "main": "dist/index.js",
   "types": "src/index.ts",
   "license": "MIT",
   "dependencies": {
     "cojson": "workspace:*",
-    "typescript": "^5.1.6",
+    "effect": "^3.1.5",
-    "ws": "^8.14.2",
+    "typescript": "^5.1.6"
-    "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae"
   },
   "scripts": {
     "dev": "tsc --watch --sourceMap --outDir dist",

packages/cojson-transport-ws/src/index.ts

@@ -0,0 +1,108 @@
+import { DisconnectedError, Peer, PingTimeoutError, SyncMessage } from "cojson";
+import { Either, Stream, Queue, Effect, Exit } from "effect";
+
+interface AnyWebSocket {
+    addEventListener(
+        type: "close",
+        listener: (event: { code: number; reason: string }) => void,
+    ): void;
+    addEventListener(
+        type: "message",
+        listener: (event: { data: string | unknown }) => void,
+    ): void;
+    addEventListener(type: "open", listener: () => void): void;
+    close(): void;
+    send(data: string): void;
+}
+
+export function createWebSocketPeer(options: {
+    id: string;
+    websocket: AnyWebSocket;
+    role: Peer["role"];
+}): Effect.Effect<Peer> {
+    return Effect.gen(function* () {
+        const ws = options.websocket;
+
+        const incoming =
+            yield* Queue.unbounded<
+                Either.Either<SyncMessage, DisconnectedError | PingTimeoutError>
+            >();
+        const outgoing = yield* Queue.unbounded<SyncMessage>();
+
+        ws.addEventListener("close", (event) => {
+            void Effect.runPromiseExit(
+                Queue.offer(
+                    incoming,
+                    Either.left(
+                        new DisconnectedError(`${event.code}: ${event.reason}`),
+                    ),
+                ),
+            ).then((e) => {
+                if (Exit.isFailure(e) && !Exit.isInterrupted(e)) {
+                    console.warn("Failed closing ws", e);
+                }
+            });
+        });
+
+        let pingTimeout: ReturnType<typeof setTimeout> | undefined;
+
+        ws.addEventListener("message", (event) => {
+            const msg = JSON.parse(event.data as string);
+
+            if (pingTimeout) {
+                clearTimeout(pingTimeout);
+            }
+
+            pingTimeout = setTimeout(() => {
+                console.debug("Ping timeout");
+                void Effect.runPromise(
+                    Queue.offer(incoming, Either.left(new PingTimeoutError())),
+                );
+                try {
+                    ws.close();
+                } catch (e) {
+                    console.error(
+                        "Error while trying to close ws on ping timeout",
+                        e,
+                    );
+                }
+            }, 2500);
+
+            if (msg.type === "ping") {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                (globalThis as any).jazzPings =
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    (globalThis as any).jazzPings || [];
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                (globalThis as any).jazzPings.push({
+                    received: Date.now(),
+                    sent: msg.time,
+                    dc: msg.dc,
+                });
+                return;
+            } else {
+                void Effect.runPromise(
+                    Queue.offer(incoming, Either.right(msg)),
+                );
+            }
+        });
+
+        ws.addEventListener("open", () => {
+            void Stream.fromQueue(outgoing).pipe(
+                Stream.runForEach((msg) =>
+                    Effect.sync(() => ws.send(JSON.stringify(msg))),
+                ),
+                Effect.runPromise,
+            );
+        });
+
+        return {
+            id: options.id,
+            incoming: Stream.fromQueue(incoming, { shutdown: true }).pipe(
+                Stream.mapEffect((either) => either),
+            ),
+            outgoing,
+            role: options.role,
+        };
+    });
+}

packages/cojson/CHANGELOG.md

@@ -1,183 +1,207 @@
 # cojson
 
+## 0.7.14
+
+### Patch Changes
+
+-   Use Effect Queues and Streams instead of custom queue implementation
+
+## 0.7.11
+
+### Patch Changes
+
+-   Fix webpack import of node:crypto module
+
+## 0.7.10
+
+### Patch Changes
+
+-   Also cache agent ID in RawControlledAccount
+
+## 0.7.9
+
+### Patch Changes
+
+-   Cache currentAgentID in RawAccount
+
 ## 0.7.0
 
 ### Minor Changes
 
-- e299c3e: New simplified API
+-   e299c3e: New simplified API
 
 ### Patch Changes
 
-- 1a35307: WIP working-ish version of LSM storage
+-   1a35307: WIP working-ish version of LSM storage
-- 96c494f: Implement profile visibility based on groups & new migration signature
+-   96c494f: Implement profile visibility based on groups & new migration signature
-- 19f52b7: Fixed bug with newRandomSessionID being called before crypto was ready
+-   19f52b7: Fixed bug with newRandomSessionID being called before crypto was ready
-- d8fe2b1: Expose experimental OPFS storage
+-   d8fe2b1: Expose experimental OPFS storage
-- 1200aae: CoJSON performance improvement
+-   1200aae: CoJSON performance improvement
-- 52675c9: Fix CoList.splice / RawCoList.append
+-   52675c9: Fix CoList.splice / RawCoList.append
-- 1a35307: Optimizations for incoming sync messages
+-   1a35307: Optimizations for incoming sync messages
-- bf0f8ec: Fix noble curves dependency
+-   bf0f8ec: Fix noble curves dependency
-- c4151fc: Support stricter TS lint rules
+-   c4151fc: Support stricter TS lint rules
-- 8636319: Factor out implementation of crypto provider and provide pure JS implementation
+-   8636319: Factor out implementation of crypto provider and provide pure JS implementation
-- 952982e: Consistent proxy based API
+-   952982e: Consistent proxy based API
-- 21771c4: Reintroduce changes from main
+-   21771c4: Reintroduce changes from main
-- 69ac514: Use effect schema much less
+-   69ac514: Use effect schema much less
-- f0f6f1b: Clean up API more & re-add jazz-nodejs
+-   f0f6f1b: Clean up API more & re-add jazz-nodejs
-- 1a44f87: Refactoring
+-   1a44f87: Refactoring
-- 63374cc: Make sure delete on CoMaps deletes keys
+-   63374cc: Make sure delete on CoMaps deletes keys
 
 ## 0.7.0-alpha.42
 
 ### Patch Changes
 
-- Fixed bug with newRandomSessionID being called before crypto was ready
+-   Fixed bug with newRandomSessionID being called before crypto was ready
 
 ## 0.7.0-alpha.39
 
 ### Patch Changes
 
-- Fix noble curves dependency
+-   Fix noble curves dependency
 
 ## 0.7.0-alpha.38
 
 ### Patch Changes
 
-- Factor out implementation of crypto provider and provide pure JS implementation
+-   Factor out implementation of crypto provider and provide pure JS implementation
 
 ## 0.7.0-alpha.37
 
 ### Patch Changes
 
-- Expose experimental OPFS storage
+-   Expose experimental OPFS storage
 
 ## 0.7.0-alpha.36
 
 ### Patch Changes
 
-- 1a35307: WIP working-ish version of LSM storage
+-   1a35307: WIP working-ish version of LSM storage
-- 1a35307: Optimizations for incoming sync messages
+-   1a35307: Optimizations for incoming sync messages
 
 ## 0.7.0-alpha.35
 
 ### Patch Changes
 
-- CoJSON performance improvement
+-   CoJSON performance improvement
 
 ## 0.7.0-alpha.29
 
 ### Patch Changes
 
-- Reintroduce changes from main
+-   Reintroduce changes from main
 
 ## 0.7.0-alpha.28
 
 ### Patch Changes
 
-- Implement profile visibility based on groups & new migration signature
+-   Implement profile visibility based on groups & new migration signature
 
 ## 0.7.0-alpha.27
 
 ### Patch Changes
 
-- Fix CoList.splice / RawCoList.append
+-   Fix CoList.splice / RawCoList.append
 
 ## 0.7.0-alpha.24
 
 ### Patch Changes
 
-- Make sure delete on CoMaps deletes keys
+-   Make sure delete on CoMaps deletes keys
 
 ## 0.7.0-alpha.11
 
 ### Patch Changes
 
-- Support stricter TS lint rules
+-   Support stricter TS lint rules
 
 ## 0.7.0-alpha.10
 
 ### Patch Changes
 
-- Clean up API more & re-add jazz-nodejs
+-   Clean up API more & re-add jazz-nodejs
 
 ## 0.7.0-alpha.7
 
 ### Patch Changes
 
-- Consistent proxy based API
+-   Consistent proxy based API
 
 ## 0.7.0-alpha.5
 
 ### Patch Changes
 
-- Refactoring
+-   Refactoring
 
 ## 0.7.0-alpha.1
 
 ### Patch Changes
 
-- Use effect schema much less
+-   Use effect schema much less
 
 ## 0.7.0-alpha.0
 
 ### Minor Changes
 
-- New simplified API
+-   New simplified API
 
 ## 0.6.6
 
 ### Patch Changes
 
-- Fix migration changes being lost on loaded account
+-   Fix migration changes being lost on loaded account
 
 ## 0.6.5
 
 ### Patch Changes
 
-- Fix loading of accounts
+-   Fix loading of accounts
 
 ## 0.6.4
 
 ### Patch Changes
 
-- IndexedDB & timer perf improvements
+-   IndexedDB & timer perf improvements
 
 ## 0.6.3
 
 ### Patch Changes
 
-- Implement passphrase based auth
+-   Implement passphrase based auth
 
 ## 0.6.2
 
 ### Patch Changes
 
-- Add peersToLoadFrom for node creation as well
+-   Add peersToLoadFrom for node creation as well
 
 ## 0.6.1
 
 ### Patch Changes
 
-- Provide localNode to AccountMigrations
+-   Provide localNode to AccountMigrations
 
 ## 0.6.0
 
 ### Minor Changes
 
-- Make addMember and removeMember take loaded Accounts instead of just IDs
+-   Make addMember and removeMember take loaded Accounts instead of just IDs
 
 ## 0.5.2
 
 ### Patch Changes
 
-- Allow account migrations to be async
+-   Allow account migrations to be async
 
 ## 0.5.1
 
 ### Patch Changes
 
-- Fix bug where accounts, profiles and data created in migrations isn't synced on account creation
+-   Fix bug where accounts, profiles and data created in migrations isn't synced on account creation
 
 ## 0.5.0
 
 ### Minor Changes
 
-- Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.
+-   Adding a lot of performance improvements to cojson, add a stresstest for the twit example and make that run smoother in a lot of ways.

packages/cojson/package.json

@@ -5,7 +5,7 @@
     "types": "src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.7.0",
+    "version": "0.7.14",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@typescript-eslint/eslint-plugin": "^6.2.1",
@@ -23,8 +23,7 @@
         "@noble/hashes": "^1.4.0",
         "@scure/base": "^1.1.1",
         "effect": "^3.1.5",
-        "hash-wasm": "^4.9.0",
+        "hash-wasm": "^4.9.0"
-        "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae"
     },
     "scripts": {
         "dev": "tsc --watch --sourceMap --outDir dist",

packages/cojson/src/coValues/account.ts

@@ -33,7 +33,12 @@ export function accountHeaderForInitialAgentSecret(
 export class RawAccount<
     Meta extends AccountMeta = AccountMeta,
 > extends RawGroup<Meta> {
+    _cachedCurrentAgentID: AgentID | undefined;
+
     currentAgentID(): AgentID {
+        if (this._cachedCurrentAgentID) {
+            return this._cachedCurrentAgentID;
+        }
         const agents = this.keys().filter((k): k is AgentID =>
             k.startsWith("sealer_"),
         );
@@ -44,6 +49,8 @@ export class RawAccount<
             );
         }
 
+        this._cachedCurrentAgentID = agents[0];
+
         return agents[0]!;
     }
 }
@@ -90,7 +97,12 @@ export class RawControlledAccount<Meta extends AccountMeta = AccountMeta>
     }
 
     currentAgentID(): AgentID {
-        return this.crypto.getAgentID(this.agentSecret);
+        if (this._cachedCurrentAgentID) {
+            return this._cachedCurrentAgentID;
+        }
+        const agentID = this.crypto.getAgentID(this.agentSecret);
+        this._cachedCurrentAgentID = agentID;
+        return agentID;
     }
 
     currentSignerID(): SignerID {

packages/cojson/src/crypto/WasmCrypto.ts

@@ -44,11 +44,13 @@ export class WasmCrypto extends CryptoProvider<Uint8Array> {
                 if ("crypto" in globalThis) {
                     resolve();
                 } else {
-                    return import("node:crypto").then(({ webcrypto }) => {
+                    return import(/*webpackIgnore: true*/ "node:crypto").then(
-                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        ({ webcrypto }) => {
-                        (globalThis as any).crypto = webcrypto;
+                            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                        resolve();
+                            (globalThis as any).crypto = webcrypto;
-                    });
+                            resolve();
+                        },
+                    );
                 }
             }),
         ]).then(([blake3instance]) => new WasmCrypto(blake3instance));

packages/cojson/src/index.ts

@@ -41,7 +41,13 @@ import type {
     BinaryCoStreamMeta,
 } from "./coValues/coStream.js";
 import type { JsonValue } from "./jsonValue.js";
-import type { SyncMessage, Peer } from "./sync.js";
+import type {
+    SyncMessage,
+    Peer,
+    IncomingSyncStream,
+    OutgoingSyncQueue,
+} from "./sync.js";
+import { DisconnectedError, PingTimeoutError } from "./sync.js";
 import type { AgentSecret } from "./crypto/crypto.js";
 import type {
     AccountID,
@@ -117,9 +123,19 @@ export {
     SyncMessage,
     isRawCoID,
     LSMStorage,
+    DisconnectedError,
+    PingTimeoutError,
 };
 
-export type { Value, FileSystem, FSErr, BlockFilename, WalFilename };
+export type {
+    Value,
+    FileSystem,
+    FSErr,
+    BlockFilename,
+    WalFilename,
+    IncomingSyncStream,
+    OutgoingSyncQueue,
+};
 
 // eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace CojsonInternalTypes {

packages/cojson/src/storage/index.ts

@@ -1,18 +1,13 @@
-import {
+import { Effect, Either, Queue, Stream, SynchronizedRef } from "effect";
-    ReadableStream,
-    WritableStream,
-    ReadableStreamDefaultReader,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
-import { Effect, Either, SynchronizedRef } from "effect";
 import { RawCoID } from "../ids.js";
 import { CoValueHeader, Transaction } from "../coValueCore.js";
 import { Signature } from "../crypto/crypto.js";
 import {
     CoValueKnownState,
+    IncomingSyncStream,
     NewContentMessage,
+    OutgoingSyncQueue,
     Peer,
-    SyncMessage,
 } from "../sync.js";
 import { CoID, RawCoValue } from "../index.js";
 import { connectedPeers } from "../streamUtils.js";
@@ -47,9 +42,6 @@ export type CoValueChunk = {
 };
 
 export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
-    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
-    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
-    fs: FS;
     currentWal: SynchronizedRef.SynchronizedRef<WH | undefined>;
     coValues: SynchronizedRef.SynchronizedRef<{
         [id: RawCoID]: CoValueChunk | undefined;
@@ -61,44 +53,28 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     >();
 
     constructor(
-        fs: FS,
+        public fs: FS,
-        fromLocalNode: ReadableStream<SyncMessage>,
+        public fromLocalNode: IncomingSyncStream,
-        toLocalNode: WritableStream<SyncMessage>,
+        public toLocalNode: OutgoingSyncQueue,
     ) {
-        this.fs = fs;
-        this.fromLocalNode = fromLocalNode.getReader();
-        this.toLocalNode = toLocalNode.getWriter();
         this.coValues = SynchronizedRef.unsafeMake({});
         this.currentWal = SynchronizedRef.unsafeMake<WH | undefined>(undefined);
 
-        void Effect.runPromise(
+        void this.fromLocalNode.pipe(
-            Effect.gen(this, function* () {
+            Stream.runForEach((msg) =>
-                let done = false;
+                Effect.gen(this, function* () {
-                while (!done) {
+                    if (msg.action === "done") {
-                    const result = yield* Effect.promise(() =>
+                        return;
-                        this.fromLocalNode.read(),
-                    );
-                    done = result.done;
-
-                    if (result.value) {
-                        if (result.value.action === "done") {
-                            continue;
-                        }
-
-                        if (result.value.action === "content") {
-                            yield* this.handleNewContent(result.value);
-                        } else {
-                            yield* this.sendNewContent(
-                                result.value.id,
-                                result.value,
-                                undefined,
-                            );
-                        }
                     }
-                }
 
-                return;
+                    if (msg.action === "content") {
-            }),
+                        yield* this.handleNewContent(msg);
+                    } else {
+                        yield* this.sendNewContent(msg.id, msg, undefined);
+                    }
+                }),
+            ),
+            Effect.runPromise,
         );
 
         setTimeout(() => this.compact(), 20000);
@@ -132,15 +108,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
             }
 
             if (!coValue) {
-                yield* Effect.promise(() =>
+                yield* Queue.offer(this.toLocalNode, {
-                    this.toLocalNode.write({
+                    id: id,
-                        id: id,
+                    action: "known",
-                        action: "known",
+                    header: false,
-                        header: false,
+                    sessions: {},
-                        sessions: {},
+                    asDependencyOf,
-                        asDependencyOf,
+                });
-                    }),
-                );
 
                 return coValues;
             }
@@ -195,17 +169,15 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
             const ourKnown: CoValueKnownState = chunkToKnownState(id, coValue);
 
-            yield* Effect.promise(() =>
+            yield* Queue.offer(this.toLocalNode, {
-                this.toLocalNode.write({
+                action: "known",
-                    action: "known",
+                ...ourKnown,
-                    ...ourKnown,
+                asDependencyOf,
-                    asDependencyOf,
+            });
-                }),
-            );
 
             for (const message of newContentMessages) {
                 if (Object.keys(message.new).length === 0) continue;
-                yield* Effect.promise(() => this.toLocalNode.write(message));
+                yield* Queue.offer(this.toLocalNode, message);
             }
 
             return { ...coValues, [id]: coValue };
@@ -452,7 +424,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         setTimeout(() => this.compact(), 5000);
     }
 
-    static asPeer<WH, RH, FS extends FileSystem<WH, RH>>({
+    static async asPeer<WH, RH, FS extends FileSystem<WH, RH>>({
         fs,
         trace,
         localNodeName = "local",
@@ -460,15 +432,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         fs: FS;
         trace?: boolean;
         localNodeName?: string;
-    }): Peer {
+    }): Promise<Peer> {
-        const [localNodeAsPeer, storageAsPeer] = connectedPeers(
+        const [localNodeAsPeer, storageAsPeer] = await Effect.runPromise(
-            localNodeName,
+            connectedPeers(localNodeName, "storage", {
-            "storage",
-            {
                 peer1role: "client",
                 peer2role: "server",
                 trace,
-            },
+            }),
         );
 
         new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);

packages/cojson/src/streamUtils.ts

@@ -1,8 +1,4 @@
-import {
+import { Console, Effect, Queue, Stream } from "effect";
-    ReadableStream,
-    TransformStream,
-    WritableStream,
-} from "isomorphic-streams";
 import { Peer, PeerID, SyncMessage } from "./sync.js";
 
 export function connectedPeers(
@@ -17,160 +13,54 @@ export function connectedPeers(
         peer1role?: Peer["role"];
         peer2role?: Peer["role"];
     } = {},
-): [Peer, Peer] {
+): Effect.Effect<[Peer, Peer]> {
-    const [inRx1, inTx1] = newStreamPair<SyncMessage>(peer1id + "_in");
+    return Effect.gen(function* () {
-    const [outRx1, outTx1] = newStreamPair<SyncMessage>(peer1id + "_out");
+        const [from1to2Rx, from1to2Tx] = yield* newQueuePair(
+            trace ? { traceAs: `${peer1id} -> ${peer2id}` } : undefined,
+        );
+        const [from2to1Rx, from2to1Tx] = yield* newQueuePair(
+            trace ? { traceAs: `${peer2id} -> ${peer1id}` } : undefined,
+        );
 
-    const [inRx2, inTx2] = newStreamPair<SyncMessage>(peer2id + "_in");
+        const peer2AsPeer: Peer = {
-    const [outRx2, outTx2] = newStreamPair<SyncMessage>(peer2id + "_out");
+            id: peer2id,
+            incoming: from2to1Rx,
+            outgoing: from1to2Tx,
+            role: peer2role,
+        };
 
-    void outRx2
+        const peer1AsPeer: Peer = {
-        .pipeThrough(
+            id: peer1id,
-            new TransformStream({
+            incoming: from1to2Rx,
-                transform(
+            outgoing: from2to1Tx,
-                    chunk: SyncMessage,
+            role: peer1role,
-                    controller: { enqueue: (msg: SyncMessage) => void },
+        };
-                ) {
-                    trace &&
-                        console.debug(
-                            `${peer2id} -> ${peer1id}`,
-                            JSON.stringify(
-                                chunk,
-                                (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                2,
-                            ),
-                        );
-                    controller.enqueue(chunk);
-                },
-            }),
-        )
-        .pipeTo(inTx1);
 
-    void outRx1
+        return [peer1AsPeer, peer2AsPeer];
-        .pipeThrough(
+    });
-            new TransformStream({
-                transform(
-                    chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void },
-                ) {
-                    trace &&
-                        console.debug(
-                            `${peer1id} -> ${peer2id}`,
-                            JSON.stringify(
-                                chunk,
-                                (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                2,
-                            ),
-                        );
-                    controller.enqueue(chunk);
-                },
-            }),
-        )
-        .pipeTo(inTx2);
-
-    const peer2AsPeer: Peer = {
-        id: peer2id,
-        incoming: inRx1,
-        outgoing: outTx1,
-        role: peer2role,
-    };
-
-    const peer1AsPeer: Peer = {
-        id: peer1id,
-        incoming: inRx2,
-        outgoing: outTx2,
-        role: peer1role,
-    };
-
-    return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newStreamPair<T>(
+export function newQueuePair(
-    pairName?: string,
+    options: { traceAs?: string } = {},
-): [ReadableStream<T>, WritableStream<T>] {
+): Effect.Effect<[Stream.Stream<SyncMessage>, Queue.Enqueue<SyncMessage>]> {
-    let queueLength = 0;
+    return Effect.gen(function* () {
-    let readerClosed = false;
+        const queue = yield* Queue.unbounded<SyncMessage>();
 
-    let resolveEnqueue: (enqueue: (item: T) => void) => void;
+        if (options.traceAs) {
-    const enqueuePromise = new Promise<(item: T) => void>((resolve) => {
+            return [Stream.fromQueue(queue).pipe(Stream.tap((msg) => Console.debug(
-        resolveEnqueue = resolve;
+                options.traceAs,
-    });
+                JSON.stringify(
-
+                    msg,
-    let resolveClose: (close: () => void) => void;
+                    (k, v) =>
-    const closePromise = new Promise<() => void>((resolve) => {
+                        k === "changes" ||
-        resolveClose = resolve;
+                        k === "encryptedChanges"
-    });
+                            ? v.slice(0, 20) + "..."
-
+                            : v,
-    let queueWasOverflowing = false;
+                    2,
-
+                ),
-    function maybeReportQueueLength() {
+            ))), queue];
-        if (queueLength >= 100) {
-            queueWasOverflowing = true;
-            if (queueLength % 100 === 0) {
-                console.warn(pairName, "overflowing queue length", queueLength);
-            }
         } else {
-            if (queueWasOverflowing) {
+            return [Stream.fromQueue(queue), queue];
-                console.debug(pairName, "ok queue length", queueLength);
-                queueWasOverflowing = false;
-            }
         }
-    }
-
-    const readable = new ReadableStream<T>({
-        async start(controller) {
-            resolveEnqueue(controller.enqueue.bind(controller));
-            resolveClose(controller.close.bind(controller));
-        },
-
-        cancel(_reason) {
-            console.log("Manually closing reader");
-            readerClosed = true;
-        },
-    }).pipeThrough(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        new TransformStream<any, any>({
-            transform(
-                chunk: SyncMessage,
-                controller: { enqueue: (msg: SyncMessage) => void },
-            ) {
-                queueLength -= 1;
-                maybeReportQueueLength();
-                controller.enqueue(chunk);
-            },
-        }),
-    ) as ReadableStream<T>;
-
-    let lastWritePromise = Promise.resolve();
-
-    const writable = new WritableStream<T>({
-        async write(chunk) {
-            queueLength += 1;
-            maybeReportQueueLength();
-            const enqueue = await enqueuePromise;
-            if (readerClosed) {
-                throw new Error("Reader closed");
-            } else {
-                // make sure write resolves before corresponding read, but make sure writes are still in order
-                await lastWritePromise;
-                lastWritePromise = new Promise((resolve) => {
-                    enqueue(chunk);
-                    resolve();
-                });
-            }
-        },
-        async abort(reason) {
-            console.debug("Manually closing writer", reason);
-            const close = await closePromise;
-            close();
-        },
     });
-
-    return [readable, writable];
 }

packages/cojson/src/sync.ts

@@ -1,13 +1,9 @@
 import { Signature } from "./crypto/crypto.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
-import { LocalNode } from "./localNode.js";
+import { LocalNode, newLoadingState } from "./localNode.js";
-import {
-    ReadableStream,
-    WritableStream,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
 import { RawCoID, SessionID } from "./ids.js";
+import { Effect, Queue, Stream } from "effect";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -60,10 +56,27 @@ export type DoneMessage = {
 
 export type PeerID = string;
 
+export class DisconnectedError extends Error {
+    readonly _tag = "DisconnectedError";
+    constructor(public message: string) {
+        super(message);
+    }
+}
+
+export class PingTimeoutError extends Error {
+    readonly _tag = "PingTimeoutError";
+}
+
+export type IncomingSyncStream = Stream.Stream<
+    SyncMessage,
+    DisconnectedError | PingTimeoutError
+>;
+export type OutgoingSyncQueue = Queue.Enqueue<SyncMessage>;
+
 export interface Peer {
     id: PeerID;
-    incoming: ReadableStream<SyncMessage>;
+    incoming: IncomingSyncStream;
-    outgoing: WritableStream<SyncMessage>;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -73,8 +86,8 @@ export interface PeerState {
     id: PeerID;
     optimisticKnownStates: { [id: RawCoID]: CoValueKnownState };
     toldKnownState: Set<RawCoID>;
-    incoming: ReadableStream<SyncMessage>;
+    incoming: IncomingSyncStream;
-    outgoing: WritableStreamDefaultWriter<SyncMessage>;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -127,25 +140,24 @@ export class SyncManager {
         });
     }
 
-    async loadFromPeers(id: RawCoID, excludePeer?: PeerID) {
+    async loadFromPeers(id: RawCoID, forPeer?: PeerID) {
-        for (const peer of this.peersInPriorityOrder()) {
+        const eligiblePeers = this.peersInPriorityOrder().filter(
-            if (peer.id === excludePeer) {
+            (peer) => peer.id !== forPeer && peer.role === "server",
-                continue;
+        );
-            }
+
-            if (peer.role !== "server") {
+        for (const peer of eligiblePeers) {
-                continue;
-            }
             // console.log("loading", id, "from", peer.id);
-            peer.outgoing
+            Effect.runPromise(
-                .write({
+                Queue.offer(peer.outgoing, {
                     action: "load",
                     id: id,
                     header: false,
                     sessions: {},
-                })
+                }),
-                .catch((e) => {
+            ).catch((e) => {
-                    console.error("Error writing to peer", e);
+                console.error("Error writing to peer", e);
-                });
+            });
+
             const coValueEntry = this.local.coValues[id];
             if (coValueEntry?.state !== "loading") {
                 continue;
@@ -297,7 +309,9 @@ export class SyncManager {
                 let lastYield = performance.now();
                 for (const [_i, piece] of newContentPieces.entries()) {
                     // console.log(
-                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
+                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${
+                    //         newContentPieces.length
+                    //     } header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
                     await this.trySendToPeer(peer, piece);
@@ -328,7 +342,7 @@ export class SyncManager {
             id: peer.id,
             optimisticKnownStates: {},
             incoming: peer.incoming,
-            outgoing: peer.outgoing.getWriter(),
+            outgoing: peer.outgoing,
             toldKnownState: new Set(),
             role: peer.role,
             delayOnError: peer.delayOnError,
@@ -354,91 +368,55 @@ export class SyncManager {
             void initialSync();
         }
 
-        const readIncoming = async () => {
+        void Effect.runPromise(
-            try {
+            peerState.incoming.pipe(
-                for await (const msg of peerState.incoming) {
+                Stream.ensuring(
-                    try {
+                    Effect.sync(() => {
-                        // await this.handleSyncMessage(msg, peerState);
+                        console.log("Peer disconnected:", peer.id);
-                        this.handleSyncMessage(msg, peerState).catch((e) => {
+                        delete this.peers[peer.id];
-                            console.error(
+                    }),
-                                new Date(),
+                ),
-                                `Error reading from peer ${peer.id}, handling msg`,
+                Stream.runForEach((msg) =>
-                                JSON.stringify(msg, (k, v) =>
+                    Effect.tryPromise({
-                                    k === "changes" || k === "encryptedChanges"
+                        try: () => this.handleSyncMessage(msg, peerState),
-                                        ? v.slice(0, 20) + "..."
+                        catch: (e) =>
-                                        : v,
+                            new Error(
-                                ),
+                                `Error reading from peer ${
-                                e,
+                                    peer.id
-                            );
+                                }, handling msg\n\n${JSON.stringify(
-                        });
+                                    msg,
-                        // await new Promise<void>((resolve) => {
+                                    (k, v) =>
-                        //     setTimeout(resolve, 0);
+                                        k === "changes" ||
-                        // });
+                                        k === "encryptedChanges"
-                    } catch (e) {
+                                            ? v.slice(0, 20) + "..."
-                        console.error(
+                                            : v,
-                            new Date(),
+                                )}`,
-                            `Error reading from peer ${peer.id}, handling msg`,
+                                { cause: e },
-                            JSON.stringify(msg, (k, v) =>
-                                k === "changes" || k === "encryptedChanges"
-                                    ? v.slice(0, 20) + "..."
-                                    : v,
                             ),
-                            e,
+                    }).pipe(
-                        );
+                        Effect.timeoutFail({
-                        if (peerState.delayOnError) {
+                            duration: 10000,
-                            await new Promise<void>((resolve) => {
+                            onTimeout: () =>
-                                setTimeout(resolve, peerState.delayOnError);
+                                new Error("Took >10s to process message"),
-                            });
+                        }),
-                        }
+                    ),
-                    }
+                ),
-                }
+                Effect.catchAll((e) =>
-            } catch (e) {
+                    Effect.logError(
-                console.error(`Error reading from peer ${peer.id}`, e);
+                        "Error in peer",
-            }
+                        peer.id,
-
+                        e.message,
-            console.log("Peer disconnected:", peer.id);
+                        typeof e.cause === "object" &&
-            delete this.peers[peer.id];
+                            e.cause instanceof Error &&
-        };
+                            e.cause.message,
-
+                    ),
-        void readIncoming();
+                ),
+            ),
+        );
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        if (!this.peers[peer.id]) {
+        return Effect.runPromise(Queue.offer(peer.outgoing, msg));
-            // already disconnected, return to drain potential queue
-            return Promise.resolve();
-        }
-
-        return new Promise<void>((resolve) => {
-            const start = Date.now();
-            peer.outgoing
-                .write(msg)
-                .then(() => {
-                    const end = Date.now();
-                    if (end - start > 1000) {
-                        // console.error(
-                        //     new Error(
-                        //         `Writing to peer "${peer.id}" took ${
-                        //             Math.round((Date.now() - start) / 100) / 10
-                        //         }s - this should never happen as write should resolve quickly or error`
-                        //     )
-                        // );
-                    } else {
-                        resolve();
-                    }
-                })
-                .catch((e) => {
-                    console.error(
-                        new Error(
-                            `Error writing to peer ${peer.id}, disconnecting`,
-                            {
-                                cause: e,
-                            },
-                        ),
-                    );
-                    delete this.peers[peer.id];
-                });
-        });
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
@@ -447,21 +425,50 @@ export class SyncManager {
 
         if (!entry) {
             // console.log(`Loading ${msg.id} from all peers except ${peer.id}`);
-            this.local
+
-                .loadCoValueCore(msg.id, {
+            // special case: we should be able to solve this much more neatly
-                    dontLoadFrom: peer.id,
+            // with an explicit state machine in the future
-                    dontWaitFor: peer.id,
+            const eligiblePeers = this.peersInPriorityOrder().filter(
-                })
+                (other) => other.id !== peer.id && peer.role === "server",
-                .catch((e) => {
+            );
-                    console.error("Error loading coValue in handleLoad", e);
+            if (eligiblePeers.length === 0) {
-                });
+                if (msg.header || Object.keys(msg.sessions).length > 0) {
+                    this.local.coValues[msg.id] = newLoadingState(
+                        new Set([peer.id]),
+                    );
+                    this.trySendToPeer(peer, {
+                        action: "known",
+                        id: msg.id,
+                        header: false,
+                        sessions: {},
+                    }).catch((e) => {
+                        console.error("Error sending known state back", e);
+                    });
+                }
+                return;
+            } else {
+                this.local
+                    .loadCoValueCore(msg.id, {
+                        dontLoadFrom: peer.id,
+                        dontWaitFor: peer.id,
+                    })
+                    .catch((e) => {
+                        console.error("Error loading coValue in handleLoad", e);
+                    });
+            }
 
             entry = this.local.coValues[msg.id]!;
         }
 
         if (entry.state === "loading") {
+            console.log(
+                "Waiting for loaded",
+                msg.id,
+                "after message from",
+                peer.id,
+            );
             const loaded = await entry.done;
-
+            console.log("Loaded", msg.id, loaded);
             if (loaded === "unavailable") {
                 peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
                 peer.toldKnownState.add(msg.id);
@@ -508,7 +515,7 @@ export class SyncManager {
                 }
             } else {
                 throw new Error(
-                    "Expected coValue entry to be created, missing subscribe?",
+                    `Expected coValue entry for ${msg.id} to be created on known state, missing subscribe?`,
                 );
             }
         }
@@ -549,7 +556,7 @@ export class SyncManager {
 
         if (!entry) {
             throw new Error(
-                "Expected coValue entry to be created, missing subscribe?",
+                `Expected coValue entry for ${msg.id} to be created on new content, missing subscribe?`,
             );
         }
 

packages/cojson/src/tests/account.test.ts

@@ -3,6 +3,7 @@ import { newRandomSessionID } from "../coValueCore.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { Effect } from "effect";
 
 const Crypto = await WasmCrypto.create();
 
@@ -52,11 +53,13 @@ test("Can create account with one node, and then load it on another", async () =
     map.set("foo", "bar", "private");
     expect(map.get("foo")).toEqual("bar");
 
-    const [node1asPeer, node2asPeer] = connectedPeers("node1", "node2", {
+    const [node1asPeer, node2asPeer] = await Effect.runPromise(connectedPeers("node1", "node2", {
         trace: true,
         peer1role: "server",
         peer2role: "client",
-    });
+    }));
+
+    console.log("After connected peers")
 
     node.syncManager.addPeer(node2asPeer);
 

packages/jazz-browser-media-images/CHANGELOG.md

@@ -1,5 +1,73 @@
 # jazz-browser-media-images
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.14
+    -   jazz-browser@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+    -   jazz-browser@0.7.13
+
+## 0.7.12
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+    -   jazz-browser@0.7.12
+
+## 0.7.11
+
+### Patch Changes
+
+-   jazz-browser@0.7.11
+-   jazz-tools@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   jazz-browser@0.7.10
+-   jazz-tools@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   jazz-browser@0.7.9
+-   jazz-tools@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+    -   jazz-browser@0.7.8
+
+## 0.7.6
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+    -   jazz-browser@0.7.6
+
+## 0.7.5
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-browser@0.7.5
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-browser-media-images/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jazz-browser-media-images",
-  "version": "0.7.3",
+  "version": "0.7.14",
   "type": "module",
   "main": "dist/index.js",
   "types": "src/index.ts",

packages/jazz-browser/CHANGELOG.md

@@ -1,5 +1,76 @@
 # jazz-browser
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+    -   jazz-tools@0.7.14
+    -   cojson-storage-indexeddb@0.7.14
+    -   cojson-transport-ws@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+
+## 0.7.12
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+    -   cojson-storage-indexeddb@0.7.11
+    -   jazz-tools@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+    -   cojson-storage-indexeddb@0.7.10
+    -   jazz-tools@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+    -   cojson-storage-indexeddb@0.7.9
+    -   jazz-tools@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+
+## 0.7.6
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+
+## 0.7.5
+
+### Patch Changes
+
+-   Ability to add seed accounts to DemoAuth
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jazz-browser",
-  "version": "0.7.3",
+  "version": "0.7.14",
   "type": "module",
   "main": "dist/index.js",
   "types": "src/index.ts",
@@ -9,8 +9,8 @@
     "@scure/bip39": "^1.3.0",
     "cojson": "workspace:*",
     "cojson-storage-indexeddb": "workspace:*",
+    "cojson-transport-ws": "workspace:*",
     "effect": "^3.1.5",
-    "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae",
     "jazz-tools": "workspace:*",
     "typescript": "^5.1.6"
   },

packages/jazz-browser/src/auth/DemoAuth.ts

@@ -15,7 +15,33 @@ export class BrowserDemoAuth<Acc extends Account> implements AuthProvider<Acc> {
         public accountSchema: CoValueClass<Acc> & typeof Account,
         public driver: BrowserDemoAuth.Driver,
         public appName: string,
-    ) {}
+        seedAccounts?: {
+            [name: string]: {
+                accountID: ID<Account>;
+                accountSecret: AgentSecret;
+            };
+        },
+    ) {
+        for (const [name, credentials] of Object.entries(seedAccounts || {})) {
+            const storageData = JSON.stringify(
+                credentials satisfies StorageData,
+            );
+            if (
+                !(
+                    localStorage["demo-auth-existing-users"]?.split(",") as
+                        | string[]
+                        | undefined
+                )?.includes(name)
+            ) {
+                localStorage["demo-auth-existing-users"] = localStorage[
+                    "demo-auth-existing-users"
+                ]
+                    ? localStorage["demo-auth-existing-users"] + "," + name
+                    : name;
+            }
+            localStorage["demo-auth-existing-users-" + name] = storageData;
+        }
+    }
 
     async createOrLoadAccount(
         getSessionFor: SessionProvider,

packages/jazz-browser/src/index.ts

@@ -1,11 +1,8 @@
-import { ReadableStream, WritableStream } from "isomorphic-streams";
 import {
     CoValue,
     ID,
-    Peer,
     AgentID,
     SessionID,
-    SyncMessage,
     cojsonInternals,
     InviteSecret,
     Account,
@@ -13,10 +10,15 @@ import {
     WasmCrypto,
     CryptoProvider,
 } from "jazz-tools";
-import { AccountID, LSMStorage } from "cojson";
+import {
+    AccountID,
+    LSMStorage,
+} from "cojson";
 import { AuthProvider } from "./auth/auth.js";
 import { OPFSFilesystem } from "./OPFSFilesystem.js";
 import { IDBStorage } from "cojson-storage-indexeddb";
+import { Effect, Queue } from "effect";
+import { createWebSocketPeer } from "cojson-transport-ws";
 export * from "./auth/auth.js";
 
 /** @category Context Creation */
@@ -29,7 +31,7 @@ export type BrowserContext<Acc extends Account> = {
 /** @category Context Creation */
 export async function createJazzBrowserContext<Acc extends Account>({
     auth,
-    peer,
+    peer: peerAddr,
     reconnectionTimeout: initialReconnectionTimeout = 500,
     storage = "indexedDB",
     crypto: customCrypto,
@@ -43,7 +45,13 @@ export async function createJazzBrowserContext<Acc extends Account>({
     const crypto = customCrypto || (await WasmCrypto.create());
     let sessionDone: () => void;
 
-    const firstWsPeer = createWebSocketPeer(peer);
+    const firstWsPeer = await Effect.runPromise(
+        createWebSocketPeer({
+            websocket: new WebSocket(peerAddr),
+            id: peerAddr + "@" + new Date().toISOString(),
+            role: "server",
+        }),
+    );
     let shouldTryToReconnect = true;
 
     let currentReconnectionTimeout = initialReconnectionTimeout;
@@ -77,7 +85,7 @@ export async function createJazzBrowserContext<Acc extends Account>({
         while (shouldTryToReconnect) {
             if (
                 Object.keys(me._raw.core.node.syncManager.peers).some(
-                    (peerId) => peerId.includes(peer),
+                    (peerId) => peerId.includes(peerAddr),
                 )
             ) {
                 // TODO: this might drain battery, use listeners instead
@@ -107,7 +115,13 @@ export async function createJazzBrowserContext<Acc extends Account>({
                 });
 
                 me._raw.core.node.syncManager.addPeer(
-                    createWebSocketPeer(peer),
+                    await Effect.runPromise(
+                        createWebSocketPeer({
+                            websocket: new WebSocket(peerAddr),
+                            id: peerAddr + "@" + new Date().toISOString(),
+                            role: "server",
+                        }),
+                    ),
                 );
             }
         }
@@ -124,9 +138,7 @@ export async function createJazzBrowserContext<Acc extends Account>({
             for (const peer of Object.values(
                 me._raw.core.node.syncManager.peers,
             )) {
-                peer.outgoing
+                void Effect.runPromise(Queue.shutdown(peer.outgoing));
-                    .close()
-                    .catch((e) => console.error("Error while closing peer", e));
             }
             sessionDone?.();
         },
@@ -207,140 +219,6 @@ export function getSessionHandleFor(
     };
 }
 
-function websocketReadableStream<T>(ws: WebSocket) {
-    ws.binaryType = "arraybuffer";
-
-    return new ReadableStream<T>({
-        start(controller) {
-            let pingTimeout: ReturnType<typeof setTimeout> | undefined;
-
-            ws.onmessage = (event) => {
-                const msg = JSON.parse(event.data);
-
-                if (pingTimeout) {
-                    clearTimeout(pingTimeout);
-                }
-
-                pingTimeout = setTimeout(() => {
-                    console.debug("Ping timeout");
-                    try {
-                        controller.close();
-                        ws.close();
-                    } catch (e) {
-                        console.error(
-                            "Error while trying to close ws on ping timeout",
-                            e,
-                        );
-                    }
-                }, 2500);
-
-                if (msg.type === "ping") {
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    (window as any).jazzPings = (window as any).jazzPings || [];
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    (window as any).jazzPings.push({
-                        received: Date.now(),
-                        sent: msg.time,
-                        dc: msg.dc,
-                    });
-                    return;
-                }
-                controller.enqueue(msg);
-            };
-            const closeListener = () => {
-                controller.close();
-                clearTimeout(pingTimeout);
-            };
-            ws.addEventListener("close", closeListener);
-            ws.addEventListener("error", () => {
-                controller.error(new Error("The WebSocket errored!"));
-                ws.removeEventListener("close", closeListener);
-            });
-        },
-
-        cancel() {
-            ws.close();
-        },
-    });
-}
-
-export function createWebSocketPeer(syncAddress: string): Peer {
-    const ws = new WebSocket(syncAddress);
-
-    const incoming = websocketReadableStream<SyncMessage>(ws);
-    const outgoing = websocketWritableStream<SyncMessage>(ws);
-
-    return {
-        id: syncAddress + "@" + new Date().toISOString(),
-        incoming,
-        outgoing,
-        role: "server",
-    };
-}
-
-function websocketWritableStream<T>(ws: WebSocket) {
-    const initialQueue = [] as T[];
-    let isOpen = false;
-
-    return new WritableStream<T>({
-        start(controller) {
-            ws.addEventListener("error", (event) => {
-                controller.error(
-                    new Error("The WebSocket errored!" + JSON.stringify(event)),
-                );
-            });
-            ws.addEventListener("close", () => {
-                controller.error(
-                    new Error("The server closed the connection unexpectedly!"),
-                );
-            });
-            ws.addEventListener("open", () => {
-                for (const item of initialQueue) {
-                    ws.send(JSON.stringify(item));
-                }
-                isOpen = true;
-            });
-        },
-
-        async write(chunk) {
-            if (isOpen) {
-                ws.send(JSON.stringify(chunk));
-                // Return immediately, since the web socket gives us no easy way to tell
-                // when the write completes.
-            } else {
-                initialQueue.push(chunk);
-            }
-        },
-
-        close() {
-            return closeWS(1000);
-        },
-
-        abort(reason) {
-            return closeWS(4000, reason && reason.message);
-        },
-    });
-
-    function closeWS(code: number, reasonString?: string) {
-        return new Promise<void>((resolve, reject) => {
-            ws.addEventListener(
-                "close",
-                (e) => {
-                    if (e.wasClean) {
-                        resolve();
-                    } else {
-                        reject(
-                            new Error("The connection was not closed cleanly"),
-                        );
-                    }
-                },
-                { once: true },
-            );
-            ws.close(code, reasonString);
-        });
-    }
-}
-
 /** @category Invite Links */
 export function createInviteLink<C extends CoValue>(
     value: C,

packages/jazz-nodejs/CHANGELOG.md

@@ -1,5 +1,69 @@
 # jazz-autosub
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+    -   jazz-tools@0.7.14
+    -   cojson-transport-ws@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+
+## 0.7.12
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+    -   cojson-transport-nodejs-ws@0.7.11
+    -   jazz-tools@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+    -   cojson-transport-nodejs-ws@0.7.10
+    -   jazz-tools@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+    -   cojson-transport-nodejs-ws@0.7.9
+    -   jazz-tools@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+
+## 0.7.6
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-nodejs/package.json

@@ -5,10 +5,11 @@
     "types": "src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.7.3",
+    "version": "0.7.14",
     "dependencies": {
         "cojson": "workspace:*",
-        "cojson-transport-nodejs-ws": "workspace:*",
+        "cojson-transport-ws": "workspace:*",
+        "effect": "^3.1.5",
         "jazz-tools": "workspace:*",
         "ws": "^8.14.2"
     },

packages/jazz-nodejs/src/index.ts

@@ -1,11 +1,8 @@
-import {
-    websocketReadableStream,
-    websocketWritableStream,
-} from "cojson-transport-nodejs-ws";
-import { WebSocket } from "ws";
-
 import { AgentSecret, Peer, SessionID, WasmCrypto } from "cojson";
+import { createWebSocketPeer } from "cojson-transport-ws";
 import { Account, CoValueClass, ID } from "jazz-tools";
+import { Effect } from "effect";
+import { WebSocket } from "ws";
 
 /** @category Context Creation */
 export async function startWorker<Acc extends Account>({
@@ -21,14 +18,13 @@ export async function startWorker<Acc extends Account>({
     syncServer?: string;
     accountSchema?: CoValueClass<Acc> & typeof Account;
 }): Promise<{ worker: Acc }> {
-    const ws = new WebSocket(peer);
+    const wsPeer: Peer = await Effect.runPromise(
-
+        createWebSocketPeer({
-    const wsPeer: Peer = {
+            id: "upstream",
-        id: "upstream",
+            websocket: new WebSocket(peer),
-        role: "server",
+            role: "server",
-        incoming: websocketReadableStream(ws),
+        }),
-        outgoing: websocketWritableStream(ws),
+    );
-    };
 
     if (!accountID) {
         throw new Error("No accountID provided");
@@ -52,17 +48,17 @@ export async function startWorker<Acc extends Account>({
         crypto: await WasmCrypto.create(),
     });
 
-    setInterval(() => {
+    setInterval(async () => {
         if (!worker._raw.core.node.syncManager.peers["upstream"]) {
             console.log(new Date(), "Reconnecting to upstream " + peer);
-            const ws = new WebSocket(peer);
 
-            const wsPeer: Peer = {
+            const wsPeer: Peer = await Effect.runPromise(
-                id: "upstream",
+                createWebSocketPeer({
-                role: "server",
+                    id: "upstream",
-                incoming: websocketReadableStream(ws),
+                    websocket: new WebSocket(peer),
-                outgoing: websocketWritableStream(ws),
+                    role: "server",
-            };
+                }),
+            );
 
             worker._raw.core.node.syncManager.addPeer(wsPeer);
         }

packages/jazz-react/CHANGELOG.md

@@ -1,5 +1,93 @@
 # jazz-react
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+    -   jazz-tools@0.7.14
+    -   jazz-browser@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+    -   jazz-browser@0.7.13
+
+## 0.7.12
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+    -   jazz-browser@0.7.12
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+    -   jazz-browser@0.7.11
+    -   jazz-tools@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+    -   jazz-browser@0.7.10
+    -   jazz-tools@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+    -   jazz-browser@0.7.9
+    -   jazz-tools@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+    -   jazz-browser@0.7.8
+
+## 0.7.7
+
+### Patch Changes
+
+-   9fdc91c: Improve compatibility with React compiler and concurrent features
+
+## 0.7.6
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+    -   jazz-browser@0.7.6
+
+## 0.7.5
+
+### Patch Changes
+
+-   Ability to add seed accounts to DemoAuth
+-   Updated dependencies
+    -   jazz-browser@0.7.5
+
+## 0.7.4
+
+### Patch Changes
+
+-   Expose auth loading state in a simple way
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jazz-react",
-  "version": "0.7.3",
+  "version": "0.7.14",
   "type": "module",
   "main": "dist/index.js",
   "types": "src/index.ts",

packages/jazz-react/src/auth/DemoAuth.tsx

@@ -1,7 +1,8 @@
-import { ReactNode, useMemo, useState } from "react";
+import { ReactNode, useEffect, useMemo, useState } from "react";
 import { BrowserDemoAuth } from "jazz-browser";
-import { Account, CoValueClass } from "jazz-tools";
+import { Account, CoValueClass, ID } from "jazz-tools";
 import { ReactAuthHook } from "./auth.js";
+import { AgentSecret } from "cojson";
 
 /** @category Auth Providers */
 export function DemoAuth<Acc extends Account = Account>({
@@ -9,13 +10,17 @@ export function DemoAuth<Acc extends Account = Account>({
     appName,
     appHostname,
     Component = DemoAuth.BasicUI,
+    seedAccounts,
 }: {
     accountSchema?: CoValueClass<Acc> & typeof Account;
     appName: string;
     appHostname?: string;
     Component?: DemoAuth.Component;
+    seedAccounts?: {
+        [name: string]: { accountID: ID<Account>; accountSecret: AgentSecret };
+    };
 }): ReactAuthHook<Acc> {
-    return function useLocalAuth() {
+    return function useLocalAuth(setJazzAuthState) {
         const [authState, setAuthState] = useState<
             | { state: "loading" }
             | {
@@ -29,6 +34,10 @@ export function DemoAuth<Acc extends Account = Account>({
 
         const [logOutCounter, setLogOutCounter] = useState(0);
 
+        useEffect(() => {
+            setJazzAuthState(authState.state);
+        }, [authState]);
+
         const auth = useMemo(() => {
             return new BrowserDemoAuth<Acc>(
                 accountSchema,
@@ -53,8 +62,9 @@ export function DemoAuth<Acc extends Account = Account>({
                     },
                 },
                 appName,
+                seedAccounts,
             );
-        }, [appName, appHostname, logOutCounter]);
+        }, [appName, appHostname, logOutCounter, seedAccounts]);
 
         const AuthUI =
             authState.state === "ready"
@@ -94,7 +104,7 @@ const DemoAuthBasicUI = ({
     signUp: (username: string) => void;
 }) => {
     const [username, setUsername] = useState<string>("");
-    const darkMode = window.matchMedia("(prefers-color-scheme: dark)").matches;
+    const darkMode = typeof window !== 'undefined' ? window.matchMedia("(prefers-color-scheme: dark)").matches : false;
 
     return (
         <div

packages/jazz-react/src/auth/PasskeyAuth.tsx

@@ -1,4 +1,4 @@
-import { useMemo, useState, ReactNode } from "react";
+import { useMemo, useState, ReactNode, useEffect } from "react";
 import { BrowserPasskeyAuth } from "jazz-browser";
 import { Account, CoValueClass } from "jazz-tools";
 import { ReactAuthHook } from "./auth.js";
@@ -15,7 +15,7 @@ export function PasskeyAuth<Acc extends Account>({
     appHostname?: string;
     Component?: PasskeyAuth.Component;
 }): ReactAuthHook<Acc> {
-    return function useLocalAuth() {
+    return function useLocalAuth(setJazzAuthState) {
         const [authState, setAuthState] = useState<
             | { state: "loading" }
             | {
@@ -26,6 +26,10 @@ export function PasskeyAuth<Acc extends Account>({
             | { state: "signedIn"; logOut: () => void }
         >({ state: "loading" });
 
+        useEffect(() => {
+            setJazzAuthState(authState.state);
+        }, [authState]);
+
         const [logOutCounter, setLogOutCounter] = useState(0);
 
         const auth = useMemo(() => {

packages/jazz-react/src/auth/PassphraseAuth.tsx

@@ -1,4 +1,4 @@
-import { useMemo, useState, ReactNode } from "react";
+import { useMemo, useState, ReactNode, useEffect } from "react";
 import { BrowserPassphraseAuth } from "jazz-browser";
 import { generateMnemonic } from "@scure/bip39";
 import { cojsonInternals } from "cojson";
@@ -19,7 +19,7 @@ export function PassphraseAuth<Acc extends Account>({
     wordlist: string[];
     Component?: PassphraseAuth.Component;
 }): ReactAuthHook<Acc> {
-    return function useLocalAuth() {
+    return function useLocalAuth(setJazzAuthState) {
         const [authState, setAuthState] = useState<
             | { state: "loading" }
             | {
@@ -32,6 +32,10 @@ export function PassphraseAuth<Acc extends Account>({
 
         const [logOutCounter, setLogOutCounter] = useState(0);
 
+        useEffect(() => {
+            setJazzAuthState(authState.state);
+        }, [authState]);
+
         const auth = useMemo(() => {
             return new BrowserPassphraseAuth<Acc>(
                 accountSchema,

packages/jazz-react/src/auth/auth.ts

@@ -2,8 +2,12 @@ import React from "react";
 import { AuthProvider } from "jazz-browser";
 import { Account } from "jazz-tools";
 
+export type AuthState = "loading" | "ready" | "signedIn";
+
 /** @category Auth Providers */
-export type ReactAuthHook<Acc extends Account> = () => {
+export type ReactAuthHook<Acc extends Account> = (
+    setJazzAuthState: (state: AuthState) => void,
+) => {
     auth: AuthProvider<Acc>;
     AuthUI: React.ReactNode;
     logOut?: () => void;

packages/jazz-react/src/index.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useRef, useState } from "react";
+import React, { useEffect, useState } from "react";
 import {
     consumeInviteLinkFromWindowLocation,
     createJazzBrowserContext,
@@ -13,11 +13,11 @@ import {
     ID,
     subscribeToCoValue,
 } from "jazz-tools";
-import { ReactAuthHook } from "./auth/auth.js";
+import { AuthState, ReactAuthHook } from "./auth/auth.js";
 
 /** @category Context & Hooks */
 export function createJazzReactContext<Acc extends Account>({
-    auth: authHook,
+    auth: useAuthHook,
     peer,
     storage = "indexedDB",
 }: {
@@ -33,10 +33,16 @@ export function createJazzReactContext<Acc extends Account>({
         | undefined
     >(undefined);
 
-    function Provider({ children }: { children: React.ReactNode }) {
+    function Provider({
+        children,
+        loading,
+    }: {
+        children: React.ReactNode;
+        loading?: React.ReactNode;
+    }) {
         const [me, setMe] = useState<Acc | undefined>();
-
+        const [authState, setAuthState] = useState<AuthState>("loading");
-        const { auth, AuthUI, logOut } = authHook();
+        const { auth, AuthUI, logOut } = useAuthHook(setAuthState);
 
         useEffect(() => {
             let done: (() => void) | undefined = undefined;
@@ -74,7 +80,8 @@ export function createJazzReactContext<Acc extends Account>({
 
         return (
             <>
-                {me && logOut ? (
+                {authState === "loading" ? loading : null}
+                {authState === "signedIn" && me && logOut ? (
                     <JazzContext.Provider
                         value={{
                             me,
@@ -83,9 +90,8 @@ export function createJazzReactContext<Acc extends Account>({
                     >
                         {children}
                     </JazzContext.Provider>
-                ) : (
+                ) : null}
-                    AuthUI
+                {authState === "ready" && AuthUI}
-                )}
             </>
         );
     }
@@ -121,24 +127,20 @@ export function createJazzReactContext<Acc extends Account>({
         id: ID<V> | undefined,
         depth: D & DepthsIn<V> = [] as D & DepthsIn<V>,
     ): DeeplyLoaded<V, D> | undefined {
-        // for some reason (at least in React 18) - if we use state directly,
+        const [state, setState] = useState<{
-        // some updates get swallowed/UI doesn't update
+            value: DeeplyLoaded<V, D> | undefined;
-        const [_, setUpdates] = useState<number>(0);
+        }>({ value: undefined });
-        const state = useRef<DeeplyLoaded<V, D> | undefined>(undefined);
         const me = React.useContext(JazzContext)?.me;
 
         useEffect(() => {
             if (!id || !me) return;
-            return subscribeToCoValue(Schema, id, me, depth, (update) => {
-                state.current = update as DeeplyLoaded<V, D>;
 
-                setUpdates((u) => {
+            return subscribeToCoValue(Schema, id, me, depth, (value) => {
-                    return u + 1;
+                setState({ value });
-                });
             });
         }, [Schema, id, me]);
 
-        return state.current;
+        return state.value;
     }
 
     function useAcceptInvite<V extends CoValue>({
@@ -181,6 +183,7 @@ export interface JazzReactContext<Acc extends Account> {
     /** @category Provider Component */
     Provider: React.FC<{
         children: React.ReactNode;
+        loading?: React.ReactNode;
     }>;
 
     /** @category Hooks */

packages/jazz-run/CHANGELOG.md

@@ -1,5 +1,69 @@
 # jazz-autosub
 
+## 0.7.14
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.14
+    -   jazz-tools@0.7.14
+    -   cojson-transport-ws@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.13
+
+## 0.7.12
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.12
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+    -   cojson-transport-nodejs-ws@0.7.11
+    -   jazz-tools@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+    -   cojson-transport-nodejs-ws@0.7.10
+    -   jazz-tools@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+    -   cojson-transport-nodejs-ws@0.7.9
+    -   jazz-tools@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.8
+
+## 0.7.6
+
+### Patch Changes
+
+-   Updated dependencies
+    -   jazz-tools@0.7.6
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-run/package.json

@@ -3,7 +3,7 @@
     "bin": "./dist/index.js",
     "type": "module",
     "license": "MIT",
-    "version": "0.7.3",
+    "version": "0.7.14",
     "scripts": {
         "lint": "eslint . --ext ts,tsx",
         "format": "prettier --write './src/**/*.{ts,tsx}'",
@@ -15,7 +15,7 @@
         "@effect/platform-node": "^0.49.2",
         "@effect/schema": "^0.66.16",
         "cojson": "workspace:*",
-        "cojson-transport-nodejs-ws": "workspace:*",
+        "cojson-transport-ws": "workspace:*",
         "effect": "^3.1.5",
         "fast-check": "^3.17.2",
         "jazz-tools": "workspace:*",

packages/jazz-run/src/index.ts

@@ -2,11 +2,8 @@
 import { Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
 import { Console, Effect } from "effect";
-import {
+import { createWebSocketPeer } from "cojson-transport-ws";
-    websocketReadableStream,
+import { WebSocket } from "ws"
-    websocketWritableStream,
-} from "cojson-transport-nodejs-ws";
-import { WebSocket } from "ws";
 import {
     Account,
     WasmCrypto,
@@ -24,23 +21,20 @@ const peer = Options.text("peer")
 const accountCreate = Command.make(
     "create",
     { name, peer },
-    ({ name, peer }) => {
+    ({ name, peer: peerAddr }) => {
         return Effect.gen(function* () {
-            const ws = new WebSocket(peer);
-
             const crypto = yield* Effect.promise(() => WasmCrypto.create());
 
+            const peer = yield* createWebSocketPeer({
+                id: "upstream",
+                websocket: new WebSocket(peerAddr),
+                role: "server",
+            });
+
             const account: Account = yield* Effect.promise(async () =>
                 Account.create({
                     creationProps: { name },
-                    peersToLoadFrom: [
+                    peersToLoadFrom: [peer],
-                        {
-                            id: "upstream",
-                            role: "server",
-                            incoming: websocketReadableStream(ws),
-                            outgoing: websocketWritableStream(ws),
-                        },
-                    ],
                     crypto,
                 }),
             );
@@ -59,7 +53,11 @@ const accountCreate = Command.make(
                 ),
             );
 
-            const ws2 = new WebSocket(peer);
+            const peer2 = yield* createWebSocketPeer({
+                id: "upstream2",
+                websocket: new WebSocket(peerAddr),
+                role: "server",
+            });
 
             yield* Effect.promise(async () =>
                 Account.become({
@@ -68,14 +66,7 @@ const accountCreate = Command.make(
                     sessionID: cojsonInternals.newRandomSessionID(
                         account.id as unknown as AccountID,
                     ),
-                    peersToLoadFrom: [
+                    peersToLoadFrom: [peer2],
-                        {
-                            id: "upstream",
-                            role: "server",
-                            incoming: websocketReadableStream(ws2),
-                            outgoing: websocketWritableStream(ws2),
-                        },
-                    ],
                     crypto,
                 }),
             );

packages/jazz-tools/CHANGELOG.md

@@ -1,5 +1,61 @@
 # jazz-autosub
 
+## 0.7.14
+
+### Patch Changes
+
+-   Use Effect Queues and Streams instead of custom queue implementation
+-   Updated dependencies
+    -   cojson@0.7.14
+
+## 0.7.13
+
+### Patch Changes
+
+-   Fix CoList.toJSON()
+
+## 0.7.12
+
+### Patch Changes
+
+-   Fix: toJSON infinitely recurses on circular CoValue structures
+
+## 0.7.11
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.11
+    -   cojson-transport-nodejs-ws@0.7.11
+
+## 0.7.10
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.10
+    -   cojson-transport-nodejs-ws@0.7.10
+
+## 0.7.9
+
+### Patch Changes
+
+-   Updated dependencies
+    -   cojson@0.7.9
+    -   cojson-transport-nodejs-ws@0.7.9
+
+## 0.7.8
+
+### Patch Changes
+
+-   Fix CoMaps not initialising properly when passing too many init options
+
+## 0.7.6
+
+### Patch Changes
+
+-   Provide way to create accounts as another account
+
 ## 0.7.3
 
 ### Patch Changes

packages/jazz-tools/package.json

@@ -5,11 +5,10 @@
     "types": "./src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.7.3",
+    "version": "0.7.14",
     "dependencies": {
         "@effect/schema": "^0.66.16",
         "cojson": "workspace:*",
-        "cojson-transport-nodejs-ws": "workspace:*",
         "effect": "^3.1.5",
         "fast-check": "^3.17.2"
     },

packages/jazz-tools/src/coValues/account.ts

@@ -1,4 +1,4 @@
-import { LocalNode } from "cojson";
+import { LocalNode, cojsonInternals } from "cojson";
 import type {
     AgentSecret,
     CoID,
@@ -12,7 +12,7 @@ import type {
     SessionID,
 } from "cojson";
 import { Context, Effect, Stream } from "effect";
-import type {
+import {
     CoMap,
     CoValue,
     CoValueClass,
@@ -61,9 +61,11 @@ export class Account extends CoValueBase implements CoValue {
                 ref: () => Profile,
                 optional: false,
             } satisfies RefEncoded<Profile>,
-            root: "json" satisfies Schema,
+            root: {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ref: () => CoMap,
-        } as any;
+                optional: true,
+            } satisfies RefEncoded<CoMap>,
+        };
     }
 
     get _owner(): Account {
@@ -214,6 +216,29 @@ export class Account extends CoValueBase implements CoValue {
         return this.fromNode(node) as A;
     }
 
+    static async createAs<A extends Account>(
+        this: CoValueClass<A> & typeof Account,
+        as: Account,
+        options: {
+            creationProps: { name: string };
+        },
+    ) {
+        // TODO: is there a cleaner way to do this?
+        const connectedPeers = await Effect.runPromise(cojsonInternals.connectedPeers(
+            "creatingAccount",
+            "createdAccount",
+            { peer1role: "server", peer2role: "client" },
+        ));
+
+        as._raw.core.node.syncManager.addPeer(connectedPeers[1]);
+
+        return this.create<A>({
+            creationProps: options.creationProps,
+            crypto: as._raw.core.node.crypto,
+            peersToLoadFrom: [connectedPeers[0]],
+        });
+    }
+
     static fromNode<A extends Account>(
         this: CoValueClass<A>,
         node: LocalNode,

packages/jazz-tools/src/coValues/coList.ts

@@ -17,7 +17,6 @@ import type {
 import {
     Account,
     Group,
-    InitValues,
     ItemsSym,
     Ref,
     SchemaInit,
@@ -35,9 +34,40 @@ import {
 import { encodeSync, decodeSync } from "@effect/schema/Schema";
 import { Effect, Stream } from "effect";
 
-/** @category CoValues */
+/**
+ * CoLists are collaborative versions of plain arrays.
+ *
+ *  * @categoryDescription Content
+ * You can access items on a `CoList` as if they were normal items on a plain array, using `[]` notation, etc.
+ *
+ * Since `CoList` is a subclass of `Array`, you can use all the normal array methods like `push`, `pop`, `splice`, etc.
+ *
+ * ```ts
+ * colorList[0];
+ * colorList[3] = "yellow";
+ * colorList.push("Kawazaki Green");
+ * colorList.splice(1, 1);
+ * ```
+ *
+ * @category CoValues
+ */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export class CoList<Item = any> extends Array<Item> implements CoValue {
+    /**
+     * Declare a `CoList` by subclassing `CoList.Of(...)` and passing the item schema using `co`.
+     *
+     * @example
+     * ```ts
+     * class ColorList extends CoList.Of(
+     *   co.string
+     * ) {}
+     * class AnimalList extends CoList.Of(
+     *   co.ref(Animal)
+     * ) {}
+     * ```
+     *
+     * @category Declaration
+     */
     static Of<Item>(item: Item): typeof CoList<Item> {
         // TODO: cache superclass for item class
         return class CoListOf extends CoList<Item> {
@@ -45,36 +75,62 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         };
     }
 
-    /** @deprecated Use UPPERCASE `CoList.Of` instead! */
+    /**
+     * @ignore
+     * @deprecated Use UPPERCASE `CoList.Of` instead! */
     static of(..._args: never): never {
         throw new Error("Can't use Array.of with CoLists");
     }
 
+    /**
+     * The ID of this `CoList`
+     * @category Content */
     id!: ID<this>;
+    /** @category Type Helpers */
     _type!: "CoList";
     static {
         this.prototype._type = "CoList";
     }
+    /** @category Internals */
     _raw!: RawCoList;
+    /** @category Internals */
     _instanceID!: string;
 
     /** @internal This is only a marker type and doesn't exist at runtime */
     [ItemsSym]!: Item;
+    /** @internal */
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     static _schema: any;
+    /** @internal */
     get _schema(): {
         [ItemsSym]: SchemaFor<Item>;
     } {
         return (this.constructor as typeof CoList)._schema;
     }
 
+    /** @category Collaboration */
     get _owner(): Account | Group {
         return this._raw.group instanceof RawAccount
             ? Account.fromRaw(this._raw.group)
             : Group.fromRaw(this._raw.group);
     }
 
-    /** @category Content */
+    /**
+     * If a `CoList`'s items are a `co.ref(...)`, you can use `coList._refs[i]` to access
+     * the `Ref` instead of the potentially loaded/null value.
+     *
+     * This allows you to always get the ID or load the value manually.
+     *
+     * @example
+     * ```ts
+     * animals._refs[0].id; // => ID<Animal>
+     * animals._refs[0].value;
+     * // => Animal | null
+     * const animal = await animals._refs[0].load();
+     * ```
+     *
+     * @category Content
+     **/
     get _refs(): {
         [idx: number]: Exclude<Item, null> extends CoValue
             ? Ref<UnCo<Exclude<Item, null>>>
@@ -115,18 +171,11 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return Account.fromNode(this._raw.core.node);
     }
 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    [InitValues]?: any;
-
     static get [Symbol.species]() {
         return Array;
     }
 
-    constructor(
+    constructor(options: { fromRaw: RawCoList } | undefined) {
-        options:
-            | { init: Item[]; owner: Account | Group }
-            | { fromRaw: RawCoList },
-    ) {
         super();
 
         Object.defineProperty(this, "_instanceID", {
@@ -134,12 +183,7 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
             enumerable: false,
         });
 
-        if ("owner" in options) {
+        if (options && "fromRaw" in options) {
-            this[InitValues] = {
-                init: options.init,
-                owner: options.owner,
-            };
-        } else if ("fromRaw" in options) {
             Object.defineProperties(this, {
                 id: {
                     value: options.fromRaw.id,
@@ -152,17 +196,48 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return new Proxy(this, CoListProxyHandler as ProxyHandler<this>);
     }
 
+    /**
+     * Create a new CoList with the given initial values and owner.
+     *
+     * The owner (a Group or Account) determines access rights to the CoMap.
+     *
+     * The CoList will immediately be persisted and synced to connected peers.
+     *
+     * @example
+     * ```ts
+     * const colours = ColorList.create(
+     *   ["red", "green", "blue"],
+     *   { owner: me }
+     * );
+     * const animals = AnimalList.create(
+     *   [cat, dog, fish],
+     *   { owner: me }
+     * );
+     * ```
+     *
+     * @category Creation
+     **/
     static create<L extends CoList>(
         this: CoValueClass<L>,
         items: UnCo<L[number]>[],
         options: { owner: Account | Group },
     ) {
-        return new this({ init: items, owner: options.owner });
+        const instance = new this({ init: items, owner: options.owner });
+        const raw = options.owner._raw.createList(
+            toRawItems(items, instance._schema[ItemsSym]),
+        );
+
+        Object.defineProperties(instance, {
+            id: {
+                value: raw.id,
+                enumerable: false,
+            },
+            _raw: { value: raw, enumerable: false },
+        });
+
+        return instance;
     }
 
-    push(...items: Item[]): number;
-    /** @private For exact type compatibility with Array superclass */
-    push(...items: Item[]): number;
     push(...items: Item[]): number {
         for (const item of toRawItems(
             items as Item[],
@@ -174,9 +249,6 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return this._raw.entries().length;
     }
 
-    unshift(...items: Item[]): number;
-    /** @private For exact type compatibility with Array superclass */
-    unshift(...items: Item[]): number;
     unshift(...items: Item[]): number {
         for (const item of toRawItems(
             items as Item[],
@@ -228,7 +300,8 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return deleted;
     }
 
-    toJSON() {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    toJSON(_key?: string, seenAbove?: ID<CoValue>[]): any[] {
         const itemDescriptor = this._schema[ItemsSym] as Schema;
         if (itemDescriptor === "json") {
             return this._raw.asArray();
@@ -237,7 +310,14 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
                 .asArray()
                 .map((e) => encodeSync(itemDescriptor.encoded)(e));
         } else if (isRefEncoded(itemDescriptor)) {
-            return this.map((item) => (item as unknown as CoValue)?.toJSON());
+            return this.map((item, idx) =>
+                seenAbove?.includes((item as CoValue)?.id)
+                    ? { _circular: (item as CoValue).id }
+                    : (item as unknown as CoValue)?.toJSON(idx + "", [
+                          ...(seenAbove || []),
+                          this.id,
+                      ]),
+            );
         } else {
             return [];
         }
@@ -247,6 +327,7 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return this.toJSON();
     }
 
+    /** @category Internals */
     static fromRaw<V extends CoList>(
         this: CoValueClass<V> & typeof CoList,
         raw: RawCoList,
@@ -254,6 +335,7 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return new this({ fromRaw: raw });
     }
 
+    /** @internal */
     static schema<V extends CoList>(
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         this: { new (...args: any): V } & typeof CoList,
@@ -263,7 +345,28 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         Object.assign(this._schema, def);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Load a `CoList` with a given ID, as a given account.
+     *
+     * `depth` specifies if item CoValue references should be loaded as well before resolving.
+     * The `DeeplyLoaded` return type guarantees that corresponding referenced CoValues are loaded to the specified depth.
+     *
+     * You can pass `[]` or for shallowly loading only this CoList, or `[itemDepth]` for recursively loading referenced CoValues.
+     *
+     * Check out the `load` methods on `CoMap`/`CoList`/`CoStream`/`Group`/`Account` to see which depth structures are valid to nest.
+     *
+     * @example
+     * ```ts
+     * const animalsWithVets =
+     *   await ListOfAnimals.load(
+     *     "co_zdsMhHtfG6VNKt7RqPUPvUtN2Ax",
+     *     me,
+     *     [{ vet: {} }]
+     *   );
+     * ```
+     *
+     * @category Subscription & Loading
+     */
     static load<L extends CoList, Depth>(
         this: CoValueClass<L>,
         id: ID<L>,
@@ -273,7 +376,13 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return loadCoValue(this, id, as, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Effectful version of `CoList.load()`.
+     *
+     * Needs to be run inside an `AccountCtx` context.
+     *
+     * @category Subscription & Loading
+     */
     static loadEf<L extends CoList, Depth>(
         this: CoValueClass<L>,
         id: ID<L>,
@@ -282,7 +391,34 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return loadCoValueEf<L, Depth>(this, id, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Load and subscribe to a `CoList` with a given ID, as a given account.
+     *
+     * Automatically also subscribes to updates to all referenced/nested CoValues as soon as they are accessed in the listener.
+     *
+     * `depth` specifies if item CoValue references should be loaded as well before calling `listener` for the first time.
+     * The `DeeplyLoaded` return type guarantees that corresponding referenced CoValues are loaded to the specified depth.
+     *
+     * You can pass `[]` or for shallowly loading only this CoList, or `[itemDepth]` for recursively loading referenced CoValues.
+     *
+     * Check out the `load` methods on `CoMap`/`CoList`/`CoStream`/`Group`/`Account` to see which depth structures are valid to nest.
+     *
+     * Returns an unsubscribe function that you should call when you no longer need updates.
+     *
+     * Also see the `useCoState` hook to reactively subscribe to a CoValue in a React component.
+     *
+     * @example
+     * ```ts
+     * const unsub = ListOfAnimals.subscribe(
+     *   "co_zdsMhHtfG6VNKt7RqPUPvUtN2Ax",
+     *   me,
+     *   { vet: {} },
+     *   (animalsWithVets) => console.log(animalsWithVets)
+     * );
+     * ```
+     *
+     * @category Subscription & Loading
+     */
     static subscribe<L extends CoList, Depth>(
         this: CoValueClass<L>,
         id: ID<L>,
@@ -293,7 +429,13 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return subscribeToCoValue<L, Depth>(this, id, as, depth, listener);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Effectful version of `CoList.subscribe()` that returns a stream of updates.
+     *
+     * Needs to be run inside an `AccountCtx` context.
+     *
+     * @category Subscription & Loading
+     */
     static subscribeEf<L extends CoList, Depth>(
         this: CoValueClass<L>,
         id: ID<L>,
@@ -302,7 +444,13 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return subscribeToCoValueEf<L, Depth>(this, id, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Given an already loaded `CoList`, ensure that items are loaded to the specified depth.
+     *
+     * Works like `CoList.load()`, but you don't need to pass the ID or the account to load as again.
+     *
+     * @category Subscription & Loading
+     */
     ensureLoaded<L extends CoList, Depth>(
         this: L,
         depth: Depth & DepthsIn<L>,
@@ -310,7 +458,15 @@ export class CoList<Item = any> extends Array<Item> implements CoValue {
         return ensureCoValueLoaded(this, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Given an already loaded `CoList`, subscribe to updates to the `CoList` and ensure that items are loaded to the specified depth.
+     *
+     * Works like `CoList.subscribe()`, but you don't need to pass the ID or the account to load as again.
+     *
+     * Returns an unsubscribe function that you should call when you no longer need updates.
+     *
+     * @category Subscription & Loading
+     **/
     subscribe<L extends CoList, Depth>(
         this: L,
         depth: Depth & DepthsIn<L>,
@@ -341,24 +497,6 @@ function toRawItems<Item>(items: Item[], itemDescriptor: Schema) {
     return rawItems;
 }
 
-function init(list: CoList) {
-    if (list[InitValues]) {
-        const { init, owner } = list[InitValues];
-        const raw = owner._raw.createList(
-            toRawItems(init, list._schema[ItemsSym]),
-        );
-
-        Object.defineProperties(list, {
-            id: {
-                value: raw.id,
-                enumerable: false,
-            },
-            _raw: { value: raw, enumerable: false },
-        });
-        delete list[InitValues];
-    }
-}
-
 const CoListProxyHandler: ProxyHandler<CoList> = {
     get(target, key, receiver) {
         if (typeof key === "string" && !isNaN(+key)) {
@@ -394,7 +532,6 @@ const CoListProxyHandler: ProxyHandler<CoList> = {
             (target.constructor as typeof CoList)._schema ||= {};
             (target.constructor as typeof CoList)._schema[ItemsSym] =
                 value[SchemaInit];
-            init(target);
             return true;
         }
         if (typeof key === "string" && !isNaN(+key)) {
@@ -423,7 +560,6 @@ const CoListProxyHandler: ProxyHandler<CoList> = {
             (target.constructor as typeof CoList)._schema ||= {};
             (target.constructor as typeof CoList)._schema[ItemsSym] =
                 descriptor.value[SchemaInit];
-            init(target);
             return true;
         } else {
             return Reflect.defineProperty(target, key, descriptor);

packages/jazz-tools/src/coValues/coMap.ts

@@ -24,7 +24,6 @@ import {
     makeRefs,
     subscriptionsScopes,
     ItemsSym,
-    InitValues,
     isRefEncoded,
     loadCoValue,
     loadCoValueEf,
@@ -42,17 +41,14 @@ type CoMapEdit<V> = {
     madeAt: Date;
 };
 
-type InitValuesFor<C extends CoMap> = {
-    init: Simplify<CoMapInit<C>>;
-    owner: Account | Group;
-};
-
 /**
  * CoMaps are collaborative versions of plain objects, mapping string-like keys to values.
  *
  * @categoryDescription Declaration
  * Declare your own CoMap schemas by subclassing `CoMap` and assigning field schemas with `co`.
  *
+ * Optional `co.ref(...)` fields must be marked with `{ optional: true }`.
+ *
  * ```ts
  * import { co, CoMap } from "jazz-tools";
  *
@@ -60,6 +56,7 @@ type InitValuesFor<C extends CoMap> = {
  *   name = co.string;
  *   age = co.number;
  *   pet = co.ref(Animal);
+ *   car = co.ref(Car, { optional: true });
  * }
  * ```
  *
@@ -103,17 +100,19 @@ export class CoMap extends CoValueBase implements CoValue {
     /**
      * If property `prop` is a `co.ref(...)`, you can use `coMaps._refs.prop` to access
      * the `Ref` instead of the potentially loaded/null value.
+     *
      * This allows you to always get the ID or load the value manually.
      *
      * @example
      * ```ts
      * person._refs.pet.id; // => ID<Animal>
      * person._refs.pet.value;
-     * // => Animal | undefined
+     * // => Animal | null
      * const pet = await person._refs.pet.load();
      * ```
      *
-     * @category Content */
+     * @category Content
+     **/
     get _refs(): {
         [Key in CoKeys<this>]: IfCo<this[Key], RefIfCoValue<this[Key]>>;
     } {
@@ -196,47 +195,67 @@ export class CoMap extends CoValueBase implements CoValue {
         return Account.fromNode(this._raw.core.node);
     }
 
-    /** @internal */
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    [InitValues]?: any;
-
     /** @internal */
     constructor(
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        options: { fromRaw: RawCoMap } | { init: any; owner: Account | Group },
+        options: { fromRaw: RawCoMap } | undefined,
     ) {
         super();
 
-        if ("owner" in options) {
+        if (options) {
-            this[InitValues] = {
+            if ("fromRaw" in options) {
-                init: options.init,
+                Object.defineProperties(this, {
-                owner: options.owner,
+                    id: {
-            } as InitValuesFor<this>;
+                        value: options.fromRaw.id as unknown as ID<this>,
-        } else if ("fromRaw" in options) {
+                        enumerable: false,
-            Object.defineProperties(this, {
+                    },
-                id: {
+                    _raw: { value: options.fromRaw, enumerable: false },
-                    value: options.fromRaw.id as unknown as ID<this>,
+                });
-                    enumerable: false,
+            } else {
-                },
+                throw new Error("Invalid CoMap constructor arguments");
-                _raw: { value: options.fromRaw, enumerable: false },
+            }
-            });
-        } else {
-            throw new Error("Invalid CoMap constructor arguments");
         }
 
         return new Proxy(this, CoMapProxyHandler as ProxyHandler<this>);
     }
 
-    /** @category Creation */
+    /**
+     * Create a new CoMap with the given initial values and owner.
+     *
+     * The owner (a Group or Account) determines access rights to the CoMap.
+     *
+     * The CoMap will immediately be persisted and synced to connected peers.
+     *
+     * @example
+     * ```ts
+     * const person = Person.create({
+     *   name: "Alice",
+     *   age: 42,
+     *   pet: cat,
+     * }, { owner: friendGroup });
+     * ```
+     *
+     * @category Creation
+     **/
     static create<M extends CoMap>(
         this: CoValueClass<M>,
         init: Simplify<CoMapInit<M>>,
         options: { owner: Account | Group },
     ) {
-        return new this({ init, owner: options.owner });
+        const instance = new this();
+        const raw = instance.rawFromInit(init, options.owner);
+        Object.defineProperties(instance, {
+            id: {
+                value: raw.id,
+                enumerable: false,
+            },
+            _raw: { value: raw, enumerable: false },
+        });
+        return instance;
     }
 
-    toJSON() {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    toJSON(_key?: string, seenAbove?: ID<CoValue>[]): any[] {
         const jsonedFields = this._raw.keys().map((key) => {
             const tKey = key as CoKeys<this>;
             const descriptor = (this._schema[tKey] ||
@@ -246,7 +265,15 @@ export class CoMap extends CoValueBase implements CoValue {
                 return [key, this._raw.get(key)];
             } else if (isRefEncoded(descriptor)) {
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const jsonedRef = (this as any)[tKey]?.toJSON();
+                if (seenAbove?.includes((this as any)[tKey]?.id)) {
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    return [key, { _circular: (this as any)[tKey]?.id }];
+                }
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const jsonedRef = (this as any)[tKey]?.toJSON(tKey, [
+                    ...(seenAbove || []),
+                    this.id,
+                ]);
                 return [key, jsonedRef];
             } else {
                 return [key, undefined];
@@ -284,6 +311,10 @@ export class CoMap extends CoValueBase implements CoValue {
                     key as keyof typeof this._schema
                 ] || this._schema[ItemsSym]) as Schema;
 
+                if (!descriptor) {
+                    continue;
+                }
+
                 if (descriptor === "json") {
                     rawInit[key] = initValue as JsonValue;
                 } else if (isRefEncoded(descriptor)) {
@@ -301,7 +332,24 @@ export class CoMap extends CoValueBase implements CoValue {
         return rawOwner.createMap(rawInit);
     }
 
-    /** @category Declaration */
+    /**
+     * Declare a Record-like CoMap schema, by extending `CoMap.Record(...)` and passing the value schema using `co`. Keys are always `string`.
+     *
+     * @example
+     * ```ts
+     * import { co, CoMap } from "jazz-tools";
+     *
+     * class ColorToFruitMap extends CoMap.Record(
+     *  co.ref(Fruit)
+     * ) {}
+     *
+     * // assume we have map: ColorToFruitMap
+     * // and strawberry: Fruit
+     * map["red"] = strawberry;
+     * ```
+     *
+     * @category Declaration
+     */
     static Record<Value>(value: IfCo<Value, Value>) {
         // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
         class RecordLikeCoMap extends CoMap {
@@ -313,7 +361,27 @@ export class CoMap extends CoValueBase implements CoValue {
         return RecordLikeCoMap;
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Load a `CoMap` with a given ID, as a given account.
+     *
+     * `depth` specifies which (if any) fields that reference other CoValues to load as well before resolving.
+     * The `DeeplyLoaded` return type guarantees that corresponding referenced CoValues are loaded to the specified depth.
+     *
+     * You can pass `[]` or `{}` for shallowly loading only this CoMap, or `{ fieldA: depthA, fieldB: depthB }` for recursively loading referenced CoValues.
+     *
+     * Check out the `load` methods on `CoMap`/`CoList`/`CoStream`/`Group`/`Account` to see which depth structures are valid to nest.
+     *
+     * @example
+     * ```ts
+     * const person = await Person.load(
+     *   "co_zdsMhHtfG6VNKt7RqPUPvUtN2Ax",
+     *   me,
+     *   { pet: {} }
+     * );
+     * ```
+     *
+     * @category Subscription & Loading
+     */
     static load<M extends CoMap, Depth>(
         this: CoValueClass<M>,
         id: ID<M>,
@@ -323,7 +391,13 @@ export class CoMap extends CoValueBase implements CoValue {
         return loadCoValue(this, id, as, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Effectful version of `CoMap.load()`.
+     *
+     * Needs to be run inside an `AccountCtx` context.
+     *
+     * @category Subscription & Loading
+     */
     static loadEf<M extends CoMap, Depth>(
         this: CoValueClass<M>,
         id: ID<M>,
@@ -332,7 +406,34 @@ export class CoMap extends CoValueBase implements CoValue {
         return loadCoValueEf<M, Depth>(this, id, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Load and subscribe to a `CoMap` with a given ID, as a given account.
+     *
+     * Automatically also subscribes to updates to all referenced/nested CoValues as soon as they are accessed in the listener.
+     *
+     * `depth` specifies which (if any) fields that reference other CoValues to load as well before calling `listener` for the first time.
+     * The `DeeplyLoaded` return type guarantees that corresponding referenced CoValues are loaded to the specified depth.
+     *
+     * You can pass `[]` or `{}` for shallowly loading only this CoMap, or `{ fieldA: depthA, fieldB: depthB }` for recursively loading referenced CoValues.
+     *
+     * Check out the `load` methods on `CoMap`/`CoList`/`CoStream`/`Group`/`Account` to see which depth structures are valid to nest.
+     *
+     * Returns an unsubscribe function that you should call when you no longer need updates.
+     *
+     * Also see the `useCoState` hook to reactively subscribe to a CoValue in a React component.
+     *
+     * @example
+     * ```ts
+     * const unsub = Person.subscribe(
+     *   "co_zdsMhHtfG6VNKt7RqPUPvUtN2Ax",
+     *   me,
+     *   { pet: {} },
+     *   (person) => console.log(person)
+     * );
+     * ```
+     *
+     * @category Subscription & Loading
+     */
     static subscribe<M extends CoMap, Depth>(
         this: CoValueClass<M>,
         id: ID<M>,
@@ -343,7 +444,13 @@ export class CoMap extends CoValueBase implements CoValue {
         return subscribeToCoValue<M, Depth>(this, id, as, depth, listener);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Effectful version of `CoMap.subscribe()` that returns a stream of updates.
+     *
+     * Needs to be run inside an `AccountCtx` context.
+     *
+     * @category Subscription & Loading
+     */
     static subscribeEf<M extends CoMap, Depth>(
         this: CoValueClass<M>,
         id: ID<M>,
@@ -352,7 +459,13 @@ export class CoMap extends CoValueBase implements CoValue {
         return subscribeToCoValueEf<M, Depth>(this, id, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Given an already loaded `CoMap`, ensure that the specified fields are loaded to the specified depth.
+     *
+     * Works like `CoMap.load()`, but you don't need to pass the ID or the account to load as again.
+     *
+     * @category Subscription & Loading
+     */
     ensureLoaded<M extends CoMap, Depth>(
         this: M,
         depth: Depth & DepthsIn<M>,
@@ -360,7 +473,15 @@ export class CoMap extends CoValueBase implements CoValue {
         return ensureCoValueLoaded(this, depth);
     }
 
-    /** @category Subscription & Loading */
+    /**
+     * Given an already loaded `CoMap`, subscribe to updates to the `CoMap` and ensure that the specified fields are loaded to the specified depth.
+     *
+     * Works like `CoMap.subscribe()`, but you don't need to pass the ID or the account to load as again.
+     *
+     * Returns an unsubscribe function that you should call when you no longer need updates.
+     *
+     * @category Subscription & Loading
+     **/
     subscribe<M extends CoMap, Depth>(
         this: M,
         depth: Depth & DepthsIn<M>,
@@ -381,30 +502,6 @@ export type CoMapInit<Map extends object> = {
         : IfCo<Map[Key], Key>]: Map[Key];
 } & { [Key in CoKeys<Map> as IfCo<Map[Key], Key>]?: Map[Key] };
 
-function tryInit(map: CoMap) {
-    if (
-        map[InitValues] &&
-        (map._schema[ItemsSym] ||
-            Object.keys(map[InitValues].init).every(
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                (key) => (map._schema as any)[key],
-            ))
-    ) {
-        const raw = map.rawFromInit(
-            map[InitValues].init,
-            map[InitValues].owner,
-        );
-        Object.defineProperties(map, {
-            id: {
-                value: raw.id,
-                enumerable: false,
-            },
-            _raw: { value: raw, enumerable: false },
-        });
-        delete map[InitValues];
-    }
-}
-
 // TODO: cache handlers per descriptor for performance?
 const CoMapProxyHandler: ProxyHandler<CoMap> = {
     get(target, key, receiver) {
@@ -447,7 +544,6 @@ const CoMapProxyHandler: ProxyHandler<CoMap> = {
             (target.constructor as typeof CoMap)._schema ||= {};
             (target.constructor as typeof CoMap)._schema[key] =
                 value[SchemaInit];
-            tryInit(target);
             return true;
         }
 
@@ -478,7 +574,6 @@ const CoMapProxyHandler: ProxyHandler<CoMap> = {
             (target.constructor as typeof CoMap)._schema ||= {};
             (target.constructor as typeof CoMap)._schema[key as string] =
                 attributes.value[SchemaInit];
-            tryInit(target);
             return true;
         } else {
             return Reflect.defineProperty(target, key, attributes);

packages/jazz-tools/src/coValues/coStream.ts

@@ -30,7 +30,6 @@ import {
     Ref,
     inspect,
     co,
-    InitValues,
     SchemaInit,
     isRefEncoded,
     loadCoValue,
@@ -93,9 +92,6 @@ export class CoStream<Item = any> extends CoValueBase implements CoValue {
         return this.perSession[this._loadedAs.sessionID!];
     }
 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    [InitValues]?: any;
-
     constructor(
         options:
             | { init: Item[]; owner: Account | Group }
@@ -103,7 +99,7 @@ export class CoStream<Item = any> extends CoValueBase implements CoValue {
     ) {
         super();
 
-        if ("fromRaw" in options) {
+        if (options && "fromRaw" in options) {
             Object.defineProperties(this, {
                 id: {
                     value: options.fromRaw.id,
@@ -111,11 +107,6 @@ export class CoStream<Item = any> extends CoValueBase implements CoValue {
                 },
                 _raw: { value: options.fromRaw, enumerable: false },
             });
-        } else {
-            this[InitValues] = {
-                init: options.init,
-                owner: options.owner,
-            };
         }
 
         return new Proxy(this, CoStreamProxyHandler as ProxyHandler<this>);
@@ -126,7 +117,21 @@ export class CoStream<Item = any> extends CoValueBase implements CoValue {
         init: S extends CoStream<infer Item> ? UnCo<Item>[] : never,
         options: { owner: Account | Group },
     ) {
-        return new this({ init, owner: options.owner });
+        const instance = new this({ init, owner: options.owner });
+        const raw = options.owner._raw.createStream();
+
+        Object.defineProperties(instance, {
+            id: {
+                value: raw.id,
+                enumerable: false,
+            },
+            _raw: { value: raw, enumerable: false },
+        });
+
+        if (init) {
+            instance.push(...init);
+        }
+        return instance;
     }
 
     push(...items: Item[]) {
@@ -317,27 +322,6 @@ function entryFromRawEntry<Item>(
     };
 }
 
-function init(stream: CoStream) {
-    const init = stream[InitValues];
-    if (!init) return;
-
-    const raw = init.owner._raw.createStream();
-
-    Object.defineProperties(stream, {
-        id: {
-            value: raw.id,
-            enumerable: false,
-        },
-        _raw: { value: raw, enumerable: false },
-    });
-
-    if (init.init) {
-        stream.push(...init.init);
-    }
-
-    delete stream[InitValues];
-}
-
 export const CoStreamProxyHandler: ProxyHandler<CoStream> = {
     get(target, key, receiver) {
         if (typeof key === "string" && key.startsWith("co_")) {
@@ -391,7 +375,6 @@ export const CoStreamProxyHandler: ProxyHandler<CoStream> = {
             (target.constructor as typeof CoStream)._schema ||= {};
             (target.constructor as typeof CoStream)._schema[ItemsSym] =
                 value[SchemaInit];
-            init(target);
             return true;
         } else {
             return Reflect.set(target, key, value, receiver);
@@ -407,7 +390,6 @@ export const CoStreamProxyHandler: ProxyHandler<CoStream> = {
             (target.constructor as typeof CoStream)._schema ||= {};
             (target.constructor as typeof CoStream)._schema[ItemsSym] =
                 descriptor.value[SchemaInit];
-            init(target);
             return true;
         } else {
             return Reflect.defineProperty(target, key, descriptor);

packages/jazz-tools/src/coValues/group.ts

@@ -114,7 +114,7 @@ export class Group extends CoValueBase implements CoValue {
             const initOwner = options.owner;
             if (!initOwner) throw new Error("No owner provided");
             if (
-                initOwner instanceof Account &&
+                initOwner._type === "Account" &&
                 isControlledAccount(initOwner)
             ) {
                 const rawOwner = initOwner._raw;

packages/jazz-tools/src/coValues/interfaces.ts

@@ -40,7 +40,7 @@ export interface CoValue {
     readonly _loadedAs: Account;
     /** @category Stringifying & Inspection */
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    toJSON(): any[] | object;
+    toJSON(key?: string, seenAbove?: ID<CoValue>[]): any[] | object | string;
     /** @category Stringifying & Inspection */
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     [inspect](): any;
@@ -68,6 +68,7 @@ export class CoValueBase implements CoValue {
     id!: ID<this>;
     _type!: string;
     _raw!: RawCoValue;
+    /** @category Internals */
     _instanceID!: string;
 
     get _owner(): Account | Group {
@@ -107,7 +108,7 @@ export class CoValueBase implements CoValue {
     }
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    toJSON(): object | any[] {
+    toJSON(): object | any[] | string {
         return {
             id: this.id,
             type: this._type,

packages/jazz-tools/src/implementation/symbols.ts

@@ -1,9 +1,6 @@
 export const SchemaInit = Symbol.for("SchemaInit");
 export type SchemaInit = typeof SchemaInit;
 
-export const InitValues = Symbol.for("InitValues");
-export type InitValues = typeof InitValues;
-
 export const ItemsSym = Symbol.for("items");
 export type ItemsSym = typeof ItemsSym;
 

packages/jazz-tools/src/index.ts

@@ -17,7 +17,7 @@ export type { ID, CoValue } from "./internal.js";
 
 export { Encoders, co } from "./internal.js";
 
-export { CoMap } from "./internal.js";
+export { CoMap, type CoMapInit } from "./internal.js";
 export { CoList } from "./internal.js";
 export { CoStream, BinaryCoStream } from "./internal.js";
 export { Group, Profile } from "./internal.js";

packages/jazz-tools/src/tests/coList.test.ts

@@ -157,11 +157,11 @@ describe("CoList resolution", async () => {
     test("Loading and availability", async () => {
         const { me, list } = await initNodeAndList();
 
-        const [initialAsPeer, secondPeer] = connectedPeers(
+        const [initialAsPeer, secondPeer] = await Effect.runPromise(connectedPeers(
             "initial",
             "second",
             { peer1role: "server", peer2role: "client" },
-        );
+        ));
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
         }
@@ -216,11 +216,11 @@ describe("CoList resolution", async () => {
     test("Subscription & auto-resolution", async () => {
         const { me, list } = await initNodeAndList();
 
-        const [initialAsPeer, secondPeer] = connectedPeers(
+        const [initialAsPeer, secondPeer] = await Effect.runPromise(connectedPeers(
             "initial",
             "second",
             { peer1role: "server", peer2role: "client" },
-        );
+        ));
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
         }

packages/jazz-tools/src/tests/coMap.test.ts

@@ -52,6 +52,22 @@ describe("Simple CoMap operations", async () => {
         expect(Object.keys(map)).toEqual(["color", "_height", "birthday"]);
     });
 
+    test("Construction with too many things provided", () => {
+        const mapWithExtra = TestMap.create(
+            {
+                color: "red",
+                _height: 10,
+                birthday: birthday,
+                name: "Hermes",
+                extra: "extra",
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            } as any,
+            { owner: me },
+        );
+
+        expect(mapWithExtra.color).toEqual("red");
+    });
+
     describe("Mutation", () => {
         test("assignment & deletion", () => {
             map.color = "blue";
@@ -237,11 +253,11 @@ describe("CoMap resolution", async () => {
 
     test("Loading and availability", async () => {
         const { me, map } = await initNodeAndMap();
-        const [initialAsPeer, secondPeer] = connectedPeers(
+        const [initialAsPeer, secondPeer] = await Effect.runPromise(connectedPeers(
             "initial",
             "second",
             { peer1role: "server", peer2role: "client" },
-        );
+        ));
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
         }
@@ -307,11 +323,11 @@ describe("CoMap resolution", async () => {
     test("Subscription & auto-resolution", async () => {
         const { me, map } = await initNodeAndMap();
 
-        const [initialAsPeer, secondAsPeer] = connectedPeers(
+        const [initialAsPeer, secondAsPeer] = await Effect.runPromise(connectedPeers(
             "initial",
             "second",
             { peer1role: "server", peer2role: "client" },
-        );
+        ));
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
         }

packages/jazz-tools/src/tests/coStream.test.ts

@@ -83,10 +83,11 @@ describe("CoStream resolution", async () => {
 
     test("Loading and availability", async () => {
         const { me, stream } = await initNodeAndStream();
-        const [initialAsPeer, secondPeer] = connectedPeers(
+        const [initialAsPeer, secondPeer] = await Effect.runPromise(
-            "initial",
+            connectedPeers("initial", "second", {
-            "second",
+                peer1role: "server",
-            { peer1role: "server", peer2role: "client" },
+                peer2role: "client",
+            }),
         );
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
@@ -175,10 +176,11 @@ describe("CoStream resolution", async () => {
     test("Subscription & auto-resolution", async () => {
         const { me, stream } = await initNodeAndStream();
 
-        const [initialAsPeer, secondAsPeer] = connectedPeers(
+        const [initialAsPeer, secondAsPeer] = await Effect.runPromise(
-            "initial",
+            connectedPeers("initial", "second", {
-            "second",
+                peer1role: "server",
-            { peer1role: "server", peer2role: "client" },
+                peer2role: "client",
+            }),
         );
         me._raw.core.node.syncManager.addPeer(secondAsPeer);
         if (!isControlledAccount(me)) {
@@ -325,10 +327,11 @@ describe("BinaryCoStream loading & Subscription", async () => {
 
     test("Loading and availability", async () => {
         const { me, stream } = await initNodeAndStream();
-        const [initialAsPeer, secondAsPeer] = connectedPeers(
+        const [initialAsPeer, secondAsPeer] = await Effect.runPromise(
-            "initial",
+            connectedPeers("initial", "second", {
-            "second",
+                peer1role: "server",
-            { peer1role: "server", peer2role: "client" },
+                peer2role: "client",
+            }),
         );
         if (!isControlledAccount(me)) {
             throw "me is not a controlled account";
@@ -357,30 +360,32 @@ describe("BinaryCoStream loading & Subscription", async () => {
     });
 
     test("Subscription", async () => {
-        const { me } = await initNodeAndStream();
-
-        const stream = BinaryCoStream.create({ owner: me });
-
-        const [initialAsPeer, secondAsPeer] = connectedPeers(
-            "initial",
-            "second",
-            { peer1role: "server", peer2role: "client" },
-        );
-        me._raw.core.node.syncManager.addPeer(secondAsPeer);
-        if (!isControlledAccount(me)) {
-            throw "me is not a controlled account";
-        }
-        const meOnSecondPeer = await Account.become({
-            accountID: me.id,
-            accountSecret: me._raw.agentSecret,
-            peersToLoadFrom: [initialAsPeer],
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            sessionID: newRandomSessionID(me.id as any),
-            crypto: Crypto,
-        });
-
         await Effect.runPromise(
             Effect.gen(function* ($) {
+                const { me } = yield* Effect.promise(() => initNodeAndStream());
+
+                const stream = BinaryCoStream.create({ owner: me });
+
+                const [initialAsPeer, secondAsPeer] = yield* connectedPeers(
+                    "initial",
+                    "second",
+                    { peer1role: "server", peer2role: "client" },
+                );
+                me._raw.core.node.syncManager.addPeer(secondAsPeer);
+                if (!isControlledAccount(me)) {
+                    throw "me is not a controlled account";
+                }
+                const meOnSecondPeer = yield* Effect.promise(() =>
+                    Account.become({
+                        accountID: me.id,
+                        accountSecret: me._raw.agentSecret,
+                        peersToLoadFrom: [initialAsPeer],
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        sessionID: newRandomSessionID(me.id as any),
+                        crypto: Crypto,
+                    }),
+                );
+
                 const queue = yield* $(Queue.unbounded<BinaryCoStream>());
 
                 BinaryCoStream.subscribe(

packages/jazz-tools/src/tests/deepLoading.test.ts

@@ -14,6 +14,7 @@ import {
     ID,
 } from "../index.js";
 import { newRandomSessionID } from "cojson/src/coValueCore.js";
+import { Effect } from "effect";
 
 class TestMap extends CoMap {
     list = co.ref(TestList);
@@ -38,10 +39,10 @@ describe("Deep loading with depth arg", async () => {
         crypto: Crypto,
     });
 
-    const [initialAsPeer, secondPeer] = connectedPeers("initial", "second", {
+    const [initialAsPeer, secondPeer] = await Effect.runPromise(connectedPeers("initial", "second", {
         peer1role: "server",
         peer2role: "client",
-    });
+    }));
     if (!isControlledAccount(me)) {
         throw "me is not a controlled account";
     }
@@ -137,9 +138,8 @@ describe("Deep loading with depth arg", async () => {
             throw new Error("map4 is undefined");
         }
         expect(map4.list[0]?.stream).not.toBe(null);
-        // TODO: we should expect null here, but apparently we don't even have the id/ref?
+        expect(map4.list[0]?.stream?.[me.id]?.value).toBe(null);
-        expect(map4.list[0]?.stream?.[me.id]?.value).not.toBeDefined();
+        expect(map4.list[0]?.stream?.byMe?.value).toBe(null);
-        expect(map4.list[0]?.stream?.byMe?.value).not.toBeDefined();
 
         const map5 = await TestMap.load(map.id, meOnSecondPeer, {
             list: [{ stream: [{}] }],
@@ -252,10 +252,10 @@ test("Deep loading a record-like coMap", async () => {
         crypto: Crypto,
     });
 
-    const [initialAsPeer, secondPeer] = connectedPeers("initial", "second", {
+    const [initialAsPeer, secondPeer] = await Effect.runPromise(connectedPeers("initial", "second", {
         peer1role: "server",
         peer2role: "client",
-    });
+    }));
     if (!isControlledAccount(me)) {
         throw "me is not a controlled account";
     }