Publish

- jazz-example-pets@0.0.14
 - jazz-example-todo@0.0.39
 - cojson@0.3.5
 - cojson-simple-sync@0.3.7
 - cojson-storage-indexeddb@0.3.5
 - cojson-storage-sqlite@0.3.7
 - jazz-browser@0.3.5
 - jazz-browser-auth-local@0.3.5
 - jazz-browser-media-images@0.3.5
 - jazz-react@0.3.5
 - jazz-react-auth-local@0.3.5
 - jazz-react-media-images@0.3.5

.github/workflows/build-and-deploy.yaml

@@ -7,8 +7,11 @@ on:
     branches: [ "main" ]
 
 jobs:
-  build-and-deploy:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        example: ["todo", "pets"]
 
     steps:
     - uses: actions/checkout@v3
@@ -17,40 +20,50 @@ jobs:
 
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 16
         cache: 'yarn'
         cache-dependency-path: yarn.lock
-    - name: Nuke Workspace
-      run: |
-        rm package.json yarn.lock;
-    - name: Yarn Build
-      run: |
-        yarn install --frozen-lockfile;
-        yarn build;
-      working-directory: ./examples/todo
 
-    - uses: satackey/action-docker-layer-caching@v0.0.11
-      continue-on-error: true
-      with:
-        key: docker-layer-caching-${{ github.workflow }}-{hash}
-        restore-keys: |
-          docker-layer-caching-${{ github.workflow }}-
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry: ghcr.io
         username: gardencmp
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Docker Build & Push
+    - name: Nuke Workspace
       run: |
-        export DOCKER_TAG=ghcr.io/gardencmp/jazz-example-todo:${{github.head_ref || github.ref_name}}-${{github.sha}}-$(date +%s) ;
-        docker build . --file Dockerfile --tag $DOCKER_TAG;
-        docker push $DOCKER_TAG;
-        echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
-      working-directory: ./examples/todo
+        rm package.json yarn.lock;
 
+    - name: Yarn Build
+      run: |
+        yarn install --frozen-lockfile;
+        yarn build;
+      working-directory: ./examples/${{ matrix.example }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v4
+      with:
+        context: ./examples/${{ matrix.example }}
+        push: true
+        tags: ghcr.io/gardencmp/${{github.event.repository.name}}-example-${{ matrix.example }}:${{github.head_ref || github.ref_name}}-${{github.sha}}-${{github.run_number}}-${{github.run_attempt}}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        example: ["todo", "pets"]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
     - uses: gacts/install-nomad@v1
     - name: Tailscale
       uses: tailscale/github-action@v1
@@ -69,13 +82,9 @@ jobs:
 
         export DOCKER_USER=gardencmp;
         export DOCKER_PASSWORD=${{ secrets.DOCKER_PULL_PAT }};
-        export DOCKER_TAG=${{ env.DOCKER_TAG }};
+        export DOCKER_TAG=ghcr.io/gardencmp/${{github.event.repository.name}}-example-${{ matrix.example }}:${{github.head_ref || github.ref_name}}-${{github.sha}}-${{github.run_number}}-${{github.run_attempt}};
 
-        for region in ${{ vars.DEPLOY_REGIONS }}
-        do
-          export REGION=$region;
-          envsubst '${DOCKER_USER} ${DOCKER_PASSWORD} ${DOCKER_TAG} ${BRANCH_SUFFIX} ${BRANCH_SUBDOMAIN} ${REGION}' < job-template.nomad > job-instance.nomad;
-          cat job-instance.nomad;
-          NOMAD_ADDR='${{ secrets.NOMAD_ADDR }}' nomad job run job-instance.nomad;
-        done
-      working-directory: ./examples/todo
+        envsubst '${DOCKER_USER} ${DOCKER_PASSWORD} ${DOCKER_TAG} ${BRANCH_SUFFIX} ${BRANCH_SUBDOMAIN}' < job-template.nomad > job-instance.nomad;
+        cat job-instance.nomad;
+        NOMAD_ADDR='http://control1v2-london:4646' nomad job run job-instance.nomad;
+      working-directory: ./examples/${{ matrix.example }}

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 yarn-error.log
-lerna-debug.log
+lerna-debug.log
+docsTmp

README.md

@@ -1,338 +1,116 @@
 # Jazz - instant sync
 
-Homepage: [jazz.tools](https://jazz.tools) — [Discord](https://discord.gg/utDMjHYg42)
+<sub>Homepage: [jazz.tools](https://jazz.tools) &mdash; Docs: [DOCS.md](./DOCS.md) &mdash; Community & support: [Discord](https://discord.gg/utDMjHYg42) &mdash; Updates: [Twitter](https://twitter.com/jazz_tools) & [Email](https://gcmp.io/news)</sub>
 
-Jazz is an open-source toolkit for *secure telepathic data.*
+ **Jazz is an open-source toolkit for building apps with *secure sync.***
 
-- Ship faster & simplify your frontend and backend
-- Get cross-device sync, real-time collaboration & offline support for free
+Quickly build and ship apps with:
 
-[Jazz Global Mesh](https://jazz.tools/mesh) is serverless sync & storage for Jazz apps. (currently free!)
+- **Cross-device sync**
+- **Collaborative features** (incl. real-time multiplayer)
+- **Instantly reacting UIs**
+- Local-first storage & offline support
+- File upload and real-time media streaming
 
+# What is *secure sync*?
 
+**Sync** means that, *instead of making API requests*, you:
 
-## What is Secure Telepathic Data?
+- **Read and write data as if it was local** &mdash; from anywhere in your app.
+- **Always have data synced to wherever it's needed, instantly:** to other devices of the same user, to other users, to your backend, to your local machine for debugging, etc.
 
-**Telepathic** means:
+**Secure** means that, *instead of relying on your API or DB for access control*, you:
 
-- **Read and write data as if it was local,** from anywhere in your app.
-- **Always have that data synced, instantly.** Across devices of the same user &mdash; or to other users (coming soon: to your backend, workers, etc.)
+- **Set fine-grained, role-based permissions in `Group`s** that are **synced along with your data**.
+- **Permissions *verifiably enforced* everywhere,** using encryption & signatures under the hood.
+- **Change roles dynamically** for evolving teams, expiring invite links and more.
 
-**Secure** means:
+# What's special about Jazz?
 
-- **Fine-grained, role-based permissions are *baked into* your data.**
-- **Permissions are enforced everywhere, locally.** (using cryptography instead of through an API)
-- Roles can be changed dynamically, supporting changing teams, invite links and more.
+Compared to other libraries and frameworks for local-first, sync-based or real-time apps, these are some of the things that make Jazz unique:
 
-## How to build an app with Jazz?
+- **Jazz is a *batteries-included,* vertically integrated toolkit,** offering everything you need to build an app, including auth, permissions, data model, sync, conflict resolution, blob storage, file uploads, real-time media streaming and more.
+- **Jazz has a *small API surface* of only a few abstractions to learn,** which combine in powerful ways to implement a broad set of features.
+- **Jazz *granularly* loads and caches *only the data that is needed*,** combining *local-first* instant UI reactivity and offline support with the on-demand data efficiency of conventional APIs
+- **Jazz supports end-to-end encryption, but doesn't require it,** allowing you to either manage your user's secret keys for them (based on existing auth flows) or letting your users
+- **Jazz is based on CoJSON, a soon-to-be *open standard,*** which means that there will be a whole ecosystem of compatible libraries and frameworks in a variety of environments &mdash; and it will be easy to achieve (secure) interop between Jazz/CoJSON-based apps and services.
 
-### Building a new app, completely with Jazz
+# Jazz Global Mesh
 
-It's still a bit early, but these are the rough steps:
+Jazz is open source and you can run your own sync & storage server, but to really provide you with everything you need, we're also running
+**[Jazz Global Mesh](https://jazz.tools/mesh)**, a globally distributed mesh of servers optimized for:
 
-1. Define your data model with [CoJSON Values](#cojson).
-2. Implement permission logic using [CoJSON Groups](#group).
-3. Hook up a user interface with [jazz-react](#jazz-react).
+ - **Ultra-low-latency sync** (with geo-aware edge caching and optimal routing)
+ - **Low-cost, reliable storage**
 
-The best example is currently the [Todo List app](#example-app-todo-list).
 
-### Gradually adding Jazz to an existing app
+**Jazz Global Mesh is free for small volumes of data** and it's the **default syncing peer,** so you can  **start building multi-user Jazz apps with persistent data in minutes,** using only frontend code!
 
-Coming soon: Jazz will support gradual adoption by integrating with your existing UI, auth and database.
+# Getting started
 
-## Example App: Todo List
+## Example App Walkthrough
 
-The best example of Jazz is currently the Todo List app.
+**For now the best tutorial is the walkthrough of the [Todo List Example App](#todo-list).**
 
-- Live version: https://example-todo.jazz.tools
-- Source code: [`./examples/todo`](./examples/todo). See the README there for a walk-through and running instructions.
+## General Scenarios
 
-# API Reference
+### Building a new, entirely sync-based React app
 
-Note: Since it's early days, this is the only source of documentation so far.
+1. Define your data model with [cojson Collaborative Values (CoValues)](./DOCS.md/#covalue).
+2. Implement permission logic using [cojson Groups](./DOCS.md/#group).
+3. Build a user interface with [jazz-react](./DOCS.md/#jazz-react)'s reactive [synced queries](./DOCS.md/#usesyncedqueryid).
 
-If you want to build something with Jazz, [join the Jazz Discord](https://discord.gg/utDMjHYg42) for encouragement and help!
+### Gradually adding sync to an existing React app
 
-## Overview: Main Packages
+Gradually migrate app features to use sync:
 
-**`cojson`**
+1. Define data model for small aspect of your app with [cojson Collaborative Values (CoValues)](./DOCS.md/#covalue).
+    - Schema adapters/importers for Prisma/Drizzle/PostgreSQL introspection coming soon.
+2. Map existing permission logic with [cojson Groups](./DOCS.md/#group) & integrate existing auth.
+    - Auth integrations coming soon.
+3. Replace some of the React state and API requests in your UI with [jazz-react](./DOCS.md/#jazz-react)'s reactive [synced queries](./DOCS.md/#usesyncedqueryid).
 
-A library implementing abstractions and protocols for "Collaborative JSON". This will soon be standardized and forms the basis of secure telepathic data.
+# Example Apps
 
-**`jazz-react`**
+## Todo List
 
-Provides you with everything you need to build react apps around CoJSON, including reactive hooks for telepathic data, local IndexedDB persistence, support for different auth providers and helpers for simple invite links for CoJSON groups.
+**A simple collaborative todo list app.**
 
-### Supporting packages
-<small>
+Live version: https://example-todo.jazz.tools
 
-**`cojson-simple-sync`**
+Source code & walkthrough: [`./examples/todo`](./examples/todo)
 
-A generic CoJSON sync server you can run locally if you don't want to use Jazz Global Mesh (the default sync backend, at `wss://sync.jazz.tools`)
+Demonstrates:
+  - Defining a data model with `CoMap`s and `CoList`s
+  - Creating data and setting permissions with `Group`s
+  - Fetching, rendering & editing data from nested `CoValue`s with reactive synced queries
 
-**`jazz-browser`**
 
-framework-agnostic primitives that allow you to use CoJSON in the browser. Used to implement `jazz-react`, will be used to implement bindings for other frameworks in the future.
+## Rate-My-Pet
 
-**`jazz-react-auth-local`** (and `jazz-browser-auth-local`): A simple auth provider that stores cryptographic keys on user devices using WebAuthentication/Passkeys. Lets you build Jazz apps completely without a backend, with end-to-end encryption by default.
+**A simple social polling app.**
 
-**`jazz-storage-indexeddb`**
+Live version: https://example-pets.jazz.tools
 
-Provides local, offline-capable persistence. Included and enabled in `jazz-react` by default.
-</small>
+Source code (walkthrough coming soon): [`./examples/pets`](./examples/pets)
 
-## `CoJSON`
+Demonstrates:
+  - Implementing per-account data streams (reactions) with `CoStream`s
+  - Implementing image upload and progressive image streaming using helpers from `jazz-react-media-images` (on top of CoJSON's `BinaryCoStreams` & `ImageDefinition` convention)
 
-CoJSON is the core implementation of secure telepathic data. It provides abstractions for Collaborative JSON values ("`CoValues`"), groups for permission management and a protocol for syncing between nodes. Our goal is to standardise CoJSON soon and port it to other languages and platforms.
 
----
+# Documentation & API Reference
 
-### `LocalNode`
+For now, docs are hosted in a single well-structured markdown file: [`./DOCS.md`](./DOCS.md).
 
-A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
+- [Package Overview](./DOCS.md/#overview)
+- [`jazz-react` API](./DOCS.md/#jazz-react)
+- [`cojson` API](./DOCS.md/#cojson)
+- [`jazz-react-media-images` API](./DOCS.md/#jazz-react-media-images)
 
-A `LocalNode` can have peers that it syncs to, for example some form of local persistence, or a sync server, such as `sync.jazz.tools` (Jazz Global Mesh).
 
-You typically get hold of a `LocalNode` using `jazz-react`'s `useJazz()`:
+In the future we'll build a dedicated docs page on the Jazz homepage.
 
-```typescript
-const { localNode } = useJazz();
-```
+----
 
-#### `LocalNode.load(id)`
-```typescript
-load<T extends ContentType>(id: CoID<T>): Promise<T>
-```
-
-Loads a CoValue's content, syncing from peers as necessary and resolving the returned promise once a first version has been loaded. See `ContentType.subscribe()` and `useTelepathicData` for listening to subsequent updates to the CoValue.
-
-#### `LocalNode.loadProfile(id)`
-```typescript
-loadProfile(accountID: AccountID): Promise<Profile>
-```
-
-Loads a profile associated with an account. `Profile` is at least a `CoMap<{string: name}>`, but might contain other, app-specific properties.
-
-#### `LocalNode.acceptInvite(valueOrGroup, inviteSecret)`
-```typescript
-acceptInvite<T extends ContentType>(
-    valueOrGroup: CoID<T>,
-    inviteSecret: InviteSecret
-): Promise<void>
-```
-
-Accepts an invite for a group, or infers the group if given the `CoID` of a value owned by that group. Resolves upon successful joining of that group, at which point you should be able to `LocalNode.load` the value.
-
-Invites can be created with `Group.createInvite(role)`.
-
-#### `LocalNode.createGroup()`
-```typescript
-createGroup(): Group
-```
-
-Creates a new group (with the current account as the group's first admin).
-
----
-
-### `Group`
-
-A CoJSON group manages permissions of its members. A `Group` object exposes those capabilities and allows you to create new CoValues owned by that group.
-
-(Internally, a `Group` is also just a `CoMap`, mapping member accounts to roles and containing some state management for making cryptographic keys available to current members)
-
-#### `Group.id`
-
-Returns the `CoID` of the `Group`.
-
-#### `Group.roleOf(accountID)`
-
-```typescript
-roleOf(accountID: AccountID): "reader" | "writer" | "admin" | undefined
-```
-
-Returns the current role of a given account.
-
-#### `Group.myRole()`
-
-```typescript
-myRole(accountID: AccountID): "reader" | "writer" | "admin" | undefined
-```
-
-Returns the role of the current account in the group.
-
-#### `Group.addMember(accountID, role)`
-
-```typescript
-addMember(
-  accountID: AccountIDOrAgentID,
-  role:  "reader" | "writer" | "admin"
-)
-```
-
-Directly grants a new member a role in the group. The current account must be an admin to be able to do so. Throws otherwise.
-
-#### `Group.createInvite(role)`
-
-```typescript
-createInvite(role: "reader" | "writer" | "admin"): InviteSecret
-```
-
-Creates an invite for new members to indirectly join the group, allowing them to grant themselves the specified role with the InviteSecret (a string starting with "inviteSecret_") - use `LocalNode.acceptInvite()` for this purpose.
-
-#### `Group.removeMember(accountID)`
-
-```typescript
-removeMember(accountID: AccountID)
-```
-
-Strips the specified member of all roles (preventing future writes) and rotates the read encryption key for that group (preventing reads of new content, including in covalues owned by this group)
-
-#### `Group.createMap(meta?)`
-```typescript
-createMap<
-    M extends { [key: string]: JsonValue },
-    Meta extends JsonObject | null = null
->(meta?: Meta): CoMap<M, Meta>
-```
-
-Creates a new `CoMap` within this group, with the specified inner content type `M` and optional static metadata.
-
-#### `Group.createList(meta?)` (coming soon)
-#### `Group.createStream(meta?)` (coming soon)
-#### `Group.createStatic(meta)` (coming soon)
-
----
-### `CoValue` ContentType: `CoMap`
-
-```typescript
-class CoMap<
-    M extends { [key: string]: JsonValue; },
-    Meta extends JsonObject | null = null,
->
-```
-
-#### `CoMap.id`
-
-```typescript
-id: CoID<CoMap<M, Meta>>
-```
-
-Returns the CoMap's (precisely typed) `CoID`
-
-#### `CoMap.keys()`
-
-```typescript
-keys(): (keyof M & string)[]
-```
-
-#### `CoMap.get(key)`
-
-```typescript
-get<K extends keyof M>(key: K): M[K] | undefined
-```
-
-Returns the current value for the given key.
-
-#### `CoMap.getLastEditor(key)`
-
-```typescript
-getLastEditor<K extends keyof M>(key: K): AccountID | undefined
-```
-
-Returns the accountID of the last account to modify the value for the given key.
-
-#### `CoMap.toJSON()`
-
-```typescript
-toJSON(): JsonObject
-```
-
-Returns a JSON representation of the state of the CoMap.
-
-#### `CoMap.subscribe(listener)`
-
-```typescript
-subscribe(
-  listener: (coMap: CoMap<M, Meta>) => void
-): () => void
-```
-Lets you subscribe to future updates to this CoMap (whether made locally or by other users). Takes a listener function that will be called with the current state for each update. Returns an unsubscribe function.
-
-Used internally by `useTelepathicData()` for reactive updates on changes to a `CoMap`.
-
-#### `CoMap.edit(editable => {...})`
-
-```typescript
-edit(changer: (editable: WriteableCoMap<M, Meta>) => void): CoMap<M, Meta>
-```
-
-Lets you apply edits to a `CoMap`, inside the changer callback, which receives a `WriteableCoMap`. A `WritableCoMap` has all the same methods as a `CoMap`, but all edits made to it with `set` or `delete` are reflected in it immediately - so it behaves mutably, whereas a `CoMap` is always immutable (you need to use `subscribe` to receive new versions of it).
-
-```typescript
-export class WriteableCoMap<
-    M extends { [key: string]: JsonValue; },
-    Meta extends JsonObject | null = null,
-> extends CoMap<M, Meta>
-```
-
-#### `WritableCoMap.set(key, value)`
-
-```typescript
-set<K extends keyof M>(
-  key: K,
-  value: M[K],
-  privacy: "private" | "trusting" = "private"
-): void
-```
-
-Sets a new value for the given key.
-
-If `privacy` is `"private"` **(default)**, both `key` and `value` are encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
-
-If `privacy` is `"trusting"`, both `key` and `value` are stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
-
-#### `WritableCoMap.delete(key)`
-
-```typescript
-delete<K extends keyof M>(
-  key: K,
-  privacy: "private" | "trusting" = "private"
-): void
-```
-
-Deletes the value for the given key (setting it to undefined).
-
-If `privacy` is `"private"` **(default)**, `key` is encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
-
-If `privacy` is `"trusting"`, `key` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
-
----
-### `CoValue` ContentType: `CoList` (not yet implemented)
-
----
-### `CoValue` ContentType: `CoStram` (not yet implemented)
-
----
-### `CoValue` ContentType: `Static` (not yet implemented)
----
-
-## `jazz-react`
----
-### `<WithJazz>`
-
-Not yet documented, see [`examples/todo`](./examples/todo/) for now
-
----
-### `useJazz()`
-
-Not yet documented, see [`examples/todo`](./examples/todo/) for now
-
----
-### `useTelepathicData(coID)`
-
-Not yet documented, see [`examples/todo`](./examples/todo/) for now
-
----
-### `useProfile(accountID)`
-
-Not yet documented, see [`examples/todo`](./examples/todo/) for now
+Copyright 2023 &mdash; Garden Computing, Inc.

examples/pets/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

examples/pets/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

examples/pets/Dockerfile

@@ -0,0 +1,4 @@
+FROM caddy:2.7.3-alpine
+LABEL org.opencontainers.image.source="https://github.com/gardencmp/jazz"
+
+COPY ./dist /usr/share/caddy/

examples/pets/README.md

@@ -0,0 +1,51 @@
+# Jazz Rate-My-Pet List Example
+
+Live version: https://example-pets.jazz.tools
+
+## Installing & running the example locally
+
+Start by checking out just the example app to a folder:
+
+```bash
+npx degit gardencmp/jazz/examples/pets jazz-example-pets
+cd jazz-example-pets
+```
+
+(This ensures that you have the example app without git history or our multi-package monorepo)
+
+Install dependencies:
+
+```bash
+npm install
+```
+
+Start the dev server:
+
+```bash
+npm run dev
+```
+
+## Structure
+
+TODO
+
+## Walkthrough
+
+### Main parts
+
+TODO
+
+### Helpers
+
+TODO
+
+## Questions / problems / feedback
+
+If you have feedback, let us know on [Discord](https://discord.gg/utDMjHYg42) or open an issue or PR to fix something that seems wrong.
+
+
+## Configuration: sync server
+
+By default, the example app uses [Jazz Global Mesh](https://jazz.tools/mesh) (`wss://sync.jazz.tools`) - so cross-device use, invites and collaboration should just work.
+
+You can also run a local sync server by running `npx cojson-simple-sync` and adding the query param `?sync=ws://localhost:4200` to the URL of the example app (for example: `http://localhost:5173/?sync=ws://localhost:4200`), or by setting the `sync` parameter of the `<WithJazz>` provider component in [./src/0_main.tsx](./src/0_main.tsx).

examples/pets/components.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "default",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "tailwind.config.js",
+    "css": "src/index.css",
+    "baseColor": "stone",
+    "cssVariables": true
+  },
+  "aliases": {
+    "components": "@/basicComponents",
+    "utils": "@/basicComponents/lib/utils"
+  }
+}

examples/pets/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/png" href="/jazz-logo.png" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Jazz Rate My Pet Example</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/2_main.tsx"></script>
+  </body>
+</html>

examples/pets/job-template.nomad

@@ -0,0 +1,56 @@
+job "example-pets$BRANCH_SUFFIX" {
+  region     = "global"
+  datacenters = ["*"]
+
+  group "static" {
+    count = 8
+
+    network {
+      port "http" {
+        to = 80
+      }
+    }
+
+    constraint {
+      attribute = "${node.class}"
+      operator  = "="
+      value     = "mesh"
+    }
+
+    spread {
+      attribute = "${node.datacenter}"
+      weight    = 100
+    }
+
+    constraint {
+      distinct_hosts = true
+    }
+
+    task "server" {
+      driver = "docker"
+
+      config {
+        image = "$DOCKER_TAG"
+        ports = ["http"]
+
+        auth = {
+          username = "$DOCKER_USER"
+          password = "$DOCKER_PASSWORD"
+        }
+      }
+
+      service {
+        tags = ["public"]
+        name = "example-pets$BRANCH_SUFFIX"
+        port = "http"
+        provider = "consul"
+      }
+
+      resources {
+        cpu    = 50 # MHz
+        memory = 50 # MB
+      }
+    }
+  }
+}
+# deploy bump 4

examples/pets/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "jazz-example-pets",
+  "private": true,
+  "version": "0.0.14",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@radix-ui/react-checkbox": "^1.0.4",
+    "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-toast": "^1.1.4",
+    "@types/qrcode": "^1.5.1",
+    "class-variance-authority": "^0.7.0",
+    "clsx": "^2.0.0",
+    "jazz-react": "^0.3.5",
+    "jazz-react-auth-local": "^0.3.5",
+    "jazz-react-media-images": "^0.3.5",
+    "lucide-react": "^0.274.0",
+    "qrcode": "^1.5.3",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-router": "^6.16.0",
+    "react-router-dom": "^6.16.0",
+    "tailwind-merge": "^1.14.0",
+    "tailwindcss-animate": "^1.0.7",
+    "uniqolor": "^1.1.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitejs/plugin-react-swc": "^3.3.2",
+    "autoprefixer": "^10.4.14",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "postcss": "^8.4.27",
+    "tailwindcss": "^3.3.3",
+    "typescript": "^5.0.2",
+    "vite": "^4.4.5"
+  }
+}

examples/pets/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

examples/pets/src/1_types.ts

@@ -0,0 +1,29 @@
+import { CoMap, CoStream, Media } from "cojson";
+
+/** Walkthrough: Defining the data model with CoJSON
+ *
+ *  Here, we define our main data model of TODO
+ *
+ *  TODO
+ **/
+
+export type PetPost = CoMap<{
+    name: string;
+    image: Media.ImageDefinition;
+    reactions: PetReactions;
+}>;
+
+export const REACTION_TYPES = [
+    "aww",
+    "love",
+    "haha",
+    "wow",
+    "tiny",
+    "chonkers",
+] as const;
+
+export type ReactionType = (typeof REACTION_TYPES)[number];
+
+export type PetReactions = CoStream<ReactionType>;
+
+/** Walkthrough: Continue with ./2_App.tsx */

examples/pets/src/2_main.tsx

@@ -0,0 +1,86 @@
+import React from "react";
+import ReactDOM from "react-dom/client";
+import { RouterProvider, createHashRouter } from "react-router-dom";
+import "./index.css";
+
+import { WithJazz, useJazz, useAcceptInvite } from "jazz-react";
+import { LocalAuth } from "jazz-react-auth-local";
+
+import {
+    Button,
+    ThemeProvider,
+    TitleAndLogo,
+} from "./basicComponents/index.ts";
+import { PrettyAuthUI } from "./components/Auth.tsx";
+import { NewPetPostForm } from "./3_NewPetPostForm.tsx";
+import { RatePetPostUI } from "./4_RatePetPostUI.tsx";
+
+/** Walkthrough: The top-level provider `<WithJazz/>`
+ *
+ *  This shows how to use the top-level provider `<WithJazz/>`,
+ *  which provides the rest of the app with a `LocalNode` (used through `useJazz` later),
+ *  based on `LocalAuth` that uses PassKeys (aka WebAuthn) to store a user's account secret
+ *  - no backend needed. */
+
+const appName = "Jazz Rate My Pet Example";
+
+const auth = LocalAuth({
+    appName,
+    Component: PrettyAuthUI,
+});
+
+ReactDOM.createRoot(document.getElementById("root")!).render(
+    <React.StrictMode>
+        <WithJazz auth={auth}>
+            <App />
+        </WithJazz>
+    </React.StrictMode>
+);
+
+/** Walkthrough: Creating pet posts & routing in `<App/>`
+ *
+ *  <App> is the main app component, handling client-side routing based
+ *  on the CoValue ID (CoID) of our PetPost, stored in the URL hash
+ *  - which can also contain invite links.
+ */
+
+export default function App() {
+    const { logOut } = useJazz();
+
+    const router = createHashRouter([
+        {
+            path: "/",
+            element: <NewPetPostForm />,
+        },
+        {
+            path: "/pet/:petPostId",
+            element: <RatePetPostUI />,
+        },
+        {
+            path: "/invite/*",
+            element: <p>Accepting invite...</p>,
+        },
+    ]);
+
+    useAcceptInvite((petPostID) => router.navigate("/pet/" + petPostID));
+
+    return (
+        <ThemeProvider>
+            <TitleAndLogo name={appName} />
+            <div className="flex flex-col h-full items-center justify-start gap-10 pt-10 pb-10 px-5">
+                <RouterProvider router={router} />
+
+                <Button
+                    onClick={() => router.navigate("/").then(logOut)}
+                    variant="outline"
+                >
+                    Log Out
+                </Button>
+            </div>
+        </ThemeProvider>
+    );
+}
+
+/** Walkthrough: continue with ./3_CreatePetPostForm.tsx */
+
+/** Walkthrough: Continue with ./1_types.ts */

examples/pets/src/3_NewPetPostForm.tsx

@@ -0,0 +1,100 @@
+import { ChangeEvent, useCallback, useState } from "react";
+import { useNavigate } from "react-router";
+
+import { CoID, CoMap, Media } from "cojson";
+import { useJazz, useSyncedQuery } from "jazz-react";
+import { createImage } from "jazz-browser-media-images";
+
+import { PetReactions } from "./1_types";
+
+import { Input, Button } from "./basicComponents";
+import { useLoadImage } from "jazz-react-media-images";
+
+/** Walkthrough: TODO
+ */
+
+type PartialPetPost = CoMap<{
+    name: string;
+    image?: Media.ImageDefinition;
+    reactions: PetReactions;
+}>;
+
+export function NewPetPostForm() {
+    const { localNode } = useJazz();
+    const navigate = useNavigate();
+
+    const [newPostId, setNewPostId] = useState<
+        CoID<PartialPetPost> | undefined
+    >(undefined);
+
+    const newPetPost = useSyncedQuery(newPostId);
+
+    const onChangeName = useCallback(
+        (name: string) => {
+            if (newPetPost) {
+                newPetPost.set({ name });
+            } else {
+                const petPostGroup = localNode.createGroup();
+                const petPost = petPostGroup.createMap<PartialPetPost>({
+                    name,
+                    reactions: petPostGroup.createStream<PetReactions>(),
+                });
+
+                setNewPostId(petPost.id);
+            }
+        },
+        [localNode, newPetPost]
+    );
+
+    const onImageSelected = useCallback(
+        async (event: ChangeEvent<HTMLInputElement>) => {
+            if (!newPetPost || !event.target.files) return;
+
+            const image = await createImage(
+                event.target.files[0],
+                newPetPost.group
+            );
+
+            newPetPost.set({ image });
+        },
+        [newPetPost]
+    );
+
+    const petImage = useLoadImage(newPetPost?.image?.id);
+
+    return (
+        <div className="flex flex-col gap-10">
+            <p>Share your pet with friends!</p>
+            <Input
+                type="text"
+                placeholder="Pet Name"
+                className="text-3xl py-6"
+                onChange={(event) => onChangeName(event.target.value)}
+                value={newPetPost?.name || ""}
+            />
+
+            {petImage ? (
+                <img
+                    className="w-80 max-w-full rounded"
+                    src={petImage.highestResSrc || petImage.placeholderDataURL}
+                />
+            ) : (
+                <Input
+                    type="file"
+                    disabled={!newPetPost?.name}
+                    onChange={onImageSelected}
+                />
+            )}
+
+            {newPetPost?.name && newPetPost?.image && (
+                <Button
+                    onClick={() => {
+                        navigate("/pet/" + newPetPost.id);
+                    }}
+                >
+                    Submit Post
+                </Button>
+            )}
+        </div>
+    );
+}

examples/pets/src/4_RatePetPostUI.tsx

@@ -0,0 +1,124 @@
+import { useParams } from "react-router";
+import { CoID, Queried } from "cojson";
+import { useSyncedQuery } from "jazz-react";
+
+import { PetPost, ReactionType, REACTION_TYPES, PetReactions } from "./1_types";
+
+import { ShareButton } from "./components/ShareButton";
+import { Button, Skeleton } from "./basicComponents";
+import { useLoadImage } from "jazz-react-media-images";
+import uniqolor from "uniqolor";
+
+/** Walkthrough: TODO
+ */
+
+const reactionEmojiMap: { [reaction in ReactionType]: string } = {
+    aww: "😍",
+    love: "❤️",
+    haha: "😂",
+    wow: "😮",
+    tiny: "🐥",
+    chonkers: "🐘",
+};
+
+export function RatePetPostUI() {
+    const petPostID = useParams<{ petPostId: CoID<PetPost> }>().petPostId;
+
+    const petPost = useSyncedQuery(petPostID);
+    const petImage = useLoadImage(petPost?.image);
+
+    return (
+        <div className="flex flex-col gap-8">
+            <div className="flex justify-between">
+                <h1 className="text-3xl font-bold">{petPost?.name}</h1>
+                <ShareButton petPost={petPost} />
+            </div>
+
+            {petImage && (
+                <img
+                    className="w-80 max-w-full rounded"
+                    src={petImage.highestResSrc || petImage.placeholderDataURL}
+                />
+            )}
+
+            <div className="flex justify-between max-w-xs flex-wrap">
+                {REACTION_TYPES.map((reactionType) => (
+                    <Button
+                        key={reactionType}
+                        variant={
+                            petPost?.reactions?.me?.last === reactionType
+                                ? "default"
+                                : "outline"
+                        }
+                        onClick={() => {
+                            petPost?.reactions?.push(reactionType);
+                        }}
+                        title={`React with ${reactionType}`}
+                        className="text-2xl px-2"
+                    >
+                        {reactionEmojiMap[reactionType]}
+                    </Button>
+                ))}
+            </div>
+
+            {petPost?.group.myRole() === "admin" && petPost.reactions && (
+                <ReactionOverview petReactions={petPost.reactions} />
+            )}
+        </div>
+    );
+}
+
+function ReactionOverview({
+    petReactions,
+}: {
+    petReactions: Queried<PetReactions>;
+}) {
+    return (
+        <div>
+            <h2>Reactions</h2>
+            <div className="flex flex-col gap-1">
+                {REACTION_TYPES.map((reactionType) => {
+                    const reactionsOfThisType = Object.values(
+                        petReactions.perAccount
+                    ).filter(({ last }) => last === reactionType);
+
+                    if (reactionsOfThisType.length === 0) return null;
+
+                    return (
+                        <div
+                            className="flex gap-2 items-center"
+                            key={reactionType}
+                        >
+                            {reactionEmojiMap[reactionType]}{" "}
+                            {reactionsOfThisType.map((reaction, idx) =>
+                                reaction.by?.profile?.name ? (
+                                    <span
+                                        className="rounded-full py-0.5 px-2 text-xs"
+                                        style={uniqueColoring(reaction.by.id)}
+                                        key={reaction.by.id}
+                                    >
+                                        {reaction.by.profile.name}
+                                    </span>
+                                ) : (
+                                    <Skeleton
+                                        className="mt-1 w-[50px] h-[1em] rounded-full"
+                                        key={idx}
+                                    />
+                                )
+                            )}
+                        </div>
+                    );
+                })}
+            </div>
+        </div>
+    );
+}
+
+function uniqueColoring(seed: string) {
+    const darkMode = window.matchMedia("(prefers-color-scheme: dark)").matches;
+
+    return {
+        color: uniqolor(seed, { lightness: darkMode ? 80 : 20 }).color,
+        background: uniqolor(seed, { lightness: darkMode ? 20 : 80 }).color,
+    };
+}

examples/pets/src/basicComponents/TitleAndLogo.tsx

@@ -0,0 +1,12 @@
+import { Toaster } from ".";
+
+export function TitleAndLogo({ name }: { name: string }) {
+    return (
+        <>
+            <div className="flex items-center gap-2 justify-center mt-5">
+                <img src="jazz-logo.png" className="h-5" /> {name}
+            </div>
+            <Toaster />
+        </>
+    );
+}

examples/pets/src/basicComponents/index.ts

@@ -0,0 +1,7 @@
+export { Button } from "./ui/button";
+export { Input } from "./ui/input";
+export { Toaster } from "./ui/toaster";
+export { useToast } from "./ui/use-toast";
+export { Skeleton } from "./ui/skeleton";
+export { TitleAndLogo } from "./TitleAndLogo";
+export { ThemeProvider } from "./themeProvider";

examples/pets/src/basicComponents/ui/button.tsx

@@ -2,7 +2,7 @@ import * as React from "react"
 import { Slot } from "@radix-ui/react-slot"
 import { cva, type VariantProps } from "class-variance-authority"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 const buttonVariants = cva(
   "inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",

examples/pets/src/basicComponents/ui/input.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 export interface InputProps
   extends React.InputHTMLAttributes<HTMLInputElement> {}

examples/pets/src/basicComponents/ui/skeleton.tsx

@@ -1,4 +1,4 @@
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 function Skeleton({
   className,

examples/pets/src/basicComponents/ui/toast.tsx

@@ -3,7 +3,7 @@ import * as ToastPrimitives from "@radix-ui/react-toast"
 import { cva, type VariantProps } from "class-variance-authority"
 import { X } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 const ToastProvider = ToastPrimitives.Provider
 

examples/pets/src/basicComponents/ui/toaster.tsx

@@ -5,8 +5,8 @@ import {
   ToastProvider,
   ToastTitle,
   ToastViewport,
-} from "@/components/ui/toast"
-import { useToast } from "@/components/ui/use-toast"
+} from "@/basicComponents/ui/toast"
+import { useToast } from "@/basicComponents/ui/use-toast"
 
 export function Toaster() {
   const { toasts } = useToast()

examples/pets/src/basicComponents/ui/use-toast.ts

@@ -4,7 +4,7 @@ import * as React from "react"
 import type {
   ToastActionElement,
   ToastProps,
-} from "@/components/ui/toast"
+} from "@/basicComponents/ui/toast"
 
 const TOAST_LIMIT = 1
 const TOAST_REMOVE_DELAY = 1000000

examples/pets/src/components/Auth.tsx

@@ -1,9 +1,10 @@
-import { LocalAuthComponent } from "jazz-react-auth-local";
 import { useState } from "react";
-import { Input } from "./ui/input";
-import { Button } from "./ui/button";
 
-export const PrettyAuthComponent: LocalAuthComponent = ({
+import { LocalAuthComponent } from "jazz-react-auth-local";
+
+import { Input, Button } from "../basicComponents";
+
+export const PrettyAuthUI: LocalAuthComponent = ({
     loading,
     logIn,
     signUp,

examples/pets/src/components/ShareButton.tsx

@@ -0,0 +1,47 @@
+import { useState } from "react";
+
+import { PetPost } from "../1_types";
+
+import { createInviteLink } from "jazz-react";
+import QRCode from "qrcode";
+
+import { useToast, Button } from "../basicComponents";
+import { Queried } from "cojson";
+
+export function ShareButton({ petPost }: { petPost?: Queried<PetPost> }) {
+    const [existingInviteLink, setExistingInviteLink] = useState<string>();
+    const { toast } = useToast();
+
+    return (
+        petPost?.group.myRole() === "admin" && (
+            <Button
+                size="sm"
+                className="py-0"
+                disabled={!petPost}
+                variant="outline"
+                onClick={async () => {
+                    let inviteLink = existingInviteLink;
+                    if (petPost && !inviteLink) {
+                        inviteLink = createInviteLink(petPost, "writer");
+                        setExistingInviteLink(inviteLink);
+                    }
+                    if (inviteLink) {
+                        const qr = await QRCode.toDataURL(inviteLink, {
+                            errorCorrectionLevel: "L",
+                        });
+                        navigator.clipboard.writeText(inviteLink).then(() =>
+                            toast({
+                                title: "Copied invite link to clipboard!",
+                                description: (
+                                    <img src={qr} className="w-20 h-20" />
+                                ),
+                            })
+                        );
+                    }
+                }}
+            >
+                Share
+            </Button>
+        )
+    );
+}

examples/pets/src/index.css

@@ -0,0 +1,76 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+ 
+@layer base {
+  :root {
+    --background: 0 0% 100%;
+    --foreground: 20 14.3% 4.1%;
+
+    --card: 0 0% 100%;
+    --card-foreground: 20 14.3% 4.1%;
+ 
+    --popover: 0 0% 100%;
+    --popover-foreground: 20 14.3% 4.1%;
+ 
+    --primary: 24 9.8% 10%;
+    --primary-foreground: 60 9.1% 97.8%;
+ 
+    --secondary: 60 4.8% 95.9%;
+    --secondary-foreground: 24 9.8% 10%;
+ 
+    --muted: 60 4.8% 95.9%;
+    --muted-foreground: 25 5.3% 44.7%;
+ 
+    --accent: 60 4.8% 95.9%;
+    --accent-foreground: 24 9.8% 10%;
+ 
+    --destructive: 0 84.2% 60.2%;
+    --destructive-foreground: 60 9.1% 97.8%;
+
+    --border: 20 5.9% 90%;
+    --input: 20 5.9% 90%;
+    --ring: 20 14.3% 4.1%;
+ 
+    --radius: 0.5rem;
+  }
+ 
+  .dark {
+    --background: 20 14.3% 4.1%;
+    --foreground: 60 9.1% 97.8%;
+ 
+    --card: 20 14.3% 4.1%;
+    --card-foreground: 60 9.1% 97.8%;
+ 
+    --popover: 20 14.3% 4.1%;
+    --popover-foreground: 60 9.1% 97.8%;
+ 
+    --primary: 60 9.1% 97.8%;
+    --primary-foreground: 24 9.8% 10%;
+ 
+    --secondary: 12 6.5% 15.1%;
+    --secondary-foreground: 60 9.1% 97.8%;
+ 
+    --muted: 12 6.5% 15.1%;
+    --muted-foreground: 24 5.4% 63.9%;
+ 
+    --accent: 12 6.5% 15.1%;
+    --accent-foreground: 60 9.1% 97.8%;
+ 
+    --destructive: 0 62.8% 30.6%;
+    --destructive-foreground: 60 9.1% 97.8%;
+ 
+    --border: 12 6.5% 15.1%;
+    --input: 12 6.5% 15.1%;
+    --ring: 24 5.7% 82.9%;
+  }
+}
+ 
+@layer base {
+  * {
+    @apply border-border;
+  }
+  body {
+    @apply bg-background text-foreground;
+  }
+}

examples/pets/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

examples/pets/tailwind.config.js

@@ -0,0 +1,76 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  darkMode: ["class"],
+  content: [
+    './pages/**/*.{ts,tsx}',
+    './components/**/*.{ts,tsx}',
+    './app/**/*.{ts,tsx}',
+    './src/**/*.{ts,tsx}',
+	],
+  theme: {
+    container: {
+      center: true,
+      padding: "2rem",
+      screens: {
+        "2xl": "1400px",
+      },
+    },
+    extend: {
+      colors: {
+        border: "hsl(var(--border))",
+        input: "hsl(var(--input))",
+        ring: "hsl(var(--ring))",
+        background: "hsl(var(--background))",
+        foreground: "hsl(var(--foreground))",
+        primary: {
+          DEFAULT: "hsl(var(--primary))",
+          foreground: "hsl(var(--primary-foreground))",
+        },
+        secondary: {
+          DEFAULT: "hsl(var(--secondary))",
+          foreground: "hsl(var(--secondary-foreground))",
+        },
+        destructive: {
+          DEFAULT: "hsl(var(--destructive))",
+          foreground: "hsl(var(--destructive-foreground))",
+        },
+        muted: {
+          DEFAULT: "hsl(var(--muted))",
+          foreground: "hsl(var(--muted-foreground))",
+        },
+        accent: {
+          DEFAULT: "hsl(var(--accent))",
+          foreground: "hsl(var(--accent-foreground))",
+        },
+        popover: {
+          DEFAULT: "hsl(var(--popover))",
+          foreground: "hsl(var(--popover-foreground))",
+        },
+        card: {
+          DEFAULT: "hsl(var(--card))",
+          foreground: "hsl(var(--card-foreground))",
+        },
+      },
+      borderRadius: {
+        lg: "var(--radius)",
+        md: "calc(var(--radius) - 2px)",
+        sm: "calc(var(--radius) - 4px)",
+      },
+      keyframes: {
+        "accordion-down": {
+          from: { height: 0 },
+          to: { height: "var(--radix-accordion-content-height)" },
+        },
+        "accordion-up": {
+          from: { height: "var(--radix-accordion-content-height)" },
+          to: { height: 0 },
+        },
+      },
+      animation: {
+        "accordion-down": "accordion-down 0.2s ease-out",
+        "accordion-up": "accordion-up 0.2s ease-out",
+      },
+    },
+  },
+  plugins: [require("tailwindcss-animate")],
+}

examples/pets/tsconfig.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2023", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}

examples/pets/tsconfig.node.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

examples/pets/vite.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react-swc'
+import path from "path";
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+  resolve: {
+    alias: {
+      "@": path.resolve(__dirname, "./src"),
+    },
+  },
+  build: {
+    minify: false
+  }
+})

examples/todo/README.md

@@ -2,343 +2,63 @@
 
 Live version: https://example-todo.jazz.tools
 
-More comprehensive guide coming soon, but these are the most important bits, with explanations:
+## Installing & running the example locally
 
-From `./src/main.tsx`
-
-```typescript
-// ...
-
-import { WithJazz } from "jazz-react";
-import { LocalAuth } from "jazz-react-auth-local";
-
-// ...
-
-ReactDOM.createRoot(document.getElementById("root")!).render(
-    <React.StrictMode>
-        <ThemeProvider>
-            <div className="flex items-center gap-2 justify-center mt-5">
-                <img src="jazz-logo.png" className="h-5" /> Jazz Todo List
-                Example
-            </div>
-            <WithJazz
-                auth={LocalAuth({
-                    appName: "Jazz Todo List Example",
-                    Component: PrettyAuthComponent,
-                })}
-            >
-                <App />
-            </WithJazz>
-        </ThemeProvider>
-    </React.StrictMode>
-);
+Start by checking out just the example app to a folder:
 
+```bash
+npx degit gardencmp/jazz/examples/todo jazz-example-todo
+cd jazz-example-todo
 ```
 
-This shows how to use the top-level component `<WithJazz/>`, which provides the rest of the app with a `LocalNode` (used through `useJazz` later), based on `LocalAuth` that uses Passkeys to store a user's account secret - no backend needed.
+(This ensures that you have the example app without git history or our multi-package monorepo)
 
-Let's move on to the main app code.
+Install dependencies:
 
----
-
-From `./src/App.tsx`
-
-```typescript
-// ...
-
-import { CoMap, CoID, AccountID } from "cojson";
-import {
-   consumeInviteLinkFromWindowLocation,
-   useJazz,
-   useProfile,
-   useTelepathicState,
-   createInviteLink
-} from "jazz-react";
-
-// ...
-
-type TaskContent = { done: boolean; text: string };
-type Task = CoMap<TaskContent>;
-
-type TodoListContent = {
-    title: string;
-    // other keys form a set of task IDs
-    [taskId: CoID<Task>]: true;
-};
-type TodoList = CoMap<TodoListContent>;
-
-// ...
+```bash
+npm install
 ```
 
-First, we define our main data model of tasks and todo lists, using CoJSON's collaborative map type, `CoMap`. We reference CoMaps of individual tasks by using them as keys inside the `TodoList` CoMap - as a makeshift solution until `CoList` is implemented.
+Start the dev server:
 
----
-
-```typescript
-// ...
-
-export default function App() {
-    const [listId, setListId] = useState<CoID<TodoList>>();
-
-    const { localNode, logOut } = useJazz();
-
-    useEffect(() => {
-        const listener = async () => {
-            const acceptedInvitation =
-                await consumeInviteLinkFromWindowLocation(localNode);
-
-            if (acceptedInvitation) {
-                setListId(acceptedInvitation.valueID as CoID<TodoList>);
-                window.location.hash = acceptedInvitation.valueID;
-                return;
-            }
-
-            setListId(window.location.hash.slice(1) as CoID<TodoList>);
-        };
-        window.addEventListener("hashchange", listener);
-        listener();
-
-        return () => {
-            window.removeEventListener("hashchange", listener);
-        };
-    }, [localNode]);
-
-    const createList = useCallback(
-        (title: string) => {
-            const listGroup = localNode.createGroup();
-            const list = listGroup.createMap<TodoListContent>();
-
-            list.edit((list) => {
-                list.set("title", title);
-            });
-
-            window.location.hash = list.id;
-        },
-        [localNode]
-    );
-
-    return (
-        <div className="flex flex-col h-full items-center justify-start gap-10 pt-10 pb-10 px-5">
-            {listId ? (
-                <TodoListComponent listId={listId} />
-            ) : (
-                <SubmittableInput
-                    onSubmit={createList}
-                    label="Create New List"
-                    placeholder="New list title"
-                />
-            )}
-            <Button
-                onClick={() => {
-                    window.location.hash = "";
-                    logOut();
-                }}
-                variant="outline"
-            >
-                Log Out
-            </Button>
-        </div>
-    );
-}
+```bash
+npm run dev
 ```
 
-`<App>` is the main app component, handling client-side routing based on the CoValue ID (`CoID`) of our `TodoList`, stored in the URL hash - which can also contain invite links, which we intercept and use with `consumeInviteLinkFromWindowLocation`.
+## Structure
 
-`createList` is the first time we see CoJSON in action: using our `localNode` (which we got from `useJazz`), we first create a group for a new todo list (which allows us to set permissions later). Then, within that group, we create a new `CoMap<TodoListContent>` with `listGroup.createMap()`.
+- [`src/basicComponents`](./src/basicComponents): simple components to build the UI, unrelated to Jazz (uses [shadcn/ui](https://ui.shadcn.com))
+- [`src/components`](./src/components/): helper components that do contain Jazz-specific logic, but aren't very relevant to understand the basics of Jazz and CoJSON
+- [`src/1_types.ts`](./src/1_types.ts),
+[`src/2_main.tsx`](./src/2_main.tsx),
+[`src/3_NewProjectForm.tsx`](./src/3_NewProjectForm.tsx),
+[`src/4_ProjectTodoTable.tsx`](./src/4_ProjectTodoTable.tsx): the main files for this example, see the walkthrough below
 
-We immediately start editing the created `list`. Within the edit callback, we can use the `set` function, to collaboratively set the key `title` to the initial title provided to `createList`.
+## Walkthrough
 
-If we have a current `listId` set, we render `<TodoListComponent>` with it, which we'll see next.
+### Main parts
 
-If we have no `listId` set, the user can use the displayed creation input to create (and open) their first list.
+1. Defining the data model with CoJSON: [`src/1_types.ts`](./src/1_types.ts)
 
----
+2. The top-level provider `<WithJazz/>` and routing: [`src/2_main.tsx`](./src/2_main.tsx)
 
-```typescript
-export function TodoListComponent({ listId }: { listId: CoID<TodoList> }) {
-    const list = useTelepathicState(listId);
+3. Creating a new todo project: [`src/3_NewProjectForm.tsx`](./src/3_NewProjectForm.tsx)
 
-    const createTask = (text: string) => {
-        if (!list) return;
-        const task = list.coValue.getGroup().createMap<TaskContent>();
+4. Reactively rendering a todo project as a table, adding and editing tasks: [`src/4_ProjectTodoTable.tsx`](./src/4_ProjectTodoTable.tsx)
 
-        task.edit((task) => {
-            task.set("text", text);
-            task.set("done", false);
-        });
+### Helpers
 
-        list.edit((list) => {
-            list.set(task.id, true);
-        });
-    };
-
-    return (
-        <div className="max-w-full w-4xl">
-            <div className="flex justify-between items-center gap-4 mb-4">
-                <h1>
-                    {list?.get("title") ? (
-                        <>
-                            {list.get("title")}{" "}
-                            <span className="text-sm">({list.id})</span>
-                        </>
-                    ) : (
-                        <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
-                    )}
-                </h1>
-                {list && <InviteButton list={list} />}
-            </div>
-            <Table>
-                <TableHeader>
-                    <TableRow>
-                        <TableHead className="w-[40px]">Done</TableHead>
-                        <TableHead>Task</TableHead>
-                    </TableRow>
-                </TableHeader>
-                <TableBody>
-                    {list &&
-                        list
-                            .keys()
-                            .filter((key): key is CoID<Task> =>
-                                key.startsWith("co_")
-                            )
-                            .map((taskId) => (
-                                <TaskRow key={taskId} taskId={taskId} />
-                            ))}
-                    <TableRow key="new">
-                        <TableCell>
-                            <Checkbox className="mt-1" disabled />
-                        </TableCell>
-                        <TableCell>
-                            <SubmittableInput
-                                onSubmit={(taskText) => createTask(taskText)}
-                                label="Add"
-                                placeholder="New task"
-                                disabled={!list}
-                            />
-                        </TableCell>
-                    </TableRow>
-                </TableBody>
-            </Table>
-        </div>
-    );
-}
-```
-
-Here in `<TodoListComponent>`, we use `useTelepathicData()` for the first time, in this case to load the CoValue for our `TodoList` and to reactively subscribe to updates to its content - whether we create edits locally, load persisted data, or receive sync updates from other devices or participants!
-
-`createTask` is similar to `createList` we saw earlier, creating a new CoMap for a new task, and then adding it as a key to our `TodoList`.
-
-As you can see, we iterate over the keys of `TodoList` and for those that look like `CoID`s (they always start with `co_`), we render a `<TaskRow>`.
-
-Below all tasks, we render a simple input for adding a task.
-
----
-
-```typescript
-function TaskRow({ taskId }: { taskId: CoID<Task> }) {
-    const task = useTelepathicState(taskId);
-
-    return (
-        <TableRow>
-            <TableCell>
-                <Checkbox
-                    className="mt-1"
-                    checked={task?.get("done")}
-                    onCheckedChange={(checked) => {
-                        task?.edit((task) => {
-                            task.set("done", !!checked);
-                        });
-                    }}
-                />
-            </TableCell>
-            <TableCell>
-                <div className="flex flex-row justify-between items-center gap-2">
-                    <span className={task?.get("done") ? "line-through" : ""}>
-                        {task?.get("text") || <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />}
-                    </span>
-                    <NameBadge accountID={task?.getLastEditor("text")} />
-                </div>
-            </TableCell>
-        </TableRow>
-    );
-}
-```
-
-`<TaskRow>` uses `useTelepathicState()` as well, to granularly load and subscribe to changes for that particular task (the only thing we let the user change is the "done" status).
-
-We also use a `<NameBadge>` helper component to render the name of the author of the task, which we get by using the collaboration feature `getLastEditor(key)` on our `Task` CoMap, which returns the accountID of the last account that changed a given key in the CoMap.
-
----
-
-```typescript
-function NameBadge({ accountID }: { accountID?: AccountID }) {
-    const profile = useProfile(accountID);
-
-    const theme = window.matchMedia("(prefers-color-scheme: dark)").matches
-        ? "dark"
-        : "light";
-
-    const brightColor = uniqolor(accountID || "", { lightness: 80 }).color;
-    const darkColor = uniqolor(accountID || "", { lightness: 20 }).color;
-
-    return (
-        profile?.get("name") && <span
-            className="rounded-full py-0.5 px-2 text-xs"
-            style={{
-                color: theme == "light" ? darkColor : brightColor,
-                background: theme == "light" ? brightColor : darkColor,
-            }}
-        >
-            {profile.get("name")}
-        </span>
-    );
-}
-```
-
-`<NameBadge>` uses `useProfile(accountID)`, which is a shorthand for loading an account's profile (which is always a `CoMap<{name: string}>`, but might have app-specific additional properties).
-
-In our case, we just display the profile name (which, by the way, is set by the `LocalAuth` provider when we first create an account).
-
----
-
-```typescript
-function InviteButton({ list }: { list: TodoList }) {
-    const [existingInviteLink, setExistingInviteLink] = useState<string>();
-    const { toast } = useToast();
-
-    return (
-        list.coValue.getGroup().myRole() === "admin" && (
-            <Button
-                size="sm"
-                className="py-0"
-                disabled={!list}
-                variant="outline"
-                onClick={() => {
-                    let inviteLink = existingInviteLink;
-                    if (list && !inviteLink) {
-                        inviteLink = createInviteLink(list, "writer");
-                        setExistingInviteLink(inviteLink);
-                    }
-                    if (inviteLink) {
-                        navigator.clipboard.writeText(inviteLink).then(() =>
-                            toast({
-                                description: "Copied invite link to clipboard!",
-                            })
-                        );
-                    }
-                }}
-            >
-                Invite
-            </Button>
-        )
-    );
-}
-```
-
-Last, we have a look at the `<InviteButton>` component, which we use inside `<TodoListComponent>`. It only becomes visible when the current user is an admin in the `TodoList`'s group. You can see how we can create an invite link using `createInviteLink(coValue, role)` that allows anyone who has it to join the group as a specified role (here, as a writer).
-
----
+- (not yet explained) Creating invite links/QR codes with `<InviteButton/>`: [`src/components/InviteButton.tsx`](./src/components/InviteButton.tsx)
 
 This is the whole Todo List app!
 
-If you have feedback, let us know on [Discord](https://discord.gg/utDMjHYg42) or open an issue or PR to fix something that seems wrong.
+## Questions / problems / feedback
+
+If you have feedback, let us know on [Discord](https://discord.gg/utDMjHYg42) or open an issue or PR to fix something that seems wrong.
+
+
+## Configuration: sync server
+
+By default, the example app uses [Jazz Global Mesh](https://jazz.tools/mesh) (`wss://sync.jazz.tools`) - so cross-device use, invites and collaboration should just work.
+
+You can also run a local sync server by running `npx cojson-simple-sync` and adding the query param `?sync=ws://localhost:4200` to the URL of the example app (for example: `http://localhost:5173/?sync=ws://localhost:4200`), or by setting the `sync` parameter of the `<WithJazz>` provider component in [./src/2_main.tsx](./src/2_main.tsx).

examples/todo/components.json

@@ -10,7 +10,7 @@
     "cssVariables": true
   },
   "aliases": {
-    "components": "@/components",
-    "utils": "@/lib/utils"
+    "components": "@/basicComponents",
+    "utils": "@/basicComponents/lib/utils"
   }
 }

examples/todo/index.html

@@ -8,6 +8,6 @@
   </head>
   <body>
     <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
+    <script type="module" src="/src/2_main.tsx"></script>
   </body>
 </html>

examples/todo/job-template.nomad

@@ -1,9 +1,9 @@
 job "example-todo$BRANCH_SUFFIX" {
-  region     = "$REGION"
-  datacenters = ["$REGION"]
+  region     = "global"
+  datacenters = ["*"]
 
   group "static" {
-    // count = 3
+    count = 8
 
     network {
       port "http" {
@@ -14,13 +14,17 @@ job "example-todo$BRANCH_SUFFIX" {
     constraint {
       attribute = "${node.class}"
       operator  = "="
-      value     = "edge"
+      value     = "mesh"
     }
 
-    // spread {
-    //   attribute = "${node.datacenter}"
-    //   weight    = 100
-    // }
+    spread {
+      attribute = "${node.datacenter}"
+      weight    = 100
+    }
+
+    constraint {
+      distinct_hosts = true
+    }
 
     task "server" {
       driver = "docker"
@@ -37,9 +41,7 @@ job "example-todo$BRANCH_SUFFIX" {
 
       service {
         tags = ["public"]
-        meta {
-          public_name = "${BRANCH_SUBDOMAIN}example-todo"
-        }
+        name = "example-todo$BRANCH_SUFFIX"
         port = "http"
         provider = "consul"
       }

examples/todo/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-todo",
   "private": true,
-  "version": "0.0.17",
+  "version": "0.0.39",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -16,14 +16,16 @@
     "@types/qrcode": "^1.5.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
-    "jazz-react": "^0.1.3",
-    "jazz-react-auth-local": "^0.1.3",
-    "lucide-react": "^0.265.0",
+    "jazz-react": "^0.3.5",
+    "jazz-react-auth-local": "^0.3.5",
+    "lucide-react": "^0.274.0",
     "qrcode": "^1.5.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-router": "^6.16.0",
+    "react-router-dom": "^6.16.0",
     "tailwind-merge": "^1.14.0",
-    "tailwindcss-animate": "^1.0.6",
+    "tailwindcss-animate": "^1.0.7",
     "uniqolor": "^1.1.0"
   },
   "devDependencies": {

examples/todo/src/1_types.ts

@@ -0,0 +1,23 @@
+import { CoMap, CoList } from "cojson";
+
+/** Walkthrough: Defining the data model with CoJSON
+ *
+ *  Here, we define our main data model of tasks, lists of tasks and projects
+ *  using CoJSON's collaborative map and list types, CoMap & CoList.
+ *
+ *  CoMap values and CoLists items can contain:
+ *  - arbitrary immutable JSON
+ *  - references to other CoValues (internally stored by their CoID)
+ **/
+
+/** An individual task which collaborators can tick or rename */
+export type Task = CoMap<{ done: boolean; text: string; }>;
+
+/** Our top level object: a project with a title, referencing a list of tasks */
+export type TodoProject = CoMap<{
+    title: string;
+    /** A collaborative, ordered list of tasks */
+    tasks: CoList<Task>;
+}>;
+
+/** Walkthrough: Continue with ./2_main.tsx */

examples/todo/src/2_main.tsx

@@ -0,0 +1,91 @@
+import React from "react";
+import ReactDOM from "react-dom/client";
+import { RouterProvider, createHashRouter } from "react-router-dom";
+import "./index.css";
+
+import { WithJazz, useJazz, useAcceptInvite } from "jazz-react";
+import { LocalAuth } from "jazz-react-auth-local";
+
+import {
+    Button,
+    ThemeProvider,
+    TitleAndLogo,
+} from "./basicComponents/index.ts";
+import { PrettyAuthUI } from "./components/Auth.tsx";
+import { NewProjectForm } from "./3_NewProjectForm.tsx";
+import { ProjectTodoTable } from "./4_ProjectTodoTable.tsx";
+
+/**
+ * Walkthrough: The top-level provider `<WithJazz/>`
+ *
+ * This shows how to use the top-level provider `<WithJazz/>`,
+ * which provides the rest of the app with a `LocalNode` (used through `useJazz` later),
+ * based on `LocalAuth` that uses Passkeys (aka WebAuthn) to store a user's account secret
+ * - no backend needed.
+ */
+
+const appName = "Jazz Todo List Example";
+
+const auth = LocalAuth({
+    appName,
+    Component: PrettyAuthUI,
+});
+
+ReactDOM.createRoot(document.getElementById("root")!).render(
+    <React.StrictMode>
+        <WithJazz auth={auth}>
+            <App />
+        </WithJazz>
+    </React.StrictMode>
+);
+
+/**
+ * Routing in `<App/>`
+ *
+ * <App> is the main app component, handling client-side routing based
+ * on the CoValue ID (CoID) of our TodoProject, stored in the URL hash
+ * - which can also contain invite links.
+ */
+
+function App() {
+    // logOut logs out the AuthProvider passed to `<WithJazz/>` above.
+    const { logOut } = useJazz();
+
+    const router = createHashRouter([
+        {
+            path: "/",
+            element: <NewProjectForm />,
+        },
+        {
+            path: "/project/:projectId",
+            element: <ProjectTodoTable />,
+        },
+        {
+            path: "/invite/*",
+            element: <p>Accepting invite...</p>
+        }
+    ]);
+
+    // `useAcceptInvite()` is a hook that accepts an invite link from the URL hash,
+    // and on success calls our callback where we navigate to the project that we were just invited to.
+    useAcceptInvite((projectID) => router.navigate("/project/" + projectID));
+
+    return (
+        <ThemeProvider>
+            <TitleAndLogo name={appName} />
+            <div className="flex flex-col h-full items-center justify-start gap-10 pt-10 pb-10 px-5">
+
+                <RouterProvider router={router} />
+
+                <Button
+                    onClick={() => router.navigate("/").then(logOut)}
+                    variant="outline"
+                >
+                    Log Out
+                </Button>
+            </div>
+        </ThemeProvider>
+    );
+}
+
+/** Walkthrough: Continue with ./3_NewProjectForm.tsx */

examples/todo/src/3_NewProjectForm.tsx

@@ -0,0 +1,48 @@
+import { useCallback } from "react";
+
+import { useJazz } from "jazz-react";
+
+import { Task, TodoProject } from "./1_types";
+
+import { SubmittableInput } from "./basicComponents";
+
+import { CoList } from "cojson";
+import { useNavigate } from "react-router";
+
+export function NewProjectForm() {
+    // A `LocalNode` represents a local view of loaded & created CoValues.
+    // It is associated with a current user account, which will determine
+    // access rights to CoValues. We get it from the top-level provider `<WithJazz/>`.
+    const { localNode } = useJazz();
+    const navigate = useNavigate();
+
+    const createProject = useCallback(
+        (title: string) => {
+            if (!title) return;
+
+            // To create a new todo project, we first create a `Group`,
+            // which is a scope for defining access rights (reader/writer/admin)
+            // of its members, which will apply to all CoValues owned by that group.
+            const projectGroup = localNode.createGroup();
+
+            // Then we create an empty todo project within that group
+            const project = projectGroup.createMap<TodoProject>({
+                title,
+                tasks: projectGroup.createList<CoList<Task>>(),
+            });
+
+            navigate("/project/" + project.id);
+        },
+        [localNode, navigate]
+    );
+
+    return (
+        <SubmittableInput
+            onSubmit={createProject}
+            label="Create New Project"
+            placeholder="New project title"
+        />
+    );
+}
+
+/** Walkthrough: continue with ./4_ProjectTodoTable.tsx */

examples/todo/src/4_ProjectTodoTable.tsx

@@ -0,0 +1,177 @@
+import { useCallback } from "react";
+
+import { CoID, Queried } from "cojson";
+import { useSyncedQuery } from "jazz-react";
+
+import { TodoProject, Task } from "./1_types";
+
+import {
+    Checkbox,
+    SubmittableInput,
+    Skeleton,
+    Table,
+    TableBody,
+    TableCell,
+    TableHead,
+    TableHeader,
+    TableRow,
+} from "./basicComponents";
+
+import { InviteButton } from "./components/InviteButton";
+import uniqolor from "uniqolor";
+import { useParams } from "react-router";
+
+/** Walkthrough: Reactively rendering a todo project as a table,
+ *               adding and editing tasks
+ *
+ *  Here in `<TodoTable/>`, we use `useSyncedQuery()` for the first time,
+ *  in this case to load the CoValue for our `TodoProject` as well as
+ *  the `ListOfTasks` referenced in it.
+ */
+
+export function ProjectTodoTable() {
+    const projectId = useParams<{ projectId: CoID<TodoProject> }>().projectId;
+
+    // `useSyncedQuery()` reactively subscribes to updates to a CoValue's
+    // content - whether we create edits locally, load persisted data, or receive
+    // sync updates from other devices or participants!
+    // It also recursively resolves and subsribes to all referenced CoValues.
+    const project = useSyncedQuery(projectId);
+
+    // `createTask` is similar to `createProject` we saw earlier, creating a new CoMap
+    // for a new task (in the same group as the project), and then
+    // adding that as an item to the project's list of tasks.
+    const createTask = useCallback(
+        (text: string) => {
+            if (!project?.tasks || !text) return;
+            const task = project.group.createMap<Task>({
+                done: false,
+                text,
+            });
+
+            // project.tasks is immutable, but `append` will create an edit
+            // that will cause useSyncedQuery to rerender this component
+            // - here and on other devices!
+            project.tasks.append(task);
+        },
+        [project?.tasks, project?.group]
+    );
+
+    return (
+        <div className="max-w-full w-4xl">
+            <div className="flex justify-between items-center gap-4 mb-4">
+                <h1>
+                    {
+                        // This is how we can access properties from the project query,
+                        // accounting for the fact that note everything might be loaded yet
+                        project?.title ? (
+                            <>
+                                {project.title}{" "}
+                                <span className="text-sm">({project.id})</span>
+                            </>
+                        ) : (
+                            <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
+                        )
+                    }
+                </h1>
+                <InviteButton value={project} />
+            </div>
+            <Table>
+                <TableHeader>
+                    <TableRow>
+                        <TableHead className="w-[40px]">Done</TableHead>
+                        <TableHead>Task</TableHead>
+                    </TableRow>
+                </TableHeader>
+                <TableBody>
+                    {project?.tasks?.map(
+                        (task) => task && <TaskRow key={task.id} task={task} />
+                    )}
+                    <NewTaskInputRow
+                        createTask={createTask}
+                        disabled={!project}
+                    />
+                </TableBody>
+            </Table>
+        </div>
+    );
+}
+
+export function TaskRow({ task }: { task: Queried<Task> | undefined }) {
+    return (
+        <TableRow>
+            <TableCell>
+                <Checkbox
+                    className="mt-1"
+                    checked={task?.done}
+                    onCheckedChange={(checked) => {
+                        // Tick or untick the task
+                        // Task is also immutable, but this will update all queries
+                        // that include this task as a reference
+                        task?.set({ done: !!checked });
+                    }}
+                />
+            </TableCell>
+            <TableCell>
+                <div className="flex flex-row justify-between items-center gap-2">
+                    {task?.text ? (
+                        <span className={task?.done ? "line-through" : ""}>
+                            {task.text}
+                        </span>
+                    ) : (
+                        <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
+                    )}
+                    {
+                        // Here we see for the first time how we can access edit history
+                        // for a CoValue, and use it to display who created the task.
+                        task?.edits.text?.by?.profile?.name ? (
+                            <span
+                                className="rounded-full py-0.5 px-2 text-xs"
+                                style={uniqueColoring(task.edits.text.by.id)}
+                            >
+                                {task.edits.text.by.profile.name}
+                            </span>
+                        ) : (
+                            <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
+                        )
+                    }
+                </div>
+            </TableCell>
+        </TableRow>
+    );
+}
+
+/** Walkthrough: This is the end of the walkthrough so far! */
+
+function NewTaskInputRow({
+    createTask,
+    disabled,
+}: {
+    createTask: (text: string) => void;
+    disabled: boolean;
+}) {
+    return (
+        <TableRow>
+            <TableCell>
+                <Checkbox className="mt-1" disabled />
+            </TableCell>
+            <TableCell>
+                <SubmittableInput
+                    onSubmit={(taskText) => createTask(taskText)}
+                    label="Add"
+                    placeholder="New task"
+                    disabled={disabled}
+                />
+            </TableCell>
+        </TableRow>
+    );
+}
+
+function uniqueColoring(seed: string) {
+    const darkMode = window.matchMedia("(prefers-color-scheme: dark)").matches;
+
+    return {
+        color: uniqolor(seed, { lightness: darkMode ? 80 : 20 }).color,
+        background: uniqolor(seed, { lightness: darkMode ? 20 : 80 }).color,
+    };
+}

examples/todo/src/App.tsx

@@ -1,248 +0,0 @@
-import { useCallback, useEffect, useState } from "react";
-
-import { CoMap, CoID, AccountID } from "cojson";
-import {
-    consumeInviteLinkFromWindowLocation,
-    useJazz,
-    useProfile,
-    useTelepathicState,
-    createInviteLink
-} from "jazz-react";
-
-import { SubmittableInput } from "./components/SubmittableInput";
-import { useToast } from "./components/ui/use-toast";
-import { Skeleton } from "./components/ui/skeleton";
-import {
-    Table, TableBody, TableCell, TableHead, TableHeader, TableRow,
-} from "@/components/ui/table";
-import { Checkbox } from "@/components/ui/checkbox";
-import { Button } from "@/components/ui/button";
-import uniqolor from "uniqolor";
-import QRCode from "qrcode";
-
-type TaskContent = { done: boolean; text: string };
-type Task = CoMap<TaskContent>;
-
-type TodoListContent = {
-    title: string;
-    // other keys form a set of task IDs
-    [taskId: CoID<Task>]: true;
-};
-type TodoList = CoMap<TodoListContent>;
-
-export default function App() {
-    const [listId, setListId] = useState<CoID<TodoList>>();
-
-    const { localNode, logOut } = useJazz();
-
-    useEffect(() => {
-        const listener = async () => {
-            const acceptedInvitation =
-                await consumeInviteLinkFromWindowLocation(localNode);
-
-            if (acceptedInvitation) {
-                setListId(acceptedInvitation.valueID as CoID<TodoList>);
-                window.location.hash = acceptedInvitation.valueID;
-                return;
-            }
-
-            setListId(window.location.hash.slice(1) as CoID<TodoList>);
-        };
-        window.addEventListener("hashchange", listener);
-        listener();
-
-        return () => {
-            window.removeEventListener("hashchange", listener);
-        };
-    }, [localNode]);
-
-    const createList = useCallback(
-        (title: string) => {
-            const listGroup = localNode.createGroup();
-            const list = listGroup.createMap<TodoListContent>();
-
-            list.edit((list) => {
-                list.set("title", title);
-            });
-
-            window.location.hash = list.id;
-        },
-        [localNode]
-    );
-
-    return (
-        <div className="flex flex-col h-full items-center justify-start gap-10 pt-10 pb-10 px-5">
-            {listId ? (
-                <TodoListComponent listId={listId} />
-            ) : (
-                <SubmittableInput
-                    onSubmit={createList}
-                    label="Create New List"
-                    placeholder="New list title"
-                />
-            )}
-            <Button
-                onClick={() => {
-                    window.location.hash = "";
-                    logOut();
-                }}
-                variant="outline"
-            >
-                Log Out
-            </Button>
-        </div>
-    );
-}
-
-export function TodoListComponent({ listId }: { listId: CoID<TodoList> }) {
-    const list = useTelepathicState(listId);
-
-    const createTask = (text: string) => {
-        if (!list) return;
-        const task = list.coValue.getGroup().createMap<TaskContent>();
-
-        task.edit((task) => {
-            task.set("text", text);
-            task.set("done", false);
-        });
-
-        list.edit((list) => {
-            list.set(task.id, true);
-        });
-    };
-
-    return (
-        <div className="max-w-full w-4xl">
-            <div className="flex justify-between items-center gap-4 mb-4">
-                <h1>
-                    {list?.get("title") ? (
-                        <>
-                            {list.get("title")}{" "}
-                            <span className="text-sm">({list.id})</span>
-                        </>
-                    ) : (
-                        <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
-                    )}
-                </h1>
-                {list && <InviteButton list={list} />}
-            </div>
-            <Table>
-                <TableHeader>
-                    <TableRow>
-                        <TableHead className="w-[40px]">Done</TableHead>
-                        <TableHead>Task</TableHead>
-                    </TableRow>
-                </TableHeader>
-                <TableBody>
-                    {list &&
-                        list
-                            .keys()
-                            .filter((key): key is CoID<Task> =>
-                                key.startsWith("co_")
-                            )
-                            .map((taskId) => (
-                                <TaskRow key={taskId} taskId={taskId} />
-                            ))}
-                    <TableRow key="new">
-                        <TableCell>
-                            <Checkbox className="mt-1" disabled />
-                        </TableCell>
-                        <TableCell>
-                            <SubmittableInput
-                                onSubmit={(taskText) => createTask(taskText)}
-                                label="Add"
-                                placeholder="New task"
-                                disabled={!list}
-                            />
-                        </TableCell>
-                    </TableRow>
-                </TableBody>
-            </Table>
-        </div>
-    );
-}
-
-function TaskRow({ taskId }: { taskId: CoID<Task> }) {
-    const task = useTelepathicState(taskId);
-
-    return (
-        <TableRow>
-            <TableCell>
-                <Checkbox
-                    className="mt-1"
-                    checked={task?.get("done")}
-                    onCheckedChange={(checked) => {
-                        task?.edit((task) => {
-                            task.set("done", !!checked);
-                        });
-                    }}
-                />
-            </TableCell>
-            <TableCell>
-                <div className="flex flex-row justify-between items-center gap-2">
-                    <span className={task?.get("done") ? "line-through" : ""}>
-                        {task?.get("text") || <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />}
-                    </span>
-                    <NameBadge accountID={task?.getLastEditor("text")} />
-                </div>
-            </TableCell>
-        </TableRow>
-    );
-}
-
-function NameBadge({ accountID }: { accountID?: AccountID }) {
-    const profile = useProfile(accountID);
-
-    const theme = window.matchMedia("(prefers-color-scheme: dark)").matches
-        ? "dark"
-        : "light";
-
-    const brightColor = uniqolor(accountID || "", { lightness: 80 }).color;
-    const darkColor = uniqolor(accountID || "", { lightness: 20 }).color;
-
-    return (
-        profile?.get("name") && <span
-            className="rounded-full py-0.5 px-2 text-xs"
-            style={{
-                color: theme == "light" ? darkColor : brightColor,
-                background: theme == "light" ? brightColor : darkColor,
-            }}
-        >
-            {profile.get("name")}
-        </span>
-    );
-}
-
-function InviteButton({ list }: { list: TodoList }) {
-    const [existingInviteLink, setExistingInviteLink] = useState<string>();
-    const { toast } = useToast();
-
-    return (
-        list.coValue.getGroup().myRole() === "admin" && (
-            <Button
-                size="sm"
-                className="py-0"
-                disabled={!list}
-                variant="outline"
-                onClick={async () => {
-                    let inviteLink = existingInviteLink;
-                    if (list && !inviteLink) {
-                        inviteLink = createInviteLink(list, "writer");
-                        setExistingInviteLink(inviteLink);
-                    }
-                    if (inviteLink) {
-                        const qr = await QRCode.toDataURL(inviteLink, { errorCorrectionLevel: 'L' });
-                        navigator.clipboard.writeText(inviteLink).then(() =>
-                            toast({
-                                title: "Copied invite link to clipboard!",
-                                description: <img src={qr} className="w-20 h-20"/>,
-                            })
-                        );
-                    }
-                }}
-            >
-                Invite
-            </Button>
-        )
-    );
-}

examples/todo/src/basicComponents/SubmittableInput.tsx

@@ -1,5 +1,5 @@
-import { Input } from "@/components/ui/input";
-import { Button } from "@/components/ui/button";
+import { Input } from "@/basicComponents/ui/input";
+import { Button } from "@/basicComponents/ui/button";
 
 export function SubmittableInput({
     onSubmit,

examples/todo/src/basicComponents/TitleAndLogo.tsx

@@ -0,0 +1,10 @@
+import { Toaster } from ".";
+
+export function TitleAndLogo({name}: {name: string}) {
+    return <>
+        <div className="flex items-center gap-2 justify-center mt-5">
+                <img src="jazz-logo.png" className="h-5" /> {name}
+            </div>
+            <Toaster />
+    </>
+}

examples/todo/src/basicComponents/index.ts

@@ -0,0 +1,17 @@
+export { Button } from "./ui/button";
+export { Checkbox } from "./ui/checkbox";
+export { Input } from "./ui/input";
+export { Skeleton } from "./ui/skeleton";
+export { Toaster } from "./ui/toaster";
+export { useToast } from "./ui/use-toast";
+export { SubmittableInput } from "./SubmittableInput";
+export { TitleAndLogo } from "./TitleAndLogo";
+export { ThemeProvider } from "./themeProvider";
+export {
+    Table,
+    TableBody,
+    TableCell,
+    TableHead,
+    TableHeader,
+    TableRow,
+} from "./ui/table";

examples/todo/src/basicComponents/lib/utils.ts

@@ -0,0 +1,6 @@
+import { type ClassValue, clsx } from "clsx"
+import { twMerge } from "tailwind-merge"
+ 
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs))
+}

examples/todo/src/basicComponents/themeProvider.tsx

@@ -0,0 +1,72 @@
+import { createContext, useContext, useEffect, useState } from "react";
+
+type ThemeProviderProps = {
+    children: React.ReactNode;
+    defaultTheme?: string;
+    storageKey?: string;
+};
+
+type ThemeProviderState = {
+    theme: string;
+    setTheme: (theme: string) => void;
+};
+
+const initialState = {
+    theme: "system",
+    setTheme: () => null,
+};
+
+const ThemeProviderContext = createContext<ThemeProviderState>(initialState);
+
+export function ThemeProvider({
+    children,
+    defaultTheme = "system",
+    storageKey = "vite-ui-theme",
+    ...props
+}: ThemeProviderProps) {
+    const [theme, setTheme] = useState(
+        () => localStorage.getItem(storageKey) || defaultTheme
+    );
+
+    useEffect(() => {
+        const root = window.document.documentElement;
+
+        root.classList.remove("light", "dark");
+
+        if (theme === "system") {
+            const systemTheme = window.matchMedia(
+                "(prefers-color-scheme: dark)"
+            ).matches
+                ? "dark"
+                : "light";
+
+            root.classList.add(systemTheme);
+            return;
+        }
+
+        root.classList.add(theme);
+    }, [theme]);
+
+    const value = {
+        theme,
+        setTheme: (theme: string) => {
+            localStorage.setItem(storageKey, theme);
+            setTheme(theme);
+        },
+    };
+
+    return (
+        <ThemeProviderContext.Provider {...props} value={value}>
+            {children}
+        </ThemeProviderContext.Provider>
+    );
+}
+
+export const useTheme = () => {
+    const context = useContext(ThemeProviderContext);
+
+    if (context === undefined)
+        throw new Error("useTheme must be used within a ThemeProvider");
+
+    return context;
+};

examples/todo/src/basicComponents/ui/button.tsx

@@ -0,0 +1,56 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/basicComponents/lib/utils"
+
+const buttonVariants = cva(
+  "inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
+  {
+    variants: {
+      variant: {
+        default: "bg-primary text-primary-foreground hover:bg-primary/90",
+        destructive:
+          "bg-destructive text-destructive-foreground hover:bg-destructive/90",
+        outline:
+          "border border-input bg-background hover:bg-accent hover:text-accent-foreground",
+        secondary:
+          "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        ghost: "hover:bg-accent hover:text-accent-foreground",
+        link: "text-primary underline-offset-4 hover:underline",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  }
+)
+
+export interface ButtonProps
+  extends React.ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof buttonVariants> {
+  asChild?: boolean
+}
+
+const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
+  ({ className, variant, size, asChild = false, ...props }, ref) => {
+    const Comp = asChild ? Slot : "button"
+    return (
+      <Comp
+        className={cn(buttonVariants({ variant, size, className }))}
+        ref={ref}
+        {...props}
+      />
+    )
+  }
+)
+Button.displayName = "Button"
+
+export { Button, buttonVariants }

examples/todo/src/basicComponents/ui/checkbox.tsx

@@ -2,7 +2,7 @@ import * as React from "react"
 import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
 import { Check } from "lucide-react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 const Checkbox = React.forwardRef<
   React.ElementRef<typeof CheckboxPrimitive.Root>,

examples/todo/src/basicComponents/ui/input.tsx

@@ -0,0 +1,25 @@
+import * as React from "react"
+
+import { cn } from "@/basicComponents/lib/utils"
+
+export interface InputProps
+  extends React.InputHTMLAttributes<HTMLInputElement> {}
+
+const Input = React.forwardRef<HTMLInputElement, InputProps>(
+  ({ className, type, ...props }, ref) => {
+    return (
+      <input
+        type={type}
+        className={cn(
+          "flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+          className
+        )}
+        ref={ref}
+        {...props}
+      />
+    )
+  }
+)
+Input.displayName = "Input"
+
+export { Input }

examples/todo/src/basicComponents/ui/skeleton.tsx

@@ -0,0 +1,15 @@
+import { cn } from "@/basicComponents/lib/utils"
+
+function Skeleton({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) {
+  return (
+    <div
+      className={cn("animate-pulse rounded-md bg-muted", className)}
+      {...props}
+    />
+  )
+}
+
+export { Skeleton }

examples/todo/src/basicComponents/ui/table.tsx

@@ -1,6 +1,6 @@
 import * as React from "react"
 
-import { cn } from "@/lib/utils"
+import { cn } from "@/basicComponents/lib/utils"
 
 const Table = React.forwardRef<
   HTMLTableElement,

examples/todo/src/basicComponents/ui/toast.tsx

@@ -0,0 +1,127 @@
+import * as React from "react"
+import * as ToastPrimitives from "@radix-ui/react-toast"
+import { cva, type VariantProps } from "class-variance-authority"
+import { X } from "lucide-react"
+
+import { cn } from "@/basicComponents/lib/utils"
+
+const ToastProvider = ToastPrimitives.Provider
+
+const ToastViewport = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Viewport>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Viewport>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Viewport
+    ref={ref}
+    className={cn(
+      "fixed top-0 z-[100] flex max-h-screen w-full flex-col-reverse p-4 sm:bottom-0 sm:right-0 sm:top-auto sm:flex-col md:max-w-[420px]",
+      className
+    )}
+    {...props}
+  />
+))
+ToastViewport.displayName = ToastPrimitives.Viewport.displayName
+
+const toastVariants = cva(
+  "group pointer-events-auto relative flex w-full items-center justify-between space-x-4 overflow-hidden rounded-md border p-6 pr-8 shadow-lg transition-all data-[swipe=cancel]:translate-x-0 data-[swipe=end]:translate-x-[var(--radix-toast-swipe-end-x)] data-[swipe=move]:translate-x-[var(--radix-toast-swipe-move-x)] data-[swipe=move]:transition-none data-[state=open]:animate-in data-[state=closed]:animate-out data-[swipe=end]:animate-out data-[state=closed]:fade-out-80 data-[state=closed]:slide-out-to-right-full data-[state=open]:slide-in-from-top-full data-[state=open]:sm:slide-in-from-bottom-full",
+  {
+    variants: {
+      variant: {
+        default: "border bg-background text-foreground",
+        destructive:
+          "destructive group border-destructive bg-destructive text-destructive-foreground",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+const Toast = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Root>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Root> &
+    VariantProps<typeof toastVariants>
+>(({ className, variant, ...props }, ref) => {
+  return (
+    <ToastPrimitives.Root
+      ref={ref}
+      className={cn(toastVariants({ variant }), className)}
+      {...props}
+    />
+  )
+})
+Toast.displayName = ToastPrimitives.Root.displayName
+
+const ToastAction = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Action>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Action>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Action
+    ref={ref}
+    className={cn(
+      "inline-flex h-8 shrink-0 items-center justify-center rounded-md border bg-transparent px-3 text-sm font-medium ring-offset-background transition-colors hover:bg-secondary focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 group-[.destructive]:border-muted/40 group-[.destructive]:hover:border-destructive/30 group-[.destructive]:hover:bg-destructive group-[.destructive]:hover:text-destructive-foreground group-[.destructive]:focus:ring-destructive",
+      className
+    )}
+    {...props}
+  />
+))
+ToastAction.displayName = ToastPrimitives.Action.displayName
+
+const ToastClose = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Close>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Close>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Close
+    ref={ref}
+    className={cn(
+      "absolute right-2 top-2 rounded-md p-1 text-foreground/50 opacity-0 transition-opacity hover:text-foreground focus:opacity-100 focus:outline-none focus:ring-2 group-hover:opacity-100 group-[.destructive]:text-red-300 group-[.destructive]:hover:text-red-50 group-[.destructive]:focus:ring-red-400 group-[.destructive]:focus:ring-offset-red-600",
+      className
+    )}
+    toast-close=""
+    {...props}
+  >
+    <X className="h-4 w-4" />
+  </ToastPrimitives.Close>
+))
+ToastClose.displayName = ToastPrimitives.Close.displayName
+
+const ToastTitle = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Title>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Title>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Title
+    ref={ref}
+    className={cn("text-sm font-semibold", className)}
+    {...props}
+  />
+))
+ToastTitle.displayName = ToastPrimitives.Title.displayName
+
+const ToastDescription = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Description>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Description>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Description
+    ref={ref}
+    className={cn("text-sm opacity-90", className)}
+    {...props}
+  />
+))
+ToastDescription.displayName = ToastPrimitives.Description.displayName
+
+type ToastProps = React.ComponentPropsWithoutRef<typeof Toast>
+
+type ToastActionElement = React.ReactElement<typeof ToastAction>
+
+export {
+  type ToastProps,
+  type ToastActionElement,
+  ToastProvider,
+  ToastViewport,
+  Toast,
+  ToastTitle,
+  ToastDescription,
+  ToastClose,
+  ToastAction,
+}

examples/todo/src/basicComponents/ui/toaster.tsx

@@ -0,0 +1,33 @@
+import {
+  Toast,
+  ToastClose,
+  ToastDescription,
+  ToastProvider,
+  ToastTitle,
+  ToastViewport,
+} from "@/basicComponents/ui/toast"
+import { useToast } from "@/basicComponents/ui/use-toast"
+
+export function Toaster() {
+  const { toasts } = useToast()
+
+  return (
+    <ToastProvider>
+      {toasts.map(function ({ id, title, description, action, ...props }) {
+        return (
+          <Toast key={id} {...props}>
+            <div className="grid gap-1">
+              {title && <ToastTitle>{title}</ToastTitle>}
+              {description && (
+                <ToastDescription>{description}</ToastDescription>
+              )}
+            </div>
+            {action}
+            <ToastClose />
+          </Toast>
+        )
+      })}
+      <ToastViewport />
+    </ToastProvider>
+  )
+}

examples/todo/src/basicComponents/ui/use-toast.ts

@@ -0,0 +1,192 @@
+// Inspired by react-hot-toast library
+import * as React from "react"
+
+import type {
+  ToastActionElement,
+  ToastProps,
+} from "@/basicComponents/ui/toast"
+
+const TOAST_LIMIT = 1
+const TOAST_REMOVE_DELAY = 1000000
+
+type ToasterToast = ToastProps & {
+  id: string
+  title?: React.ReactNode
+  description?: React.ReactNode
+  action?: ToastActionElement
+}
+
+const actionTypes = {
+  ADD_TOAST: "ADD_TOAST",
+  UPDATE_TOAST: "UPDATE_TOAST",
+  DISMISS_TOAST: "DISMISS_TOAST",
+  REMOVE_TOAST: "REMOVE_TOAST",
+} as const
+
+let count = 0
+
+function genId() {
+  count = (count + 1) % Number.MAX_VALUE
+  return count.toString()
+}
+
+type ActionType = typeof actionTypes
+
+type Action =
+  | {
+      type: ActionType["ADD_TOAST"]
+      toast: ToasterToast
+    }
+  | {
+      type: ActionType["UPDATE_TOAST"]
+      toast: Partial<ToasterToast>
+    }
+  | {
+      type: ActionType["DISMISS_TOAST"]
+      toastId?: ToasterToast["id"]
+    }
+  | {
+      type: ActionType["REMOVE_TOAST"]
+      toastId?: ToasterToast["id"]
+    }
+
+interface State {
+  toasts: ToasterToast[]
+}
+
+const toastTimeouts = new Map<string, ReturnType<typeof setTimeout>>()
+
+const addToRemoveQueue = (toastId: string) => {
+  if (toastTimeouts.has(toastId)) {
+    return
+  }
+
+  const timeout = setTimeout(() => {
+    toastTimeouts.delete(toastId)
+    dispatch({
+      type: "REMOVE_TOAST",
+      toastId: toastId,
+    })
+  }, TOAST_REMOVE_DELAY)
+
+  toastTimeouts.set(toastId, timeout)
+}
+
+export const reducer = (state: State, action: Action): State => {
+  switch (action.type) {
+    case "ADD_TOAST":
+      return {
+        ...state,
+        toasts: [action.toast, ...state.toasts].slice(0, TOAST_LIMIT),
+      }
+
+    case "UPDATE_TOAST":
+      return {
+        ...state,
+        toasts: state.toasts.map((t) =>
+          t.id === action.toast.id ? { ...t, ...action.toast } : t
+        ),
+      }
+
+    case "DISMISS_TOAST": {
+      const { toastId } = action
+
+      // ! Side effects ! - This could be extracted into a dismissToast() action,
+      // but I'll keep it here for simplicity
+      if (toastId) {
+        addToRemoveQueue(toastId)
+      } else {
+        state.toasts.forEach((toast) => {
+          addToRemoveQueue(toast.id)
+        })
+      }
+
+      return {
+        ...state,
+        toasts: state.toasts.map((t) =>
+          t.id === toastId || toastId === undefined
+            ? {
+                ...t,
+                open: false,
+              }
+            : t
+        ),
+      }
+    }
+    case "REMOVE_TOAST":
+      if (action.toastId === undefined) {
+        return {
+          ...state,
+          toasts: [],
+        }
+      }
+      return {
+        ...state,
+        toasts: state.toasts.filter((t) => t.id !== action.toastId),
+      }
+  }
+}
+
+const listeners: Array<(state: State) => void> = []
+
+let memoryState: State = { toasts: [] }
+
+function dispatch(action: Action) {
+  memoryState = reducer(memoryState, action)
+  listeners.forEach((listener) => {
+    listener(memoryState)
+  })
+}
+
+type Toast = Omit<ToasterToast, "id">
+
+function toast({ ...props }: Toast) {
+  const id = genId()
+
+  const update = (props: ToasterToast) =>
+    dispatch({
+      type: "UPDATE_TOAST",
+      toast: { ...props, id },
+    })
+  const dismiss = () => dispatch({ type: "DISMISS_TOAST", toastId: id })
+
+  dispatch({
+    type: "ADD_TOAST",
+    toast: {
+      ...props,
+      id,
+      open: true,
+      onOpenChange: (open) => {
+        if (!open) dismiss()
+      },
+    },
+  })
+
+  return {
+    id: id,
+    dismiss,
+    update,
+  }
+}
+
+function useToast() {
+  const [state, setState] = React.useState<State>(memoryState)
+
+  React.useEffect(() => {
+    listeners.push(setState)
+    return () => {
+      const index = listeners.indexOf(setState)
+      if (index > -1) {
+        listeners.splice(index, 1)
+      }
+    }
+  }, [state])
+
+  return {
+    ...state,
+    toast,
+    dismiss: (toastId?: string) => dispatch({ type: "DISMISS_TOAST", toastId }),
+  }
+}
+
+export { useToast, toast }

examples/todo/src/components/Auth.tsx

@@ -0,0 +1,48 @@
+import { useState } from "react";
+
+import { LocalAuthComponent } from "jazz-react-auth-local";
+
+import { Input, Button } from "../basicComponents";
+
+export const PrettyAuthUI: LocalAuthComponent = ({
+    loading,
+    logIn,
+    signUp,
+}) => {
+    const [username, setUsername] = useState<string>("");
+
+    return (
+        <div className="w-full h-full flex items-center justify-center p-5">
+            {loading ? (
+                <div>Loading...</div>
+            ) : (
+                <div className="w-72 flex flex-col gap-4">
+                    <form
+                        className="w-72 flex flex-col gap-2"
+                        onSubmit={(e) => {
+                            e.preventDefault();
+                            signUp(username);
+                        }}
+                    >
+                        <Input
+                            placeholder="Display name"
+                            value={username}
+                            onChange={(e) => setUsername(e.target.value)}
+                            autoComplete="webauthn"
+                            className="text-base"
+                        />
+                        <Button asChild>
+                            <Input
+                                type="submit"
+                                value="Sign Up as new account"
+                            />
+                        </Button>
+                    </form>
+                    <Button onClick={logIn}>
+                        Log In with existing account
+                    </Button>
+                </div>
+            )}
+        </div>
+    );
+};

examples/todo/src/components/InviteButton.tsx

@@ -0,0 +1,45 @@
+import { useState } from "react";
+
+import { createInviteLink } from "jazz-react";
+import QRCode from "qrcode";
+
+import { useToast, Button } from "../basicComponents";
+import { CoValue, Queried } from "cojson";
+
+export function InviteButton<T extends CoValue>({ value }: { value: T | Queried<T> | undefined }) {
+    const [existingInviteLink, setExistingInviteLink] = useState<string>();
+    const { toast } = useToast();
+
+    return (
+        value?.group?.myRole() === "admin" && (
+            <Button
+                size="sm"
+                className="py-0"
+                disabled={!value.group || !value.id}
+                variant="outline"
+                onClick={async () => {
+                    let inviteLink = existingInviteLink;
+                    if (value.group && value.id && !inviteLink) {
+                        inviteLink = createInviteLink(value, "writer");
+                        setExistingInviteLink(inviteLink);
+                    }
+                    if (inviteLink) {
+                        const qr = await QRCode.toDataURL(inviteLink, {
+                            errorCorrectionLevel: "L",
+                        });
+                        navigator.clipboard.writeText(inviteLink).then(() =>
+                            toast({
+                                title: "Copied invite link to clipboard!",
+                                description: (
+                                    <img src={qr} className="w-20 h-20" />
+                                ),
+                            })
+                        );
+                    }
+                }}
+            >
+                Invite
+            </Button>
+        )
+    );
+}

examples/todo/src/index.css

@@ -1,12 +1,7 @@
 @tailwind base;
 @tailwind components;
 @tailwind utilities;
-
-:root, body, #root {
-  width: 100%;
-  height: 100%;
-}
-
+ 
 @layer base {
   :root {
     --background: 0 0% 100%;
@@ -14,63 +9,63 @@
 
     --card: 0 0% 100%;
     --card-foreground: 20 14.3% 4.1%;
-
+ 
     --popover: 0 0% 100%;
     --popover-foreground: 20 14.3% 4.1%;
-
+ 
     --primary: 24 9.8% 10%;
     --primary-foreground: 60 9.1% 97.8%;
-
+ 
     --secondary: 60 4.8% 95.9%;
     --secondary-foreground: 24 9.8% 10%;
-
+ 
     --muted: 60 4.8% 95.9%;
     --muted-foreground: 25 5.3% 44.7%;
-
+ 
     --accent: 60 4.8% 95.9%;
     --accent-foreground: 24 9.8% 10%;
-
+ 
     --destructive: 0 84.2% 60.2%;
     --destructive-foreground: 60 9.1% 97.8%;
 
     --border: 20 5.9% 90%;
     --input: 20 5.9% 90%;
     --ring: 20 14.3% 4.1%;
-
+ 
     --radius: 0.5rem;
   }
-
+ 
   .dark {
     --background: 20 14.3% 4.1%;
     --foreground: 60 9.1% 97.8%;
-
+ 
     --card: 20 14.3% 4.1%;
     --card-foreground: 60 9.1% 97.8%;
-
+ 
     --popover: 20 14.3% 4.1%;
     --popover-foreground: 60 9.1% 97.8%;
-
+ 
     --primary: 60 9.1% 97.8%;
     --primary-foreground: 24 9.8% 10%;
-
+ 
     --secondary: 12 6.5% 15.1%;
     --secondary-foreground: 60 9.1% 97.8%;
-
+ 
     --muted: 12 6.5% 15.1%;
     --muted-foreground: 24 5.4% 63.9%;
-
+ 
     --accent: 12 6.5% 15.1%;
     --accent-foreground: 60 9.1% 97.8%;
-
+ 
     --destructive: 0 62.8% 30.6%;
     --destructive-foreground: 60 9.1% 97.8%;
-
+ 
     --border: 12 6.5% 15.1%;
     --input: 12 6.5% 15.1%;
     --ring: 24 5.7% 82.9%;
   }
 }
-
+ 
 @layer base {
   * {
     @apply border-border;

examples/todo/src/main.tsx

@@ -1,35 +0,0 @@
-import React from "react";
-import ReactDOM from "react-dom/client";
-
-import { WithJazz } from "jazz-react";
-import { LocalAuth } from "jazz-react-auth-local";
-
-import { PrettyAuthComponent } from "./components/prettyAuth.tsx";
-import { ThemeProvider } from "./components/themeProvider.tsx";
-import { Toaster } from "./components/ui/toaster.tsx";
-import App from "./App.tsx";
-import "./index.css";
-
-ReactDOM.createRoot(document.getElementById("root")!).render(
-    <React.StrictMode>
-        <ThemeProvider>
-            <div className="flex items-center gap-2 justify-center mt-5">
-                <img src="jazz-logo.png" className="h-5" /> Jazz Todo List
-                Example
-            </div>
-            <WithJazz
-                auth={LocalAuth({
-                    appName: "Jazz Todo List Example",
-                    Component: PrettyAuthComponent,
-                })}
-                syncAddress={
-                    new URLSearchParams(window.location.search).get("sync") ||
-                    undefined
-                }
-            >
-                <App />
-                <Toaster />
-            </WithJazz>
-        </ThemeProvider>
-    </React.StrictMode>
-);

generateDocs.ts

@@ -0,0 +1,787 @@
+import { readFile, writeFile } from "fs/promises";
+import { Application, JSONOutput, ReflectionKind } from "typedoc";
+
+const manuallyIgnore = new Set(["CojsonInternalTypes"]);
+
+async function main() {
+    // Application.bootstrap also exists, which will not load plugins
+    // Also accepts an array of option readers if you want to disable
+    // TypeDoc's tsconfig.json/package.json/typedoc.json option readers
+    const packageDocs = Object.entries({
+        "jazz-react": "index.tsx",
+        cojson: "index.ts",
+        "jazz-react-media-images": "index.tsx",
+        "jazz-browser": "index.ts",
+        "jazz-browser-media-images": "index.ts",
+    }).map(async ([packageName, entryPoint]) => {
+        const app = await Application.bootstrapWithPlugins({
+            entryPoints: [`packages/${packageName}/src/${entryPoint}`],
+            tsconfig: `packages/${packageName}/tsconfig.json`,
+            sort: ["required-first"],
+            groupOrder: ["Functions", "Classes", "TypeAliases", "Namespaces"],
+            categorizeByGroup: false
+        });
+
+        const project = await app.convert();
+
+        if (!project) {
+            throw new Error("Failed to convert project" + packageName);
+        }
+        // Alternatively generate JSON output
+        await app.generateJson(project, `docsTmp/${packageName}.json`);
+
+        const docs = JSON.parse(
+            await readFile(`docsTmp/${packageName}.json`, "utf8")
+        ) as JSONOutput.ProjectReflection;
+
+        return (
+            `# ${packageName}\n\n` +
+            docs
+                .groups!.map((group) => {
+                    return group.children
+                        ?.flatMap((childId) => {
+                            const child = docs.children!.find(
+                                (child) => child.id === childId
+                            )!;
+
+                            if (
+                                manuallyIgnore.has(child.name) ||
+                                child.comment?.blockTags?.some(
+                                    (tag) =>
+                                        tag.tag === "@deprecated" ||
+                                        tag.tag === "@internal" ||
+                                        tag.tag === "@ignore"
+                                ) ||
+                                child.signatures?.every((signature) =>
+                                    signature.comment?.blockTags?.some(
+                                        (tag) =>
+                                            tag.tag === "@deprecated" ||
+                                            tag.tag === "@internal" ||
+                                            tag.tag === "@ignore"
+                                    )
+                                )
+                            ) {
+                                return [];
+                            }
+
+                            return (
+                                `## \`${renderChildName(
+                                    child
+                                )}\`\n\n<sup>(${group.title
+                                    .toLowerCase()
+                                    .replace("bles", "ble")
+                                    .replace("ces", "ce")
+                                    .replace(/es$/, "")
+                                    .replace(
+                                        "ns",
+                                        "n"
+                                    )} in \`${packageName}\`)</sup>\n\n` +
+                                renderChildType(child) +
+                                (child.kind === ReflectionKind.Class ||
+                                child.kind === ReflectionKind.Interface ||
+                                child.kind === ReflectionKind.Namespace
+                                    ? renderSummary(child.comment) +
+                                      renderExamples(child.comment) +
+                                      (child.categories || child.groups)
+                                          ?.map((category) =>
+                                              renderChildCategory(child, category)
+                                          )
+                                          .join("<br/>\n\n")
+                                    : child.kind === ReflectionKind.Function
+                                    ? renderSummary(
+                                          child.signatures?.[0].comment
+                                      ) +
+                                      renderParamComments(
+                                          child.signatures?.[0].parameters || []
+                                      ) +
+                                      renderExamples(
+                                          child.signatures?.[0].comment
+                                      ) +
+                                      "\n\n"
+                                    : "TODO: doc generator not implemented yet " +
+                                      child.kind)
+                            );
+                        })
+                        .join("\n\n----\n\n");
+                })
+                .join("\n\n----\n\n")
+        );
+
+        function renderSummary(comment?: JSONOutput.Comment): string {
+            if (comment) {
+                return (
+                    comment.summary
+                        .map((token) =>
+                            token.kind === "text" || token.kind === "code"
+                                ? token.text
+                                : ""
+                        )
+                        .join("") +
+                    "\n\n" +
+                    "\n\n"
+                );
+            } else {
+                return "TODO: document\n\n";
+            }
+        }
+
+        function renderExamples(comment?: JSONOutput.Comment): string {
+            return (comment?.blockTags || [])
+                .map((blockTag) =>
+                    blockTag.tag === "@example"
+                        ? "##### Example:\n\n" +
+                          blockTag.content
+                              .map((token) =>
+                                  token.kind === "text" || token.kind === "code"
+                                      ? token.text
+                                      : ""
+                              )
+                              .join("") +
+                          "\n\n"
+                        : ""
+                )
+                .join("");
+        }
+
+        function renderParamComments(params: JSONOutput.ParameterReflection[]) {
+            const paramDocs = params.flatMap((param) => {
+                if (param.type?.type === "reflection") {
+                    return param.type.declaration.children?.flatMap((child) => {
+                        if (
+                            child.name === "children" &&
+                            child.type?.type === "reference" &&
+                            child.type?.name === "ReactNode"
+                        ) {
+                            return [];
+                        }
+                        return (
+                            `| \`${param.name}.${child.name}${
+                                child.flags.isOptional || child.defaultValue
+                                    ? "?"
+                                    : ""
+                            }\` | ` +
+                            (child.comment
+                                ? child.comment.summary
+                                      .map((token) =>
+                                          token.kind === "text" ||
+                                          token.kind === "code"
+                                              ? token.text
+                                              : ""
+                                      )
+                                      .join("")
+                                : "TODO: document") +
+                            " |"
+                        );
+                    });
+                } else {
+                    const comment = param.comment;
+                    return [
+                        `| \`${param.name}${
+                            param.flags.isOptional || param.defaultValue
+                                ? "?"
+                                : ""
+                        }\` | ` +
+                            (comment
+                                ? comment.summary
+                                      .map((token) =>
+                                          token.kind === "text" ||
+                                          token.kind === "code"
+                                              ? token.text
+                                              : ""
+                                      )
+                                      .join("")
+                                : "TODO: document ") +
+                            " |",
+                    ];
+                }
+            });
+
+            if (paramDocs.length) {
+                return `### Parameters:\n\n| name | description |\n| ----: | ---- |\n${paramDocs.join(
+                    "\n"
+                )}\n\n`;
+            }
+        }
+
+        function renderChildName(child: JSONOutput.DeclarationReflection) {
+            if (child.signatures) {
+                if (
+                    child.signatures[0].type?.type === "reference" &&
+                    child.signatures[0].type.qualifiedName ===
+                        "React.JSX.Element"
+                ) {
+                    return `<${child.name}/>`;
+                } else {
+                    return (
+                        child.name +
+                        `(${(child.signatures[0].parameters || [])
+                            .map(renderParamSimple)
+                            .join(", ")})`
+                    );
+                }
+            } else {
+                return child.name;
+            }
+        }
+
+        function renderChildType(
+            child: JSONOutput.DeclarationReflection
+        ): string {
+            const isClass = child.kind === ReflectionKind.Class;
+            const isTypeAlias = child.kind === ReflectionKind.TypeAlias;
+            const isInterface = child.kind === ReflectionKind.Interface;
+            const isNamespace = child.kind === ReflectionKind.Namespace;
+            const isFunction = !!child.signatures;
+
+            const kind = isClass
+                ? "class"
+                : isTypeAlias
+                ? "type"
+                : isFunction
+                ? "function"
+                : isInterface
+                ? "interface"
+                : isNamespace
+                ? "namespace"
+                : "";
+
+            return (
+                "```typescript\n" +
+                `export ${kind} ${child.name}` +
+                ((child.typeParameters || child.signatures?.[0].typeParameter)
+                    ? "<" +
+                      (child.typeParameters || child.signatures?.[0].typeParameter || []).map(renderTypeParam).join(", ") +
+                      ">"
+                    : "") +
+                (child.extendedTypes
+                    ? " extends " +
+                      child.extendedTypes.map(renderType).join(", ")
+                    : "") +
+                (child.implementedTypes
+                    ? " implements " +
+                      child.implementedTypes.map(renderType).join(", ")
+                    : "") +
+                (isClass || isInterface || isNamespace
+                    ? " {...}"
+                    : isTypeAlias
+                    ? ` = ${renderType(child.type)}`
+                    : child.signatures
+                    ? `(${(child.signatures[0].parameters || [])
+                          .map(renderParam)
+                          .join(", ")}): ${renderType(
+                          child.signatures[0].type
+                      )}`
+                    : "") +
+                "\n```\n"
+            );
+        }
+
+        function renderChildCategory(
+            child: JSONOutput.DeclarationReflection,
+            category: JSONOutput.ReflectionGroup
+        ): string {
+            return (
+                `### \`${child.name}\`: ${category.title.replace(/[^d]+\./, "")}\n\n` +
+                category.children
+                    ?.map((memberId) => {
+                        const member = child.children!.find(
+                            (member) => member.id === memberId
+                        )!;
+
+                        if (member.kind === 2048 || member.kind === 512) {
+                            if (
+                                member.signatures?.every(
+                                    (sig) =>
+                                        sig.comment?.modifierTags?.includes(
+                                            "@internal"
+                                        ) ||
+                                        sig.comment?.modifierTags?.includes(
+                                            "@deprecated"
+                                        )
+                                )
+                            ) {
+                                return "";
+                            } else {
+                                return documentConstructorOrMethod(
+                                    member,
+                                    child
+                                );
+                            }
+                        } else if (
+                            member.kind === 1024 ||
+                            member.kind === 262144
+                        ) {
+                            if (
+                                member.comment?.modifierTags?.includes(
+                                    "@internal"
+                                ) ||
+                                member.comment?.modifierTags?.includes(
+                                    "@deprecated"
+                                )
+                            ) {
+                                return "";
+                            } else {
+                                return documentProperty(member, child);
+                            }
+                        } else if (member.kind === 2097152) {
+                            if (
+                                member.comment?.modifierTags?.includes(
+                                    "@internal"
+                                ) ||
+                                member.comment?.modifierTags?.includes(
+                                    "@deprecated"
+                                )
+                            ) {
+                                return "";
+                            } else {
+                                return documentProperty(
+                                    { ...member, flags: { isStatic: true } },
+                                    child
+                                );
+                            }
+                        } else {
+                            return "Unknown member kind " + member.kind;
+                        }
+                    })
+                    .join("\n\n")
+            );
+        }
+
+        function renderType(t?: JSONOutput.SomeType): string {
+            if (!t) return "";
+            if (t.type === "reference") {
+                return (
+                    t.name +
+                    (t.typeArguments
+                        ? "<" + t.typeArguments.map(renderType).join(", ") + ">"
+                        : "")
+                );
+            } else if (t.type === "intrinsic") {
+                return t.name;
+            } else if (t.type === "literal") {
+                return JSON.stringify(t.value);
+            } else if (t.type === "union") {
+                const seen = new Set<string>();
+                return t.types
+                    .flatMap((t) => {
+                        const rendered =
+                            t.type === "intersection" || t.type === "union"
+                                ? `(${renderType(t)})`
+                                : renderType(t);
+
+                        if (seen.has(rendered)) {
+                            return [];
+                        } else {
+                            seen.add(rendered);
+                            return [rendered];
+                        }
+                    })
+                    .join(" | ");
+            } else if (t.type === "intersection") {
+                const seen = new Set<string>();
+                return t.types
+                    .flatMap((t) => {
+                        const rendered =
+                            t.type === "intersection" || t.type === "union"
+                                ? `(${renderType(t)})`
+                                : renderType(t);
+
+                        if (seen.has(rendered)) {
+                            return [];
+                        } else {
+                            seen.add(rendered);
+                            return [rendered];
+                        }
+                    })
+                    .join(" & ");
+            } else if (t.type === "indexedAccess") {
+                return (
+                    renderType(t.objectType) +
+                    "[" +
+                    renderType(t.indexType) +
+                    "]"
+                );
+            } else if (t.type === "reflection") {
+                if (t.declaration.indexSignature) {
+                    return (
+                        `{${
+                            t.declaration.children
+                                ? t.declaration.children
+                                      .map(
+                                          (child) =>
+                                              `  ${child.name}${
+                                                  child.flags.isOptional
+                                                      ? "?"
+                                                      : ""
+                                              }: ${indentEnd(
+                                                  renderType(child.type)
+                                              )},`
+                                      )
+                                      .join("\n")
+                                : ""
+                        }\n  [` +
+                        t.declaration.indexSignature?.parameters?.[0].name +
+                        ": " +
+                        renderType(
+                            t.declaration.indexSignature?.parameters?.[0].type
+                        ) +
+                        "]: " +
+                        indentEnd(
+                            renderType(t.declaration.indexSignature?.type)
+                        ) +
+                        " }"
+                    );
+                } else if (t.declaration.children) {
+                    return `{\n${t.declaration.children
+                        .map((child) =>
+                            child.signatures
+                                ? child.signatures
+                                      .map(
+                                          (signature) =>
+                                              `  ${child.name}(${
+                                                  signature.parameters
+                                                      ? "\n  " +
+                                                        indent(
+                                                            signature.parameters
+                                                                .map((p) =>
+                                                                    indentEnd(
+                                                                        renderParam(
+                                                                            p
+                                                                        )
+                                                                    )
+                                                                )
+                                                                .join(",\n  ")
+                                                        ) +
+                                                        "\n  )"
+                                                      : "()"
+                                              }: ${indentEnd(
+                                                  renderType(signature.type)
+                                              )}`
+                                      )
+                                      .join("\n") + ",\n"
+                                : `  ${child.name}${
+                                      child.flags.isOptional ? "?" : ""
+                                  }: ${indentEnd(renderType(child.type))},\n`
+                        )
+                        .join("")}}`;
+                } else if (t.declaration.signatures) {
+                    return t.declaration.signatures
+                        .map(
+                            (signature) =>
+                                `(${(signature.parameters || [])
+                                    .map(renderParam)
+                                    .join(", ")}) => ${renderType(
+                                    signature.type
+                                )}`
+                        )
+                        .join("\n");
+                } else {
+                    return "COMPLEX_TYPE_REFLECTION";
+                }
+            } else if (t.type === "array") {
+                return renderType(t.elementType) + "[]";
+            } else if (t.type === "tuple") {
+                return `[${t.elements?.map(renderType).join(", ")}]`;
+            } else if (t.type === "templateLiteral") {
+                const matchingNamedType = docs.children?.find(
+                    (child) =>
+                        child.variant === "declaration" &&
+                        child.type?.type === "templateLiteral" &&
+                        child.type.head === t.head &&
+                        child.type.tail.every(
+                            (piece, i) => piece[1] === t.tail[i][1]
+                        )
+                );
+
+                if (matchingNamedType) {
+                    return matchingNamedType.name;
+                } else {
+                    if (
+                        t.head === "sealerSecret_z" &&
+                        t.tail[0][1] === "/signerSecret_z"
+                    ) {
+                        return "AgentSecret";
+                    } else if (
+                        t.head === "sealer_z" &&
+                        t.tail[0][1] === "/signer_z"
+                    ) {
+                        if (t.tail[1] && t.tail[1][1] === "_session_z") {
+                            return "SessionID";
+                        } else {
+                            return "AgentID";
+                        }
+                    } else {
+                        return (
+                            "`" +
+                            t.head +
+                            t.tail
+                                .map(
+                                    (bit) =>
+                                        "${" + renderType(bit[0]) + "}" + bit[1]
+                                )
+                                .join("") +
+                            "`"
+                        );
+                    }
+                }
+            } else if (t.type === "conditional") {
+                const trueRendered = renderType(t.trueType);
+                const falseRendered = renderType(t.falseType);
+
+                if (
+                    trueRendered.includes("\n") ||
+                    falseRendered.includes("\n")
+                ) {
+                    return (
+                        renderType(t.checkType) +
+                        " extends " +
+                        renderType(t.extendsType) +
+                        "\n  ? " +
+                        indentEnd(renderType(t.trueType)) +
+                        "\n  : " +
+                        indentEnd(renderType(t.falseType))
+                    );
+                } else {
+                    return (
+                        renderType(t.checkType) +
+                        " extends " +
+                        renderType(t.extendsType) +
+                        " ? " +
+                        renderType(t.trueType) +
+                        " : " +
+                        renderType(t.falseType)
+                    );
+                }
+            } else if (t.type === "inferred") {
+                return "infer " + t.name;
+            } else if (t.type === "typeOperator") {
+                return t.operator + " " + renderType(t.target);
+            } else if (t.type === "mapped") {
+                return `{\n  [${t.parameter} in ${renderType(
+                    t.parameterType
+                )}]: ${indentEnd(renderType(t.templateType))}\n}`;
+            } else {
+                return "COMPLEX_TYPE_" + t.type;
+            }
+        }
+
+        // function renderTemplateLiteral(tempLit: JSONOutput.TemplateLiteralType) {
+        //     return tempLit.head + tempLit.tail.map((piece) => piece[0] + piece[1]).join("");
+        // }
+
+        // function resolveTemplateLiteralPieceType(t: SomeType): string {
+        //     if (t.type === "string") {
+        //         return "${string}"
+        //     }
+        //     if (t.type === "reference") {
+        //         const referencedType = docs.children?.find(
+        //             (child) => child.name === t.name
+        //         );
+
+        //     }
+        // }
+
+        function renderTypeParam(
+            t?: JSONOutput.TypeParameterReflection
+        ): string {
+            if (!t) return "";
+            return t.name + (t.type ? " extends " + renderType(t.type) : "");
+        }
+
+        function renderParam(param: JSONOutput.ParameterReflection) {
+            return param.name === "__namedParameters"
+                ? renderType(param.type)
+                : `${param.name}: ${renderType(param.type)}`;
+        }
+
+        function renderParamSimple(param: JSONOutput.ParameterReflection) {
+            return param.name === "__namedParameters" &&
+                param.type?.type === "reflection"
+                ? `{${param.type?.declaration.children
+                      ?.map(
+                          (child) =>
+                              child.name + (child.flags.isOptional ? "?" : "")
+                      )
+                      .join(", ")}}${
+                      param.flags.isOptional || param.defaultValue ? "?" : ""
+                  }`
+                : param.name +
+                      (param.flags.isOptional || param.defaultValue ? "?" : "");
+        }
+
+        function documentConstructorOrMethod(
+            member: JSONOutput.DeclarationReflection,
+            child: JSONOutput.DeclarationReflection
+        ) {
+            const isInClass = child.kind === 128;
+            const isInTypeDef = child.kind === 2097152;
+            const isInInterface = child.kind === 256;
+            const isInNamespace = child.kind === 4;
+            const isInFunction = !!child.signatures;
+
+            const inKind = isInClass
+                ? "class"
+                : isInTypeDef
+                ? "type"
+                : isInFunction
+                ? "function"
+                : isInInterface
+                ? "interface"
+                : isInNamespace
+                ? "namespace"
+                : "";
+
+            const stem =
+                member.name === "constructor"
+                    ? "new " + child.name + "</code></b>"
+                    : (member.flags.isStatic ? child.name : "") +
+                      "." +
+                      member.name +
+                      "";
+
+            return member.signatures
+                ?.map((signature) => {
+                    return (
+                        `<details>\n<summary><b><code>${stem}(${(
+                            signature?.parameters?.map(renderParamSimple) || []
+                        ).join(", ")})</code></b> ${
+                            member.inheritedFrom
+                                ? "<sub><sup>from <code>" +
+                                  member.inheritedFrom.name.split(".")[0] +
+                                  "</code></sup></sub> "
+                                : ""
+                        } ${
+                            signature?.comment
+                                ? ""
+                                : "<sub><sup>(undocumented)</sup></sub>"
+                        }</summary>\n\n` +
+                        ("```typescript\n" +
+                            `${inKind} ${child.name}${
+                                child.typeParameters
+                                    ? `<${child.typeParameters
+                                          .map((t) => t.name)
+                                          .join(", ")}>`
+                                    : ""
+                            } {\n\n${indent(
+                                `${member.name}${
+                                    signature.typeParameter
+                                        ? `<${signature.typeParameter
+                                              .map(renderTypeParam)
+                                              .join(", ")}>`
+                                        : ""
+                                }(${
+                                    (
+                                        signature.parameters?.map(
+                                            (param) =>
+                                                `\n  ${param.name}${
+                                                    param.flags.isOptional ||
+                                                    param.defaultValue
+                                                        ? "?"
+                                                        : ""
+                                                }: ${indentEnd(
+                                                    renderType(param.type)
+                                                )}${
+                                                    param.defaultValue
+                                                        ? ` = ${param.defaultValue}`
+                                                        : ""
+                                                }`
+                                        ) || []
+                                    ).join(",") +
+                                    (signature.parameters?.length ? "\n" : "")
+                                }): ${renderType(signature.type)} {...}`
+                            )}\n\n}\n` +
+                            "```\n" +
+                            renderSummary(signature.comment)) +
+                        renderParamComments(signature.parameters || []) +
+                        renderExamples(signature.comment) +
+                        "</details>\n\n"
+                    );
+                })
+                .join("\n\n");
+        }
+
+        function documentProperty(
+            member: JSONOutput.DeclarationReflection,
+            child: JSONOutput.DeclarationReflection
+        ) {
+            const isInClass = child.kind === 128;
+            const isInTypeDef = child.kind === 2097152;
+            const isInInterface = child.kind === 256;
+            const isInNamespace = child.kind === 4;
+            const isInFunction = !!child.signatures;
+
+            const inKind = isInClass
+                ? "class"
+                : isInTypeDef
+                ? "type"
+                : isInFunction
+                ? "function"
+                : isInInterface
+                ? "interface"
+                : isInNamespace
+                ? "namespace"
+                : "";
+
+            const stem = member.flags.isStatic ? child.name : "";
+            return (
+                `<details>\n<summary><b><code>${stem}.${
+                    member.name
+                }</code></b> ${
+                    member.inheritedFrom
+                        ? "<sub><sup>from <code>" +
+                          member.inheritedFrom.name.split(".")[0] +
+                          "</code></sup></sub> "
+                        : ""
+                } ${
+                    member.comment ? "" : "<sub><sup>(undocumented)</sup></sub>"
+                }</summary>\n\n` +
+                "```typescript\n" +
+                `${inKind} ${child.name}${
+                    child.typeParameters
+                        ? `<${child.typeParameters
+                              .map((t) => t.name)
+                              .join(", ")}>`
+                        : ""
+                } {\n\n${indent(
+                    `${member.getSignature ? "get " : ""}${member.name}${
+                        member.getSignature ? "()" : ""
+                    }: ${renderType(member.type || member.getSignature?.type)}${
+                        member.getSignature ? " {...}" : ""
+                    }`
+                )}` +
+                "\n\n}\n```\n" +
+                renderSummary(member.comment) +
+                renderExamples(member.comment) +
+                "</details>\n\n"
+            );
+        }
+    });
+
+    const docsContent = await readFile("./DOCS.md", "utf8");
+
+    await writeFile(
+        "./DOCS.md",
+        docsContent.slice(
+            0,
+            docsContent.indexOf("<!-- AUTOGENERATED DOCS AFTER THIS POINT -->")
+        ) +
+            "<!-- AUTOGENERATED DOCS AFTER THIS POINT -->\n" +
+            (await Promise.all(packageDocs)).join("\n\n\n")
+    );
+}
+
+function indent(text: string): string {
+    return text
+        .split("\n")
+        .map((line) => "  " + line)
+        .join("\n");
+}
+
+function indentEnd(text: string): string {
+    return text
+        .split("\n")
+        .map((line, i) => (i === 0 ? line : "  " + line))
+        .join("\n");
+}
+
+main().catch(console.error);

package.json

@@ -7,6 +7,14 @@
   ],
   "dependencies": {},
   "devDependencies": {
-    "lerna": "^7.1.5"
+    "lerna": "^7.1.5",
+    "ts-node": "^10.9.1",
+    "typedoc": "^0.25.1"
+  },
+  "scripts": {
+    "build-all": "lerna run build",
+    "updated": "lerna updated --include-merged-tags",
+    "publish-all": "yarn run gen-docs && lerna publish --include-merged-tags",
+    "gen-docs": "ts-node generateDocs.ts"
   }
 }

packages/cojson-simple-sync/.gitignore

@@ -170,4 +170,5 @@ dist
 
 .DS_Store
 
-out
+out
+sync.db*

packages/cojson-simple-sync/package.json

@@ -4,7 +4,7 @@
     "types": "src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.1.3",
+    "version": "0.3.7",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@types/ws": "^8.5.5",
@@ -16,7 +16,8 @@
         "typescript": "5.0.2"
     },
     "dependencies": {
-        "cojson": "^0.1.3",
+        "cojson": "^0.3.5",
+        "cojson-storage-sqlite": "^0.3.7",
         "ws": "^8.13.0"
     },
     "scripts": {
@@ -30,5 +31,6 @@
     "jest": {
         "preset": "ts-jest",
         "testEnvironment": "node"
-    }
+    },
+    "gitHead": "33c27053293b4801b968c61d5c4c989f93a67d13"
 }

packages/cojson-simple-sync/src/index.ts

@@ -1,11 +1,11 @@
 import { AnonymousControlledAccount, LocalNode, cojsonInternals } from "cojson";
-import { WebSocketServer, createWebSocketStream } from "ws";
-import { Duplex } from "node:stream";
-import { TransformStream } from "node:stream/web"
+import { WebSocketServer } from "ws";
+import { SQLiteStorage } from "cojson-storage-sqlite";
+import { websocketReadableStream, websocketWritableStream } from "./websocketStreams.js";
 
 const wss = new WebSocketServer({ port: 4200 });
 
-console.log("COJSON sync server listening on port " + wss.options.port)
+console.log("COJSON sync server listening on port " + wss.options.port);
 
 const agentSecret = cojsonInternals.newRandomAgentSecret();
 const agentID = cojsonInternals.getAgentID(agentSecret);
@@ -15,6 +15,10 @@ const localNode = new LocalNode(
     cojsonInternals.newRandomSessionID(agentID)
 );
 
+SQLiteStorage.asPeer({ filename: "./sync.db" })
+    .then((storage) => localNode.syncManager.addPeer(storage))
+    .catch((e) => console.error(e));
+
 wss.on("connection", function connection(ws, req) {
     const pinging = setInterval(() => {
         ws.send(
@@ -30,29 +34,6 @@ wss.on("connection", function connection(ws, req) {
         clearInterval(pinging);
     });
 
-
-    const duplexStream = createWebSocketStream(ws, {
-        decodeStrings: false,
-        readableObjectMode: true,
-        writableObjectMode: true,
-        encoding: "utf-8",
-        defaultEncoding: "utf-8",
-    });
-
-    const { readable: incomingStrings, writable: outgoingStrings } = Duplex.toWeb(duplexStream);
-
-    const toJSON = new TransformStream({
-        transform: (chunk, controller) => {
-            controller.enqueue(JSON.parse(chunk));
-        }
-    })
-
-    const fromJSON = new TransformStream({
-        transform: (chunk, controller) => {
-            controller.enqueue(JSON.stringify(chunk));
-        }
-    });
-
     const clientAddress =
         (req.headers["x-forwarded-for"] as string | undefined)
             ?.split(",")[0]
@@ -60,14 +41,12 @@ wss.on("connection", function connection(ws, req) {
 
     const clientId = clientAddress + "@" + new Date().toISOString();
 
-    localNode.sync.addPeer({
+    localNode.syncManager.addPeer({
         id: clientId,
         role: "client",
-        incoming: incomingStrings.pipeThrough(toJSON),
-        outgoing: fromJSON.writable,
+        incoming: websocketReadableStream(ws),
+        outgoing: websocketWritableStream(ws),
     });
 
-    void fromJSON.readable.pipeTo(outgoingStrings);
-
     ws.on("error", (e) => console.error(`Error on connection ${clientId}:`, e));
 });

packages/cojson-simple-sync/src/websocketStreams.ts

@@ -0,0 +1,86 @@
+import { WebSocket } from "ws";
+import { WritableStream, ReadableStream } from "isomorphic-streams";
+
+export function websocketReadableStream<T>(ws: WebSocket) {
+    ws.binaryType = "arraybuffer";
+
+    return new ReadableStream<T>({
+        start(controller) {
+            ws.addEventListener("message", (event) => {
+                if (typeof event.data !== "string")
+                    return console.warn(
+                        "Got non-string message from client",
+                        event.data
+                    );
+                const msg = JSON.parse(event.data);
+                if (msg.type === "ping") {
+                    // console.debug(
+                    //     "Got ping from",
+                    //     msg.dc,
+                    //     "latency",
+                    //     Date.now() - msg.time,
+                    //     "ms"
+                    // );
+                    return;
+                }
+                controller.enqueue(msg);
+            });
+            ws.addEventListener("close", () => controller.close());
+            ws.addEventListener("error", () =>
+                controller.error(new Error("The WebSocket errored!"))
+            );
+        },
+
+        cancel() {
+            ws.close();
+        },
+    });
+}
+
+export function websocketWritableStream<T>(ws: WebSocket) {
+    return new WritableStream<T>({
+        start(controller) {
+            ws.addEventListener("close", () =>
+                controller.error(
+                    new Error("The WebSocket closed unexpectedly!")
+                )
+            );
+            ws.addEventListener("error", () =>
+                controller.error(new Error("The WebSocket errored!"))
+            );
+
+            if (ws.readyState === WebSocket.OPEN) {
+                return;
+            }
+
+            return new Promise((resolve) => ws.once("open", resolve));
+        },
+
+        write(chunk) {
+            ws.send(JSON.stringify(chunk));
+            // Return immediately, since the web socket gives us no easy way to tell
+            // when the write completes.
+        },
+
+        close() {
+            return closeWS(1000);
+        },
+
+        abort(reason) {
+            return closeWS(4000, reason && reason.message);
+        },
+    });
+
+    function closeWS(code: number, reasonString?: string) {
+        return new Promise<void>((resolve, reject) => {
+            ws.onclose = (e) => {
+                if (e.wasClean) {
+                    resolve();
+                } else {
+                    reject(new Error("The connection was not closed cleanly"));
+                }
+            };
+            ws.close(code, reasonString);
+        });
+    }
+}

packages/cojson-storage-indexeddb/package.json

@@ -1,11 +1,11 @@
 {
-  "name": "jazz-storage-indexeddb",
-  "version": "0.1.3",
+  "name": "cojson-storage-indexeddb",
+  "version": "0.3.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.1.3",
+    "cojson": "^0.3.5",
     "typescript": "^5.1.6"
   },
   "devDependencies": {
@@ -18,5 +18,6 @@
     "lint": "eslint src/**/*.ts",
     "build": "npm run lint && rm -rf ./dist && tsc --declaration --sourceMap --outDir dist",
     "prepublishOnly": "npm run build"
-  }
+  },
+  "gitHead": "33c27053293b4801b968c61d5c4c989f93a67d13"
 }

packages/cojson-storage-indexeddb/src/index.test.ts

@@ -12,7 +12,7 @@ test.skip("Should be able to initialize and load from empty DB", async () => {
         )
     );
 
-    node.sync.addPeer(await IDBStorage.asPeer({ trace: true }));
+    node.syncManager.addPeer(await IDBStorage.asPeer({ trace: true }));
 
     console.log("yay!");
 
@@ -20,7 +20,7 @@ test.skip("Should be able to initialize and load from empty DB", async () => {
 
     await new Promise((resolve) => setTimeout(resolve, 200));
 
-    expect(node.sync.peers["storage"]).toBeDefined();
+    expect(node.syncManager.peers["storage"]).toBeDefined();
 });
 
 test("Should be able to sync data to database and then load that from a new node", async () => {
@@ -33,7 +33,7 @@ test("Should be able to sync data to database and then load that from a new node
         )
     );
 
-    node1.sync.addPeer(
+    node1.syncManager.addPeer(
         await IDBStorage.asPeer({ trace: true, localNodeName: "node1" })
     );
 
@@ -56,7 +56,7 @@ test("Should be able to sync data to database and then load that from a new node
         )
     );
 
-    node2.sync.addPeer(
+    node2.syncManager.addPeer(
         await IDBStorage.asPeer({ trace: true, localNodeName: "node2" })
     );
 

packages/cojson-storage-indexeddb/src/index.ts

@@ -1,4 +1,11 @@
-import { cojsonInternals, SessionID, SyncMessage, Peer, CojsonInternalTypes } from "cojson";
+import {
+    cojsonInternals,
+    SessionID,
+    SyncMessage,
+    Peer,
+    CojsonInternalTypes,
+    MAX_RECOMMENDED_TX_SIZE,
+} from "cojson";
 import {
     ReadableStream,
     WritableStream,
@@ -18,6 +25,7 @@ type SessionRow = {
     sessionID: SessionID;
     lastIdx: number;
     lastSignature: CojsonInternalTypes.Signature;
+    bytesSinceLastSignature?: number;
 };
 
 type StoredSessionRow = SessionRow & { rowID: number };
@@ -28,6 +36,12 @@ type TransactionRow = {
     tx: CojsonInternalTypes.Transaction;
 };
 
+type SignatureAfterRow = {
+    ses: number;
+    idx: number;
+    signature: CojsonInternalTypes.Signature;
+};
+
 export class IDBStorage {
     db: IDBDatabase;
     fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
@@ -49,7 +63,7 @@ export class IDBStorage {
                 done = result.done;
 
                 if (result.value) {
-                    this.handleSyncMessage(result.value);
+                    await this.handleSyncMessage(result.value);
                 }
             }
         })();
@@ -82,42 +96,63 @@ export class IDBStorage {
         toLocalNode: WritableStream<SyncMessage>
     ) {
         const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
-            const request = indexedDB.open("jazz-storage", 1);
+            const request = indexedDB.open("jazz-storage", 4);
             request.onerror = () => {
                 reject(request.error);
             };
             request.onsuccess = () => {
                 resolve(request.result);
             };
-            request.onupgradeneeded = () => {
+            request.onupgradeneeded = async (ev) => {
                 const db = request.result;
+                if (ev.oldVersion === 0) {
+                    const coValues = db.createObjectStore("coValues", {
+                        autoIncrement: true,
+                        keyPath: "rowID",
+                    });
 
-                const coValues = db.createObjectStore("coValues", {
-                    autoIncrement: true,
-                    keyPath: "rowID",
-                });
-
-                coValues.createIndex("coValuesById", "id", {
-                    unique: true,
-                });
-
-                const sessions = db.createObjectStore("sessions", {
-                    autoIncrement: true,
-                    keyPath: "rowID",
-                });
-
-                sessions.createIndex("sessionsByCoValue", "coValue");
-                sessions.createIndex(
-                    "uniqueSessions",
-                    ["coValue", "sessionID"],
-                    {
+                    coValues.createIndex("coValuesById", "id", {
                         unique: true,
-                    }
-                );
+                    });
 
-                db.createObjectStore("transactions", {
-                    keyPath: ["ses", "idx"],
-                });
+                    const sessions = db.createObjectStore("sessions", {
+                        autoIncrement: true,
+                        keyPath: "rowID",
+                    });
+
+                    sessions.createIndex("sessionsByCoValue", "coValue");
+                    sessions.createIndex(
+                        "uniqueSessions",
+                        ["coValue", "sessionID"],
+                        {
+                            unique: true,
+                        }
+                    );
+
+                    db.createObjectStore("transactions", {
+                        keyPath: ["ses", "idx"],
+                    });
+                }
+                if (ev.oldVersion <= 1) {
+                    db.createObjectStore("signatureAfter", {
+                        keyPath: ["ses", "idx"],
+                    });
+                }
+                if (ev.oldVersion !== 0 && ev.oldVersion <= 3) {
+                    // fix embarrassing off-by-one error for transaction indices
+                    console.log("Migration: fixing off-by-one error");
+                    const transaction = (ev.target as unknown as {transaction: IDBTransaction}).transaction;
+
+                    const txsStore = transaction.objectStore("transactions");
+                    const txs = await promised(txsStore.getAll());
+
+                    for (const tx of txs) {
+                        await promised(txsStore.delete([tx.ses, tx.idx]));
+                        tx.idx -= 1;
+                        await promised(txsStore.add(tx));
+                    }
+                    console.log("Migration: fixing off-by-one error - done");
+                }
             };
         });
 
@@ -147,10 +182,12 @@ export class IDBStorage {
             coValues,
             sessions,
             transactions,
+            signatureAfter,
         }: {
             coValues: IDBObjectStore;
             sessions: IDBObjectStore;
             transactions: IDBObjectStore;
+            signatureAfter: IDBObjectStore;
         },
         asDependencyOf?: CojsonInternalTypes.RawCoID
     ) {
@@ -170,12 +207,14 @@ export class IDBStorage {
             sessions: {},
         };
 
-        const newContent: CojsonInternalTypes.NewContentMessage = {
-            action: "content",
-            id: theirKnown.id,
-            header: theirKnown.header ? undefined : coValueRow?.header,
-            new: {},
-        };
+        const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
+            {
+                action: "content",
+                id: theirKnown.id,
+                header: theirKnown.header ? undefined : coValueRow?.header,
+                new: {},
+            },
+        ];
 
         for (const sessionRow of allOurSessions) {
             ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -187,6 +226,21 @@ export class IDBStorage {
                 const firstNewTxIdx =
                     theirKnown.sessions[sessionRow.sessionID] || 0;
 
+                const signaturesAndIdxs = await promised<SignatureAfterRow[]>(
+                    signatureAfter.getAll(
+                        IDBKeyRange.bound(
+                            [sessionRow.rowID, firstNewTxIdx],
+                            [sessionRow.rowID, Infinity]
+                        )
+                    )
+                );
+
+                // console.log(
+                //     theirKnown.id,
+                //     "signaturesAndIdxs",
+                //     JSON.stringify(signaturesAndIdxs)
+                // );
+
                 const newTxInSession = await promised<TransactionRow[]>(
                     transactions.getAll(
                         IDBKeyRange.bound(
@@ -196,36 +250,83 @@ export class IDBStorage {
                     )
                 );
 
-                newContent.new[sessionRow.sessionID] = {
-                    after: firstNewTxIdx,
-                    lastSignature: sessionRow.lastSignature,
-                    newTransactions: newTxInSession.map((row) => row.tx),
-                };
+                let idx = firstNewTxIdx;
+
+                // console.log(
+                //     theirKnown.id,
+                //     "newTxInSession",
+                //     newTxInSession.length
+                // );
+
+                for (const tx of newTxInSession) {
+                    let sessionEntry =
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ];
+                    if (!sessionEntry) {
+                        sessionEntry = {
+                            after: idx,
+                            lastSignature: "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
+                            newTransactions: [],
+                        };
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ] = sessionEntry;
+                    }
+
+                    sessionEntry.newTransactions.push(tx.tx);
+
+                    if (
+                        signaturesAndIdxs[0] &&
+                        idx === signaturesAndIdxs[0].idx
+                    ) {
+                        sessionEntry.lastSignature =
+                            signaturesAndIdxs[0].signature;
+                        signaturesAndIdxs.shift();
+                        newContentPieces.push({
+                            action: "content",
+                            id: theirKnown.id,
+                            new: {},
+                        });
+                    } else if (
+                        idx ===
+                        firstNewTxIdx + newTxInSession.length - 1
+                    ) {
+                        sessionEntry.lastSignature = sessionRow.lastSignature;
+                    }
+                    idx += 1;
+                }
             }
         }
 
         const dependedOnCoValues =
             coValueRow?.header.ruleset.type === "group"
-                ? Object.values(newContent.new).flatMap((sessionEntry) =>
-                      sessionEntry.newTransactions.flatMap((tx) => {
-                          if (tx.privacy !== "trusting") return [];
-                          return tx.changes
-                              .map(
-                                  (change) =>
-                                      change &&
-                                      typeof change === "object" &&
-                                      "op" in change &&
-                                      change.op === "set" &&
-                                      "key" in change &&
-                                      change.key
-                              )
-                              .filter(
-                                  (key): key is CojsonInternalTypes.RawCoID =>
-                                      typeof key === "string" &&
-                                      key.startsWith("co_")
-                              );
-                      })
-                  )
+                ? newContentPieces
+                      .flatMap((piece) => Object.values(piece.new))
+                      .flatMap((sessionEntry) =>
+                          sessionEntry.newTransactions.flatMap((tx) => {
+                              if (tx.privacy !== "trusting") return [];
+                              // TODO: avoid parse here?
+                              return cojsonInternals
+                                  .parseJSON(tx.changes)
+                                  .map(
+                                      (change) =>
+                                          change &&
+                                          typeof change === "object" &&
+                                          "op" in change &&
+                                          change.op === "set" &&
+                                          "key" in change &&
+                                          change.key
+                                  )
+                                  .filter(
+                                      (
+                                          key
+                                      ): key is CojsonInternalTypes.RawCoID =>
+                                          typeof key === "string" &&
+                                          key.startsWith("co_")
+                                  );
+                          })
+                      )
                 : coValueRow?.header.ruleset.type === "ownedByGroup"
                 ? [coValueRow?.header.ruleset.group]
                 : [];
@@ -233,7 +334,7 @@ export class IDBStorage {
         for (const dependedOnCoValue of dependedOnCoValues) {
             await this.sendNewContentAfter(
                 { id: dependedOnCoValue, header: false, sessions: {} },
-                { coValues, sessions, transactions },
+                { coValues, sessions, transactions, signatureAfter },
                 asDependencyOf || theirKnown.id
             );
         }
@@ -244,8 +345,15 @@ export class IDBStorage {
             asDependencyOf,
         });
 
-        if (newContent.header || Object.keys(newContent.new).length > 0) {
-            await this.toLocalNode.write(newContent);
+        const nonEmptyNewContentPieces = newContentPieces.filter(
+            (piece) => piece.header || Object.keys(piece.new).length > 0
+        );
+
+        // console.log(theirKnown.id, nonEmptyNewContentPieces);
+
+        for (const piece of nonEmptyNewContentPieces) {
+            await this.toLocalNode.write(piece);
+            await new Promise((resolve) => setTimeout(resolve, 0));
         }
     }
 
@@ -254,7 +362,7 @@ export class IDBStorage {
     }
 
     async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
-        const { coValues, sessions, transactions } =
+        const { coValues, sessions, transactions, signatureAfter } =
             this.inTransaction("readwrite");
 
         let storedCoValueRowID = (
@@ -325,18 +433,39 @@ export class IDBStorage {
                 const actuallyNewOffset =
                     (sessionRow?.lastIdx || 0) -
                     (msg.new[sessionID]?.after || 0);
+
                 const actuallyNewTransactions =
                     newTransactions.slice(actuallyNewOffset);
 
+                let newBytesSinceLastSignature =
+                    (sessionRow?.bytesSinceLastSignature || 0) +
+                    actuallyNewTransactions.reduce(
+                        (sum, tx) =>
+                            sum +
+                            (tx.privacy === "private"
+                                ? tx.encryptedChanges.length
+                                : tx.changes.length),
+                        0
+                    );
+
+                const newLastIdx =
+                    (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+
+                let shouldWriteSignature = false;
+
+                if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+                    shouldWriteSignature = true;
+                    newBytesSinceLastSignature = 0;
+                }
+
                 let nextIdx = sessionRow?.lastIdx || 0;
 
                 const sessionUpdate = {
                     coValue: storedCoValueRowID,
                     sessionID: sessionID,
-                    lastIdx:
-                        (sessionRow?.lastIdx || 0) +
-                        actuallyNewTransactions.length,
+                    lastIdx: newLastIdx,
                     lastSignature: msg.new[sessionID]!.lastSignature,
+                    bytesSinceLastSignature: newBytesSinceLastSignature,
                 };
 
                 const sessionRowID = (await promised(
@@ -350,8 +479,18 @@ export class IDBStorage {
                     )
                 )) as number;
 
+                if (shouldWriteSignature) {
+                    await promised(
+                        signatureAfter.put({
+                            ses: sessionRowID,
+                            // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
+                            idx: newLastIdx - 1,
+                            signature: msg.new[sessionID]!.lastSignature,
+                        } satisfies SignatureAfterRow)
+                    );
+                }
+
                 for (const newTransaction of actuallyNewTransactions) {
-                    nextIdx++;
                     await promised(
                         transactions.add({
                             ses: sessionRowID,
@@ -359,6 +498,7 @@ export class IDBStorage {
                             tx: newTransaction,
                         } satisfies TransactionRow)
                     );
+                    nextIdx++;
                 }
             }
         }
@@ -382,9 +522,10 @@ export class IDBStorage {
         coValues: IDBObjectStore;
         sessions: IDBObjectStore;
         transactions: IDBObjectStore;
+        signatureAfter: IDBObjectStore;
     } {
         const tx = this.db.transaction(
-            ["coValues", "sessions", "transactions"],
+            ["coValues", "sessions", "transactions", "signatureAfter"],
             mode
         );
 
@@ -401,8 +542,9 @@ export class IDBStorage {
         const coValues = tx.objectStore("coValues");
         const sessions = tx.objectStore("sessions");
         const transactions = tx.objectStore("transactions");
+        const signatureAfter = tx.objectStore("signatureAfter");
 
-        return { coValues, sessions, transactions };
+        return { coValues, sessions, transactions, signatureAfter };
     }
 }
 

packages/cojson-storage-sqlite/.eslintrc.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+    extends: [
+        'eslint:recommended',
+        'plugin:@typescript-eslint/recommended',
+    ],
+    parser: '@typescript-eslint/parser',
+    plugins: ['@typescript-eslint'],
+    parserOptions: {
+        project: "./tsconfig.json",
+    },
+    root: true,
+    rules: {
+        "no-unused-vars": "off",
+        "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }],
+        // "@typescript-eslint/no-floating-promises": "error",
+    },
+};

packages/cojson-storage-sqlite/.gitignore

@@ -0,0 +1,171 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+\*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+\*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+\*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+\*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.cache
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+.cache/
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+.cache
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.\*
+
+.DS_Store

packages/cojson-storage-sqlite/.npmignore

@@ -0,0 +1,2 @@
+coverage
+node_modules

packages/cojson-storage-sqlite/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "cojson-storage-sqlite",
+  "type": "module",
+  "version": "0.3.7",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "license": "MIT",
+  "dependencies": {
+    "better-sqlite3": "^8.5.2",
+    "cojson": "^0.3.5",
+    "typescript": "^5.1.6"
+  },
+  "scripts": {
+    "lint": "eslint src/**/*.ts",
+    "build": "npm run lint && rm -rf ./dist && tsc --declaration --sourceMap --outDir dist",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.4"
+  },
+  "gitHead": "33c27053293b4801b968c61d5c4c989f93a67d13"
+}

packages/cojson-storage-sqlite/src/index.ts

@@ -0,0 +1,598 @@
+import {
+    cojsonInternals,
+    SyncMessage,
+    Peer,
+    CojsonInternalTypes,
+    SessionID,
+    MAX_RECOMMENDED_TX_SIZE,
+} from "cojson";
+import {
+    ReadableStream,
+    WritableStream,
+    ReadableStreamDefaultReader,
+    WritableStreamDefaultWriter,
+} from "isomorphic-streams";
+
+import Database, { Database as DatabaseT } from "better-sqlite3";
+
+type CoValueRow = {
+    id: CojsonInternalTypes.RawCoID;
+    header: string;
+};
+
+type StoredCoValueRow = CoValueRow & { rowID: number };
+
+type SessionRow = {
+    coValue: number;
+    sessionID: SessionID;
+    lastIdx: number;
+    lastSignature: CojsonInternalTypes.Signature;
+    bytesSinceLastSignature?: number;
+};
+
+type StoredSessionRow = SessionRow & { rowID: number };
+
+type TransactionRow = {
+    ses: number;
+    idx: number;
+    tx: string;
+};
+
+type SignatureAfterRow = {
+    ses: number;
+    idx: number;
+    signature: CojsonInternalTypes.Signature;
+};
+
+export class SQLiteStorage {
+    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
+    db: DatabaseT;
+
+    constructor(
+        db: DatabaseT,
+        fromLocalNode: ReadableStream<SyncMessage>,
+        toLocalNode: WritableStream<SyncMessage>
+    ) {
+        this.db = db;
+        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode.getWriter();
+
+        (async () => {
+            let done = false;
+            while (!done) {
+                const result = await this.fromLocalNode.read();
+                done = result.done;
+
+                if (result.value) {
+                    await this.handleSyncMessage(result.value);
+                }
+            }
+        })();
+    }
+
+    static async asPeer({
+        filename,
+        trace,
+        localNodeName = "local",
+    }: {
+        filename: string;
+        trace?: boolean;
+        localNodeName?: string;
+    }): Promise<Peer> {
+        const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
+            localNodeName,
+            "storage",
+            { peer1role: "client", peer2role: "server", trace }
+        );
+
+        await SQLiteStorage.open(
+            filename,
+            localNodeAsPeer.incoming,
+            localNodeAsPeer.outgoing
+        );
+
+        return storageAsPeer;
+    }
+
+    static async open(
+        filename: string,
+        fromLocalNode: ReadableStream<SyncMessage>,
+        toLocalNode: WritableStream<SyncMessage>
+    ) {
+        const db = Database(filename);
+        db.pragma("journal_mode = WAL");
+
+        const oldVersion = (
+            db.pragma("user_version") as [{ user_version: number }]
+        )[0].user_version as number;
+
+        console.log("DB version", oldVersion);
+
+        if (oldVersion === 0) {
+            console.log("Migration 0 -> 1: Basic schema");
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS transactions (
+                    ses INTEGER,
+                    idx INTEGER,
+                    tx TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`
+            ).run();
+
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS sessions (
+                    rowID INTEGER PRIMARY KEY,
+                    coValue INTEGER NOT NULL,
+                    sessionID TEXT NOT NULL,
+                    lastIdx INTEGER,
+                    lastSignature TEXT,
+                    UNIQUE (sessionID, coValue)
+                );`
+            ).run();
+
+            db.prepare(
+                `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`
+            ).run();
+
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS coValues (
+                    rowID INTEGER PRIMARY KEY,
+                    id TEXT NOT NULL UNIQUE,
+                    header TEXT NOT NULL UNIQUE
+                );`
+            ).run();
+
+            db.prepare(
+                `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`
+            ).run();
+
+            db.pragma("user_version = 1");
+            console.log("Migration 0 -> 1: Basic schema - done");
+        }
+
+        if (oldVersion <= 1) {
+            // fix embarrassing off-by-one error for transaction indices
+            console.log(
+                "Migration 1 -> 2: Fix off-by-one error for transaction indices"
+            );
+
+            const txs = db
+                .prepare(`SELECT * FROM transactions`)
+                .all() as TransactionRow[];
+
+            for (const tx of txs) {
+                db.prepare(
+                    `DELETE FROM transactions WHERE ses = ? AND idx = ?`
+                ).run(tx.ses, tx.idx);
+                tx.idx -= 1;
+                db.prepare(
+                    `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`
+                ).run(tx.ses, tx.idx, tx.tx);
+            }
+
+            db.pragma("user_version = 2");
+            console.log(
+                "Migration 1 -> 2: Fix off-by-one error for transaction indices - done"
+            );
+        }
+
+        if (oldVersion <= 2) {
+            console.log("Migration 2 -> 3: Add signatureAfter");
+
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS signatureAfter (
+                    ses INTEGER,
+                    idx INTEGER,
+                    signature TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`
+            ).run();
+
+            db.prepare(
+                `ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;`
+            ).run();
+
+            db.pragma("user_version = 3");
+            console.log("Migration 2 -> 3: Add signatureAfter - done");
+        }
+
+        return new SQLiteStorage(db, fromLocalNode, toLocalNode);
+    }
+
+    async handleSyncMessage(msg: SyncMessage) {
+        switch (msg.action) {
+            case "load":
+                await this.handleLoad(msg);
+                break;
+            case "content":
+                await this.handleContent(msg);
+                break;
+            case "known":
+                await this.handleKnown(msg);
+                break;
+            case "done":
+                await this.handleDone(msg);
+                break;
+        }
+    }
+
+    async sendNewContentAfter(
+        theirKnown: CojsonInternalTypes.CoValueKnownState,
+        asDependencyOf?: CojsonInternalTypes.RawCoID
+    ) {
+        const coValueRow = (await this.db
+            .prepare(`SELECT * FROM coValues WHERE id = ?`)
+            .get(theirKnown.id)) as StoredCoValueRow | undefined;
+
+        const allOurSessions = coValueRow
+            ? (this.db
+                  .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
+                  .all(coValueRow.rowID) as StoredSessionRow[])
+            : [];
+
+        const ourKnown: CojsonInternalTypes.CoValueKnownState = {
+            id: theirKnown.id,
+            header: !!coValueRow,
+            sessions: {},
+        };
+
+        let parsedHeader;
+
+        try {
+            parsedHeader = (coValueRow?.header &&
+                JSON.parse(coValueRow.header)) as
+                | CojsonInternalTypes.CoValueHeader
+                | undefined;
+        } catch (e) {
+            console.warn(
+                theirKnown.id,
+                "Invalid JSON in header",
+                e,
+                coValueRow?.header
+            );
+            return;
+        }
+
+        const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
+            {
+                action: "content",
+                id: theirKnown.id,
+                header: theirKnown.header ? undefined : parsedHeader,
+                new: {},
+            },
+        ];
+
+        for (const sessionRow of allOurSessions) {
+            ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+
+            if (
+                sessionRow.lastIdx >
+                (theirKnown.sessions[sessionRow.sessionID] || 0)
+            ) {
+                const firstNewTxIdx =
+                    theirKnown.sessions[sessionRow.sessionID] || 0;
+
+                const signaturesAndIdxs = this.db
+                    .prepare<[number, number]>(
+                        `SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?`
+                    )
+                    .all(
+                        sessionRow.rowID,
+                        firstNewTxIdx
+                    ) as SignatureAfterRow[];
+
+                // console.log(
+                //     theirKnown.id,
+                //     "signaturesAndIdxs",
+                //     JSON.stringify(signaturesAndIdxs)
+                // );
+
+                const newTxInSession = this.db
+                    .prepare<[number, number]>(
+                        `SELECT * FROM transactions WHERE ses = ? AND idx >= ?`
+                    )
+                    .all(sessionRow.rowID, firstNewTxIdx) as TransactionRow[];
+
+                let idx = firstNewTxIdx;
+
+                // console.log(
+                //     theirKnown.id,
+                //     "newTxInSession",
+                //     newTxInSession.length
+                // );
+
+                for (const tx of newTxInSession) {
+                    let sessionEntry =
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ];
+                    if (!sessionEntry) {
+                        sessionEntry = {
+                            after: idx,
+                            lastSignature:
+                                "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
+                            newTransactions: [],
+                        };
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ] = sessionEntry;
+                    }
+
+                    let parsedTx;
+
+                    try {
+                        parsedTx = JSON.parse(tx.tx);
+                    } catch (e) {
+                        console.warn(
+                            theirKnown.id,
+                            "Invalid JSON in transaction",
+                            e,
+                            tx.tx
+                        );
+                        break;
+                    }
+
+                    sessionEntry.newTransactions.push(parsedTx);
+
+                    if (
+                        signaturesAndIdxs[0] &&
+                        idx === signaturesAndIdxs[0].idx
+                    ) {
+                        sessionEntry.lastSignature =
+                            signaturesAndIdxs[0].signature;
+                        signaturesAndIdxs.shift();
+                        newContentPieces.push({
+                            action: "content",
+                            id: theirKnown.id,
+                            new: {},
+                        });
+                    } else if (
+                        idx ===
+                        firstNewTxIdx + newTxInSession.length - 1
+                    ) {
+                        sessionEntry.lastSignature = sessionRow.lastSignature;
+                    }
+                    idx += 1;
+                }
+            }
+        }
+
+        const dependedOnCoValues =
+            parsedHeader?.ruleset.type === "group"
+                ? newContentPieces
+                      .flatMap((piece) => Object.values(piece.new))
+                      .flatMap((sessionEntry) =>
+                          sessionEntry.newTransactions.flatMap((tx) => {
+                              if (tx.privacy !== "trusting") return [];
+                              // TODO: avoid parsing here?
+                              let parsedChanges;
+
+                              try {
+                                  parsedChanges = cojsonInternals.parseJSON(
+                                      tx.changes
+                                  );
+                              } catch (e) {
+                                  console.warn(
+                                      theirKnown.id,
+                                      "Invalid JSON in transaction",
+                                      e,
+                                      tx.changes
+                                  );
+                                  return [];
+                              }
+
+                              return parsedChanges
+                                  .map(
+                                      (change) =>
+                                          change &&
+                                          typeof change === "object" &&
+                                          "op" in change &&
+                                          change.op === "set" &&
+                                          "key" in change &&
+                                          change.key
+                                  )
+                                  .filter(
+                                      (
+                                          key
+                                      ): key is CojsonInternalTypes.RawCoID =>
+                                          typeof key === "string" &&
+                                          key.startsWith("co_")
+                                  );
+                          })
+                      )
+                : parsedHeader?.ruleset.type === "ownedByGroup"
+                ? [parsedHeader?.ruleset.group]
+                : [];
+
+        for (const dependedOnCoValue of dependedOnCoValues) {
+            await this.sendNewContentAfter(
+                { id: dependedOnCoValue, header: false, sessions: {} },
+                asDependencyOf || theirKnown.id
+            );
+        }
+
+        await this.toLocalNode.write({
+            action: "known",
+            ...ourKnown,
+            asDependencyOf,
+        });
+
+        const nonEmptyNewContentPieces = newContentPieces.filter(
+            (piece) => piece.header || Object.keys(piece.new).length > 0
+        );
+
+        // console.log(theirKnown.id, nonEmptyNewContentPieces);
+
+        for (const piece of nonEmptyNewContentPieces) {
+            await this.toLocalNode.write(piece);
+            await new Promise((resolve) => setTimeout(resolve, 0));
+        }
+    }
+
+    handleLoad(msg: CojsonInternalTypes.LoadMessage) {
+        return this.sendNewContentAfter(msg);
+    }
+
+    async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
+        let storedCoValueRowID = (
+            this.db
+                .prepare<CojsonInternalTypes.RawCoID>(
+                    `SELECT rowID FROM coValues WHERE id = ?`
+                )
+                .get(msg.id) as StoredCoValueRow | undefined
+        )?.rowID;
+
+        if (storedCoValueRowID === undefined) {
+            const header = msg.header;
+            if (!header) {
+                console.error("Expected to be sent header first");
+                await this.toLocalNode.write({
+                    action: "known",
+                    id: msg.id,
+                    header: false,
+                    sessions: {},
+                    isCorrection: true,
+                });
+                return;
+            }
+
+            storedCoValueRowID = this.db
+                .prepare<[CojsonInternalTypes.RawCoID, string]>(
+                    `INSERT INTO coValues (id, header) VALUES (?, ?)`
+                )
+                .run(msg.id, JSON.stringify(header)).lastInsertRowid as number;
+        }
+
+        const ourKnown: CojsonInternalTypes.CoValueKnownState = {
+            id: msg.id,
+            header: true,
+            sessions: {},
+        };
+        let invalidAssumptions = false;
+
+        this.db.transaction(() => {
+            const allOurSessions = (
+                this.db
+                    .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
+                    .all(storedCoValueRowID!) as StoredSessionRow[]
+            ).reduce((acc, row) => {
+                acc[row.sessionID] = row;
+                return acc;
+            }, {} as { [sessionID: string]: StoredSessionRow });
+
+            for (const sessionID of Object.keys(msg.new) as SessionID[]) {
+                const sessionRow = allOurSessions[sessionID];
+                if (sessionRow) {
+                    ourKnown.sessions[sessionRow.sessionID] =
+                        sessionRow.lastIdx;
+                }
+
+                if (
+                    (sessionRow?.lastIdx || 0) <
+                    (msg.new[sessionID]?.after || 0)
+                ) {
+                    invalidAssumptions = true;
+                } else {
+                    const newTransactions =
+                        msg.new[sessionID]?.newTransactions || [];
+
+                    const actuallyNewOffset =
+                        (sessionRow?.lastIdx || 0) -
+                        (msg.new[sessionID]?.after || 0);
+
+                    const actuallyNewTransactions =
+                        newTransactions.slice(actuallyNewOffset);
+
+                    let newBytesSinceLastSignature =
+                        (sessionRow?.bytesSinceLastSignature || 0) +
+                        actuallyNewTransactions.reduce(
+                            (sum, tx) =>
+                                sum +
+                                (tx.privacy === "private"
+                                    ? tx.encryptedChanges.length
+                                    : tx.changes.length),
+                            0
+                        );
+
+                    const newLastIdx =
+                        (sessionRow?.lastIdx || 0) +
+                        actuallyNewTransactions.length;
+
+                    let shouldWriteSignature = false;
+
+                    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+                        shouldWriteSignature = true;
+                        newBytesSinceLastSignature = 0;
+                    }
+
+                    let nextIdx = sessionRow?.lastIdx || 0;
+
+                    const sessionUpdate = {
+                        coValue: storedCoValueRowID!,
+                        sessionID: sessionID,
+                        lastIdx: newLastIdx,
+                        lastSignature: msg.new[sessionID]!.lastSignature,
+                        bytesSinceLastSignature: newBytesSinceLastSignature,
+                    };
+
+                    const upsertedSession = this.db
+                        .prepare<[number, string, number, string, number]>(
+                            `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
+                            RETURNING rowID`
+                        )
+                        .get(
+                            sessionUpdate.coValue,
+                            sessionUpdate.sessionID,
+                            sessionUpdate.lastIdx,
+                            sessionUpdate.lastSignature,
+                            sessionUpdate.bytesSinceLastSignature
+                        ) as { rowID: number };
+
+                    const sessionRowID = upsertedSession.rowID;
+
+                    if (shouldWriteSignature) {
+                        this.db
+                            .prepare<[number, number, string]>(
+                                `INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)`
+                            )
+                            .run(
+                                sessionRowID,
+                                // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
+                                newLastIdx - 1,
+                                msg.new[sessionID]!.lastSignature
+                            );
+                    }
+
+                    for (const newTransaction of actuallyNewTransactions) {
+                        this.db
+                            .prepare<[number, number, string]>(
+                                `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`
+                            )
+                            .run(
+                                sessionRowID,
+                                nextIdx,
+                                JSON.stringify(newTransaction)
+                            );
+                        nextIdx++;
+                    }
+                }
+            }
+        })();
+
+        if (invalidAssumptions) {
+            await this.toLocalNode.write({
+                action: "known",
+                ...ourKnown,
+                isCorrection: invalidAssumptions,
+            });
+        }
+    }
+
+    handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
+        return this.sendNewContentAfter(msg);
+    }
+
+    handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
+}

packages/cojson-storage-sqlite/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext"],
+    "module": "esnext",
+    "target": "ES2020",
+    "moduleResolution": "bundler",
+    "moduleDetection": "force",
+    "strict": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUncheckedIndexedAccess": true,
+    "esModuleInterop": true,
+  },
+  "include": ["./src/**/*"],
+}

packages/cojson/.eslintrc.cjs

@@ -9,6 +9,7 @@ module.exports = {
     parserOptions: {
         project: "./tsconfig.json",
     },
+    ignorePatterns: [".eslint.cjs", "**/tests/*"],
     root: true,
     rules: {
         "no-unused-vars": "off",

packages/cojson/package.json

@@ -5,7 +5,7 @@
     "types": "dist/index.d.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.1.3",
+    "version": "0.3.5",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@typescript-eslint/eslint-plugin": "^6.2.1",
@@ -19,9 +19,8 @@
     "dependencies": {
         "@noble/ciphers": "^0.1.3",
         "@noble/curves": "^1.1.0",
-        "@noble/hashes": "^1.3.1",
         "@scure/base": "^1.1.1",
-        "fast-json-stable-stringify": "https://github.com/tirithen/fast-json-stable-stringify#7a3dcf2",
+        "hash-wasm": "^4.9.0",
         "isomorphic-streams": "https://github.com/sgwilym/isomorphic-streams.git#aa9394781bfc92f8d7c981be7daf8af4b4cd4fae"
     },
     "scripts": {
@@ -51,5 +50,6 @@
             "/node_modules/",
             "/dist/"
         ]
-    }
+    },
+    "gitHead": "33c27053293b4801b968c61d5c4c989f93a67d13"
 }

packages/cojson/src/account.ts

@@ -1,5 +1,5 @@
-import { CoValueHeader } from "./coValue.js";
-import { CoID } from "./contentType.js";
+import { CoValueHeader } from "./coValueCore.js";
+import { CoID } from "./coValue.js";
 import {
     AgentSecret,
     SealerID,
@@ -13,7 +13,8 @@ import {
     getAgentSignerSecret,
 } from "./crypto.js";
 import { AgentID } from "./ids.js";
-import { CoMap, LocalNode } from "./index.js";
+import { CoMap } from "./coValues/coMap.js";
+import { LocalNode } from "./localNode.js";
 import { Group, GroupContent } from "./group.js";
 
 export function accountHeaderForInitialAgentSecret(
@@ -31,13 +32,13 @@ export function accountHeaderForInitialAgentSecret(
     };
 }
 
-export class Account extends Group {
+export class AccountGroup extends Group {
     get id(): AccountID {
-        return this.groupMap.id as AccountID;
+        return this.underlyingMap.id as AccountID;
     }
 
     getCurrentAgentID(): AgentID {
-        const agents = this.groupMap
+        const agents = this.underlyingMap
             .keys()
             .filter((k): k is AgentID => k.startsWith("sealer_"));
 
@@ -52,7 +53,7 @@ export class Account extends Group {
 }
 
 export interface GeneralizedControlledAccount {
-    id: AccountIDOrAgentID;
+    id: AccountID | AgentID;
     agentSecret: AgentSecret;
 
     currentAgentID: () => AgentID;
@@ -62,8 +63,9 @@ export interface GeneralizedControlledAccount {
     currentSealerSecret: () => SealerSecret;
 }
 
+/** @hidden */
 export class ControlledAccount
-    extends Account
+    extends AccountGroup
     implements GeneralizedControlledAccount
 {
     agentSecret: AgentSecret;
@@ -99,6 +101,7 @@ export class ControlledAccount
     }
 }
 
+/** @hidden */
 export class AnonymousControlledAccount
     implements GeneralizedControlledAccount
 {
@@ -133,15 +136,12 @@ export class AnonymousControlledAccount
     }
 }
 
-export type AccountContent = GroupContent & { profile: CoID<Profile> };
+export type AccountContent = { profile: Profile } & GroupContent;
 export type AccountMeta = { type: "account" };
-export type AccountID = CoID<CoMap<AccountContent, AccountMeta>>;
+export type Account = CoMap<AccountContent, AccountMeta>;
+export type AccountID = CoID<Account>;
 
-export type AccountIDOrAgentID = AgentID | AccountID;
-export type AccountOrAgentID = AgentID | Account;
-export type AccountOrAgentSecret = AgentSecret | Account;
-
-export function isAccountID(id: AccountIDOrAgentID): id is AccountID {
+export function isAccountID(id: AccountID | AgentID): id is AccountID {
     return id.startsWith("co_");
 }
 

packages/cojson/src/base64url.test.ts

@@ -0,0 +1,32 @@
+import { base64URLtoBytes, bytesToBase64url } from "./base64url";
+
+const txt = new TextEncoder();
+
+test("Test our Base64 URL encoding and decoding", () => {
+    // tests from the RFC
+
+    expect(base64URLtoBytes("")).toEqual(new Uint8Array([]));
+    expect(bytesToBase64url(new Uint8Array([]))).toEqual("");
+
+    expect(bytesToBase64url(txt.encode("f"))).toEqual("Zg==");
+    expect(bytesToBase64url(txt.encode("fo"))).toEqual("Zm8=");
+    expect(bytesToBase64url(txt.encode("foo"))).toEqual("Zm9v");
+    expect(bytesToBase64url(txt.encode("foob"))).toEqual("Zm9vYg==");
+    expect(bytesToBase64url(txt.encode("fooba"))).toEqual("Zm9vYmE=");
+    expect(bytesToBase64url(txt.encode("foobar"))).toEqual("Zm9vYmFy");
+    // reverse
+    expect(base64URLtoBytes("Zg==")).toEqual(txt.encode("f"));
+    expect(base64URLtoBytes("Zm8=")).toEqual(txt.encode("fo"));
+    expect(base64URLtoBytes("Zm9v")).toEqual(txt.encode("foo"));
+    expect(base64URLtoBytes("Zm9vYg==")).toEqual(txt.encode("foob"));
+    expect(base64URLtoBytes("Zm9vYmE=")).toEqual(txt.encode("fooba"));
+    expect(base64URLtoBytes("Zm9vYmFy")).toEqual(txt.encode("foobar"));
+
+    expect(base64URLtoBytes("V2hhdCBkb2VzIDIgKyAyLjEgZXF1YWw_PyB-IDQ=")).toEqual(
+        txt.encode("What does 2 + 2.1 equal?? ~ 4")
+    );
+    // reverse
+    expect(
+        bytesToBase64url(txt.encode("What does 2 + 2.1 equal?? ~ 4"))
+    ).toEqual("V2hhdCBkb2VzIDIgKyAyLjEgZXF1YWw_PyB-IDQ=");
+});

packages/cojson/src/base64url.ts

@@ -0,0 +1,68 @@
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export function base64URLtoBytes(base64: string) {
+    base64 = base64.replace(/=/g, "");
+    const n = base64.length;
+    const rem = n % 4;
+    const k = rem && rem - 1; // how many bytes the last base64 chunk encodes
+    const m = (n >> 2) * 3 + k; // total encoded bytes
+
+    const encoded = new Uint8Array(n + 3);
+    encoder.encodeInto(base64 + "===", encoded);
+
+    for (let i = 0, j = 0; i < n; i += 4, j += 3) {
+        const x =
+            (lookup[encoded[i]!]! << 18) +
+            (lookup[encoded[i + 1]!]! << 12) +
+            (lookup[encoded[i + 2]!]! << 6) +
+            lookup[encoded[i + 3]!]!;
+        encoded[j] = x >> 16;
+        encoded[j + 1] = (x >> 8) & 0xff;
+        encoded[j + 2] = x & 0xff;
+    }
+    return new Uint8Array(encoded.buffer, 0, m);
+}
+
+export function bytesToBase64url(bytes: Uint8Array) {
+    // const before = performance.now();
+    const m = bytes.length;
+    const k = m % 3;
+    const n = Math.floor(m / 3) * 4 + (k && k + 1);
+    const N = Math.ceil(m / 3) * 4;
+    const encoded = new Uint8Array(N);
+
+    for (let i = 0, j = 0; j < m; i += 4, j += 3) {
+        const y =
+            (bytes[j]! << 16) + (bytes[j + 1]! << 8) + (bytes[j + 2]! | 0);
+        encoded[i] = encodeLookup[y >> 18]!;
+        encoded[i + 1] = encodeLookup[(y >> 12) & 0x3f]!;
+        encoded[i + 2] = encodeLookup[(y >> 6) & 0x3f]!;
+        encoded[i + 3] = encodeLookup[y & 0x3f]!;
+    }
+
+    let base64 = decoder.decode(new Uint8Array(encoded.buffer, 0, n));
+    if (k === 1) base64 += "==";
+    if (k === 2) base64 += "=";
+    // const after = performance.now();
+    // console.log(
+    //     "bytesToBase64url bandwidth in MB/s for length",
+    //     (1000 * bytes.length / (after - before)) / (1024 * 1024),
+    //     bytes.length
+    // );
+    return base64;
+}
+
+const alphabet =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+
+const lookup = new Uint8Array(128);
+for (const [i, a] of Array.from(alphabet).entries()) {
+    lookup[a.charCodeAt(0)] = i;
+}
+lookup["=".charCodeAt(0)] = 0;
+
+const encodeLookup = new Uint8Array(64);
+for (const [i, a] of Array.from(alphabet).entries()) {
+    encodeLookup[i] = a.charCodeAt(0);
+}

packages/cojson/src/coValue.ts

@@ -1,589 +1,95 @@
-import { randomBytes } from "@noble/hashes/utils";
-import { ContentType } from "./contentType.js";
-import { Static } from "./contentTypes/static.js";
-import { CoStream } from "./contentTypes/coStream.js";
-import { CoMap } from "./contentTypes/coMap.js";
-import {
-    Encrypted,
-    Hash,
-    KeySecret,
-    Signature,
-    StreamingHash,
-    unseal,
-    shortHash,
-    sign,
-    verify,
-    encryptForTransaction,
-    decryptForTransaction,
-    KeyID,
-    decryptKeySecret,
-    getAgentSignerID,
-    getAgentSealerID,
-} from "./crypto.js";
 import { JsonObject, JsonValue } from "./jsonValue.js";
-import { base58 } from "@scure/base";
+import { RawCoID } from "./ids.js";
+import { CoMap } from "./coValues/coMap.js";
 import {
-    PermissionsDef as RulesetDef,
-    determineValidTransactions,
-    isKeyForKeyField,
-} from "./permissions.js";
-import { Group, expectGroupContent } from "./group.js";
-import { LocalNode } from "./node.js";
-import { CoValueKnownState, NewContentMessage } from "./sync.js";
-import { RawCoID, SessionID, TransactionID } from "./ids.js";
-import { CoList } from "./contentTypes/coList.js";
-import {
-    AccountID,
-    AccountIDOrAgentID,
-    GeneralizedControlledAccount,
-} from "./account.js";
+    BinaryCoStream,
+    BinaryCoStreamMeta,
+    CoStream,
+} from "./coValues/coStream.js";
+import { CoList } from "./coValues/coList.js";
+import { CoValueCore } from "./coValueCore.js";
+import { Group } from "./group.js";
 
-export type CoValueHeader = {
-    type: ContentType["type"];
-    ruleset: RulesetDef;
+export type CoID<T extends CoValue> = RawCoID & {
+    readonly __type: T;
+};
+
+export interface CoValue {
+    /** The `CoValue`'s (precisely typed) `CoID` */
+    id: CoID<this>;
+    core: CoValueCore;
+    /** Specifies which kind of `CoValue` this is */
+    type: string;
+    /** The `CoValue`'s (precisely typed) static metadata */
     meta: JsonObject | null;
-    createdAt: `2${string}` | null;
-    uniqueness: `z${string}` | null;
-};
-
-export function idforHeader(header: CoValueHeader): RawCoID {
-    const hash = shortHash(header);
-    return `co_z${hash.slice("shortHash_z".length)}`;
+    /** The `Group` this `CoValue` belongs to (determining permissions) */
+    group: Group;
+    /** Returns an immutable JSON presentation of this `CoValue` */
+    toJSON(): JsonValue;
+    atTime(time: number): this;
+    /** Lets you subscribe to future updates to this CoValue (whether made locally or by other users).
+     *
+     * Takes a listener function that will be called with the current state for each update.
+     *
+     * Returns an unsubscribe function.
+     *
+     * Used internally by `useTelepathicData()` for reactive updates on changes to a `CoValue`. */
+    subscribe(listener: (coValue: this) => void): () => void;
 }
 
-export function accountOrAgentIDfromSessionID(
-    sessionID: SessionID
-): AccountIDOrAgentID {
-    return sessionID.split("_session")[0] as AccountIDOrAgentID;
+export type AnyCoMap = CoMap<
+    { [key: string]: JsonValue | CoValue | undefined },
+    JsonObject | null
+>;
+
+export type AnyCoList = CoList<JsonValue | CoValue, JsonObject | null>;
+
+export type AnyCoStream = CoStream<JsonValue | CoValue, JsonObject | null>;
+
+export type AnyBinaryCoStream = BinaryCoStream<BinaryCoStreamMeta>;
+
+
+export type AnyCoValue =
+    | AnyCoMap
+    | AnyCoList
+    | AnyCoStream
+    | AnyBinaryCoStream
+
+export function expectMap(
+    content: CoValue
+): AnyCoMap {
+    if (content.type !== "comap") {
+        throw new Error("Expected map");
+    }
+
+    return content as AnyCoMap;
 }
 
-export function newRandomSessionID(accountID: AccountIDOrAgentID): SessionID {
-    return `${accountID}_session_z${base58.encode(randomBytes(8))}`;
+export function expectList(
+    content: CoValue
+): AnyCoList {
+    if (content.type !== "colist") {
+        throw new Error("Expected list");
+    }
+
+    return content as AnyCoList;
 }
 
-type SessionLog = {
-    transactions: Transaction[];
-    lastHash?: Hash;
-    streamingHash: StreamingHash;
-    lastSignature: Signature;
-};
-
-export type PrivateTransaction = {
-    privacy: "private";
-    madeAt: number;
-    keyUsed: KeyID;
-    encryptedChanges: Encrypted<
-        JsonValue[],
-        { in: RawCoID; tx: TransactionID }
-    >;
-};
-
-export type TrustingTransaction = {
-    privacy: "trusting";
-    madeAt: number;
-    changes: JsonValue[];
-};
-
-export type Transaction = PrivateTransaction | TrustingTransaction;
-
-export type DecryptedTransaction = {
-    txID: TransactionID;
-    changes: JsonValue[];
-    madeAt: number;
-};
-
-const readKeyCache = new WeakMap<CoValue, { [id: KeyID]: KeySecret }>();
-
-export class CoValue {
-    id: RawCoID;
-    node: LocalNode;
-    header: CoValueHeader;
-    _sessions: { [key: SessionID]: SessionLog };
-    _cachedContent?: ContentType;
-    listeners: Set<(content?: ContentType) => void> = new Set();
-
-    constructor(
-        header: CoValueHeader,
-        node: LocalNode,
-        internalInitSessions: { [key: SessionID]: SessionLog } = {}
-    ) {
-        this.id = idforHeader(header);
-        this.header = header;
-        this._sessions = internalInitSessions;
-        this.node = node;
-
-        if (header.ruleset.type == "ownedByGroup") {
-            this.node
-                .expectCoValueLoaded(header.ruleset.group)
-                .subscribe((_groupUpdate) => {
-                    this._cachedContent = undefined;
-                    const newContent = this.getCurrentContent();
-                    for (const listener of this.listeners) {
-                        listener(newContent);
-                    }
-                });
-        }
+export function expectStream(
+    content: CoValue
+): AnyCoStream {
+    if (content.type !== "costream") {
+        throw new Error("Expected stream");
     }
 
-    get sessions(): Readonly<{ [key: SessionID]: SessionLog }> {
-        return this._sessions;
-    }
-
-    testWithDifferentAccount(
-        account: GeneralizedControlledAccount,
-        ownSessionID: SessionID
-    ): CoValue {
-        const newNode = this.node.testWithDifferentAccount(
-            account,
-            ownSessionID
-        );
-
-        return newNode.expectCoValueLoaded(this.id);
-    }
-
-    knownState(): CoValueKnownState {
-        return {
-            id: this.id,
-            header: true,
-            sessions: Object.fromEntries(
-                Object.entries(this.sessions).map(([k, v]) => [
-                    k,
-                    v.transactions.length,
-                ])
-            ),
-        };
-    }
-
-    get meta(): JsonValue {
-        return this.header?.meta ?? null;
-    }
-
-    nextTransactionID(): TransactionID {
-        const sessionID = this.node.ownSessionID;
-        return {
-            sessionID,
-            txIndex: this.sessions[sessionID]?.transactions.length || 0,
-        };
-    }
-
-    tryAddTransactions(
-        sessionID: SessionID,
-        newTransactions: Transaction[],
-        givenExpectedNewHash: Hash | undefined,
-        newSignature: Signature
-    ): boolean {
-        const signerID = getAgentSignerID(
-            this.node.resolveAccountAgent(
-                accountOrAgentIDfromSessionID(sessionID),
-                "Expected to know signer of transaction"
-            )
-        );
-
-        if (!signerID) {
-            console.warn(
-                "Unknown agent",
-                accountOrAgentIDfromSessionID(sessionID)
-            );
-            return false;
-        }
-
-        const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
-            sessionID,
-            newTransactions
-        );
-
-        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
-            console.warn("Invalid hash", {
-                expectedNewHash,
-                givenExpectedNewHash,
-            });
-            return false;
-        }
-
-        if (!verify(newSignature, expectedNewHash, signerID)) {
-            console.warn(
-                "Invalid signature",
-                newSignature,
-                expectedNewHash,
-                signerID
-            );
-            return false;
-        }
-
-        const transactions = this.sessions[sessionID]?.transactions ?? [];
-
-        transactions.push(...newTransactions);
-
-        this._sessions[sessionID] = {
-            transactions,
-            lastHash: expectedNewHash,
-            streamingHash: newStreamingHash,
-            lastSignature: newSignature,
-        };
-
-        this._cachedContent = undefined;
-
-        const content = this.getCurrentContent();
-
-        for (const listener of this.listeners) {
-            listener(content);
-        }
-
-        return true;
-    }
-
-    subscribe(listener: (content?: ContentType) => void): () => void {
-        this.listeners.add(listener);
-        listener(this.getCurrentContent());
-
-        return () => {
-            this.listeners.delete(listener);
-        };
-    }
-
-    expectedNewHashAfter(
-        sessionID: SessionID,
-        newTransactions: Transaction[]
-    ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
-        const streamingHash =
-            this.sessions[sessionID]?.streamingHash.clone() ??
-            new StreamingHash();
-        for (const transaction of newTransactions) {
-            streamingHash.update(transaction);
-        }
-
-        const newStreamingHash = streamingHash.clone();
-
-        return {
-            expectedNewHash: streamingHash.digest(),
-            newStreamingHash,
-        };
-    }
-
-    makeTransaction(
-        changes: JsonValue[],
-        privacy: "private" | "trusting"
-    ): boolean {
-        const madeAt = Date.now();
-
-        let transaction: Transaction;
-
-        if (privacy === "private") {
-            const { secret: keySecret, id: keyID } = this.getCurrentReadKey();
-
-            if (!keySecret) {
-                throw new Error(
-                    "Can't make transaction without read key secret"
-                );
-            }
-
-            transaction = {
-                privacy: "private",
-                madeAt,
-                keyUsed: keyID,
-                encryptedChanges: encryptForTransaction(changes, keySecret, {
-                    in: this.id,
-                    tx: this.nextTransactionID(),
-                }),
-            };
-        } else {
-            transaction = {
-                privacy: "trusting",
-                madeAt,
-                changes,
-            };
-        }
-
-        const sessionID = this.node.ownSessionID;
-
-        const { expectedNewHash } = this.expectedNewHashAfter(sessionID, [
-            transaction,
-        ]);
-
-        const signature = sign(
-            this.node.account.currentSignerSecret(),
-            expectedNewHash
-        );
-
-        const success = this.tryAddTransactions(
-            sessionID,
-            [transaction],
-            expectedNewHash,
-            signature
-        );
-
-        if (success) {
-            void this.node.sync.syncCoValue(this);
-        }
-
-        return success;
-    }
-
-    getCurrentContent(): ContentType {
-        if (this._cachedContent) {
-            return this._cachedContent;
-        }
-
-        if (this.header.type === "comap") {
-            this._cachedContent = new CoMap(this);
-        } else if (this.header.type === "colist") {
-            this._cachedContent = new CoList(this);
-        } else if (this.header.type === "costream") {
-            this._cachedContent = new CoStream(this);
-        } else if (this.header.type === "static") {
-            this._cachedContent = new Static(this);
-        } else {
-            throw new Error(`Unknown coValue type ${this.header.type}`);
-        }
-
-        return this._cachedContent;
-    }
-
-    getValidSortedTransactions(): DecryptedTransaction[] {
-        const validTransactions = determineValidTransactions(this);
-
-        const allTransactions: DecryptedTransaction[] = validTransactions
-            .map(({ txID, tx }) => {
-                if (tx.privacy === "trusting") {
-                    return {
-                        txID,
-                        madeAt: tx.madeAt,
-                        changes: tx.changes,
-                    };
-                } else {
-                    const readKey = this.getReadKey(tx.keyUsed);
-
-                    if (!readKey) {
-                        return undefined;
-                    } else {
-                        const decrytedChanges = decryptForTransaction(
-                            tx.encryptedChanges,
-                            readKey,
-                            {
-                                in: this.id,
-                                tx: txID,
-                            }
-                        );
-
-                        if (!decrytedChanges) {
-                            console.error(
-                                "Failed to decrypt transaction despite having key"
-                            );
-                            return undefined;
-                        }
-                        return {
-                            txID,
-                            madeAt: tx.madeAt,
-                            changes: decrytedChanges,
-                        };
-                    }
-                }
-            })
-            .filter((x): x is Exclude<typeof x, undefined> => !!x);
-        allTransactions.sort(
-            (a, b) =>
-                a.madeAt - b.madeAt ||
-                (a.txID.sessionID < b.txID.sessionID ? -1 : 1) ||
-                a.txID.txIndex - b.txID.txIndex
-        );
-
-        return allTransactions;
-    }
-
-    getCurrentReadKey(): { secret: KeySecret | undefined; id: KeyID } {
-        if (this.header.ruleset.type === "group") {
-            const content = expectGroupContent(this.getCurrentContent());
-
-            const currentKeyId = content.get("readKey");
-
-            if (!currentKeyId) {
-                throw new Error("No readKey set");
-            }
-
-            const secret = this.getReadKey(currentKeyId);
-
-            return {
-                secret: secret,
-                id: currentKeyId,
-            };
-        } else if (this.header.ruleset.type === "ownedByGroup") {
-            return this.node
-                .expectCoValueLoaded(this.header.ruleset.group)
-                .getCurrentReadKey();
-        } else {
-            throw new Error(
-                "Only groups or values owned by groups have read secrets"
-            );
-        }
-    }
-
-    getReadKey(keyID: KeyID): KeySecret | undefined {
-        if (readKeyCache.get(this)?.[keyID]) {
-            return readKeyCache.get(this)?.[keyID];
-        }
-        if (this.header.ruleset.type === "group") {
-            const content = expectGroupContent(this.getCurrentContent());
-
-            // Try to find key revelation for us
-
-            const readKeyEntry = content.getLastEntry(
-                `${keyID}_for_${this.node.account.id}`
-            );
-
-            if (readKeyEntry) {
-                const revealer = accountOrAgentIDfromSessionID(
-                    readKeyEntry.txID.sessionID
-                );
-                const revealerAgent = this.node.resolveAccountAgent(
-                    revealer,
-                    "Expected to know revealer"
-                );
-
-                const secret = unseal(
-                    readKeyEntry.value,
-                    this.node.account.currentSealerSecret(),
-                    getAgentSealerID(revealerAgent),
-                    {
-                        in: this.id,
-                        tx: readKeyEntry.txID,
-                    }
-                );
-
-                if (secret) {
-                    let cache = readKeyCache.get(this);
-                    if (!cache) {
-                        cache = {};
-                        readKeyCache.set(this, cache);
-                    }
-                    cache[keyID] = secret;
-
-                    return secret as KeySecret;
-                }
-            }
-
-            // Try to find indirect revelation through previousKeys
-
-            for (const field of content.keys()) {
-                if (isKeyForKeyField(field) && field.startsWith(keyID)) {
-                    const encryptingKeyID = field.split("_for_")[1] as KeyID;
-                    const encryptingKeySecret =
-                        this.getReadKey(encryptingKeyID);
-
-                    if (!encryptingKeySecret) {
-                        continue;
-                    }
-
-                    const encryptedPreviousKey = content.get(field)!;
-
-                    const secret = decryptKeySecret(
-                        {
-                            encryptedID: keyID,
-                            encryptingID: encryptingKeyID,
-                            encrypted: encryptedPreviousKey,
-                        },
-                        encryptingKeySecret
-                    );
-
-                    if (secret) {
-                        let cache = readKeyCache.get(this);
-                        if (!cache) {
-                            cache = {};
-                            readKeyCache.set(this, cache);
-                        }
-                        cache[keyID] = secret;
-
-                        return secret as KeySecret;
-                    } else {
-                        console.error(
-                            `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`
-                        );
-                    }
-                }
-            }
-
-            return undefined;
-        } else if (this.header.ruleset.type === "ownedByGroup") {
-            return this.node
-                .expectCoValueLoaded(this.header.ruleset.group)
-                .getReadKey(keyID);
-        } else {
-            throw new Error(
-                "Only groups or values owned by groups have read secrets"
-            );
-        }
-    }
-
-    getGroup(): Group {
-        if (this.header.ruleset.type !== "ownedByGroup") {
-            throw new Error("Only values owned by groups have groups");
-        }
-
-        return new Group(
-            expectGroupContent(
-                this.node
-                    .expectCoValueLoaded(this.header.ruleset.group)
-                    .getCurrentContent()
-            ),
-            this.node
-        );
-    }
-
-    getTx(txID: TransactionID): Transaction | undefined {
-        return this.sessions[txID.sessionID]?.transactions[txID.txIndex];
-    }
-
-    newContentSince(
-        knownState: CoValueKnownState | undefined
-    ): NewContentMessage | undefined {
-        const newContent: NewContentMessage = {
-            action: "content",
-            id: this.id,
-            header: knownState?.header ? undefined : this.header,
-            new: Object.fromEntries(
-                Object.entries(this.sessions)
-                    .map(([sessionID, log]) => {
-                        const newTransactions = log.transactions.slice(
-                            knownState?.sessions[sessionID as SessionID] || 0
-                        );
-
-                        if (
-                            newTransactions.length === 0 ||
-                            !log.lastHash ||
-                            !log.lastSignature
-                        ) {
-                            return undefined;
-                        }
-
-                        return [
-                            sessionID,
-                            {
-                                after:
-                                    knownState?.sessions[
-                                        sessionID as SessionID
-                                    ] || 0,
-                                newTransactions,
-                                lastSignature: log.lastSignature,
-                            },
-                        ];
-                    })
-                    .filter((x): x is Exclude<typeof x, undefined> => !!x)
-            ),
-        };
-
-        if (!newContent.header && Object.keys(newContent.new).length === 0) {
-            return undefined;
-        }
-
-        return newContent;
-    }
-
-    getDependedOnCoValues(): RawCoID[] {
-        return this.header.ruleset.type === "group"
-            ? expectGroupContent(this.getCurrentContent())
-                  .keys()
-                  .filter((k): k is AccountID => k.startsWith("co_"))
-            : this.header.ruleset.type === "ownedByGroup"
-            ? [this.header.ruleset.group]
-            : [];
-    }
+    return content as AnyCoStream;
+}
+
+export function isCoValue(value: JsonValue | CoValue | undefined) : value is CoValue {
+    return (
+        value instanceof CoMap ||
+        value instanceof CoList ||
+        value instanceof CoStream ||
+        value instanceof BinaryCoStream
+    );
 }

packages/cojson/src/coValueCore.ts

@@ -0,0 +1,818 @@
+import { randomBytes } from "@noble/hashes/utils";
+import { AnyCoValue, CoValue } from "./coValue.js";
+import { BinaryCoStream, CoStream } from "./coValues/coStream.js";
+import { CoMap } from "./coValues/coMap.js";
+import {
+    Encrypted,
+    Hash,
+    KeySecret,
+    Signature,
+    StreamingHash,
+    unseal,
+    shortHash,
+    sign,
+    verify,
+    encryptForTransaction,
+    KeyID,
+    decryptKeySecret,
+    getAgentSignerID,
+    getAgentSealerID,
+    decryptRawForTransaction,
+} from "./crypto.js";
+import { JsonObject, JsonValue } from "./jsonValue.js";
+import { base58 } from "@scure/base";
+import {
+    PermissionsDef as RulesetDef,
+    determineValidTransactions,
+    isKeyForKeyField,
+} from "./permissions.js";
+import { Group, expectGroupContent } from "./group.js";
+import { LocalNode } from "./localNode.js";
+import { CoValueKnownState, NewContentMessage } from "./sync.js";
+import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
+import { CoList } from "./coValues/coList.js";
+import { AccountID, GeneralizedControlledAccount } from "./account.js";
+import { Stringified, stableStringify } from "./jsonStringify.js";
+
+export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
+
+export type CoValueHeader = {
+    type: AnyCoValue["type"];
+    ruleset: RulesetDef;
+    meta: JsonObject | null;
+    createdAt: `2${string}` | null;
+    uniqueness: `z${string}` | null;
+};
+
+export function idforHeader(header: CoValueHeader): RawCoID {
+    const hash = shortHash(header);
+    return `co_z${hash.slice("shortHash_z".length)}`;
+}
+
+export function accountOrAgentIDfromSessionID(
+    sessionID: SessionID
+): AccountID | AgentID {
+    return sessionID.split("_session")[0] as AccountID | AgentID;
+}
+
+export function newRandomSessionID(accountID: AccountID | AgentID): SessionID {
+    return `${accountID}_session_z${base58.encode(randomBytes(8))}`;
+}
+
+type SessionLog = {
+    transactions: Transaction[];
+    lastHash?: Hash;
+    streamingHash: StreamingHash;
+    signatureAfter: { [txIdx: number]: Signature | undefined };
+    lastSignature: Signature;
+};
+
+export type PrivateTransaction = {
+    privacy: "private";
+    madeAt: number;
+    keyUsed: KeyID;
+    encryptedChanges: Encrypted<
+        JsonValue[],
+        { in: RawCoID; tx: TransactionID }
+    >;
+};
+
+export type TrustingTransaction = {
+    privacy: "trusting";
+    madeAt: number;
+    changes: Stringified<JsonValue[]>;
+};
+
+export type Transaction = PrivateTransaction | TrustingTransaction;
+
+export type DecryptedTransaction = {
+    txID: TransactionID;
+    changes: Stringified<JsonValue[]>;
+    madeAt: number;
+};
+
+const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
+
+export class CoValueCore {
+    id: RawCoID;
+    node: LocalNode;
+    header: CoValueHeader;
+    _sessions: { [key: SessionID]: SessionLog };
+    _cachedContent?: CoValue;
+    listeners: Set<(content?: CoValue) => void> = new Set();
+    _decryptionCache: {
+        [key: Encrypted<JsonValue[], JsonValue>]:
+            | Stringified<JsonValue[]>
+            | undefined;
+    } = {};
+
+    constructor(
+        header: CoValueHeader,
+        node: LocalNode,
+        internalInitSessions: { [key: SessionID]: SessionLog } = {}
+    ) {
+        this.id = idforHeader(header);
+        this.header = header;
+        this._sessions = internalInitSessions;
+        this.node = node;
+
+        if (header.ruleset.type == "ownedByGroup") {
+            this.node
+                .expectCoValueLoaded(header.ruleset.group)
+                .subscribe((_groupUpdate) => {
+                    this._cachedContent = undefined;
+                    const newContent = this.getCurrentContent();
+                    for (const listener of this.listeners) {
+                        listener(newContent);
+                    }
+                });
+        }
+    }
+
+    get sessions(): Readonly<{ [key: SessionID]: SessionLog }> {
+        return this._sessions;
+    }
+
+    testWithDifferentAccount(
+        account: GeneralizedControlledAccount,
+        currentSessionID: SessionID
+    ): CoValueCore {
+        const newNode = this.node.testWithDifferentAccount(
+            account,
+            currentSessionID
+        );
+
+        return newNode.expectCoValueLoaded(this.id);
+    }
+
+    knownState(): CoValueKnownState {
+        return {
+            id: this.id,
+            header: true,
+            sessions: Object.fromEntries(
+                Object.entries(this.sessions).map(([k, v]) => [
+                    k,
+                    v.transactions.length,
+                ])
+            ),
+        };
+    }
+
+    get meta(): JsonValue {
+        return this.header?.meta ?? null;
+    }
+
+    nextTransactionID(): TransactionID {
+        const sessionID = this.node.currentSessionID;
+        return {
+            sessionID,
+            txIndex: this.sessions[sessionID]?.transactions.length || 0,
+        };
+    }
+
+    tryAddTransactions(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        givenExpectedNewHash: Hash | undefined,
+        newSignature: Signature
+    ): boolean {
+        const signerID = getAgentSignerID(
+            this.node.resolveAccountAgent(
+                accountOrAgentIDfromSessionID(sessionID),
+                "Expected to know signer of transaction"
+            )
+        );
+
+        if (!signerID) {
+            console.warn(
+                "Unknown agent",
+                accountOrAgentIDfromSessionID(sessionID)
+            );
+            return false;
+        }
+
+        // const beforeHash = performance.now();
+        const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
+            sessionID,
+            newTransactions
+        );
+        // const afterHash = performance.now();
+        // console.log(
+        //     "Hashing took",
+        //     afterHash - beforeHash
+        // );
+
+        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
+            console.warn("Invalid hash", {
+                expectedNewHash,
+                givenExpectedNewHash,
+            });
+            return false;
+        }
+
+        // const beforeVerify = performance.now();
+        if (!verify(newSignature, expectedNewHash, signerID)) {
+            console.warn(
+                "Invalid signature in",
+                this.id,
+                newSignature,
+                expectedNewHash,
+                signerID
+            );
+            return false;
+        }
+        // const afterVerify = performance.now();
+        // console.log(
+        //     "Verify took",
+        //     afterVerify - beforeVerify
+        // );
+
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
+
+        return true;
+    }
+
+    async tryAddTransactionsAsync(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        givenExpectedNewHash: Hash | undefined,
+        newSignature: Signature
+    ): Promise<boolean> {
+        const signerID = getAgentSignerID(
+            this.node.resolveAccountAgent(
+                accountOrAgentIDfromSessionID(sessionID),
+                "Expected to know signer of transaction"
+            )
+        );
+
+        if (!signerID) {
+            console.warn(
+                "Unknown agent",
+                accountOrAgentIDfromSessionID(sessionID)
+            );
+            return false;
+        }
+
+        const nTxBefore = this.sessions[sessionID]?.transactions.length ?? 0;
+
+        // const beforeHash = performance.now();
+        const { expectedNewHash, newStreamingHash } =
+            await this.expectedNewHashAfterAsync(sessionID, newTransactions);
+        // const afterHash = performance.now();
+        // console.log(
+        //     "Hashing took",
+        //     afterHash - beforeHash
+        // );
+
+        const nTxAfter = this.sessions[sessionID]?.transactions.length ?? 0;
+
+        if (nTxAfter !== nTxBefore) {
+            const newTransactionLengthBefore = newTransactions.length;
+            newTransactions = newTransactions.slice(nTxAfter - nTxBefore);
+            console.warn("Transactions changed while async hashing", {
+                nTxBefore,
+                nTxAfter,
+                newTransactionLengthBefore,
+                remainingNewTransactions: newTransactions.length,
+            });
+        }
+
+        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
+            console.warn("Invalid hash", {
+                expectedNewHash,
+                givenExpectedNewHash,
+            });
+            return false;
+        }
+
+        // const beforeVerify = performance.now();
+        if (!verify(newSignature, expectedNewHash, signerID)) {
+            console.warn(
+                "Invalid signature in",
+                this.id,
+                newSignature,
+                expectedNewHash,
+                signerID
+            );
+            return false;
+        }
+        // const afterVerify = performance.now();
+        // console.log(
+        //     "Verify took",
+        //     afterVerify - beforeVerify
+        // );
+
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
+
+        return true;
+    }
+
+    private doAddTransactions(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        newSignature: Signature,
+        expectedNewHash: Hash,
+        newStreamingHash: StreamingHash
+    ) {
+        const transactions = this.sessions[sessionID]?.transactions ?? [];
+        transactions.push(...newTransactions);
+
+        const signatureAfter = this.sessions[sessionID]?.signatureAfter ?? {};
+
+        const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
+            (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
+            -1
+        );
+
+        const sizeOfTxsSinceLastInbetweenSignature = transactions
+            .slice(lastInbetweenSignatureIdx + 1)
+            .reduce(
+                (sum, tx) =>
+                    sum +
+                    (tx.privacy === "private"
+                        ? tx.encryptedChanges.length
+                        : tx.changes.length),
+                0
+            );
+
+        if (sizeOfTxsSinceLastInbetweenSignature > 100 * 1024) {
+            // console.log(
+            //     "Saving inbetween signature for tx ",
+            //     sessionID,
+            //     transactions.length - 1,
+            //     sizeOfTxsSinceLastInbetweenSignature
+            // );
+            signatureAfter[transactions.length - 1] = newSignature;
+        }
+
+        this._sessions[sessionID] = {
+            transactions,
+            lastHash: expectedNewHash,
+            streamingHash: newStreamingHash,
+            lastSignature: newSignature,
+            signatureAfter: signatureAfter,
+        };
+
+        this._cachedContent = undefined;
+
+        if (this.listeners.size > 0) {
+            const content = this.getCurrentContent();
+            for (const listener of this.listeners) {
+                listener(content);
+            }
+        }
+    }
+
+    subscribe(listener: (content?: CoValue) => void): () => void {
+        this.listeners.add(listener);
+        listener(this.getCurrentContent());
+
+        return () => {
+            this.listeners.delete(listener);
+        };
+    }
+
+    expectedNewHashAfter(
+        sessionID: SessionID,
+        newTransactions: Transaction[]
+    ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
+        const streamingHash =
+            this.sessions[sessionID]?.streamingHash.clone() ??
+            new StreamingHash();
+        for (const transaction of newTransactions) {
+            streamingHash.update(transaction);
+        }
+
+        const newStreamingHash = streamingHash.clone();
+
+        return {
+            expectedNewHash: streamingHash.digest(),
+            newStreamingHash,
+        };
+    }
+
+    async expectedNewHashAfterAsync(
+        sessionID: SessionID,
+        newTransactions: Transaction[]
+    ): Promise<{ expectedNewHash: Hash; newStreamingHash: StreamingHash }> {
+        const streamingHash =
+            this.sessions[sessionID]?.streamingHash.clone() ??
+            new StreamingHash();
+        let before = performance.now();
+        for (const transaction of newTransactions) {
+            streamingHash.update(transaction);
+            const after = performance.now();
+            if (after - before > 1) {
+                // console.log("Hashing blocked for", after - before);
+                await new Promise((resolve) => setTimeout(resolve, 0));
+                before = performance.now();
+            }
+        }
+
+        const newStreamingHash = streamingHash.clone();
+
+        return {
+            expectedNewHash: streamingHash.digest(),
+            newStreamingHash,
+        };
+    }
+
+    makeTransaction(
+        changes: JsonValue[],
+        privacy: "private" | "trusting"
+    ): boolean {
+        const madeAt = Date.now();
+
+        let transaction: Transaction;
+
+        if (privacy === "private") {
+            const { secret: keySecret, id: keyID } = this.getCurrentReadKey();
+
+            if (!keySecret) {
+                throw new Error(
+                    "Can't make transaction without read key secret"
+                );
+            }
+
+            const encrypted = encryptForTransaction(changes, keySecret, {
+                in: this.id,
+                tx: this.nextTransactionID(),
+            });
+
+            this._decryptionCache[encrypted] = stableStringify(changes);
+
+            transaction = {
+                privacy: "private",
+                madeAt,
+                keyUsed: keyID,
+                encryptedChanges: encrypted,
+            };
+        } else {
+            transaction = {
+                privacy: "trusting",
+                madeAt,
+                changes: stableStringify(changes),
+            };
+        }
+
+        const sessionID = this.node.currentSessionID;
+
+        const { expectedNewHash } = this.expectedNewHashAfter(sessionID, [
+            transaction,
+        ]);
+
+        const signature = sign(
+            this.node.account.currentSignerSecret(),
+            expectedNewHash
+        );
+
+        const success = this.tryAddTransactions(
+            sessionID,
+            [transaction],
+            expectedNewHash,
+            signature
+        );
+
+        if (success) {
+            void this.node.syncManager.syncCoValue(this);
+        }
+
+        return success;
+    }
+
+    getCurrentContent(): CoValue {
+        if (this._cachedContent) {
+            return this._cachedContent;
+        }
+
+        if (this.header.type === "comap") {
+            this._cachedContent = new CoMap(this);
+        } else if (this.header.type === "colist") {
+            this._cachedContent = new CoList(this);
+        } else if (this.header.type === "costream") {
+            if (this.header.meta && this.header.meta.type === "binary") {
+                this._cachedContent = new BinaryCoStream(this);
+            } else {
+                this._cachedContent = new CoStream(this);
+            }
+        } else {
+            throw new Error(`Unknown coValue type ${this.header.type}`);
+        }
+
+        return this._cachedContent;
+    }
+
+    getValidSortedTransactions(): DecryptedTransaction[] {
+        const validTransactions = determineValidTransactions(this);
+
+        const allTransactions: DecryptedTransaction[] = validTransactions
+            .map(({ txID, tx }) => {
+                if (tx.privacy === "trusting") {
+                    return {
+                        txID,
+                        madeAt: tx.madeAt,
+                        changes: tx.changes,
+                    };
+                } else {
+                    const readKey = this.getReadKey(tx.keyUsed);
+
+                    if (!readKey) {
+                        return undefined;
+                    } else {
+                        let decrytedChanges =
+                            this._decryptionCache[tx.encryptedChanges];
+
+                        if (!decrytedChanges) {
+                            decrytedChanges = decryptRawForTransaction(
+                                tx.encryptedChanges,
+                                readKey,
+                                {
+                                    in: this.id,
+                                    tx: txID,
+                                }
+                            );
+                            this._decryptionCache[tx.encryptedChanges] =
+                                decrytedChanges;
+                        }
+
+                        if (!decrytedChanges) {
+                            console.error(
+                                "Failed to decrypt transaction despite having key"
+                            );
+                            return undefined;
+                        }
+                        return {
+                            txID,
+                            madeAt: tx.madeAt,
+                            changes: decrytedChanges,
+                        };
+                    }
+                }
+            })
+            .filter((x): x is Exclude<typeof x, undefined> => !!x);
+        allTransactions.sort(
+            (a, b) =>
+                a.madeAt - b.madeAt ||
+                (a.txID.sessionID < b.txID.sessionID ? -1 : 1) ||
+                a.txID.txIndex - b.txID.txIndex
+        );
+
+        return allTransactions;
+    }
+
+    getCurrentReadKey(): { secret: KeySecret | undefined; id: KeyID } {
+        if (this.header.ruleset.type === "group") {
+            const content = expectGroupContent(this.getCurrentContent());
+
+            const currentKeyId = content.get("readKey");
+
+            if (!currentKeyId) {
+                throw new Error("No readKey set");
+            }
+
+            const secret = this.getReadKey(currentKeyId);
+
+            return {
+                secret: secret,
+                id: currentKeyId,
+            };
+        } else if (this.header.ruleset.type === "ownedByGroup") {
+            return this.node
+                .expectCoValueLoaded(this.header.ruleset.group)
+                .getCurrentReadKey();
+        } else {
+            throw new Error(
+                "Only groups or values owned by groups have read secrets"
+            );
+        }
+    }
+
+    getReadKey(keyID: KeyID): KeySecret | undefined {
+        if (readKeyCache.get(this)?.[keyID]) {
+            return readKeyCache.get(this)?.[keyID];
+        }
+        if (this.header.ruleset.type === "group") {
+            const content = expectGroupContent(this.getCurrentContent());
+
+            // Try to find key revelation for us
+
+            const lastReadyKeyEdit = content.lastEditAt(
+                `${keyID}_for_${this.node.account.id}`
+            );
+
+            if (lastReadyKeyEdit?.value) {
+                const revealer = lastReadyKeyEdit.by;
+                const revealerAgent = this.node.resolveAccountAgent(
+                    revealer,
+                    "Expected to know revealer"
+                );
+
+                const secret = unseal(
+                    lastReadyKeyEdit.value,
+                    this.node.account.currentSealerSecret(),
+                    getAgentSealerID(revealerAgent),
+                    {
+                        in: this.id,
+                        tx: lastReadyKeyEdit.tx,
+                    }
+                );
+
+                if (secret) {
+                    let cache = readKeyCache.get(this);
+                    if (!cache) {
+                        cache = {};
+                        readKeyCache.set(this, cache);
+                    }
+                    cache[keyID] = secret;
+
+                    return secret as KeySecret;
+                }
+            }
+
+            // Try to find indirect revelation through previousKeys
+
+            for (const field of content.keys()) {
+                if (isKeyForKeyField(field) && field.startsWith(keyID)) {
+                    const encryptingKeyID = field.split("_for_")[1] as KeyID;
+                    const encryptingKeySecret =
+                        this.getReadKey(encryptingKeyID);
+
+                    if (!encryptingKeySecret) {
+                        continue;
+                    }
+
+                    const encryptedPreviousKey = content.get(field)!;
+
+                    const secret = decryptKeySecret(
+                        {
+                            encryptedID: keyID,
+                            encryptingID: encryptingKeyID,
+                            encrypted: encryptedPreviousKey,
+                        },
+                        encryptingKeySecret
+                    );
+
+                    if (secret) {
+                        let cache = readKeyCache.get(this);
+                        if (!cache) {
+                            cache = {};
+                            readKeyCache.set(this, cache);
+                        }
+                        cache[keyID] = secret;
+
+                        return secret as KeySecret;
+                    } else {
+                        console.error(
+                            `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`
+                        );
+                    }
+                }
+            }
+
+            return undefined;
+        } else if (this.header.ruleset.type === "ownedByGroup") {
+            return this.node
+                .expectCoValueLoaded(this.header.ruleset.group)
+                .getReadKey(keyID);
+        } else {
+            throw new Error(
+                "Only groups or values owned by groups have read secrets"
+            );
+        }
+    }
+
+    getGroup(): Group {
+        if (this.header.ruleset.type !== "ownedByGroup") {
+            throw new Error("Only values owned by groups have groups");
+        }
+
+        return new Group(
+            expectGroupContent(
+                this.node
+                    .expectCoValueLoaded(this.header.ruleset.group)
+                    .getCurrentContent()
+            ),
+            this.node
+        );
+    }
+
+    getTx(txID: TransactionID): Transaction | undefined {
+        return this.sessions[txID.sessionID]?.transactions[txID.txIndex];
+    }
+
+    newContentSince(
+        knownState: CoValueKnownState | undefined
+    ): NewContentMessage[] | undefined {
+        let currentPiece: NewContentMessage = {
+            action: "content",
+            id: this.id,
+            header: knownState?.header ? undefined : this.header,
+            new: {},
+        };
+
+        const pieces = [currentPiece];
+
+        const sentState: CoValueKnownState["sessions"] = {
+            ...knownState?.sessions,
+        };
+
+        let newTxsWereAdded = true;
+        let pieceSize = 0;
+        while (newTxsWereAdded) {
+            newTxsWereAdded = false;
+
+            for (const [sessionID, log] of Object.entries(this.sessions) as [
+                SessionID,
+                SessionLog
+            ][]) {
+                const nextKnownSignatureIdx = Object.keys(log.signatureAfter)
+                    .map(Number)
+                    .sort((a, b) => a - b)
+                    .find((idx) => idx >= (sentState[sessionID] ?? -1));
+
+                const txsToAdd = log.transactions.slice(
+                    sentState[sessionID] ?? 0,
+                    nextKnownSignatureIdx === undefined
+                        ? undefined
+                        : nextKnownSignatureIdx + 1
+                );
+
+                if (txsToAdd.length === 0) continue;
+
+                newTxsWereAdded = true;
+
+                const oldPieceSize = pieceSize;
+                pieceSize += txsToAdd.reduce(
+                    (sum, tx) =>
+                        sum +
+                        (tx.privacy === "private"
+                            ? tx.encryptedChanges.length
+                            : tx.changes.length),
+                    0
+                );
+
+                if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+                    currentPiece = {
+                        action: "content",
+                        id: this.id,
+                        header: undefined,
+                        new: {},
+                    };
+                    pieces.push(currentPiece);
+                    pieceSize = pieceSize - oldPieceSize;
+                }
+
+                let sessionEntry = currentPiece.new[sessionID];
+                if (!sessionEntry) {
+                    sessionEntry = {
+                        after: sentState[sessionID] ?? 0,
+                        newTransactions: [],
+                        lastSignature: "WILL_BE_REPLACED" as Signature,
+                    };
+                    currentPiece.new[sessionID] = sessionEntry;
+                }
+
+                sessionEntry.newTransactions.push(...txsToAdd);
+                sessionEntry.lastSignature =
+                    nextKnownSignatureIdx === undefined
+                        ? log.lastSignature!
+                        : log.signatureAfter[nextKnownSignatureIdx]!;
+
+                sentState[sessionID] =
+                    (sentState[sessionID] || 0) + txsToAdd.length;
+            }
+        }
+
+        const piecesWithContent = pieces.filter(
+            (piece) => Object.keys(piece.new).length > 0 || piece.header
+        );
+
+        if (piecesWithContent.length === 0) {
+            return undefined;
+        }
+
+        return piecesWithContent;
+    }
+
+    getDependedOnCoValues(): RawCoID[] {
+        return this.header.ruleset.type === "group"
+            ? expectGroupContent(this.getCurrentContent())
+                  .keys()
+                  .filter((k): k is AccountID => k.startsWith("co_"))
+            : this.header.ruleset.type === "ownedByGroup"
+            ? [this.header.ruleset.group]
+            : [];
+    }
+}

packages/cojson/src/coValues/coList.ts

@@ -0,0 +1,596 @@
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { CoID, CoValue, isCoValue } from "../coValue.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
+import { AgentID, SessionID, TransactionID } from "../ids.js";
+import { Group } from "../group.js";
+import { AccountID } from "../account.js";
+import { parseJSON } from "../jsonStringify.js";
+
+type OpID = TransactionID & { changeIdx: number };
+
+type InsertionOpPayload<T extends JsonValue | CoValue> =
+    | {
+          op: "pre";
+          value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+          before: OpID | "end";
+      }
+    | {
+          op: "app";
+          value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+          after: OpID | "start";
+      };
+
+type DeletionOpPayload = {
+    op: "del";
+    insertion: OpID;
+};
+
+export type ListOpPayload<T extends JsonValue | CoValue> =
+    | InsertionOpPayload<T>
+    | DeletionOpPayload;
+
+type InsertionEntry<T extends JsonValue | CoValue> = {
+    madeAt: number;
+    predecessors: OpID[];
+    successors: OpID[];
+} & InsertionOpPayload<T>;
+
+type DeletionEntry = {
+    madeAt: number;
+    deletionID: OpID;
+} & DeletionOpPayload;
+
+export class CoListView<
+    Item extends JsonValue | CoValue,
+    Meta extends JsonObject | null = null
+> implements CoValue
+{
+    /** @category 6. Meta */
+    id: CoID<this>;
+    /** @category 6. Meta */
+    type = "colist" as const;
+    /** @category 6. Meta */
+    core: CoValueCore;
+    /** @internal */
+    afterStart: OpID[];
+    /** @internal */
+    beforeEnd: OpID[];
+    /** @internal */
+    insertions: {
+        [sessionID: SessionID]: {
+            [txIdx: number]: {
+                [changeIdx: number]: InsertionEntry<Item>;
+            };
+        };
+    };
+    /** @internal */
+    deletionsByInsertion: {
+        [deletedSessionID: SessionID]: {
+            [deletedTxIdx: number]: {
+                [deletedChangeIdx: number]: DeletionEntry[];
+            };
+        };
+    };
+    /** @category 6. Meta */
+    readonly _item!: Item;
+
+    /** @internal */
+    constructor(core: CoValueCore) {
+        this.id = core.id as CoID<this>;
+        this.core = core;
+        this.afterStart = [];
+        this.beforeEnd = [];
+        this.insertions = {};
+        this.deletionsByInsertion = {};
+
+        this.insertions = {};
+        this.deletionsByInsertion = {};
+        this.afterStart = [];
+        this.beforeEnd = [];
+
+        for (const {
+            txID,
+            changes,
+            madeAt,
+        } of this.core.getValidSortedTransactions()) {
+            for (const [changeIdx, changeUntyped] of parseJSON(
+                changes
+            ).entries()) {
+                const change = changeUntyped as ListOpPayload<Item>;
+
+                if (change.op === "pre" || change.op === "app") {
+                    let sessionEntry = this.insertions[txID.sessionID];
+                    if (!sessionEntry) {
+                        sessionEntry = {};
+                        this.insertions[txID.sessionID] = sessionEntry;
+                    }
+                    let txEntry = sessionEntry[txID.txIndex];
+                    if (!txEntry) {
+                        txEntry = {};
+                        sessionEntry[txID.txIndex] = txEntry;
+                    }
+                    txEntry[changeIdx] = {
+                        madeAt,
+                        predecessors: [],
+                        successors: [],
+                        ...change,
+                    };
+                    if (change.op === "pre") {
+                        if (change.before === "end") {
+                            this.beforeEnd.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        } else {
+                            const beforeEntry =
+                                this.insertions[change.before.sessionID]?.[
+                                    change.before.txIndex
+                                ]?.[change.before.changeIdx];
+                            if (!beforeEntry) {
+                                throw new Error(
+                                    "Not yet implemented: insertion before missing op " +
+                                        change.before
+                                );
+                            }
+                            beforeEntry.predecessors.splice(0, 0, {
+                                ...txID,
+                                changeIdx,
+                            });
+                        }
+                    } else {
+                        if (change.after === "start") {
+                            this.afterStart.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        } else {
+                            const afterEntry =
+                                this.insertions[change.after.sessionID]?.[
+                                    change.after.txIndex
+                                ]?.[change.after.changeIdx];
+                            if (!afterEntry) {
+                                throw new Error(
+                                    "Not yet implemented: insertion after missing op " +
+                                        change.after
+                                );
+                            }
+                            afterEntry.successors.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        }
+                    }
+                } else if (change.op === "del") {
+                    let sessionEntry =
+                        this.deletionsByInsertion[change.insertion.sessionID];
+                    if (!sessionEntry) {
+                        sessionEntry = {};
+                        this.deletionsByInsertion[change.insertion.sessionID] =
+                            sessionEntry;
+                    }
+                    let txEntry = sessionEntry[change.insertion.txIndex];
+                    if (!txEntry) {
+                        txEntry = {};
+                        sessionEntry[change.insertion.txIndex] = txEntry;
+                    }
+                    let changeEntry = txEntry[change.insertion.changeIdx];
+                    if (!changeEntry) {
+                        changeEntry = [];
+                        txEntry[change.insertion.changeIdx] = changeEntry;
+                    }
+                    changeEntry.push({
+                        madeAt,
+                        deletionID: {
+                            ...txID,
+                            changeIdx,
+                        },
+                        ...change,
+                    });
+                } else {
+                    throw new Error(
+                        "Unknown list operation " +
+                            (change as { op: unknown }).op
+                    );
+                }
+            }
+        }
+    }
+
+    /** @category 6. Meta */
+    get meta(): Meta {
+        return this.core.header.meta as Meta;
+    }
+
+    /** @category 6. Meta */
+    get group(): Group {
+        return this.core.getGroup();
+    }
+
+    /**
+     * Not yet implemented
+     *
+     * @category 4. Time travel
+     */
+    atTime(_time: number): this {
+        throw new Error("Not yet implemented");
+    }
+
+    /**
+     * Get the item currently at `idx`.
+     *
+     * @category 1. Reading
+     */
+    get(
+        idx: number
+    ):
+        | (Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>)
+        | undefined {
+        const entry = this.entries()[idx];
+        if (!entry) {
+            return undefined;
+        }
+        return entry.value;
+    }
+
+    /**
+     * Returns the current items in the CoList as an array.
+     *
+     * @category 1. Reading
+     **/
+    asArray(): (Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>)[] {
+        return this.entries().map((entry) => entry.value);
+    }
+
+    /** @internal */
+     entries(): {
+        value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+        madeAt: number;
+        opID: OpID;
+    }[] {
+        const arr: {
+            value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+            madeAt: number;
+            opID: OpID;
+        }[] = [];
+        for (const opID of this.afterStart) {
+            this.fillArrayFromOpID(opID, arr);
+        }
+        for (const opID of this.beforeEnd) {
+            this.fillArrayFromOpID(opID, arr);
+        }
+        return arr;
+    }
+
+    /** @internal */
+    private fillArrayFromOpID(
+        opID: OpID,
+        arr: {
+            value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+            madeAt: number;
+            opID: OpID;
+        }[]
+    ) {
+        const entry =
+            this.insertions[opID.sessionID]?.[opID.txIndex]?.[opID.changeIdx];
+        if (!entry) {
+            throw new Error("Missing op " + opID);
+        }
+        for (const predecessor of entry.predecessors) {
+            this.fillArrayFromOpID(predecessor, arr);
+        }
+        const deleted =
+            (this.deletionsByInsertion[opID.sessionID]?.[opID.txIndex]?.[
+                opID.changeIdx
+            ]?.length || 0) > 0;
+        if (!deleted) {
+            arr.push({
+                value: entry.value,
+                madeAt: entry.madeAt,
+                opID,
+            });
+        }
+        for (const successor of entry.successors) {
+            this.fillArrayFromOpID(successor, arr);
+        }
+    }
+
+    /**
+     * Returns the current items in the CoList as an array. (alias of `asArray`)
+     *
+     * @category 1. Reading
+     */
+    toJSON(): (Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>)[] {
+        return this.asArray();
+    }
+
+    /** @category 5. Edit history */
+    editAt(idx: number):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+          }
+        | undefined {
+        const entry = this.entries()[idx];
+        if (!entry) {
+            return undefined;
+        }
+        const madeAt = new Date(entry.madeAt);
+        const by = accountOrAgentIDfromSessionID(entry.opID.sessionID);
+        const value = entry.value;
+        return {
+            by,
+            tx: {
+                sessionID: entry.opID.sessionID,
+                txIndex: entry.opID.txIndex,
+            },
+            at: madeAt,
+            value,
+        };
+    }
+
+    /** @category 5. Edit history */
+    deletionEdits(): {
+        by: AccountID | AgentID;
+        tx: TransactionID;
+        at: Date;
+        // TODO: add indices that are now before and after the deleted item
+    }[] {
+        const edits: {
+            by: AccountID | AgentID;
+            tx: TransactionID;
+            at: Date;
+        }[] = [];
+
+        for (const sessionID in this.deletionsByInsertion) {
+            const sessionEntry =
+                this.deletionsByInsertion[sessionID as SessionID];
+            for (const txIdx in sessionEntry) {
+                const txEntry = sessionEntry[Number(txIdx)];
+                for (const changeIdx in txEntry) {
+                    const changeEntry = txEntry[Number(changeIdx)];
+                    for (const deletion of changeEntry || []) {
+                        const madeAt = new Date(deletion.madeAt);
+                        const by = accountOrAgentIDfromSessionID(
+                            deletion.deletionID.sessionID
+                        );
+                        edits.push({
+                            by,
+                            tx: deletion.deletionID,
+                            at: madeAt,
+                        });
+                    }
+                }
+            }
+        }
+
+        return edits;
+    }
+
+    /** @category 3. Subscription */
+    subscribe(listener: (coList: this) => void): () => void {
+        return this.core.subscribe((content) => {
+            listener(content as this);
+        });
+    }
+}
+
+export class CoList<
+        Item extends JsonValue | CoValue,
+        Meta extends JsonObject | null = null
+    >
+    extends CoListView<Item, Meta>
+    implements CoValue
+{
+    /** Returns a new version of this CoList with `item` appended after the item currently at index `after`.
+     *
+     * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `item` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Editing
+     **/
+    append(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        after?: number,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        const entries = this.entries();
+        after =
+            after === undefined
+                ? entries.length > 0
+                    ? entries.length - 1
+                    : 0
+                : 0;
+        let opIDBefore;
+        if (entries.length > 0) {
+            const entryBefore = entries[after];
+            if (!entryBefore) {
+                throw new Error("Invalid index " + after);
+            }
+            opIDBefore = entryBefore.opID;
+        } else {
+            if (after !== 0) {
+                throw new Error("Invalid index " + after);
+            }
+            opIDBefore = "start";
+        }
+        this.core.makeTransaction(
+            [
+                {
+                    op: "app",
+                    value: isCoValue(item) ? item.id : item,
+                    after: opIDBefore,
+                },
+            ],
+            privacy
+        );
+
+        return new CoList(this.core) as this;
+    }
+
+    /**
+     * Returns a new version of this CoList with `item` prepended before the item currently at index `before`.
+     *
+     * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `item` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Editing
+     */
+    prepend(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        before?: number,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        const entries = this.entries();
+        before = before === undefined ? 0 : before;
+        let opIDAfter;
+        if (entries.length > 0) {
+            const entryAfter = entries[before];
+            if (entryAfter) {
+                opIDAfter = entryAfter.opID;
+            } else {
+                if (before !== entries.length) {
+                    throw new Error("Invalid index " + before);
+                }
+                opIDAfter = "end";
+            }
+        } else {
+            if (before !== 0) {
+                throw new Error("Invalid index " + before);
+            }
+            opIDAfter = "end";
+        }
+        this.core.makeTransaction(
+            [
+                {
+                    op: "pre",
+                    value: isCoValue(item) ? item.id : item,
+                    before: opIDAfter,
+                },
+            ],
+            privacy
+        );
+
+        return new CoList(this.core) as this;
+    }
+
+    /** Returns a new version of this CoList with the item at index `at` deleted from the list.
+     *
+     * If `privacy` is `"private"` **(default)**, the fact of this deletion is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, the fact of this deletion is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Editing
+     **/
+    delete(at: number, privacy: "private" | "trusting" = "private"): this {
+        const entries = this.entries();
+        const entry = entries[at];
+        if (!entry) {
+            throw new Error("Invalid index " + at);
+        }
+        this.core.makeTransaction(
+            [
+                {
+                    op: "del",
+                    insertion: entry.opID,
+                },
+            ],
+            privacy
+        );
+
+        return new CoList(this.core) as this;
+    }
+
+    /** @category 2. Editing */
+    mutate(mutator: (mutable: MutableCoList<Item, Meta>) => void): this {
+        const mutable = new MutableCoList<Item, Meta>(this.core);
+        mutator(mutable);
+        return new CoList(this.core) as this;
+    }
+
+    /** @deprecated Use `mutate` instead. */
+    edit(mutator: (mutable: MutableCoList<Item, Meta>) => void): this {
+        return this.mutate(mutator);
+    }
+}
+
+export class MutableCoList<
+        Item extends JsonValue | CoValue,
+        Meta extends JsonObject | null = null
+    >
+    extends CoListView<Item, Meta>
+    implements CoValue
+{
+    /** Appends `item` after the item currently at index `after`.
+     *
+     * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `item` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Mutating
+     **/
+    append(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        after?: number,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const listAfter = CoList.prototype.append.call(
+            this,
+            item,
+            after,
+            privacy
+        ) as CoList<Item, Meta>;
+        this.afterStart = listAfter.afterStart;
+        this.beforeEnd = listAfter.beforeEnd;
+        this.insertions = listAfter.insertions;
+        this.deletionsByInsertion = listAfter.deletionsByInsertion;
+    }
+
+    /** Prepends `item` before the item currently at index `before`.
+     *
+     * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `item` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * * @category 2. Mutating
+     **/
+    prepend(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        before?: number,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const listAfter = CoList.prototype.prepend.call(
+            this,
+            item,
+            before,
+            privacy
+        ) as CoList<Item, Meta>;
+        this.afterStart = listAfter.afterStart;
+        this.beforeEnd = listAfter.beforeEnd;
+        this.insertions = listAfter.insertions;
+        this.deletionsByInsertion = listAfter.deletionsByInsertion;
+    }
+
+    /** Deletes the item at index `at` from the list.
+     *
+     * If `privacy` is `"private"` **(default)**, the fact of this deletion is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, the fact of this deletion is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * * @category 2. Mutating
+     **/
+    delete(at: number, privacy: "private" | "trusting" = "private"): void {
+        const listAfter = CoList.prototype.delete.call(
+            this,
+            at,
+            privacy
+        ) as CoList<Item, Meta>;
+        this.afterStart = listAfter.afterStart;
+        this.beforeEnd = listAfter.beforeEnd;
+        this.insertions = listAfter.insertions;
+        this.deletionsByInsertion = listAfter.deletionsByInsertion;
+    }
+}

packages/cojson/src/coValues/coMap.ts

@@ -0,0 +1,446 @@
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { AgentID, TransactionID } from "../ids.js";
+import { CoID, CoValue, isCoValue } from "../coValue.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
+import { AccountID } from "../account.js";
+import { Group } from "../group.js";
+import { parseJSON } from "../jsonStringify.js";
+
+type MapOp<K extends string, V extends JsonValue | CoValue | undefined> = {
+    txID: TransactionID;
+    madeAt: number;
+    changeIdx: number;
+} & MapOpPayload<K, V>;
+// TODO: add after TransactionID[] for conflicts/ordering
+
+export type MapOpPayload<
+    K extends string,
+    V extends JsonValue | CoValue | undefined
+> =
+    | {
+          op: "set";
+          key: K;
+          value: V extends CoValue ? CoID<V> : Exclude<V, CoValue>;
+      }
+    | {
+          op: "del";
+          key: K;
+      };
+
+export class CoMapView<
+    Shape extends { [key: string]: JsonValue | CoValue | undefined },
+    Meta extends JsonObject | null = null
+> implements CoValue
+{
+    /** @category 6. Meta */
+    id: CoID<this>;
+    /** @category 6. Meta */
+    type = "comap" as const;
+    /** @category 6. Meta */
+    core: CoValueCore;
+    /** @internal */
+    ops: {
+        [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
+    };
+    /** @internal */
+    atTimeFilter?: number = undefined;
+    /** @category 6. Meta */
+    readonly _shape!: Shape;
+
+    /** @internal */
+    constructor(core: CoValueCore) {
+        this.id = core.id as CoID<this>;
+        this.core = core;
+        this.ops = {};
+
+        for (const {
+            txID,
+            changes,
+            madeAt,
+        } of core.getValidSortedTransactions()) {
+            for (const [changeIdx, changeUntyped] of parseJSON(
+                changes
+            ).entries()) {
+                const change = changeUntyped as MapOpPayload<
+                    keyof Shape & string,
+                    Shape[keyof Shape & string]
+                >;
+                let entries = this.ops[change.key];
+                if (!entries) {
+                    entries = [];
+                    this.ops[change.key] = entries;
+                }
+                entries.push({
+                    txID,
+                    madeAt,
+                    changeIdx,
+                    ...(change as MapOpPayload<
+                        keyof Shape & string,
+                        Shape[keyof Shape & string]
+                    >),
+                });
+            }
+        }
+    }
+
+    /** @category 6. Meta */
+    get meta(): Meta {
+        return this.core.header.meta as Meta;
+    }
+
+    /** @category 6. Meta */
+    get group(): Group {
+        return this.core.getGroup();
+    }
+
+    /** @category 4. Time travel */
+    atTime(time: number): this {
+        const clone = Object.create(this) as this;
+        clone.id = this.id;
+        clone.type = this.type;
+        clone.core = this.core;
+        clone.ops = this.ops;
+        clone.atTimeFilter = time;
+        return clone;
+    }
+
+    /** @internal */
+    timeFilteredOps<K extends keyof Shape & string>(
+        key: K
+    ): MapOp<K, Shape[K]>[] | undefined {
+        if (this.atTimeFilter) {
+            return this.ops[key]?.filter(
+                (op) => op.madeAt <= this.atTimeFilter!
+            );
+        } else {
+            return this.ops[key];
+        }
+    }
+
+    /**
+     * Get all keys currently in the map.
+     *
+     * @category 1. Reading */
+    keys(): (keyof Shape & string)[] {
+        const keys = Object.keys(this.ops) as (keyof Shape & string)[];
+
+        if (this.atTimeFilter) {
+            return keys.filter((key) => {
+                this.timeFilteredOps(key)?.length;
+            });
+        } else {
+            return keys;
+        }
+    }
+
+    /**
+     * Returns the current value for the given key.
+     *
+     * @category 1. Reading
+     **/
+    get<K extends keyof Shape & string>(
+        key: K
+    ):
+        | (Shape[K] extends CoValue
+              ? CoID<Shape[K]>
+              : Exclude<Shape[K], CoValue>)
+        | undefined {
+        const ops = this.timeFilteredOps(key);
+        if (!ops) {
+            return undefined;
+        }
+
+        const includeUntil = this.atTimeFilter;
+        const lastEntry = includeUntil
+            ? ops.findLast((entry) => entry.madeAt <= includeUntil)
+            : ops[ops.length - 1]!;
+
+        if (lastEntry?.op === "del") {
+            return undefined;
+        } else {
+            return lastEntry?.value;
+        }
+    }
+
+    /** @category 1. Reading */
+    asObject(): {
+        [K in keyof Shape & string]: Shape[K] extends CoValue
+            ? CoID<Shape[K]>
+            : Exclude<Shape[K], CoValue>;
+    } {
+        const object: Partial<{
+            [K in keyof Shape & string]: Shape[K] extends CoValue
+                ? CoID<Shape[K]>
+                : Exclude<Shape[K], CoValue>;
+        }> = {};
+
+        for (const key of this.keys()) {
+            const value = this.get(key);
+            if (value !== undefined) {
+                object[key] = value;
+            }
+        }
+
+        return object as {
+            [K in keyof Shape & string]: Shape[K] extends CoValue
+                ? CoID<Shape[K]>
+                : Exclude<Shape[K], CoValue>;
+        };
+    }
+
+    /** @category 1. Reading */
+    toJSON(): {
+        [K in keyof Shape & string]: Shape[K] extends CoValue
+            ? CoID<Shape[K]>
+            : Exclude<Shape[K], CoValue>;
+    } {
+        return this.asObject();
+    }
+
+    /** @category 5. Edit history */
+    nthEditAt<K extends keyof Shape & string>(
+        key: K,
+        n: number
+    ):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value?: Shape[K] extends CoValue
+                  ? CoID<Shape[K]>
+                  : Exclude<Shape[K], CoValue>;
+          }
+        | undefined {
+        const ops = this.timeFilteredOps(key);
+        if (!ops || ops.length <= n) {
+            return undefined;
+        }
+
+        const entry = ops[n]!;
+
+        if (this.atTimeFilter && entry.madeAt > this.atTimeFilter) {
+            return undefined;
+        }
+
+        return {
+            by: accountOrAgentIDfromSessionID(entry.txID.sessionID),
+            tx: entry.txID,
+            at: new Date(entry.madeAt),
+            value: entry.op === "del" ? undefined : entry.value,
+        };
+    }
+
+    /** @category 5. Edit history */
+    lastEditAt<K extends keyof Shape & string>(
+        key: K
+    ):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value?: Shape[K] extends CoValue
+                  ? CoID<Shape[K]>
+                  : Exclude<Shape[K], CoValue>;
+          }
+        | undefined {
+        const ops = this.timeFilteredOps(key);
+        if (!ops || ops.length === 0) {
+            return undefined;
+        }
+        return this.nthEditAt(key, ops.length - 1);
+    }
+
+    /** @category 5. Edit history */
+    *editsAt<K extends keyof Shape & string>(key: K) {
+        const ops = this.timeFilteredOps(key);
+        if (!ops) {
+            return;
+        }
+
+        for (let i = 0; i < ops.length; i++) {
+            yield this.nthEditAt(key, i)!;
+        }
+    }
+
+    /** @category 3. Subscription */
+    subscribe(listener: (coMap: this) => void): () => void {
+        return this.core.subscribe((content) => {
+            listener(content as this);
+        });
+    }
+}
+
+/** A collaborative map with precise shape `Shape` and optional static metadata `Meta` */
+export class CoMap<
+        Shape extends { [key: string]: JsonValue | CoValue | undefined },
+        Meta extends JsonObject | null = null
+    >
+    extends CoMapView<Shape, Meta>
+    implements CoValue
+{
+
+
+    /** Returns a new version of this CoMap with a new value for the given key.
+     *
+     * If `privacy` is `"private"` **(default)**, both `key` and `value` are encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, both `key` and `value` are stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Editing
+     **/
+    set<K extends keyof Shape & string>(
+        key: K,
+        value: Shape[K] extends CoValue ? Shape[K] | CoID<Shape[K]> : Shape[K],
+        privacy?: "private" | "trusting"
+    ): this;
+    set(
+        kv: {
+            [K in keyof Shape & string]?: Shape[K] extends CoValue
+                ? Shape[K] | CoID<Shape[K]>
+                : Shape[K];
+        },
+        privacy?: "private" | "trusting"
+    ): this;
+    set<K extends keyof Shape & string>(
+        ...args:
+            | [
+                  {
+                      [K in keyof Shape & string]?: Shape[K] extends CoValue
+                          ? Shape[K] | CoID<Shape[K]>
+                          : Shape[K];
+                  },
+                  ("private" | "trusting")?
+              ]
+            | [
+                  K,
+                  Shape[K] extends CoValue
+                      ? Shape[K] | CoID<Shape[K]>
+                      : Shape[K],
+                  ("private" | "trusting")?
+              ]
+    ): this {
+        if (typeof args[0] === "string") {
+            const [key, value, privacy = "private"] = args;
+            this.core.makeTransaction(
+                [
+                    {
+                        op: "set",
+                        key,
+                        value: isCoValue(value) ? value.id : value,
+                    },
+                ],
+                privacy
+            );
+        } else {
+            const [kv, privacy = "private"] = args as [
+                {
+                    [K in keyof Shape & string]: Shape[K] extends CoValue
+                        ? Shape[K] | CoID<Shape[K]>
+                        : Shape[K];
+                },
+                "private" | "trusting" | undefined
+            ];
+
+            for (const [key, value] of Object.entries(kv)) {
+                this.core.makeTransaction(
+                    [
+                        {
+                            op: "set",
+                            key,
+                            value: isCoValue(value) ? value.id : value,
+                        },
+                    ],
+                    privacy
+                );
+            }
+        }
+
+        return new CoMap(this.core) as this;
+    }
+
+    /** Returns a new version of this CoMap with the given key deleted (setting it to undefined).
+     *
+     * If `privacy` is `"private"` **(default)**, `key` is encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `key` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Editing
+     **/
+    delete(
+        key: keyof Shape & string,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        this.core.makeTransaction(
+            [
+                {
+                    op: "del",
+                    key,
+                },
+            ],
+            privacy
+        );
+
+        return new CoMap(this.core) as this;
+    }
+
+    /** @category 2. Editing */
+    mutate(mutator: (mutable: MutableCoMap<Shape, Meta>) => void): this {
+        const mutable = new MutableCoMap<Shape, Meta>(this.core);
+        mutator(mutable);
+        return new CoMap(this.core) as this;
+    }
+
+    /** @deprecated Use `mutate` instead. */
+    edit(mutator: (mutable: MutableCoMap<Shape, Meta>) => void): this {
+        return this.mutate(mutator);
+    }
+}
+
+export class MutableCoMap<
+        Shape extends { [key: string]: JsonValue | CoValue | undefined },
+        Meta extends JsonObject | null = null
+    >
+    extends CoMapView<Shape, Meta>
+    implements CoValue
+{
+    /** Sets a new value for the given key.
+     *
+     * If `privacy` is `"private"` **(default)**, both `key` and `value` are encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, both `key` and `value` are stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     *
+     * @category 2. Mutation
+     */
+    set<K extends keyof Shape & string>(
+        key: K,
+        value: Shape[K] extends CoValue ? Shape[K] | CoID<Shape[K]> : Shape[K],
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        // eslint-disable-next-line @typescript-eslint/ban-types
+        const after = (CoMap.prototype.set as Function).call(
+            this,
+            key,
+            value,
+            privacy
+        ) as CoMap<Shape, Meta>;
+        this.ops = after.ops;
+    }
+
+    /** Deletes the value for the given key (setting it to undefined).
+     *
+     * If `privacy` is `"private"` **(default)**, `key` is encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
+     *
+     * If `privacy` is `"trusting"`, `key` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers.
+     * @category 2. Mutation
+     */
+    delete(
+        key: keyof Shape & string,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const after = CoMap.prototype.delete.call(this, key, privacy) as CoMap<
+            Shape,
+            Meta
+        >;
+        this.ops = after.ops;
+    }
+}

packages/cojson/src/coValues/coStream.ts

@@ -0,0 +1,480 @@
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { CoValue, CoID, isCoValue } from "../coValue.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
+import { Group } from "../group.js";
+import { AgentID, SessionID, TransactionID } from "../ids.js";
+import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
+import { AccountID } from "../account.js";
+import { parseJSON } from "../jsonStringify.js";
+
+export type BinaryStreamInfo = {
+    mimeType: string;
+    fileName?: string;
+    totalSizeBytes?: number;
+};
+
+export type BinaryStreamStart = {
+    type: "start";
+} & BinaryStreamInfo;
+
+export type BinaryStreamChunk = {
+    type: "chunk";
+    chunk: `binary_U${string}`;
+};
+
+export type BinaryStreamEnd = {
+    type: "end";
+};
+
+export type BinaryCoStreamMeta = JsonObject & { type: "binary" };
+
+export type BinaryStreamItem =
+    | BinaryStreamStart
+    | BinaryStreamChunk
+    | BinaryStreamEnd;
+
+export type CoStreamItem<Item extends JsonValue | CoValue> = {
+    value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+    tx: TransactionID;
+    madeAt: number;
+};
+
+export class CoStreamView<
+    Item extends JsonValue | CoValue,
+    Meta extends JsonObject | null = null
+> implements CoValue
+{
+    id: CoID<this>;
+    type = "costream" as const;
+    core: CoValueCore;
+    items: {
+        [key: SessionID]: CoStreamItem<Item>[];
+    };
+    readonly _item!: Item;
+
+    constructor(core: CoValueCore) {
+        this.id = core.id as CoID<this>;
+        this.core = core;
+        this.items = {};
+        this.fillFromCoValue();
+    }
+
+    get meta(): Meta {
+        return this.core.header.meta as Meta;
+    }
+
+    get group(): Group {
+        return this.core.getGroup();
+    }
+
+    /** Not yet implemented */
+    atTime(_time: number): this {
+        throw new Error("Not yet implemented");
+    }
+
+    /** @internal */
+    protected fillFromCoValue() {
+        this.items = {};
+
+        for (const {
+            txID,
+            madeAt,
+            changes,
+        } of this.core.getValidSortedTransactions()) {
+            for (const changeUntyped of parseJSON(changes)) {
+                const change = changeUntyped as Item extends CoValue
+                    ? CoID<Item>
+                    : Exclude<Item, CoValue>;
+                let entries = this.items[txID.sessionID];
+                if (!entries) {
+                    entries = [];
+                    this.items[txID.sessionID] = entries;
+                }
+                entries.push({ value: change, madeAt, tx: txID });
+            }
+        }
+    }
+
+    getSingleStream():
+        | (Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>)[]
+        | undefined {
+        if (Object.keys(this.items).length === 0) {
+            return undefined;
+        } else if (Object.keys(this.items).length !== 1) {
+            throw new Error(
+                "CoStream.getSingleStream() can only be called when there is exactly one stream"
+            );
+        }
+
+        return Object.values(this.items)[0]?.map((item) => item.value);
+    }
+
+    sessions(): SessionID[] {
+        return Object.keys(this.items) as SessionID[];
+    }
+
+    accounts(): Set<AccountID | AgentID> {
+        return new Set(this.sessions().map(accountOrAgentIDfromSessionID));
+    }
+
+    nthItemIn(
+        sessionID: SessionID,
+        n: number
+    ):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+          }
+        | undefined {
+        const items = this.items[sessionID];
+        if (!items) return;
+
+        const item = items[n];
+        if (!item) return;
+
+        return {
+            by: accountOrAgentIDfromSessionID(sessionID),
+            tx: item.tx,
+            at: new Date(item.madeAt),
+            value: item.value,
+        };
+    }
+
+    lastItemIn(sessionID: SessionID):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+          }
+        | undefined {
+        const items = this.items[sessionID];
+        if (!items) return;
+        return this.nthItemIn(sessionID, items.length - 1);
+    }
+
+    *itemsIn(sessionID: SessionID) {
+        const items = this.items[sessionID];
+        if (!items) return;
+        for (const item of items) {
+            yield {
+                by: accountOrAgentIDfromSessionID(sessionID),
+                tx: item.tx,
+                at: new Date(item.madeAt),
+                value: item.value,
+            };
+        }
+    }
+
+    lastItemBy(account: AccountID | AgentID):
+        | {
+              by: AccountID | AgentID;
+              tx: TransactionID;
+              at: Date;
+              value: Item extends CoValue ? CoID<Item> : Exclude<Item, CoValue>;
+          }
+        | undefined {
+        let latestItem:
+            | {
+                  by: AccountID | AgentID;
+                  tx: TransactionID;
+                  at: Date;
+                  value: Item extends CoValue
+                      ? CoID<Item>
+                      : Exclude<Item, CoValue>;
+              }
+            | undefined;
+
+        for (const sessionID of Object.keys(this.items)) {
+            if (sessionID.startsWith(account)) {
+                const item = this.lastItemIn(sessionID as SessionID);
+                if (!item) continue;
+                if (!latestItem || item.at > latestItem.at) {
+                    latestItem = {
+                        by: item.by,
+                        tx: item.tx,
+                        at: item.at,
+                        value: item.value,
+                    };
+                }
+            }
+        }
+
+        return latestItem;
+    }
+
+    *itemsBy(account: AccountID | AgentID) {
+        // TODO: this can be made more lazy without a huge collect and sort
+        const items = [
+            ...Object.keys(this.items).flatMap((sessionID) =>
+                sessionID.startsWith(account)
+                    ? [...this.itemsIn(sessionID as SessionID)].map((item) => ({
+                          in: sessionID as SessionID,
+                          ...item,
+                      }))
+                    : []
+            ),
+        ];
+
+        items.sort((a, b) => a.at.getTime() - b.at.getTime());
+
+        for (const item of items) {
+            yield item;
+        }
+    }
+
+    toJSON(): {
+        [key: SessionID]: (Item extends CoValue
+            ? CoID<Item>
+            : Exclude<Item, CoValue>)[];
+    } {
+        return Object.fromEntries(
+            Object.entries(this.items).map(([sessionID, items]) => [
+                sessionID,
+                items.map((item) => item.value),
+            ])
+        );
+    }
+
+    subscribe(listener: (coStream: this) => void): () => void {
+        return this.core.subscribe((content) => {
+            listener(content as this);
+        });
+    }
+}
+
+export class CoStream<
+        Item extends JsonValue | CoValue,
+        Meta extends JsonObject | null = null
+    >
+    extends CoStreamView<Item, Meta>
+    implements CoValue
+{
+    push(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        this.core.makeTransaction([isCoValue(item) ? item.id : item], privacy);
+        return new CoStream(this.core) as this;
+    }
+
+    mutate(mutator: (mutable: MutableCoStream<Item, Meta>) => void): this {
+        const mutable = new MutableCoStream<Item, Meta>(this.core);
+        mutator(mutable);
+        return new CoStream(this.core) as this;
+    }
+
+    /** @deprecated Use `mutate` instead. */
+    edit(mutator: (mutable: MutableCoStream<Item, Meta>) => void): this {
+        return this.mutate(mutator);
+    }
+}
+
+export class MutableCoStream<
+        Item extends JsonValue | CoValue,
+        Meta extends JsonObject | null = null
+    >
+    extends CoStreamView<Item, Meta>
+    implements CoValue
+{
+    push(
+        item: Item extends CoValue ? Item | CoID<Item> : Item,
+        privacy: "private" | "trusting" = "private"
+    ) {
+        this.core.makeTransaction([isCoValue(item) ? item.id : item], privacy);
+        this.fillFromCoValue();
+    }
+}
+
+const binary_U_prefixLength = 8; // "binary_U".length;
+
+export class BinaryCoStreamView<
+        Meta extends BinaryCoStreamMeta = { type: "binary" }
+    >
+    extends CoStreamView<BinaryStreamItem, Meta>
+    implements CoValue
+{
+    id!: CoID<this>;
+
+    getBinaryChunks(
+        allowUnfinished?: boolean
+    ):
+        | (BinaryStreamInfo & { chunks: Uint8Array[]; finished: boolean })
+        | undefined {
+        // const before = performance.now();
+        const items = this.getSingleStream();
+
+        if (!items) return;
+
+        const start = items[0];
+
+        if (start?.type !== "start") {
+            console.error("Invalid binary stream start", start);
+            return;
+        }
+
+        const end = items[items.length - 1];
+
+        if (end?.type !== "end" && !allowUnfinished) return;
+
+        const chunks: Uint8Array[] = [];
+
+        let finished = false;
+        // let totalLength = 0;
+
+        for (const item of items.slice(1)) {
+            if (item.type === "end") {
+                finished = true;
+                break;
+            }
+
+            if (item.type !== "chunk") {
+                console.error("Invalid binary stream chunk", item);
+                return undefined;
+            }
+
+            const chunk = base64URLtoBytes(
+                item.chunk.slice(binary_U_prefixLength)
+            );
+            // totalLength += chunk.length;
+            chunks.push(chunk);
+        }
+
+        // const after = performance.now();
+        // console.log(
+        //     "getBinaryChunks bandwidth in MB/s",
+        //     (1000 * totalLength) / (after - before) / (1024 * 1024)
+        // );
+
+        return {
+            mimeType: start.mimeType,
+            fileName: start.fileName,
+            totalSizeBytes: start.totalSizeBytes,
+            chunks,
+            finished,
+        };
+    }
+}
+
+export class BinaryCoStream<
+        Meta extends BinaryCoStreamMeta = { type: "binary" }
+    >
+    extends BinaryCoStreamView<Meta>
+    implements CoValue
+{
+    /** @internal */
+    push(
+        item: BinaryStreamItem,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        this.core.makeTransaction([item], privacy);
+        return new BinaryCoStream(this.core) as this;
+    }
+
+    startBinaryStream(
+        settings: BinaryStreamInfo,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        return this.push(
+            {
+                type: "start",
+                ...settings,
+            } satisfies BinaryStreamStart,
+            privacy
+        );
+    }
+
+    pushBinaryStreamChunk(
+        chunk: Uint8Array,
+        privacy: "private" | "trusting" = "private"
+    ): this {
+        // const before = performance.now();
+        return this.push(
+            {
+                type: "chunk",
+                chunk: `binary_U${bytesToBase64url(chunk)}`,
+            } satisfies BinaryStreamChunk,
+            privacy
+        );
+        // const after = performance.now();
+        // console.log(
+        //     "pushBinaryStreamChunk bandwidth in MB/s",
+        //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        // );
+    }
+
+    endBinaryStream(privacy: "private" | "trusting" = "private"): this {
+        return this.push(
+            {
+                type: "end",
+            } satisfies BinaryStreamEnd,
+            privacy
+        );
+    }
+
+    mutate(mutator: (mutable: MutableBinaryCoStream<Meta>) => void): this {
+        const mutable = new MutableBinaryCoStream<Meta>(this.core);
+        mutator(mutable);
+        return new BinaryCoStream(this.core) as this;
+    }
+
+    /** @deprecated Use `mutate` instead. */
+    edit(mutator: (mutable: MutableBinaryCoStream<Meta>) => void): this {
+        return this.mutate(mutator);
+    }
+}
+
+export class MutableBinaryCoStream<
+        Meta extends BinaryCoStreamMeta = { type: "binary" }
+    >
+    extends BinaryCoStreamView<Meta>
+    implements CoValue
+{
+    /** @internal */
+    push(item: BinaryStreamItem, privacy: "private" | "trusting" = "private") {
+        MutableCoStream.prototype.push.call(this, item, privacy);
+    }
+
+    startBinaryStream(
+        settings: BinaryStreamInfo,
+        privacy: "private" | "trusting" = "private"
+    ) {
+        this.push(
+            {
+                type: "start",
+                ...settings,
+            } satisfies BinaryStreamStart,
+            privacy
+        );
+    }
+
+    pushBinaryStreamChunk(
+        chunk: Uint8Array,
+        privacy: "private" | "trusting" = "private"
+    ) {
+        // const before = performance.now();
+        this.push(
+            {
+                type: "chunk",
+                chunk: `binary_U${bytesToBase64url(chunk)}`,
+            } satisfies BinaryStreamChunk,
+            privacy
+        );
+        // const after = performance.now();
+        // console.log(
+        //     "pushBinaryStreamChunk bandwidth in MB/s",
+        //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        // );
+    }
+
+    endBinaryStream(privacy: "private" | "trusting" = "private") {
+        this.push(
+            {
+                type: "end",
+            } satisfies BinaryStreamEnd,
+            privacy
+        );
+    }
+}

packages/cojson/src/contentType.test.ts

@@ -1,175 +0,0 @@
-import { accountOrAgentIDfromSessionID } from "./coValue.js";
-import { createdNowUnique } from "./crypto.js";
-import { LocalNode } from "./node.js";
-import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
-
-test("Empty COJSON Map works", () => {
-    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
-
-    const coValue = node.createCoValue({
-        type: "comap",
-        ruleset: { type: "unsafeAllowAll" },
-        meta: null,
-        ...createdNowUnique(),
-    });
-
-    const content = coValue.getCurrentContent();
-
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    expect(content.type).toEqual("comap");
-    expect([...content.keys()]).toEqual([]);
-    expect(content.toJSON()).toEqual({});
-});
-
-test("Can insert and delete Map entries in edit()", () => {
-    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
-
-    const coValue = node.createCoValue({
-        type: "comap",
-        ruleset: { type: "unsafeAllowAll" },
-        meta: null,
-        ...createdNowUnique(),
-    });
-
-    const content = coValue.getCurrentContent();
-
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    expect(content.type).toEqual("comap");
-
-    content.edit((editable) => {
-        editable.set("hello", "world", "trusting");
-        expect(editable.get("hello")).toEqual("world");
-        editable.set("foo", "bar", "trusting");
-        expect(editable.get("foo")).toEqual("bar");
-        expect([...editable.keys()]).toEqual(["hello", "foo"]);
-        editable.delete("foo", "trusting");
-        expect(editable.get("foo")).toEqual(undefined);
-    });
-});
-
-test("Can get map entry values at different points in time", () => {
-    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
-
-    const coValue = node.createCoValue({
-        type: "comap",
-        ruleset: { type: "unsafeAllowAll" },
-        meta: null,
-        ...createdNowUnique(),
-    });
-
-    const content = coValue.getCurrentContent();
-
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    expect(content.type).toEqual("comap");
-
-    content.edit((editable) => {
-        const beforeA = Date.now();
-        while (Date.now() < beforeA + 10) {}
-        editable.set("hello", "A", "trusting");
-        const beforeB = Date.now();
-        while (Date.now() < beforeB + 10) {}
-        editable.set("hello", "B", "trusting");
-        const beforeC = Date.now();
-        while (Date.now() < beforeC + 10) {}
-        editable.set("hello", "C", "trusting");
-        expect(editable.get("hello")).toEqual("C");
-        expect(editable.getAtTime("hello", Date.now())).toEqual("C");
-        expect(editable.getAtTime("hello", beforeA)).toEqual(undefined);
-        expect(editable.getAtTime("hello", beforeB)).toEqual("A");
-        expect(editable.getAtTime("hello", beforeC)).toEqual("B");
-    });
-});
-
-test("Can get all historic values of key", () => {
-    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
-
-    const coValue = node.createCoValue({
-        type: "comap",
-        ruleset: { type: "unsafeAllowAll" },
-        meta: null,
-        ...createdNowUnique(),
-    });
-
-    const content = coValue.getCurrentContent();
-
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    expect(content.type).toEqual("comap");
-
-    content.edit((editable) => {
-        editable.set("hello", "A", "trusting");
-        const txA = editable.getLastTxID("hello");
-        editable.set("hello", "B", "trusting");
-        const txB = editable.getLastTxID("hello");
-        editable.delete("hello", "trusting");
-        const txDel = editable.getLastTxID("hello");
-        editable.set("hello", "C", "trusting");
-        const txC = editable.getLastTxID("hello");
-        expect(editable.getHistory("hello")).toEqual([
-            {
-                txID: txA,
-                value: "A",
-                at: txA && coValue.getTx(txA)?.madeAt,
-            },
-            {
-                txID: txB,
-                value: "B",
-                at: txB && coValue.getTx(txB)?.madeAt,
-            },
-            {
-                txID: txDel,
-                value: undefined,
-                at: txDel && coValue.getTx(txDel)?.madeAt,
-            },
-            {
-                txID: txC,
-                value: "C",
-                at: txC && coValue.getTx(txC)?.madeAt,
-            },
-        ]);
-    });
-});
-
-test("Can get last tx ID for a key", () => {
-    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
-
-    const coValue = node.createCoValue({
-        type: "comap",
-        ruleset: { type: "unsafeAllowAll" },
-        meta: null,
-        ...createdNowUnique(),
-    });
-
-    const content = coValue.getCurrentContent();
-
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    expect(content.type).toEqual("comap");
-
-    content.edit((editable) => {
-        expect(editable.getLastTxID("hello")).toEqual(undefined);
-        editable.set("hello", "A", "trusting");
-        const sessionID = editable.getLastTxID("hello")?.sessionID;
-        expect(sessionID && accountOrAgentIDfromSessionID(sessionID)).toEqual(
-            node.account.id
-        );
-        expect(editable.getLastTxID("hello")?.txIndex).toEqual(0);
-        editable.set("hello", "B", "trusting");
-        expect(editable.getLastTxID("hello")?.txIndex).toEqual(1);
-        editable.set("hello", "C", "trusting");
-        expect(editable.getLastTxID("hello")?.txIndex).toEqual(2);
-    });
-});

packages/cojson/src/contentType.ts

@@ -1,26 +0,0 @@
-import { JsonObject, JsonValue } from "./jsonValue.js";
-import { RawCoID } from "./ids.js";
-import { CoMap } from "./contentTypes/coMap.js";
-import { CoStream } from "./contentTypes/coStream.js";
-import { Static } from "./contentTypes/static.js";
-import { CoList } from "./contentTypes/coList.js";
-
-export type CoID<T extends ContentType> = RawCoID & {
-    readonly __type: T;
-};
-
-export type ContentType =
-    | CoMap<{ [key: string]: JsonValue }, JsonObject | null>
-    | CoList<JsonValue, JsonObject | null>
-    | CoStream<JsonValue, JsonObject | null>
-    | Static<JsonObject>;
-
-export function expectMap(
-    content: ContentType
-): CoMap<{ [key: string]: string }, JsonObject | null> {
-    if (content.type !== "comap") {
-        throw new Error("Expected map");
-    }
-
-    return content as CoMap<{ [key: string]: string }, JsonObject | null>;
-}

packages/cojson/src/contentTypes/coList.ts

@@ -1,24 +0,0 @@
-import { JsonObject, JsonValue } from '../jsonValue.js';
-import { CoID } from '../contentType.js';
-import { CoValue } from '../coValue.js';
-
-export class CoList<T extends JsonValue, Meta extends JsonObject | null = null> {
-    id: CoID<CoList<T, Meta>>;
-    type = "colist" as const;
-    coValue: CoValue;
-
-    constructor(coValue: CoValue) {
-        this.id = coValue.id as CoID<CoList<T, Meta>>;
-        this.coValue = coValue;
-    }
-
-    toJSON(): JsonObject {
-        throw new Error("Method not implemented.");
-    }
-
-    subscribe(listener: (coMap: CoList<T, Meta>) => void): () => void {
-        return this.coValue.subscribe((content) => {
-            listener(content as CoList<T, Meta>);
-        });
-    }
-}

packages/cojson/src/contentTypes/coMap.ts

@@ -1,221 +0,0 @@
-import { JsonObject, JsonValue } from '../jsonValue.js';
-import { TransactionID } from '../ids.js';
-import { CoID } from '../contentType.js';
-import { CoValue, accountOrAgentIDfromSessionID } from '../coValue.js';
-import { AccountID, isAccountID } from '../account.js';
-
-type MapOp<K extends string, V extends JsonValue> = {
-    txID: TransactionID;
-    madeAt: number;
-    changeIdx: number;
-} & MapOpPayload<K, V>;
-// TODO: add after TransactionID[] for conflicts/ordering
-
-export type MapOpPayload<K extends string, V extends JsonValue> = {
-    op: "set";
-    key: K;
-    value: V;
-} |
-{
-    op: "del";
-    key: K;
-};
-
-export type MapK<M extends { [key: string]: JsonValue; }> = keyof M & string;
-export type MapV<M extends { [key: string]: JsonValue; }> = M[MapK<M>];
-export type MapM<M extends { [key: string]: JsonValue; }> = {
-    [KK in MapK<M>]: M[KK];
-}
-
-export class CoMap<
-    M extends { [key: string]: JsonValue; },
-    Meta extends JsonObject | null = null,
-> {
-    id: CoID<CoMap<MapM<M>, Meta>>;
-    coValue: CoValue;
-    type = "comap" as const;
-    ops: {
-        [KK in MapK<M>]?: MapOp<KK, M[KK]>[];
-    };
-
-    constructor(coValue: CoValue) {
-        this.id = coValue.id as CoID<CoMap<MapM<M>, Meta>>;
-        this.coValue = coValue;
-        this.ops = {};
-
-        this.fillOpsFromCoValue();
-    }
-
-    protected fillOpsFromCoValue() {
-        this.ops = {};
-
-        for (const { txID, changes, madeAt } of this.coValue.getValidSortedTransactions()) {
-            for (const [changeIdx, changeUntyped] of (
-                changes
-            ).entries()) {
-                const change = changeUntyped as MapOpPayload<MapK<M>, MapV<M>>;
-                let entries = this.ops[change.key];
-                if (!entries) {
-                    entries = [];
-                    this.ops[change.key] = entries;
-                }
-                entries.push({
-                    txID,
-                    madeAt,
-                    changeIdx,
-                    ...(change as MapOpPayload<MapK<M>, MapV<M>>),
-                });
-            }
-        }
-    }
-
-    keys(): MapK<M>[] {
-        return Object.keys(this.ops) as MapK<M>[];
-    }
-
-    get<K extends MapK<M>>(key: K): M[K] | undefined {
-        const ops = this.ops[key];
-        if (!ops) {
-            return undefined;
-        }
-
-        const lastEntry = ops[ops.length - 1]!;
-
-        if (lastEntry.op === "del") {
-            return undefined;
-        } else {
-            return lastEntry.value;
-        }
-    }
-
-    getAtTime<K extends MapK<M>>(key: K, time: number): M[K] | undefined {
-        const ops = this.ops[key];
-        if (!ops) {
-            return undefined;
-        }
-
-        const lastOpBeforeOrAtTime = ops.findLast((op) => op.madeAt <= time);
-
-        if (!lastOpBeforeOrAtTime) {
-            return undefined;
-        }
-
-        if (lastOpBeforeOrAtTime.op === "del") {
-            return undefined;
-        } else {
-            return lastOpBeforeOrAtTime.value;
-        }
-    }
-
-    getLastEditor<K extends MapK<M>>(key: K): AccountID | undefined {
-        const tx  = this.getLastTxID(key);
-        if (!tx) {
-            return undefined;
-        }
-        const accountID = accountOrAgentIDfromSessionID(tx.sessionID);
-        if (isAccountID(accountID)) {
-            return accountID;
-        } else {
-            return undefined;
-        }
-    }
-
-    getLastTxID<K extends MapK<M>>(key: K): TransactionID | undefined {
-        const ops = this.ops[key];
-        if (!ops) {
-            return undefined;
-        }
-
-        const lastEntry = ops[ops.length - 1]!;
-
-        return lastEntry.txID;
-    }
-
-    getLastEntry<K extends MapK<M>>(key: K): { at: number; txID: TransactionID; value: M[K]; } | undefined {
-        const ops = this.ops[key];
-        if (!ops) {
-            return undefined;
-        }
-
-        const lastEntry = ops[ops.length - 1]!;
-
-        if (lastEntry.op === "del") {
-            return undefined;
-        } else {
-            return { at: lastEntry.madeAt, txID: lastEntry.txID, value: lastEntry.value };
-        }
-    }
-
-    getHistory<K extends MapK<M>>(key: K): { at: number; txID: TransactionID; value: M[K] | undefined; }[] {
-        const ops = this.ops[key];
-        if (!ops) {
-            return [];
-        }
-
-        const history: { at: number; txID: TransactionID; value: M[K] | undefined; }[] = [];
-
-        for (const op of ops) {
-            if (op.op === "del") {
-                history.push({ at: op.madeAt, txID: op.txID, value: undefined });
-            } else {
-                history.push({ at: op.madeAt, txID: op.txID, value: op.value });
-            }
-        }
-
-        return history;
-    }
-
-    toJSON(): JsonObject {
-        const json: JsonObject = {};
-
-        for (const key of this.keys()) {
-            const value = this.get(key);
-            if (value !== undefined) {
-                json[key] = value;
-            }
-        }
-
-        return json;
-    }
-
-    edit(changer: (editable: WriteableCoMap<M, Meta>) => void): CoMap<M, Meta> {
-        const editable = new WriteableCoMap<M, Meta>(this.coValue);
-        changer(editable);
-        return new CoMap(this.coValue);
-    }
-
-    subscribe(listener: (coMap: CoMap<M, Meta>) => void): () => void {
-        return this.coValue.subscribe((content) => {
-            listener(content as CoMap<M, Meta>);
-        });
-    }
-}
-
-export class WriteableCoMap<
-    M extends { [key: string]: JsonValue; },
-    Meta extends JsonObject | null = null,
-
-> extends CoMap<M, Meta> {
-    set<K extends MapK<M>>(key: K, value: M[K], privacy: "private" | "trusting" = "private"): void {
-        this.coValue.makeTransaction([
-            {
-                op: "set",
-                key,
-                value,
-            },
-        ], privacy);
-
-        this.fillOpsFromCoValue();
-    }
-
-    delete(key: MapK<M>, privacy: "private" | "trusting" = "private"): void {
-        this.coValue.makeTransaction([
-            {
-                op: "del",
-                key,
-            },
-        ], privacy);
-
-        this.fillOpsFromCoValue();
-    }
-}

packages/cojson/src/contentTypes/coStream.ts

@@ -1,24 +0,0 @@
-import { JsonObject, JsonValue } from '../jsonValue.js';
-import { CoID } from '../contentType.js';
-import { CoValue } from '../coValue.js';
-
-export class CoStream<T extends JsonValue, Meta extends JsonObject | null = null> {
-    id: CoID<CoStream<T, Meta>>;
-    type = "costream" as const;
-    coValue: CoValue;
-
-    constructor(coValue: CoValue) {
-        this.id = coValue.id as CoID<CoStream<T, Meta>>;
-        this.coValue = coValue;
-    }
-
-    toJSON(): JsonObject {
-        throw new Error("Method not implemented.");
-    }
-
-    subscribe(listener: (coMap: CoStream<T, Meta>) => void): () => void {
-        return this.coValue.subscribe((content) => {
-            listener(content as CoStream<T, Meta>);
-        });
-    }
-}

packages/cojson/src/contentTypes/static.ts

@@ -1,22 +0,0 @@
-import { JsonObject } from '../jsonValue.js';
-import { CoID } from '../contentType.js';
-import { CoValue } from '../coValue.js';
-
-export class Static<T extends JsonObject> {
-    id: CoID<Static<T>>;
-    type = "static" as const;
-    coValue: CoValue;
-
-    constructor(coValue: CoValue) {
-        this.id = coValue.id as CoID<Static<T>>;
-        this.coValue = coValue;
-    }
-
-    toJSON(): JsonObject {
-        throw new Error("Method not implemented.");
-    }
-
-    subscribe(_listener: (coMap: Static<T>) => void): () => void {
-        throw new Error("Method not implemented.");
-    }
-}