Publish

- jazz-example-pets@0.0.9
 - jazz-example-todo@0.0.34
 - cojson@0.2.3
 - cojson-simple-sync@0.2.6
 - cojson-storage-sqlite@0.2.6
 - jazz-browser@0.2.5
 - jazz-browser-auth-local@0.2.5
 - jazz-browser-media-images@0.2.5
 - jazz-react@0.2.5
 - jazz-react-auth-local@0.2.5
 - jazz-react-media-images@0.2.5
 - jazz-storage-indexeddb@0.2.5

.github/workflows/build-and-deploy.yaml

@@ -7,8 +7,11 @@ on:
     branches: [ "main" ]
 
 jobs:
-  build-and-deploy:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        example: ["todo", "pets"]
 
     steps:
     - uses: actions/checkout@v3
@@ -17,40 +20,50 @@ jobs:
 
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 16
         cache: 'yarn'
         cache-dependency-path: yarn.lock
-    - name: Nuke Workspace
-      run: |
-        rm package.json yarn.lock;
-    - name: Yarn Build
-      run: |
-        yarn install --frozen-lockfile;
-        yarn build;
-      working-directory: ./examples/todo
 
-    - uses: satackey/action-docker-layer-caching@v0.0.11
-      continue-on-error: true
-      with:
-        key: docker-layer-caching-${{ github.workflow }}-{hash}
-        restore-keys: |
-          docker-layer-caching-${{ github.workflow }}-
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry: ghcr.io
         username: gardencmp
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Docker Build & Push
+    - name: Nuke Workspace
       run: |
-        export DOCKER_TAG=ghcr.io/gardencmp/jazz-example-todo:${{github.head_ref || github.ref_name}}-${{github.sha}}-$(date +%s) ;
-        docker build . --file Dockerfile --tag $DOCKER_TAG;
-        docker push $DOCKER_TAG;
-        echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
-      working-directory: ./examples/todo
+        rm package.json yarn.lock;
 
+    - name: Yarn Build
+      run: |
+        yarn install --frozen-lockfile;
+        yarn build;
+      working-directory: ./examples/${{ matrix.example }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v4
+      with:
+        context: ./examples/${{ matrix.example }}
+        push: true
+        tags: ghcr.io/gardencmp/${{github.event.repository.name}}-example-${{ matrix.example }}:${{github.head_ref || github.ref_name}}-${{github.sha}}-${{github.run_number}}-${{github.run_attempt}}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        example: ["todo", "pets"]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
     - uses: gacts/install-nomad@v1
     - name: Tailscale
       uses: tailscale/github-action@v1
@@ -69,9 +82,9 @@ jobs:
 
         export DOCKER_USER=gardencmp;
         export DOCKER_PASSWORD=${{ secrets.DOCKER_PULL_PAT }};
-        export DOCKER_TAG=${{ env.DOCKER_TAG }};
+        export DOCKER_TAG=ghcr.io/gardencmp/${{github.event.repository.name}}-example-${{ matrix.example }}:${{github.head_ref || github.ref_name}}-${{github.sha}}-${{github.run_number}}-${{github.run_attempt}};
 
         envsubst '${DOCKER_USER} ${DOCKER_PASSWORD} ${DOCKER_TAG} ${BRANCH_SUFFIX} ${BRANCH_SUBDOMAIN}' < job-template.nomad > job-instance.nomad;
         cat job-instance.nomad;
         NOMAD_ADDR='http://control1v2-london:4646' nomad job run job-instance.nomad;
-      working-directory: ./examples/todo
+      working-directory: ./examples/${{ matrix.example }}

README.md

@@ -79,4 +79,13 @@ framework-agnostic primitives that allow you to use CoJSON in the browser. Used
 **`jazz-storage-indexeddb`**
 
 Provides local, offline-capable persistence. Included and enabled in `jazz-react` by default.
+
+**`jazz-react-media-images`**  → [DOCS](./DOCS.md#jazz-react-media-images)
+
+TODO: document
+
+**`jazz-browser-media-images`**  → [DOCS](./DOCS.md#jazz-browser-media-images)
+
+TODO: document
+
 </small>

examples/pets/README.md

@@ -1,14 +1,14 @@
-# Jazz Todo List Example
+# Jazz Rate-My-Pet List Example
 
-Live version: https://example-todo.jazz.tools
+Live version: https://example-pets.jazz.tools
 
 ## Installing & running the example locally
 
 Start by checking out just the example app to a folder:
 
 ```bash
-npx degit gardencmp/jazz/examples/todo jazz-example-todo
-cd jazz-example-todo
+npx degit gardencmp/jazz/examples/pets jazz-example-pets
+cd jazz-example-pets
 ```
 
 (This ensures that you have the example app without git history or our multi-package monorepo)
@@ -27,31 +27,17 @@ npm run dev
 
 ## Structure
 
-- [`src/basicComponents`](./src/basicComponents) contains simple components to build the UI, unrelated to Jazz (powered by [shadcn/ui](https://ui.shadcn.com))
-- [`src/components`](./src/components/) contains helper components that do contain Jazz-specific logic, but are not super relevant to understand the basics of Jazz and CoJSON
-- [`src/0_main.tsx`](./src/0_main.tsx), [`src/1_types.ts`](./src/1_types.ts), [`src/2_App.tsx`](./src/2_App.tsx), [`src/3_TodoTable.tsx`](./src/3_TodoTable.tsx), [`src/router.ts`](./src/router.ts) - the main files for this example, see the walkthrough below
+TODO
 
 ## Walkthrough
 
 ### Main parts
 
-- The top-level provider `<WithJazz/>`: [`src/0_main.tsx`](./src/0_main.tsx)
-
-- Defining the data model with CoJSON: [`src/1_types.ts`](./src/1_types.ts)
-
-- Creating todo projects & routing in `<App/>`: [`src/2_App.tsx`](./src/2_App.tsx)
-
-- Reactively rendering a todo project as a table, adding and editing tasks: [`src/3_TodoTable.tsx`](./src/3_TodoTable.tsx)
+TODO
 
 ### Helpers
 
-- Getting user profiles in `<NameBadge/>`: [`src/components/NameBadge.tsx`](./src/components/NameBadge.tsx)
-
-- (not yet commented) Creating invite links/QR codes with `<InviteButton/>`: [`src/components/InviteButton.tsx`](./src/components/InviteButton.tsx)
-
-- (not yet commented) `location.hash`-based routing and accepting invite links with `useSimpleHashRouterThatAcceptsInvites()` in [`src/router.ts`](./src/router.ts)
-
-This is the whole Todo List app!
+TODO
 
 ## Questions / problems / feedback
 

examples/pets/job-template.nomad

@@ -1,4 +1,4 @@
-job "example-todo$BRANCH_SUFFIX" {
+job "example-pets$BRANCH_SUFFIX" {
   region     = "global"
   datacenters = ["*"]
 
@@ -41,7 +41,7 @@ job "example-todo$BRANCH_SUFFIX" {
 
       service {
         tags = ["public"]
-        name = "example-todo$BRANCH_SUFFIX"
+        name = "example-pets$BRANCH_SUFFIX"
         port = "http"
         provider = "consul"
       }

examples/pets/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-pets",
   "private": true,
-  "version": "0.0.4",
+  "version": "0.0.9",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -16,9 +16,9 @@
     "@types/qrcode": "^1.5.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
-    "jazz-react": "^0.2.0",
-    "jazz-react-auth-local": "^0.2.0",
-    "jazz-react-media-images": "^0.2.0",
+    "jazz-react": "^0.2.5",
+    "jazz-react-auth-local": "^0.2.5",
+    "jazz-react-media-images": "^0.2.5",
     "lucide-react": "^0.274.0",
     "qrcode": "^1.5.3",
     "react": "^18.2.0",

examples/pets/src/1_types.ts

@@ -1,4 +1,4 @@
-import { CoMap, CoID, CoStream, Media } from "cojson";
+import { CoMap, CoStream, Media } from "cojson";
 
 /** Walkthrough: Defining the data model with CoJSON
  *
@@ -9,8 +9,8 @@ import { CoMap, CoID, CoStream, Media } from "cojson";
 
 export type PetPost = CoMap<{
     name: string;
-    image: CoID<Media.ImageDefinition>;
-    reactions: CoID<PetReactions>;
+    image: Media.ImageDefinition;
+    reactions: PetReactions;
 }>;
 
 export const REACTION_TYPES = [

examples/pets/src/3_CreatePetPostForm.tsx

@@ -1,7 +1,7 @@
 import { ChangeEvent, useCallback, useState } from "react";
 
-import { CoID } from "cojson";
-import { useJazz, useTelepathicState } from "jazz-react";
+import { CoID, CoMap, Media } from "cojson";
+import { useJazz, useTelepathicQuery } from "jazz-react";
 import { createImage } from "jazz-browser-media-images";
 
 import { PetPost, PetReactions } from "./1_types";
@@ -12,6 +12,12 @@ import { useLoadImage } from "jazz-react-media-images";
 /** Walkthrough: TODO
  */
 
+type PartialPetPost = CoMap<{
+    name: string;
+    image?: Media.ImageDefinition;
+    reactions: PetReactions;
+}>
+
 export function CreatePetPostForm({
     onCreate,
 }: {
@@ -19,30 +25,27 @@ export function CreatePetPostForm({
 }) {
     const { localNode } = useJazz();
 
-    const [newPostId, setNewPostId] = useState<CoID<PetPost> | undefined>(
+    const [newPostId, setNewPostId] = useState<CoID<PartialPetPost> | undefined>(
         undefined
     );
 
-    const newPetPost = useTelepathicState(newPostId);
+    const newPetPost = useTelepathicQuery(newPostId);
 
     const onChangeName = useCallback(
         (name: string) => {
-            let petPost = newPetPost;
-            if (!petPost) {
+            if (newPetPost) {
+                newPetPost.edit((petPost) => {
+                    petPost.set("name", name);
+                });
+            } else {
                 const petPostGroup = localNode.createGroup();
-                petPost = petPostGroup.createMap<PetPost>();
-                const petReactions = petPostGroup.createStream<PetReactions>();
-
-                petPost = petPost.edit((petPost) => {
-                    petPost.set("reactions", petReactions.id);
+                const petPost = petPostGroup.createMap<PartialPetPost>({
+                    name,
+                    reactions: petPostGroup.createStream<PetReactions>(),
                 });
 
                 setNewPostId(petPost.id);
             }
-
-            petPost.edit((petPost) => {
-                petPost.set("name", name);
-            });
         },
         [localNode, newPetPost]
     );
@@ -51,19 +54,19 @@ export function CreatePetPostForm({
         async (event: ChangeEvent<HTMLInputElement>) => {
             if (!newPetPost || !event.target.files) return;
 
-            const imageDefinition = await createImage(
+            const image = await createImage(
                 event.target.files[0],
                 newPetPost.group
             );
 
             newPetPost.edit((petPost) => {
-                petPost.set("image", imageDefinition.id);
+                petPost.set("image", image);
             });
         },
         [newPetPost]
     );
 
-    const petImage = useLoadImage(newPetPost?.get("image"));
+    const petImage = useLoadImage(newPetPost?.image?.id);
 
     return (
         <div className="flex flex-col gap-10">
@@ -73,7 +76,7 @@ export function CreatePetPostForm({
                 placeholder="Pet Name"
                 className="text-3xl py-6"
                 onChange={(event) => onChangeName(event.target.value)}
-                value={newPetPost?.get("name") || ""}
+                value={newPetPost?.name || ""}
             />
 
             {petImage ? (
@@ -84,15 +87,15 @@ export function CreatePetPostForm({
             ) : (
                 <Input
                     type="file"
-                    disabled={!newPetPost?.get("name")}
+                    disabled={!newPetPost?.name}
                     onChange={onImageSelected}
                 />
             )}
 
-            {newPetPost?.get("name") && newPetPost?.get("image") && (
+            {newPetPost?.name && newPetPost?.image && (
                 <Button
                     onClick={() => {
-                        onCreate(newPetPost.id);
+                        onCreate(newPetPost.id as CoID<PetPost>);
                     }}
                 >
                     Submit Post

examples/pets/src/4_RatePetPostUI.tsx

@@ -1,12 +1,12 @@
-import { AccountID, CoID } from "cojson";
-import { useTelepathicState } from "jazz-react";
+import { CoID, Queried } from "cojson";
+import { useTelepathicQuery } from "jazz-react";
 
-import { PetPost, PetReactions, ReactionType, REACTION_TYPES } from "./1_types";
+import { PetPost, ReactionType, REACTION_TYPES, PetReactions } from "./1_types";
 
 import { ShareButton } from "./components/ShareButton";
-import { NameBadge } from "./components/NameBadge";
-import { Button } from "./basicComponents";
+import { Button, Skeleton } from "./basicComponents";
 import { useLoadImage } from "jazz-react-media-images";
+import uniqolor from "uniqolor";
 
 /** Walkthrough: TODO
  */
@@ -21,14 +21,13 @@ const reactionEmojiMap: { [reaction in ReactionType]: string } = {
 };
 
 export function RatePetPostUI({ petPostID }: { petPostID: CoID<PetPost> }) {
-    const petPost = useTelepathicState(petPostID);
-    const petReactions = useTelepathicState(petPost?.get("reactions"));
-    const petImage = useLoadImage(petPost?.get("image"));
+    const petPost = useTelepathicQuery(petPostID);
+    const petImage = useLoadImage(petPost?.image);
 
     return (
         <div className="flex flex-col gap-8">
             <div className="flex justify-between">
-                <h1 className="text-3xl font-bold">{petPost?.get("name")}</h1>
+                <h1 className="text-3xl font-bold">{petPost?.name}</h1>
                 <ShareButton petPost={petPost} />
             </div>
 
@@ -44,12 +43,12 @@ export function RatePetPostUI({ petPostID }: { petPostID: CoID<PetPost> }) {
                     <Button
                         key={reactionType}
                         variant={
-                            petReactions?.getLastItemFromMe() === reactionType
+                            petPost?.reactions?.me?.last === reactionType
                                 ? "default"
                                 : "outline"
                         }
                         onClick={() => {
-                            petReactions?.edit((reactions) => {
+                            petPost?.reactions?.edit((reactions) => {
                                 reactions.push(reactionType);
                             });
                         }}
@@ -61,26 +60,28 @@ export function RatePetPostUI({ petPostID }: { petPostID: CoID<PetPost> }) {
                 ))}
             </div>
 
-            {petPost?.group.myRole() === "admin" && petReactions && (
-                <ReactionOverview petReactions={petReactions} />
+            {petPost?.group.myRole() === "admin" && petPost.reactions && (
+                <ReactionOverview petReactions={petPost.reactions} />
             )}
         </div>
     );
 }
 
-function ReactionOverview({ petReactions }: { petReactions: PetReactions }) {
+function ReactionOverview({
+    petReactions,
+}: {
+    petReactions: Queried<PetReactions>;
+}) {
     return (
         <div>
             <h2>Reactions</h2>
             <div className="flex flex-col gap-1">
                 {REACTION_TYPES.map((reactionType) => {
-                    const accountsWithThisReaction = Object.entries(
-                        petReactions.getLastItemsPerAccount()
-                    ).flatMap(([accountID, reaction]) =>
-                        reaction === reactionType ? [accountID] : []
-                    );
+                    const reactionsOfThisType = Object.values(
+                        petReactions.perAccount
+                    ).filter(({ last }) => last === reactionType);
 
-                    if (accountsWithThisReaction.length === 0) return null;
+                    if (reactionsOfThisType.length === 0) return null;
 
                     return (
                         <div
@@ -88,12 +89,18 @@ function ReactionOverview({ petReactions }: { petReactions: PetReactions }) {
                             key={reactionType}
                         >
                             {reactionEmojiMap[reactionType]}{" "}
-                            {accountsWithThisReaction.map((accountID) => (
-                                <NameBadge
-                                    key={accountID}
-                                    accountID={accountID as AccountID}
-                                />
-                            ))}
+                            {reactionsOfThisType.map((reaction) =>
+                                reaction.by?.profile?.name ? (
+                                    <span
+                                        className="rounded-full py-0.5 px-2 text-xs"
+                                        style={uniqueColoring(reaction.by.id)}
+                                    >
+                                        {reaction.by.profile.name}
+                                    </span>
+                                ) : (
+                                    <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
+                                )
+                            )}
                         </div>
                     );
                 })}
@@ -101,3 +108,12 @@ function ReactionOverview({ petReactions }: { petReactions: PetReactions }) {
         </div>
     );
 }
+
+function uniqueColoring(seed: string) {
+    const darkMode = window.matchMedia("(prefers-color-scheme: dark)").matches;
+
+    return {
+        color: uniqolor(seed, { lightness: darkMode ? 80 : 20 }).color,
+        background: uniqolor(seed, { lightness: darkMode ? 20 : 80 }).color,
+    };
+}

examples/pets/src/components/NameBadge.tsx

@@ -1,46 +0,0 @@
-import { AccountID } from "cojson";
-import { useProfile } from "jazz-react";
-
-import { Skeleton } from "@/basicComponents";
-import uniqolor from "uniqolor";
-
-/** Walkthrough: Getting user profiles in `<NameBadge/>`
- *
- *  `<NameBadge/>` uses `useProfile(accountID)`, which is a shorthand for
- *  useTelepathicState on an account's profile.
- *
- *  Profiles are always a `CoMap<{name: string}>`, but they might have app-specific
- *  additional properties).
- *
- *  In our case, we just display the profile name (which is set by the LocalAuth
- *  provider when we first create an account).
- */
-
-export function NameBadge({ accountID }: { accountID?: AccountID }) {
-    const profile = useProfile(accountID);
-
-    return accountID && profile?.get("name") ? (
-        <span
-            className="rounded-full py-0.5 px-2 text-xs"
-            style={randomUserColor(accountID)}
-        >
-            {profile.get("name")}
-        </span>
-    ) : (
-        <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
-    );
-}
-
-function randomUserColor(accountID: AccountID) {
-    const theme = window.matchMedia("(prefers-color-scheme: dark)").matches
-        ? "dark"
-        : "light";
-
-    const brightColor = uniqolor(accountID || "", { lightness: 80 }).color;
-    const darkColor = uniqolor(accountID || "", { lightness: 20 }).color;
-
-    return {
-        color: theme == "light" ? darkColor : brightColor,
-        background: theme == "light" ? brightColor : darkColor,
-    };
-}

examples/pets/src/components/ShareButton.tsx

@@ -6,8 +6,9 @@ import { createInviteLink } from "jazz-react";
 import QRCode from "qrcode";
 
 import { useToast, Button } from "../basicComponents";
+import { Queried } from "cojson";
 
-export function ShareButton({ petPost }: { petPost?: PetPost }) {
+export function ShareButton({ petPost }: { petPost?: Queried<PetPost> }) {
     const [existingInviteLink, setExistingInviteLink] = useState<string>();
     const { toast } = useToast();
 

examples/pets/src/router.ts

@@ -1,8 +1,8 @@
 import { useCallback, useEffect, useState } from "react";
-import { CoID, LocalNode, CoValueImpl } from "cojson";
+import { CoID, LocalNode, CoValue } from "cojson";
 import { consumeInviteLinkFromWindowLocation } from "jazz-react";
 
-export function useSimpleHashRouterThatAcceptsInvites<C extends CoValueImpl>(
+export function useSimpleHashRouterThatAcceptsInvites<C extends CoValue>(
     localNode: LocalNode
 ) {
     const [currentValueId, setCurrentValueId] = useState<CoID<C>>();

examples/todo/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-todo",
   "private": true,
-  "version": "0.0.29",
+  "version": "0.0.34",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -16,8 +16,8 @@
     "@types/qrcode": "^1.5.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
-    "jazz-react": "^0.2.0",
-    "jazz-react-auth-local": "^0.2.0",
+    "jazz-react": "^0.2.5",
+    "jazz-react-auth-local": "^0.2.5",
     "lucide-react": "^0.274.0",
     "qrcode": "^1.5.3",
     "react": "^18.2.0",

examples/todo/src/1_types.ts

@@ -1,4 +1,4 @@
-import { CoMap, CoList, CoID } from "cojson";
+import { CoMap, CoList } from "cojson";
 
 /** Walkthrough: Defining the data model with CoJSON
  *
@@ -16,13 +16,11 @@ import { CoMap, CoList, CoID } from "cojson";
 /** An individual task which collaborators can tick or rename */
 export type Task = CoMap<{ done: boolean; text: string; }>;
 
-/** A collaborative, ordered list of task references */
-export type ListOfTasks = CoList<CoID<Task>>;
-
 /** Our top level object: a project with a title, referencing a list of tasks */
 export type TodoProject = CoMap<{
     title: string;
-    tasks: CoID<ListOfTasks>;
+    /** A collaborative, ordered list of tasks */
+    tasks: CoList<Task>;
 }>;
 
 /** Walkthrough: Continue with ./2_App.tsx */

examples/todo/src/2_App.tsx

@@ -2,12 +2,13 @@ import { useCallback } from "react";
 
 import { useJazz } from "jazz-react";
 
-import { TodoProject, ListOfTasks } from "./1_types";
+import { Task, TodoProject } from "./1_types";
 
 import { SubmittableInput, Button } from "./basicComponents";
 
 import { useSimpleHashRouterThatAcceptsInvites } from "./router";
 import { TodoTable } from "./3_TodoTable";
+import { CoList } from "cojson";
 
 /** Walkthrough: Creating todo projects & routing in `<App/>`
  *
@@ -35,15 +36,10 @@ export default function App() {
             // of its members, which will apply to all CoValues owned by that group.
             const projectGroup = localNode.createGroup();
 
-            // Then we create an empty todo project and list of tasks within that group.
-            const project = projectGroup.createMap<TodoProject>();
-            const tasks = projectGroup.createList<ListOfTasks>();
-
-            // We edit the todo project to initialise it.
-            // Inside the `.edit` callback we can mutate a CoValue
-            project.edit((project) => {
-                project.set("title", title);
-                project.set("tasks", tasks.id);
+            // Then we create an empty todo project
+            const project = projectGroup.createMap<TodoProject>({
+                title,
+                tasks: projectGroup.createList<CoList<Task>>()
             });
 
             navigateToProjectId(project.id);

examples/todo/src/3_TodoTable.tsx

@@ -1,7 +1,7 @@
 import { useCallback } from "react";
 
-import { CoID } from "cojson";
-import { useTelepathicState } from "jazz-react";
+import { CoID, Queried } from "cojson";
+import { useTelepathicQuery } from "jazz-react";
 
 import { TodoProject, Task } from "./1_types";
 
@@ -18,7 +18,7 @@ import {
 } from "./basicComponents";
 
 import { InviteButton } from "./components/InviteButton";
-import { NameBadge } from "./components/NameBadge";
+import uniqolor from "uniqolor";
 
 /** Walkthrough: Reactively rendering a todo project as a table,
  *               adding and editing tasks
@@ -32,27 +32,24 @@ export function TodoTable({ projectId }: { projectId: CoID<TodoProject> }) {
     // `useTelepathicData()` reactively subscribes to updates to a CoValue's
     // content - whether we create edits locally, load persisted data, or receive
     // sync updates from other devices or participants!
-    const project = useTelepathicState(projectId);
-    const projectTasks = useTelepathicState(project?.get("tasks"));
+    const project = useTelepathicQuery(projectId);
 
     // `createTask` is similar to `createProject` we saw earlier, creating a new CoMap
     // for a new task (in the same group as the list of tasks/the project), and then
     // adding it as an item to the project's list of tasks.
     const createTask = useCallback(
         (text: string) => {
-            if (!projectTasks || !text) return;
-            const task = projectTasks.group.createMap<Task>();
-
-            task.edit((task) => {
-                task.set("text", text);
-                task.set("done", false);
+            if (!project?.tasks || !text) return;
+            const task = project?.group.createMap<Task>({
+                text,
+                done: false,
             });
 
-            projectTasks.edit((projectTasks) => {
-                projectTasks.push(task.id);
+            project?.tasks.edit((tasks) => {
+                tasks.push(task);
             });
         },
-        [projectTasks]
+        [project?.tasks, project?.group]
     );
 
     return (
@@ -60,11 +57,11 @@ export function TodoTable({ projectId }: { projectId: CoID<TodoProject> }) {
             <div className="flex justify-between items-center gap-4 mb-4">
                 <h1>
                     {
-                        // This is how we can access properties from the project,
+                        // This is how we can access properties from the project query,
                         // accounting for the fact that it might not be loaded yet
-                        project?.get("title") ? (
+                        project?.title ? (
                             <>
-                                {project.get("title")}{" "}
+                                {project.title}{" "}
                                 <span className="text-sm">({project.id})</span>
                             </>
                         ) : (
@@ -72,7 +69,7 @@ export function TodoTable({ projectId }: { projectId: CoID<TodoProject> }) {
                         )
                     }
                 </h1>
-                <InviteButton list={project} />
+                <InviteButton value={project} />
             </div>
             <Table>
                 <TableHeader>
@@ -82,14 +79,9 @@ export function TodoTable({ projectId }: { projectId: CoID<TodoProject> }) {
                     </TableRow>
                 </TableHeader>
                 <TableBody>
-                    {
-                        // Here, we iterate over the items of our `ListOfTasks`
-                        // and render a `<TaskRow>` for each.
-
-                        projectTasks?.map((taskId: CoID<Task>) => (
-                            <TaskRow key={taskId} taskId={taskId} />
-                        ))
-                    }
+                    {project?.tasks?.map(
+                        (task) => task && <TaskRow key={task.id} task={task} />
+                    )}
                     <NewTaskInputRow
                         createTask={createTask}
                         disabled={!project}
@@ -100,17 +92,13 @@ export function TodoTable({ projectId }: { projectId: CoID<TodoProject> }) {
     );
 }
 
-export function TaskRow({ taskId }: { taskId: CoID<Task> }) {
-    // `<TaskRow/>` uses `useTelepathicState()` as well, to granularly load and
-    // subscribe to changes for that particular task.
-    const task = useTelepathicState(taskId);
-
+export function TaskRow({ task }: { task: Queried<Task> | undefined }) {
     return (
         <TableRow>
             <TableCell>
                 <Checkbox
                     className="mt-1"
-                    checked={task?.get("done")}
+                    checked={task?.done}
                     onCheckedChange={(checked) => {
                         // (the only thing we let the user change is the "done" status)
                         task?.edit((task) => {
@@ -121,16 +109,23 @@ export function TaskRow({ taskId }: { taskId: CoID<Task> }) {
             </TableCell>
             <TableCell>
                 <div className="flex flex-row justify-between items-center gap-2">
-                    <span className={task?.get("done") ? "line-through" : ""}>
-                        {task?.get("text") || (
-                            <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
-                        )}
-                    </span>
-                    {/* We also use a `<NameBadge/>` helper component to render the name
-                        of the author of the task. We get the author by using the collaboration
-                        feature `whoEdited(key)` on our `Task` CoMap, which returns the accountID
-                        of the last account that changed a given key in the CoMap. */}
-                    <NameBadge accountID={task?.whoEdited("text")} />
+                    {task?.text ? (
+                        <span className={task?.done ? "line-through" : ""}>
+                            {task.text}
+                        </span>
+                    ) : (
+                        <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
+                    )}
+                    {task?.edits.text?.by?.profile?.name ? (
+                        <span
+                            className="rounded-full py-0.5 px-2 text-xs"
+                            style={uniqueColoring(task.edits.text.by.id)}
+                        >
+                            {task.edits.text.by.profile.name}
+                        </span>
+                    ) : (
+                        <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
+                    )}
                 </div>
             </TableCell>
         </TableRow>
@@ -160,3 +155,12 @@ function NewTaskInputRow({
         </TableRow>
     );
 }
+
+function uniqueColoring(seed: string) {
+    const darkMode = window.matchMedia("(prefers-color-scheme: dark)").matches;
+
+    return {
+        color: uniqolor(seed, { lightness: darkMode ? 80 : 20 }).color,
+        background: uniqolor(seed, { lightness: darkMode ? 20 : 80 }).color,
+    };
+}

examples/todo/src/components/InviteButton.tsx

@@ -1,27 +1,26 @@
 import { useState } from "react";
 
-import { TodoProject } from "../1_types";
-
 import { createInviteLink } from "jazz-react";
 import QRCode from "qrcode";
 
 import { useToast, Button } from "../basicComponents";
+import { CoValue, Queried } from "cojson";
 
-export function InviteButton({ list }: { list?: TodoProject }) {
+export function InviteButton<T extends CoValue>({ value }: { value: T | Queried<T> | undefined }) {
     const [existingInviteLink, setExistingInviteLink] = useState<string>();
     const { toast } = useToast();
 
     return (
-        list?.group.myRole() === "admin" && (
+        value?.group?.myRole() === "admin" && (
             <Button
                 size="sm"
                 className="py-0"
-                disabled={!list}
+                disabled={!value.group || !value.id}
                 variant="outline"
                 onClick={async () => {
                     let inviteLink = existingInviteLink;
-                    if (list && !inviteLink) {
-                        inviteLink = createInviteLink(list, "writer");
+                    if (value.group && value.id && !inviteLink) {
+                        inviteLink = createInviteLink(value, "writer");
                         setExistingInviteLink(inviteLink);
                     }
                     if (inviteLink) {

examples/todo/src/components/NameBadge.tsx

@@ -1,46 +0,0 @@
-import { AccountID } from "cojson";
-import { useProfile } from "jazz-react";
-
-import { Skeleton } from "@/basicComponents";
-import uniqolor from "uniqolor";
-
-/** Walkthrough: Getting user profiles in `<NameBadge/>`
- *
- *  `<NameBadge/>` uses `useProfile(accountID)`, which is a shorthand for
- *  useTelepathicState on an account's profile.
- *
- *  Profiles are always a `CoMap<{name: string}>`, but they might have app-specific
- *  additional properties).
- *
- *  In our case, we just display the profile name (which is set by the LocalAuth
- *  provider when we first create an account).
- */
-
-export function NameBadge({ accountID }: { accountID?: AccountID }) {
-    const profile = useProfile(accountID);
-
-    return accountID && profile?.get("name") ? (
-        <span
-            className="rounded-full py-0.5 px-2 text-xs"
-            style={randomUserColor(accountID)}
-        >
-            {profile.get("name")}
-        </span>
-    ) : (
-        <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
-    );
-}
-
-function randomUserColor(accountID: AccountID) {
-    const theme = window.matchMedia("(prefers-color-scheme: dark)").matches
-        ? "dark"
-        : "light";
-
-    const brightColor = uniqolor(accountID || "", { lightness: 80 }).color;
-    const darkColor = uniqolor(accountID || "", { lightness: 20 }).color;
-
-    return {
-        color: theme == "light" ? darkColor : brightColor,
-        background: theme == "light" ? brightColor : darkColor,
-    };
-}

examples/todo/src/router.ts

@@ -1,8 +1,8 @@
 import { useCallback, useEffect, useState } from "react";
-import { CoID, LocalNode, CoValueImpl } from "cojson";
+import { CoID, LocalNode, CoValue } from "cojson";
 import { consumeInviteLinkFromWindowLocation } from "jazz-react";
 
-export function useSimpleHashRouterThatAcceptsInvites<C extends CoValueImpl>(
+export function useSimpleHashRouterThatAcceptsInvites<C extends CoValue>(
     localNode: LocalNode
 ) {
     const [currentValueId, setCurrentValueId] = useState<CoID<C>>();

generateDocs.ts

@@ -11,11 +11,19 @@ async function main() {
         cojson: "index.ts",
         "jazz-react": "index.tsx",
         "jazz-browser": "index.ts",
+        "jazz-browser-media-images": "index.ts",
+        "jazz-react-media-images": "index.tsx",
     }).map(async ([packageName, entryPoint]) => {
         const app = await Application.bootstrapWithPlugins({
             entryPoints: [`packages/${packageName}/src/${entryPoint}`],
             tsconfig: `packages/${packageName}/tsconfig.json`,
             sort: ["required-first"],
+            groupOrder: [
+                "Functions",
+                "Classes",
+                "TypeAliases",
+                "Namespaces"
+            ]
         });
 
         const project = await app.convert();
@@ -61,7 +69,14 @@ async function main() {
                                               renderChildGroup(child, group)
                                           )
                                           .join("\n\n")
-                                    : "TODO: doc generator not implemented yet")
+                                    : child.kind === 4
+                                    ? child.groups
+                                          ?.map((group) =>
+                                              renderChildGroup(child, group)
+                                          )
+                                          .join("\n\n")
+                                    : "TODO: doc generator not implemented yet " +
+                                      child.kind)
                             );
                         })
                         .join("\n\n----\n\n");
@@ -129,6 +144,7 @@ async function main() {
             const isClass = child.kind === 128;
             const isTypeDef = child.kind === 2097152;
             const isInterface = child.kind === 256;
+            const isNamespace = child.kind === 4;
             const isFunction = !!child.signatures;
             return (
                 "```typescript\n" +
@@ -141,6 +157,8 @@ async function main() {
                         ? "function"
                         : isInterface
                         ? "interface"
+                        : isNamespace
+                        ? "namespace"
                         : ""
                 } ${child.name}` +
                 (child.typeParameters
@@ -156,7 +174,7 @@ async function main() {
                     ? " implements " +
                       child.implementedTypes.map(renderType).join(", ")
                     : "") +
-                (isClass || isInterface
+                (isClass || isInterface || isNamespace
                     ? " {...}"
                     : isTypeDef
                     ? ` = ${renderType(child.type)}`
@@ -184,20 +202,43 @@ async function main() {
                         )!;
 
                         if (member.kind === 2048 || member.kind === 512) {
-                            if (member.signatures?.every(sig => sig.comment?.modifierTags?.includes("@internal"))) {
-                                return ""
+                            if (
+                                member.signatures?.every((sig) =>
+                                    sig.comment?.modifierTags?.includes(
+                                        "@internal"
+                                    )
+                                )
+                            ) {
+                                return "";
                             } else {
-                                return documentConstructorOrMethod(member, child);
+                                return documentConstructorOrMethod(
+                                    member,
+                                    child
+                                );
                             }
                         } else if (
                             member.kind === 1024 ||
                             member.kind === 262144
                         ) {
-                            if (member.comment?.modifierTags?.includes("@internal")) {
-                                return ""
+                            if (
+                                member.comment?.modifierTags?.includes(
+                                    "@internal"
+                                )
+                            ) {
+                                return "";
                             } else {
                                 return documentProperty(member, child);
                             }
+                        } else if (member.kind === 2097152) {
+                            if (
+                                member.comment?.modifierTags?.includes(
+                                    "@internal"
+                                )
+                            ) {
+                                return "";
+                            } else {
+                                return documentProperty({...member, flags: {isStatic: true}}, child);
+                            }
                         } else {
                             return "Unknown member kind " + member.kind;
                         }
@@ -233,7 +274,14 @@ async function main() {
             } else if (t.type === "reflection") {
                 if (t.declaration.indexSignature) {
                     return (
-                        "{ [" +
+                        `{ ${t.declaration.children?t.declaration.children
+                            .map(
+                                (child) =>
+                                    `${child.name}${
+                                        child.flags.isOptional ? "?" : ""
+                                    }: ${renderType(child.type)}`
+                            )
+                            .join(", ") + ", " : ""}[` +
                         t.declaration.indexSignature?.parameters?.[0].name +
                         ": " +
                         renderType(
@@ -267,6 +315,8 @@ async function main() {
                 }
             } else if (t.type === "array") {
                 return renderType(t.elementType) + "[]";
+            } else if (t.type === "tuple") {
+                return `[${t.elements?.map(renderType).join(", ")}]`;
             } else if (t.type === "templateLiteral") {
                 const matchingNamedType = docs.children?.find(
                     (child) =>
@@ -296,7 +346,7 @@ async function main() {
                             return "AgentID";
                         }
                     } else {
-                        return "TEMPLATE_LITERAL";
+                        return "`" + t.head + t.tail.map(bit => "${" + renderType(bit[0]) + "}" + bit[1]).join("") + "`";
                     }
                 }
             } else {
@@ -418,9 +468,7 @@ async function main() {
                           member.inheritedFrom.name.split(".")[0] +
                           "</code>) "
                         : ""
-                } ${
-                    member.comment ? "" : "(undocumented)"
-                }</summary>\n\n` +
+                } ${member.comment ? "" : "(undocumented)"}</summary>\n\n` +
                 "```typescript\n" +
                 `${member.getSignature ? "get " : ""}${stem}.${member.name}${
                     member.getSignature ? "()" : ""

packages/cojson-simple-sync/package.json

@@ -4,7 +4,7 @@
     "types": "src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.2.0",
+    "version": "0.2.6",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@types/ws": "^8.5.5",
@@ -16,8 +16,8 @@
         "typescript": "5.0.2"
     },
     "dependencies": {
-        "cojson": "^0.2.0",
-        "cojson-storage-sqlite": "^0.2.0",
+        "cojson": "^0.2.3",
+        "cojson-storage-sqlite": "^0.2.6",
         "ws": "^8.13.0"
     },
     "scripts": {

packages/cojson-storage-sqlite/package.json

@@ -1,13 +1,13 @@
 {
   "name": "cojson-storage-sqlite",
   "type": "module",
-  "version": "0.2.0",
+  "version": "0.2.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
     "better-sqlite3": "^8.5.2",
-    "cojson": "^0.2.0",
+    "cojson": "^0.2.3",
     "typescript": "^5.1.6"
   },
   "scripts": {

packages/cojson-storage-sqlite/src/index.ts

@@ -4,8 +4,7 @@ import {
     Peer,
     CojsonInternalTypes,
     SessionID,
-    // CojsonInternalTypes,
-    // SessionID,
+    MAX_RECOMMENDED_TX_SIZE,
 } from "cojson";
 import {
     ReadableStream,
@@ -15,7 +14,6 @@ import {
 } from "isomorphic-streams";
 
 import Database, { Database as DatabaseT } from "better-sqlite3";
-import { RawCoID } from "cojson/dist/ids";
 
 type CoValueRow = {
     id: CojsonInternalTypes.RawCoID;
@@ -29,6 +27,7 @@ type SessionRow = {
     sessionID: SessionID;
     lastIdx: number;
     lastSignature: CojsonInternalTypes.Signature;
+    bytesSinceLastSignature?: number;
 };
 
 type StoredSessionRow = SessionRow & { rowID: number };
@@ -39,6 +38,12 @@ type TransactionRow = {
     tx: string;
 };
 
+type SignatureAfterRow = {
+    ses: number;
+    idx: number;
+    signature: CojsonInternalTypes.Signature;
+};
+
 export class SQLiteStorage {
     fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
     toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
@@ -60,7 +65,7 @@ export class SQLiteStorage {
                 done = result.done;
 
                 if (result.value) {
-                    this.handleSyncMessage(result.value);
+                    await this.handleSyncMessage(result.value);
                 }
             }
         })();
@@ -98,41 +103,99 @@ export class SQLiteStorage {
         const db = Database(filename);
         db.pragma("journal_mode = WAL");
 
-        db.prepare(
-            `CREATE TABLE IF NOT EXISTS transactions (
-                ses INTEGER,
-                idx INTEGER,
-                tx TEXT NOT NULL ,
-                PRIMARY KEY (ses, idx)
-            ) WITHOUT ROWID;`
-        ).run();
+        const oldVersion = (
+            db.pragma("user_version") as [{ user_version: number }]
+        )[0].user_version as number;
 
-        db.prepare(
-            `CREATE TABLE IF NOT EXISTS sessions (
-                rowID INTEGER PRIMARY KEY,
-                coValue INTEGER NOT NULL,
-                sessionID TEXT NOT NULL,
-                lastIdx INTEGER,
-                lastSignature TEXT,
-                UNIQUE (sessionID, coValue)
-            );`
-        ).run();
+        console.log("DB version", oldVersion);
 
-        db.prepare(
-            `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`
-        ).run();
+        if (oldVersion === 0) {
+            console.log("Migration 0 -> 1: Basic schema");
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS transactions (
+                    ses INTEGER,
+                    idx INTEGER,
+                    tx TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`
+            ).run();
 
-        db.prepare(
-            `CREATE TABLE IF NOT EXISTS coValues (
-                rowID INTEGER PRIMARY KEY,
-                id TEXT NOT NULL UNIQUE,
-                header TEXT NOT NULL UNIQUE
-            );`
-        ).run();
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS sessions (
+                    rowID INTEGER PRIMARY KEY,
+                    coValue INTEGER NOT NULL,
+                    sessionID TEXT NOT NULL,
+                    lastIdx INTEGER,
+                    lastSignature TEXT,
+                    UNIQUE (sessionID, coValue)
+                );`
+            ).run();
 
-        db.prepare(
-            `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`
-        ).run();
+            db.prepare(
+                `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`
+            ).run();
+
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS coValues (
+                    rowID INTEGER PRIMARY KEY,
+                    id TEXT NOT NULL UNIQUE,
+                    header TEXT NOT NULL UNIQUE
+                );`
+            ).run();
+
+            db.prepare(
+                `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`
+            ).run();
+
+            db.pragma("user_version = 1");
+            console.log("Migration 0 -> 1: Basic schema - done");
+        }
+
+        if (oldVersion <= 1) {
+            // fix embarrassing off-by-one error for transaction indices
+            console.log(
+                "Migration 1 -> 2: Fix off-by-one error for transaction indices"
+            );
+
+            const txs = db
+                .prepare(`SELECT * FROM transactions`)
+                .all() as TransactionRow[];
+
+            for (const tx of txs) {
+                db.prepare(
+                    `DELETE FROM transactions WHERE ses = ? AND idx = ?`
+                ).run(tx.ses, tx.idx);
+                tx.idx -= 1;
+                db.prepare(
+                    `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`
+                ).run(tx.ses, tx.idx, tx.tx);
+            }
+
+            db.pragma("user_version = 2");
+            console.log(
+                "Migration 1 -> 2: Fix off-by-one error for transaction indices - done"
+            );
+        }
+
+        if (oldVersion <= 2) {
+            console.log("Migration 2 -> 3: Add signatureAfter");
+
+            db.prepare(
+                `CREATE TABLE IF NOT EXISTS signatureAfter (
+                    ses INTEGER,
+                    idx INTEGER,
+                    signature TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`
+            ).run();
+
+            db.prepare(
+                `ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;`
+            ).run();
+
+            db.pragma("user_version = 3");
+            console.log("Migration 2 -> 3: Add signatureAfter - done");
+        }
 
         return new SQLiteStorage(db, fromLocalNode, toLocalNode);
     }
@@ -179,12 +242,14 @@ export class SQLiteStorage {
             | CojsonInternalTypes.CoValueHeader
             | undefined;
 
-        const newContent: CojsonInternalTypes.NewContentMessage = {
-            action: "content",
-            id: theirKnown.id,
-            header: theirKnown.header ? undefined : parsedHeader,
-            new: {},
-        };
+            const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
+                {
+                    action: "content",
+                    id: theirKnown.id,
+                    header: theirKnown.header ? undefined : parsedHeader,
+                    new: {},
+                },
+            ];
 
         for (const sessionRow of allOurSessions) {
             ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -196,25 +261,77 @@ export class SQLiteStorage {
                 const firstNewTxIdx =
                     theirKnown.sessions[sessionRow.sessionID] || 0;
 
+                const signaturesAndIdxs = this.db
+                    .prepare<[number, number]>(
+                        `SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?`
+                    )
+                    .all(sessionRow.rowID, firstNewTxIdx) as SignatureAfterRow[];
+
+                // console.log(
+                //     theirKnown.id,
+                //     "signaturesAndIdxs",
+                //     JSON.stringify(signaturesAndIdxs)
+                // );
+
                 const newTxInSession = this.db
                     .prepare<[number, number]>(
-                        `SELECT * FROM transactions WHERE ses = ? AND idx > ?`
+                        `SELECT * FROM transactions WHERE ses = ? AND idx >= ?`
                     )
                     .all(sessionRow.rowID, firstNewTxIdx) as TransactionRow[];
 
-                newContent.new[sessionRow.sessionID] = {
-                    after: firstNewTxIdx,
-                    lastSignature: sessionRow.lastSignature,
-                    newTransactions: newTxInSession.map((row) =>
-                        JSON.parse(row.tx)
-                    ),
-                };
+                let idx = firstNewTxIdx;
+
+                // console.log(
+                //     theirKnown.id,
+                //     "newTxInSession",
+                //     newTxInSession.length
+                // );
+
+                for (const tx of newTxInSession) {
+                    let sessionEntry =
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ];
+                    if (!sessionEntry) {
+                        sessionEntry = {
+                            after: idx,
+                            lastSignature: "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
+                            newTransactions: [],
+                        };
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ] = sessionEntry;
+                    }
+
+                    sessionEntry.newTransactions.push(JSON.parse(tx.tx));
+
+                    if (
+                        signaturesAndIdxs[0] &&
+                        idx === signaturesAndIdxs[0].idx
+                    ) {
+                        sessionEntry.lastSignature =
+                            signaturesAndIdxs[0].signature;
+                        signaturesAndIdxs.shift();
+                        newContentPieces.push({
+                            action: "content",
+                            id: theirKnown.id,
+                            new: {},
+                        });
+                    } else if (
+                        idx ===
+                        firstNewTxIdx + newTxInSession.length - 1
+                    ) {
+                        sessionEntry.lastSignature = sessionRow.lastSignature;
+                    }
+                    idx += 1;
+                }
             }
         }
 
         const dependedOnCoValues =
             parsedHeader?.ruleset.type === "group"
-                ? Object.values(newContent.new).flatMap((sessionEntry) =>
+                ? newContentPieces
+                .flatMap((piece) => Object.values(piece.new)).flatMap((sessionEntry) =>
                       sessionEntry.newTransactions.flatMap((tx) => {
                           if (tx.privacy !== "trusting") return [];
                           // TODO: avoid parsing here?
@@ -253,8 +370,15 @@ export class SQLiteStorage {
             asDependencyOf,
         });
 
-        if (newContent.header || Object.keys(newContent.new).length > 0) {
-            await this.toLocalNode.write(newContent);
+        const nonEmptyNewContentPieces = newContentPieces.filter(
+            (piece) => piece.header || Object.keys(piece.new).length > 0
+        );
+
+        // console.log(theirKnown.id, nonEmptyNewContentPieces);
+
+        for (const piece of nonEmptyNewContentPieces) {
+            await this.toLocalNode.write(piece);
+            await new Promise((resolve) => setTimeout(resolve, 0));
         }
     }
 
@@ -265,7 +389,9 @@ export class SQLiteStorage {
     async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
         let storedCoValueRowID = (
             this.db
-                .prepare<RawCoID>(`SELECT rowID FROM coValues WHERE id = ?`)
+                .prepare<CojsonInternalTypes.RawCoID>(
+                    `SELECT rowID FROM coValues WHERE id = ?`
+                )
                 .get(msg.id) as StoredCoValueRow | undefined
         )?.rowID;
 
@@ -284,7 +410,7 @@ export class SQLiteStorage {
             }
 
             storedCoValueRowID = this.db
-                .prepare<[RawCoID, string]>(
+                .prepare<[CojsonInternalTypes.RawCoID, string]>(
                     `INSERT INTO coValues (id, header) VALUES (?, ?)`
                 )
                 .run(msg.id, JSON.stringify(header)).lastInsertRowid as number;
@@ -326,37 +452,72 @@ export class SQLiteStorage {
                     const actuallyNewOffset =
                         (sessionRow?.lastIdx || 0) -
                         (msg.new[sessionID]?.after || 0);
+
                     const actuallyNewTransactions =
                         newTransactions.slice(actuallyNewOffset);
 
+                    let newBytesSinceLastSignature =
+                        (sessionRow?.bytesSinceLastSignature || 0) +
+                        actuallyNewTransactions.reduce(
+                            (sum, tx) =>
+                                sum +
+                                (tx.privacy === "private"
+                                    ? tx.encryptedChanges.length
+                                    : tx.changes.length),
+                            0
+                        );
+
+                    const newLastIdx =
+                        (sessionRow?.lastIdx || 0) +
+                        actuallyNewTransactions.length;
+
+                    let shouldWriteSignature = false;
+
+                    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+                        shouldWriteSignature = true;
+                        newBytesSinceLastSignature = 0;
+                    }
+
                     let nextIdx = sessionRow?.lastIdx || 0;
 
                     const sessionUpdate = {
                         coValue: storedCoValueRowID!,
                         sessionID: sessionID,
-                        lastIdx:
-                            (sessionRow?.lastIdx || 0) +
-                            actuallyNewTransactions.length,
+                        lastIdx: newLastIdx,
                         lastSignature: msg.new[sessionID]!.lastSignature,
+                        bytesSinceLastSignature: newBytesSinceLastSignature,
                     };
 
                     const upsertedSession = this.db
-                        .prepare<[number, string, number, string]>(
-                            `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature) VALUES (?, ?, ?, ?)
-                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature
+                        .prepare<[number, string, number, string, number]>(
+                            `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
                             RETURNING rowID`
                         )
                         .get(
                             sessionUpdate.coValue,
                             sessionUpdate.sessionID,
                             sessionUpdate.lastIdx,
-                            sessionUpdate.lastSignature
+                            sessionUpdate.lastSignature,
+                            sessionUpdate.bytesSinceLastSignature,
                         ) as { rowID: number };
 
                     const sessionRowID = upsertedSession.rowID;
 
+                    if (shouldWriteSignature) {
+                        this.db
+                            .prepare<[number, number, string]>(
+                                `INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)`
+                            )
+                            .run(
+                                sessionRowID,
+                                // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
+                                newLastIdx - 1,
+                                msg.new[sessionID]!.lastSignature
+                            );
+                    }
+
                     for (const newTransaction of actuallyNewTransactions) {
-                        nextIdx++;
                         this.db
                             .prepare<[number, number, string]>(
                                 `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`
@@ -366,6 +527,7 @@ export class SQLiteStorage {
                                 nextIdx,
                                 JSON.stringify(newTransaction)
                             );
+                        nextIdx++;
                     }
                 }
             }

packages/cojson/.eslintrc.cjs

@@ -9,6 +9,7 @@ module.exports = {
     parserOptions: {
         project: "./tsconfig.json",
     },
+    ignorePatterns: [".eslint.cjs", "**/tests/*"],
     root: true,
     rules: {
         "no-unused-vars": "off",

packages/cojson/package.json

@@ -5,7 +5,7 @@
     "types": "dist/index.d.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.2.0",
+    "version": "0.2.3",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@typescript-eslint/eslint-plugin": "^6.2.1",

packages/cojson/src/account.ts

@@ -32,7 +32,7 @@ export function accountHeaderForInitialAgentSecret(
     };
 }
 
-export class Account extends Group {
+export class AccountGroup extends Group {
     get id(): AccountID {
         return this.underlyingMap.id as AccountID;
     }
@@ -65,7 +65,7 @@ export interface GeneralizedControlledAccount {
 
 /** @hidden */
 export class ControlledAccount
-    extends Account
+    extends AccountGroup
     implements GeneralizedControlledAccount
 {
     agentSecret: AgentSecret;
@@ -136,10 +136,10 @@ export class AnonymousControlledAccount
     }
 }
 
-export type AccountContent = GroupContent & { profile: CoID<Profile> };
+export type AccountContent = { profile: Profile } & GroupContent;
 export type AccountMeta = { type: "account" };
-export type AccountMap = CoMap<AccountContent, AccountMeta>;
-export type AccountID = CoID<AccountMap>;
+export type Account = CoMap<AccountContent, AccountMeta>;
+export type AccountID = CoID<Account>;
 
 export function isAccountID(id: AccountID | AgentID): id is AccountID {
     return id.startsWith("co_");

packages/cojson/src/coValue.ts

@@ -1,17 +1,32 @@
 import { JsonObject, JsonValue } from "./jsonValue.js";
 import { RawCoID } from "./ids.js";
 import { CoMap } from "./coValues/coMap.js";
-import { BinaryCoStream, BinaryCoStreamMeta, CoStream } from "./coValues/coStream.js";
+import {
+    BinaryCoStream,
+    BinaryCoStreamMeta,
+    CoStream,
+} from "./coValues/coStream.js";
 import { Static } from "./coValues/static.js";
 import { CoList } from "./coValues/coList.js";
 import { CoValueCore } from "./coValueCore.js";
 import { Group } from "./group.js";
 
-export type CoID<T extends CoValueImpl> = RawCoID & {
+export type CoID<T extends CoValue> = RawCoID & {
     readonly __type: T;
 };
 
-export interface ReadableCoValue extends CoValue {
+export interface CoValue {
+    /** The `CoValue`'s (precisely typed) `CoID` */
+    id: CoID<this>;
+    core: CoValueCore;
+    /** Specifies which kind of `CoValue` this is */
+    type: string;
+    /** The `CoValue`'s (precisely typed) static metadata */
+    meta: JsonObject | null;
+    /** The `Group` this `CoValue` belongs to (determining permissions) */
+    group: Group;
+    /** Returns an immutable JSON presentation of this `CoValue` */
+    toJSON(): JsonValue;
     /** Lets you subscribe to future updates to this CoValue (whether made locally or by other users).
      *
      * Takes a listener function that will be called with the current state for each update.
@@ -19,42 +34,37 @@ export interface ReadableCoValue extends CoValue {
      * Returns an unsubscribe function.
      *
      * Used internally by `useTelepathicData()` for reactive updates on changes to a `CoValue`. */
-    subscribe(listener: (coValue: CoValueImpl) => void): () => void;
+    subscribe(listener: (coValue: this) => void): () => void;
     /** Lets you apply edits to a `CoValue`, inside the changer callback, which receives a `WriteableCoValue`.
      *
      *  A `WritableCoValue` has all the same methods as a `CoValue`, but all edits made to it (with its additional mutator methods)
      *  are reflected in it immediately - so it behaves mutably, whereas a `CoValue` is always immutable
      *  (you need to use `subscribe` to receive new versions of it). */
-    edit?:
-        | ((changer: (editable: WriteableCoValue) => void) => CoValueImpl)
-        | undefined;
+    edit?: ((changer: (editable: CoValue) => void) => this) | undefined;
 }
 
-export interface CoValue {
-    /** The `CoValue`'s (precisely typed) `CoID` */
-    id: CoID<CoValueImpl>;
-    core: CoValueCore;
-    /** Specifies which kind of `CoValue` this is */
-    type: CoValueImpl["type"];
-    /** The `CoValue`'s (precisely typed) static metadata */
-    meta: JsonObject | null;
-    /** The `Group` this `CoValue` belongs to (determining permissions) */
-    group: Group;
-    /** Returns an immutable JSON presentation of this `CoValue` */
-    toJSON(): JsonValue;
-}
+export type AnyCoMap = CoMap<
+    { [key: string]: JsonValue | CoValue | undefined },
+    JsonObject | null
+>;
 
-export interface WriteableCoValue extends CoValue {}
+export type AnyCoList = CoList<JsonValue | CoValue, JsonObject | null>;
 
-export type CoValueImpl =
-    | CoMap<{ [key: string]: JsonValue | undefined; }, JsonObject | null>
-    | CoList<JsonValue, JsonObject | null>
-    | CoStream<JsonValue, JsonObject | null>
-    | BinaryCoStream<BinaryCoStreamMeta>
-    | Static<JsonObject>;
+export type AnyCoStream = CoStream<JsonValue | CoValue, JsonObject | null>;
+
+export type AnyBinaryCoStream = BinaryCoStream<BinaryCoStreamMeta>;
+
+export type AnyStatic = Static<JsonObject>;
+
+export type AnyCoValue =
+    | AnyCoMap
+    | AnyCoList
+    | AnyCoStream
+    | AnyBinaryCoStream
+    | AnyStatic;
 
 export function expectMap(
-    content: CoValueImpl
+    content: CoValue
 ): CoMap<{ [key: string]: string }, JsonObject | null> {
     if (content.type !== "comap") {
         throw new Error("Expected map");
@@ -62,3 +72,19 @@ export function expectMap(
 
     return content as CoMap<{ [key: string]: string }, JsonObject | null>;
 }
+
+export function isCoValueImpl(
+    value: JsonValue | AnyCoValue | undefined
+): value is AnyCoValue {
+    return (
+        value instanceof CoMap ||
+        value instanceof CoList ||
+        value instanceof CoStream ||
+        value instanceof BinaryCoStream ||
+        value instanceof Static
+    );
+}
+
+export function isCoValue(value: JsonValue | CoValue | undefined) : value is CoValue {
+    return isCoValueImpl(value as AnyCoValue);
+}

packages/cojson/src/coValueCore.ts

@@ -1,5 +1,5 @@
 import { randomBytes } from "@noble/hashes/utils";
-import { CoValueImpl } from "./coValue.js";
+import { AnyCoValue } from "./coValue.js";
 import { Static } from "./coValues/static.js";
 import { BinaryCoStream, CoStream } from "./coValues/coStream.js";
 import { CoMap } from "./coValues/coMap.js";
@@ -14,7 +14,6 @@ import {
     sign,
     verify,
     encryptForTransaction,
-    decryptForTransaction,
     KeyID,
     decryptKeySecret,
     getAgentSignerID,
@@ -33,14 +32,13 @@ import { LocalNode } from "./node.js";
 import { CoValueKnownState, NewContentMessage } from "./sync.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
 import { CoList } from "./coValues/coList.js";
-import {
-    AccountID,
-    GeneralizedControlledAccount,
-} from "./account.js";
-import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
+import { AccountID, GeneralizedControlledAccount } from "./account.js";
+import { Stringified, stableStringify } from "./jsonStringify.js";
+
+export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
 
 export type CoValueHeader = {
-    type: CoValueImpl["type"];
+    type: AnyCoValue["type"];
     ruleset: RulesetDef;
     meta: JsonObject | null;
     createdAt: `2${string}` | null;
@@ -66,6 +64,7 @@ type SessionLog = {
     transactions: Transaction[];
     lastHash?: Hash;
     streamingHash: StreamingHash;
+    signatureAfter: { [txIdx: number]: Signature | undefined };
     lastSignature: Signature;
 };
 
@@ -79,7 +78,6 @@ export type PrivateTransaction = {
     >;
 };
 
-
 export type TrustingTransaction = {
     privacy: "trusting";
     madeAt: number;
@@ -101,9 +99,13 @@ export class CoValueCore {
     node: LocalNode;
     header: CoValueHeader;
     _sessions: { [key: SessionID]: SessionLog };
-    _cachedContent?: CoValueImpl;
-    listeners: Set<(content?: CoValueImpl) => void> = new Set();
-    _decryptionCache: {[key: Encrypted<JsonValue[], JsonValue>]: Stringified<JsonValue[]> | undefined} = {}
+    _cachedContent?: AnyCoValue;
+    listeners: Set<(content?: AnyCoValue) => void> = new Set();
+    _decryptionCache: {
+        [key: Encrypted<JsonValue[], JsonValue>]:
+            | Stringified<JsonValue[]>
+            | undefined;
+    } = {};
 
     constructor(
         header: CoValueHeader,
@@ -212,7 +214,8 @@ export class CoValueCore {
         // const beforeVerify = performance.now();
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
-                "Invalid signature",
+                "Invalid signature in",
+                this.id,
                 newSignature,
                 expectedNewHash,
                 signerID
@@ -225,25 +228,13 @@ export class CoValueCore {
         //     afterVerify - beforeVerify
         // );
 
-        const transactions = this.sessions[sessionID]?.transactions ?? [];
-
-        transactions.push(...newTransactions);
-
-        this._sessions[sessionID] = {
-            transactions,
-            lastHash: expectedNewHash,
-            streamingHash: newStreamingHash,
-            lastSignature: newSignature,
-        };
-
-        this._cachedContent = undefined;
-
-        if (this.listeners.size > 0) {
-            const content = this.getCurrentContent();
-            for (const listener of this.listeners) {
-                listener(content);
-            }
-        }
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
 
         return true;
     }
@@ -272,10 +263,8 @@ export class CoValueCore {
         const nTxBefore = this.sessions[sessionID]?.transactions.length ?? 0;
 
         // const beforeHash = performance.now();
-        const { expectedNewHash, newStreamingHash } = await this.expectedNewHashAfterAsync(
-            sessionID,
-            newTransactions
-        );
+        const { expectedNewHash, newStreamingHash } =
+            await this.expectedNewHashAfterAsync(sessionID, newTransactions);
         // const afterHash = performance.now();
         // console.log(
         //     "Hashing took",
@@ -286,7 +275,7 @@ export class CoValueCore {
 
         if (nTxAfter !== nTxBefore) {
             const newTransactionLengthBefore = newTransactions.length;
-            newTransactions = newTransactions.slice((nTxAfter - nTxBefore));
+            newTransactions = newTransactions.slice(nTxAfter - nTxBefore);
             console.warn("Transactions changed while async hashing", {
                 nTxBefore,
                 nTxAfter,
@@ -306,7 +295,8 @@ export class CoValueCore {
         // const beforeVerify = performance.now();
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
-                "Invalid signature",
+                "Invalid signature in",
+                this.id,
                 newSignature,
                 expectedNewHash,
                 signerID
@@ -319,15 +309,61 @@ export class CoValueCore {
         //     afterVerify - beforeVerify
         // );
 
-        const transactions = this.sessions[sessionID]?.transactions ?? [];
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
 
+        return true;
+    }
+
+    private doAddTransactions(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        newSignature: Signature,
+        expectedNewHash: Hash,
+        newStreamingHash: StreamingHash
+    ) {
+        const transactions = this.sessions[sessionID]?.transactions ?? [];
         transactions.push(...newTransactions);
 
+        const signatureAfter = this.sessions[sessionID]?.signatureAfter ?? {};
+
+        const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
+            (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
+            -1
+        );
+
+        const sizeOfTxsSinceLastInbetweenSignature = transactions
+            .slice(lastInbetweenSignatureIdx + 1)
+            .reduce(
+                (sum, tx) =>
+                    sum +
+                    (tx.privacy === "private"
+                        ? tx.encryptedChanges.length
+                        : tx.changes.length),
+                0
+            );
+
+        if (sizeOfTxsSinceLastInbetweenSignature > 100 * 1024) {
+            // console.log(
+            //     "Saving inbetween signature for tx ",
+            //     sessionID,
+            //     transactions.length - 1,
+            //     sizeOfTxsSinceLastInbetweenSignature
+            // );
+            signatureAfter[transactions.length - 1] = newSignature;
+        }
+
         this._sessions[sessionID] = {
             transactions,
             lastHash: expectedNewHash,
             streamingHash: newStreamingHash,
             lastSignature: newSignature,
+            signatureAfter: signatureAfter,
         };
 
         this._cachedContent = undefined;
@@ -338,11 +374,9 @@ export class CoValueCore {
                 listener(content);
             }
         }
-
-        return true;
     }
 
-    subscribe(listener: (content?: CoValueImpl) => void): () => void {
+    subscribe(listener: (content?: AnyCoValue) => void): () => void {
         this.listeners.add(listener);
         listener(this.getCurrentContent());
 
@@ -379,10 +413,10 @@ export class CoValueCore {
             new StreamingHash();
         let before = performance.now();
         for (const transaction of newTransactions) {
-            streamingHash.update(transaction)
+            streamingHash.update(transaction);
             const after = performance.now();
             if (after - before > 1) {
-                console.log("Hashing blocked for", after - before);
+                // console.log("Hashing blocked for", after - before);
                 await new Promise((resolve) => setTimeout(resolve, 0));
                 before = performance.now();
             }
@@ -459,7 +493,7 @@ export class CoValueCore {
         return success;
     }
 
-    getCurrentContent(): CoValueImpl {
+    getCurrentContent(): AnyCoValue {
         if (this._cachedContent) {
             return this._cachedContent;
         }
@@ -500,7 +534,8 @@ export class CoValueCore {
                     if (!readKey) {
                         return undefined;
                     } else {
-                        let decrytedChanges = this._decryptionCache[tx.encryptedChanges];
+                        let decrytedChanges =
+                            this._decryptionCache[tx.encryptedChanges];
 
                         if (!decrytedChanges) {
                             decrytedChanges = decryptRawForTransaction(
@@ -511,7 +546,8 @@ export class CoValueCore {
                                     tx: txID,
                                 }
                             );
-                            this._decryptionCache[tx.encryptedChanges] = decrytedChanges;
+                            this._decryptionCache[tx.encryptedChanges] =
+                                decrytedChanges;
                         }
 
                         if (!decrytedChanges) {
@@ -683,47 +719,95 @@ export class CoValueCore {
 
     newContentSince(
         knownState: CoValueKnownState | undefined
-    ): NewContentMessage | undefined {
-        const newContent: NewContentMessage = {
+    ): NewContentMessage[] | undefined {
+        let currentPiece: NewContentMessage = {
             action: "content",
             id: this.id,
             header: knownState?.header ? undefined : this.header,
-            new: Object.fromEntries(
-                Object.entries(this.sessions)
-                    .map(([sessionID, log]) => {
-                        const newTransactions = log.transactions.slice(
-                            knownState?.sessions[sessionID as SessionID] || 0
-                        );
-
-                        if (
-                            newTransactions.length === 0 ||
-                            !log.lastHash ||
-                            !log.lastSignature
-                        ) {
-                            return undefined;
-                        }
-
-                        return [
-                            sessionID,
-                            {
-                                after:
-                                    knownState?.sessions[
-                                        sessionID as SessionID
-                                    ] || 0,
-                                newTransactions,
-                                lastSignature: log.lastSignature,
-                            },
-                        ];
-                    })
-                    .filter((x): x is Exclude<typeof x, undefined> => !!x)
-            ),
+            new: {},
         };
 
-        if (!newContent.header && Object.keys(newContent.new).length === 0) {
+        const pieces = [currentPiece];
+
+        const sentState: CoValueKnownState["sessions"] = {
+            ...knownState?.sessions,
+        };
+
+        let newTxsWereAdded = true;
+        let pieceSize = 0;
+        while (newTxsWereAdded) {
+            newTxsWereAdded = false;
+
+            for (const [sessionID, log] of Object.entries(this.sessions) as [
+                SessionID,
+                SessionLog
+            ][]) {
+                const nextKnownSignatureIdx = Object.keys(log.signatureAfter)
+                    .map(Number)
+                    .sort((a, b) => a - b)
+                    .find((idx) => idx >= (sentState[sessionID] ?? -1));
+
+                const txsToAdd = log.transactions.slice(
+                    sentState[sessionID] ?? 0,
+                    nextKnownSignatureIdx === undefined
+                        ? undefined
+                        : nextKnownSignatureIdx + 1
+                );
+
+                if (txsToAdd.length === 0) continue;
+
+                newTxsWereAdded = true;
+
+                const oldPieceSize = pieceSize;
+                pieceSize += txsToAdd.reduce(
+                    (sum, tx) =>
+                        sum +
+                        (tx.privacy === "private"
+                            ? tx.encryptedChanges.length
+                            : tx.changes.length),
+                    0
+                );
+
+                if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+                    currentPiece = {
+                        action: "content",
+                        id: this.id,
+                        header: undefined,
+                        new: {},
+                    };
+                    pieces.push(currentPiece);
+                    pieceSize = pieceSize - oldPieceSize;
+                }
+
+                let sessionEntry = currentPiece.new[sessionID];
+                if (!sessionEntry) {
+                    sessionEntry = {
+                        after: sentState[sessionID] ?? 0,
+                        newTransactions: [],
+                        lastSignature: "WILL_BE_REPLACED" as Signature
+                    };
+                    currentPiece.new[sessionID] = sessionEntry;
+                }
+
+                sessionEntry.newTransactions.push(...txsToAdd);
+                sessionEntry.lastSignature = nextKnownSignatureIdx === undefined
+                ? log.lastSignature!
+                : log.signatureAfter[nextKnownSignatureIdx]!
+
+                sentState[sessionID] =
+                    (sentState[sessionID] || 0) + txsToAdd.length;
+            }
+        }
+
+        const piecesWithContent = pieces.filter(
+            (piece) => Object.keys(piece.new).length > 0 || piece.header
+        );
+
+        if (piecesWithContent.length === 0) {
             return undefined;
         }
 
-        return newContent;
+        return piecesWithContent;
     }
 
     getDependedOnCoValues(): RawCoID[] {

packages/cojson/src/coValues/coList.ts

@@ -1,5 +1,5 @@
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { CoID, ReadableCoValue, WriteableCoValue } from "../coValue.js";
+import { CoID, CoValue, isCoValue } from "../coValue.js";
 import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { SessionID, TransactionID } from "../ids.js";
 import { Group } from "../group.js";
@@ -8,15 +8,15 @@ import { parseJSON } from "../jsonStringify.js";
 
 type OpID = TransactionID & { changeIdx: number };
 
-type InsertionOpPayload<T extends JsonValue> =
+type InsertionOpPayload<T extends JsonValue | CoValue> =
     | {
           op: "pre";
-          value: T;
+          value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
           before: OpID | "end";
       }
     | {
           op: "app";
-          value: T;
+          value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
           after: OpID | "start";
       };
 
@@ -25,11 +25,11 @@ type DeletionOpPayload = {
     insertion: OpID;
 };
 
-export type ListOpPayload<T extends JsonValue> =
+export type ListOpPayload<T extends JsonValue | CoValue> =
     | InsertionOpPayload<T>
     | DeletionOpPayload;
 
-type InsertionEntry<T extends JsonValue> = {
+type InsertionEntry<T extends JsonValue | CoValue> = {
     madeAt: number;
     predecessors: OpID[];
     successors: OpID[];
@@ -40,10 +40,12 @@ type DeletionEntry = {
     deletionID: OpID;
 } & DeletionOpPayload;
 
-export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
-    implements ReadableCoValue
+export class CoList<
+    T extends JsonValue | CoValue,
+    Meta extends JsonObject | null = null
+> implements CoValue
 {
-    id: CoID<CoList<T, Meta>>;
+    id: CoID<this>;
     type = "colist" as const;
     core: CoValueCore;
     /** @internal */
@@ -69,7 +71,7 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
 
     /** @internal */
     constructor(core: CoValueCore) {
-        this.id = core.id as CoID<CoList<T, Meta>>;
+        this.id = core.id as CoID<this>;
         this.core = core;
         this.afterStart = [];
         this.beforeEnd = [];
@@ -99,7 +101,9 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
             changes,
             madeAt,
         } of this.core.getValidSortedTransactions()) {
-            for (const [changeIdx, changeUntyped] of parseJSON(changes).entries()) {
+            for (const [changeIdx, changeUntyped] of parseJSON(
+                changes
+            ).entries()) {
                 const change = changeUntyped as ListOpPayload<T>;
 
                 if (change.op === "pre" || change.op === "app") {
@@ -201,7 +205,9 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
     }
 
     /** Get the item currently at `idx`. */
-    get(idx: number): T | undefined {
+    get(
+        idx: number
+    ): (T extends CoValue ? CoID<T> : Exclude<T, CoValue>) | undefined {
         const entry = this.entries()[idx];
         if (!entry) {
             return undefined;
@@ -210,12 +216,20 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
     }
 
     /** Returns the current items in the CoList as an array. */
-    asArray(): T[] {
+    asArray(): (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)[] {
         return this.entries().map((entry) => entry.value);
     }
 
-    entries(): { value: T; madeAt: number; opID: OpID }[] {
-        const arr: { value: T; madeAt: number; opID: OpID }[] = [];
+    entries(): {
+        value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+        madeAt: number;
+        opID: OpID;
+    }[] {
+        const arr: {
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+            madeAt: number;
+            opID: OpID;
+        }[] = [];
         for (const opID of this.afterStart) {
             this.fillArrayFromOpID(opID, arr);
         }
@@ -228,7 +242,11 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
     /** @internal */
     private fillArrayFromOpID(
         opID: OpID,
-        arr: { value: T; madeAt: number; opID: OpID }[]
+        arr: {
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+            madeAt: number;
+            opID: OpID;
+        }[]
     ) {
         const entry =
             this.insertions[opID.sessionID]?.[opID.txIndex]?.[opID.changeIdx];
@@ -269,23 +287,42 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
     }
 
     /** Returns the current items in the CoList as an array. (alias of `asArray`) */
-    toJSON(): T[] {
+    toJSON(): (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)[] {
         return this.asArray();
     }
 
-    map<U>(mapper: (value: T, idx: number) => U): U[] {
+    map<U>(
+        mapper: (
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>,
+            idx: number
+        ) => U
+    ): U[] {
         return this.entries().map((entry, idx) => mapper(entry.value, idx));
     }
 
-    filter<U extends T>(predicate: (value: T, idx: number) => value is U): U[];
-    filter(predicate: (value: T, idx: number) => boolean): T[] {
+    filter<U extends T extends CoValue ? CoID<T> : Exclude<T, CoValue>>(
+        predicate: (
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>,
+            idx: number
+        ) => value is U
+    ): U[];
+    filter(
+        predicate: (
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>,
+            idx: number
+        ) => boolean
+    ): (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)[] {
         return this.entries()
             .filter((entry, idx) => predicate(entry.value, idx))
             .map((entry) => entry.value);
     }
 
     reduce<U>(
-        reducer: (accumulator: U, value: T, idx: number) => U,
+        reducer: (
+            accumulator: U,
+            value: T extends CoValue ? CoID<T> : Exclude<T, CoValue>,
+            idx: number
+        ) => U,
         initialValue: U
     ): U {
         return this.entries().reduce(
@@ -294,32 +331,28 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
         );
     }
 
-    subscribe(listener: (coMap: CoList<T, Meta>) => void): () => void {
+    subscribe(listener: (coList: this) => void): () => void {
         return this.core.subscribe((content) => {
-            listener(content as CoList<T, Meta>);
+            listener(content as this);
         });
     }
 
-    edit(
-        changer: (editable: WriteableCoList<T, Meta>) => void
-    ): CoList<T, Meta> {
+    edit(changer: (editable: WriteableCoList<T, Meta>) => void): this {
         const editable = new WriteableCoList<T, Meta>(this.core);
         changer(editable);
-        return new CoList(this.core);
+        return new CoList(this.core) as this;
     }
 }
 
 export class WriteableCoList<
-        T extends JsonValue,
+        T extends JsonValue | CoValue,
         Meta extends JsonObject | null = null
     >
     extends CoList<T, Meta>
-    implements WriteableCoValue
+    implements CoValue
 {
     /** @internal */
-    edit(
-        _changer: (editable: WriteableCoList<T, Meta>) => void
-    ): CoList<T, Meta> {
+    edit(_changer: (editable: WriteableCoList<T, Meta>) => void): this {
         throw new Error("Already editing.");
     }
 
@@ -330,7 +363,7 @@ export class WriteableCoList<
      * If `privacy` is `"trusting"`, both `value` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers. */
     append(
         after: number,
-        value: T,
+        value: T extends CoValue ? T | CoID<T> : T,
         privacy: "private" | "trusting" = "private"
     ): void {
         const entries = this.entries();
@@ -351,7 +384,7 @@ export class WriteableCoList<
             [
                 {
                     op: "app",
-                    value,
+                    value: isCoValue(value) ? value.id : value,
                     after: opIDBefore,
                 },
             ],
@@ -366,7 +399,10 @@ export class WriteableCoList<
      * If `privacy` is `"private"` **(default)**, both `value` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
      *
      * If `privacy` is `"trusting"`, both `value` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers. */
-    push(value: T, privacy: "private" | "trusting" = "private"): void {
+    push(
+        value: T extends CoValue ? T | CoID<T> : T,
+        privacy: "private" | "trusting" = "private"
+    ): void {
         // TODO: optimize
         const entries = this.entries();
         this.append(
@@ -385,7 +421,7 @@ export class WriteableCoList<
      */
     prepend(
         before: number,
-        value: T,
+        value: T extends CoValue ? T | CoID<T> : T,
         privacy: "private" | "trusting" = "private"
     ): void {
         const entries = this.entries();
@@ -410,7 +446,7 @@ export class WriteableCoList<
             [
                 {
                     op: "pre",
-                    value,
+                    value: isCoValue(value) ? value.id : value,
                     before: opIDAfter,
                 },
             ],

packages/cojson/src/coValues/coMap.ts

@@ -1,48 +1,49 @@
-import { JsonObject, JsonValue } from '../jsonValue.js';
-import { TransactionID } from '../ids.js';
-import { CoID, ReadableCoValue, WriteableCoValue } from '../coValue.js';
-import { CoValueCore, accountOrAgentIDfromSessionID } from '../coValueCore.js';
-import { AccountID, isAccountID } from '../account.js';
-import { Group } from '../group.js';
-import { parseJSON } from '../jsonStringify.js';
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { TransactionID } from "../ids.js";
+import { CoID, CoValue, isCoValue } from "../coValue.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
+import { AccountID, isAccountID } from "../account.js";
+import { Group } from "../group.js";
+import { parseJSON } from "../jsonStringify.js";
 
-type MapOp<K extends string, V extends JsonValue | undefined> = {
+type MapOp<K extends string, V extends JsonValue | CoValue | undefined> = {
     txID: TransactionID;
     madeAt: number;
     changeIdx: number;
 } & MapOpPayload<K, V>;
 // TODO: add after TransactionID[] for conflicts/ordering
 
-export type MapOpPayload<K extends string, V extends JsonValue | undefined> = {
-    op: "set";
-    key: K;
-    value: V;
-} |
-{
-    op: "del";
-    key: K;
-};
-
-export type MapK<M extends { [key: string]: JsonValue | undefined; }> = keyof M & string;
-export type MapV<M extends { [key: string]: JsonValue | undefined; }> = M[MapK<M>];
-
+export type MapOpPayload<
+    K extends string,
+    V extends JsonValue | CoValue | undefined
+> =
+    | {
+          op: "set";
+          key: K;
+          value: V extends CoValue ? CoID<V> : Exclude<V, CoValue>;
+      }
+    | {
+          op: "del";
+          key: K;
+      };
 
 /** A collaborative map with precise shape `M` and optional static metadata `Meta` */
 export class CoMap<
-    M extends { [key: string]: JsonValue | undefined; },
-    Meta extends JsonObject | null = null,
-> implements ReadableCoValue {
-    id: CoID<CoMap<M, Meta>>;
+    M extends { [key: string]: JsonValue | CoValue | undefined },
+    Meta extends JsonObject | null = null
+> implements CoValue
+{
+    id: CoID<this>;
     type = "comap" as const;
     core: CoValueCore;
     /** @internal */
     ops: {
-        [KK in MapK<M>]?: MapOp<KK, M[KK]>[];
+        [Key in keyof M & string]?: MapOp<Key, M[Key]>[];
     };
 
     /** @internal */
     constructor(core: CoValueCore) {
-        this.id = core.id as CoID<CoMap<M, Meta>>;
+        this.id = core.id as CoID<this>;
         this.core = core;
         this.ops = {};
 
@@ -61,11 +62,18 @@ export class CoMap<
     protected fillOpsFromCoValue() {
         this.ops = {};
 
-        for (const { txID, changes, madeAt } of this.core.getValidSortedTransactions()) {
-            for (const [changeIdx, changeUntyped] of (
-                parseJSON(changes)
+        for (const {
+            txID,
+            changes,
+            madeAt,
+        } of this.core.getValidSortedTransactions()) {
+            for (const [changeIdx, changeUntyped] of parseJSON(
+                changes
             ).entries()) {
-                const change = changeUntyped as MapOpPayload<MapK<M>, MapV<M>>;
+                const change = changeUntyped as MapOpPayload<
+                    keyof M & string,
+                    M[keyof M & string]
+                >;
                 let entries = this.ops[change.key];
                 if (!entries) {
                     entries = [];
@@ -75,18 +83,25 @@ export class CoMap<
                     txID,
                     madeAt,
                     changeIdx,
-                    ...(change as MapOpPayload<MapK<M>, MapV<M>>),
+                    ...(change as MapOpPayload<
+                        keyof M & string,
+                        M[keyof M & string]
+                    >),
                 });
             }
         }
     }
 
-    keys(): MapK<M>[] {
-        return Object.keys(this.ops) as MapK<M>[];
+    keys(): (keyof M & string)[] {
+        return Object.keys(this.ops) as (keyof M & string)[];
     }
 
     /** Returns the current value for the given key. */
-    get<K extends MapK<M>>(key: K): M[K] | undefined {
+    get<K extends keyof M & string>(
+        key: K
+    ):
+        | (M[K] extends CoValue ? CoID<M[K]> : Exclude<M[K], CoValue>)
+        | undefined {
         const ops = this.ops[key];
         if (!ops) {
             return undefined;
@@ -101,7 +116,12 @@ export class CoMap<
         }
     }
 
-    getAtTime<K extends MapK<M>>(key: K, time: number): M[K] | undefined {
+    getAtTime<K extends keyof M & string>(
+        key: K,
+        time: number
+    ):
+        | (M[K] extends CoValue ? CoID<M[K]> : Exclude<M[K], CoValue>)
+        | undefined {
         const ops = this.ops[key];
         if (!ops) {
             return undefined;
@@ -121,8 +141,8 @@ export class CoMap<
     }
 
     /** Returns the accountID of the last account to modify the value for the given key. */
-    whoEdited<K extends MapK<M>>(key: K): AccountID | undefined {
-        const tx  = this.getLastTxID(key);
+    whoEdited<K extends keyof M & string>(key: K): AccountID | undefined {
+        const tx = this.getLastTxID(key);
         if (!tx) {
             return undefined;
         }
@@ -134,7 +154,7 @@ export class CoMap<
         }
     }
 
-    getLastTxID<K extends MapK<M>>(key: K): TransactionID | undefined {
+    getLastTxID<K extends keyof M & string>(key: K): TransactionID | undefined {
         const ops = this.ops[key];
         if (!ops) {
             return undefined;
@@ -145,7 +165,15 @@ export class CoMap<
         return lastEntry.txID;
     }
 
-    getLastEntry<K extends MapK<M>>(key: K): { at: number; txID: TransactionID; value: M[K]; } | undefined {
+    getLastEntry<K extends keyof M & string>(
+        key: K
+    ):
+        | {
+              at: number;
+              txID: TransactionID;
+              value: M[K] extends CoValue ? CoID<M[K]> : Exclude<M[K], CoValue>;
+          }
+        | undefined {
         const ops = this.ops[key];
         if (!ops) {
             return undefined;
@@ -156,21 +184,43 @@ export class CoMap<
         if (lastEntry.op === "del") {
             return undefined;
         } else {
-            return { at: lastEntry.madeAt, txID: lastEntry.txID, value: lastEntry.value };
+            return {
+                at: lastEntry.madeAt,
+                txID: lastEntry.txID,
+                value: lastEntry.value,
+            };
         }
     }
 
-    getHistory<K extends MapK<M>>(key: K): { at: number; txID: TransactionID; value: M[K] | undefined; }[] {
+    getHistory<K extends keyof M & string>(
+        key: K
+    ): {
+        at: number;
+        txID: TransactionID;
+        value:
+            | (M[K] extends CoValue ? CoID<M[K]> : Exclude<M[K], CoValue>)
+            | undefined;
+    }[] {
         const ops = this.ops[key];
         if (!ops) {
             return [];
         }
 
-        const history: { at: number; txID: TransactionID; value: M[K] | undefined; }[] = [];
+        const history: {
+            at: number;
+            txID: TransactionID;
+            value:
+                | (M[K] extends CoValue ? CoID<M[K]> : Exclude<M[K], CoValue>)
+                | undefined;
+        }[] = [];
 
         for (const op of ops) {
             if (op.op === "del") {
-                history.push({ at: op.madeAt, txID: op.txID, value: undefined });
+                history.push({
+                    at: op.madeAt,
+                    txID: op.txID,
+                    value: undefined,
+                });
             } else {
                 history.push({ at: op.madeAt, txID: op.txID, value: op.value });
             }
@@ -192,25 +242,28 @@ export class CoMap<
         return json;
     }
 
-    subscribe(listener: (coMap: CoMap<M, Meta>) => void): () => void {
+    subscribe(listener: (coMap: this) => void): () => void {
         return this.core.subscribe((content) => {
-            listener(content as CoMap<M, Meta>);
+            listener(content as this);
         });
     }
 
-    edit(changer: (editable: WriteableCoMap<M, Meta>) => void): CoMap<M, Meta> {
+    edit(changer: (editable: WriteableCoMap<M, Meta>) => void): this {
         const editable = new WriteableCoMap<M, Meta>(this.core);
         changer(editable);
-        return new CoMap(this.core);
+        return new CoMap(this.core) as this;
     }
 }
 
 export class WriteableCoMap<
-    M extends { [key: string]: JsonValue | undefined; },
-    Meta extends JsonObject | null = null,
-> extends CoMap<M, Meta> implements WriteableCoValue {
+        M extends { [key: string]: JsonValue | CoValue | undefined },
+        Meta extends JsonObject | null = null
+    >
+    extends CoMap<M, Meta>
+    implements CoValue
+{
     /** @internal */
-    edit(_changer: (editable: WriteableCoMap<M, Meta>) => void): CoMap<M, Meta> {
+    edit(_changer: (editable: WriteableCoMap<M, Meta>) => void): this {
         throw new Error("Already editing.");
     }
 
@@ -219,14 +272,21 @@ export class WriteableCoMap<
      * If `privacy` is `"private"` **(default)**, both `key` and `value` are encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
      *
      * If `privacy` is `"trusting"`, both `key` and `value` are stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers. */
-    set<K extends MapK<M>>(key: K, value: M[K], privacy: "private" | "trusting" = "private"): void {
-        this.core.makeTransaction([
-            {
-                op: "set",
-                key,
-                value,
-            },
-        ], privacy);
+    set<K extends keyof M & string>(
+        key: K,
+        value: M[K] extends CoValue ? M[K] | CoID<M[K]> : M[K],
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        this.core.makeTransaction(
+            [
+                {
+                    op: "set",
+                    key,
+                    value: isCoValue(value) ? value.id : value,
+                },
+            ],
+            privacy
+        );
 
         this.fillOpsFromCoValue();
     }
@@ -236,13 +296,19 @@ export class WriteableCoMap<
      * If `privacy` is `"private"` **(default)**, `key` is encrypted in the transaction, only readable by other members of the group this `CoMap` belongs to. Not even sync servers can see the content in plaintext.
      *
      * If `privacy` is `"trusting"`, `key` is stored in plaintext in the transaction, visible to everyone who gets a hold of it, including sync servers. */
-    delete(key: MapK<M>, privacy: "private" | "trusting" = "private"): void {
-        this.core.makeTransaction([
-            {
-                op: "del",
-                key,
-            },
-        ], privacy);
+    delete(
+        key: keyof M & string,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        this.core.makeTransaction(
+            [
+                {
+                    op: "del",
+                    key,
+                },
+            ],
+            privacy
+        );
 
         this.fillOpsFromCoValue();
     }

packages/cojson/src/coValues/coStream.ts

@@ -1,14 +1,13 @@
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { CoID, ReadableCoValue, WriteableCoValue } from "../coValue.js";
+import { CoValue, CoID, isCoValue } from "../coValue.js";
 import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { Group } from "../group.js";
 import { SessionID } from "../ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
-import { AccountID } from "../index.js";
-import { isAccountID } from "../account.js";
+import { AccountID, isAccountID } from "../account.js";
 import { parseJSON } from "../jsonStringify.js";
 
-export type BinaryChunkInfo = {
+export type BinaryStreamInfo = {
     mimeType: string;
     fileName?: string;
     totalSizeBytes?: number;
@@ -16,7 +15,7 @@ export type BinaryChunkInfo = {
 
 export type BinaryStreamStart = {
     type: "start";
-} & BinaryChunkInfo;
+} & BinaryStreamInfo;
 
 export type BinaryStreamChunk = {
     type: "chunk";
@@ -34,20 +33,25 @@ export type BinaryStreamItem =
     | BinaryStreamChunk
     | BinaryStreamEnd;
 
+export type CoStreamItem<T extends JsonValue | CoValue> = {
+    item: T extends CoValue ? CoID<T> : Exclude<T, CoValue>;
+    madeAt: number;
+};
+
 export class CoStream<
-    T extends JsonValue,
+    T extends JsonValue | CoValue,
     Meta extends JsonObject | null = null
-> implements ReadableCoValue
+> implements CoValue
 {
-    id: CoID<CoStream<T, Meta>>;
+    id: CoID<this>;
     type = "costream" as const;
     core: CoValueCore;
     items: {
-        [key: SessionID]: {item: T, madeAt: number}[];
+        [key: SessionID]: CoStreamItem<T>[];
     };
 
     constructor(core: CoValueCore) {
-        this.id = core.id as CoID<CoStream<T, Meta>>;
+        this.id = core.id as CoID<this>;
         this.core = core;
         this.items = {};
         this.fillFromCoValue();
@@ -71,18 +75,22 @@ export class CoStream<
             changes,
         } of this.core.getValidSortedTransactions()) {
             for (const changeUntyped of parseJSON(changes)) {
-                const change = changeUntyped as T;
+                const change = changeUntyped as T extends CoValue
+                    ? CoID<T>
+                    : Exclude<T, CoValue>;
                 let entries = this.items[txID.sessionID];
                 if (!entries) {
                     entries = [];
                     this.items[txID.sessionID] = entries;
                 }
-                entries.push({item: change, madeAt});
+                entries.push({ item: change, madeAt });
             }
         }
     }
 
-    getSingleStream(): T[] | undefined {
+    getSingleStream():
+        | (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)[]
+        | undefined {
         if (Object.keys(this.items).length === 0) {
             return undefined;
         } else if (Object.keys(this.items).length !== 1) {
@@ -91,36 +99,54 @@ export class CoStream<
             );
         }
 
-        return Object.values(this.items)[0]?.map(item => item.item);
+        return Object.values(this.items)[0]?.map((item) => item.item);
     }
 
-    getLastItemsPerAccount(): {[account: AccountID]: T | undefined} {
-        const result: {[account: AccountID]: {item: T, madeAt: number} | undefined} = {};
+    getLastItemsPerAccount(): {
+        [account: AccountID]:
+            | (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)
+            | undefined;
+    } {
+        const result: { [account: AccountID]: CoStreamItem<T> | undefined } =
+            {};
 
         for (const [sessionID, items] of Object.entries(this.items)) {
-            const account = accountOrAgentIDfromSessionID(sessionID as SessionID);
+            const account = accountOrAgentIDfromSessionID(
+                sessionID as SessionID
+            );
             if (!isAccountID(account)) continue;
             if (items.length > 0) {
                 const lastItemOfSession = items[items.length - 1]!;
-                if (!result[account] || lastItemOfSession.madeAt > result[account]!.madeAt) {
+                if (
+                    !result[account] ||
+                    lastItemOfSession.madeAt > result[account]!.madeAt
+                ) {
                     result[account] = lastItemOfSession;
                 }
             }
         }
 
-        return Object.fromEntries(Object.entries(result).map(([account, item]) =>
-            [account, item?.item]
-        ));
+        return Object.fromEntries(
+            Object.entries(result).map(([account, item]) => [
+                account,
+                item?.item,
+            ])
+        );
     }
 
-    getLastItemFrom(account: AccountID): T | undefined {
-        let lastItem: {item: T, madeAt: number} | undefined;
+    getLastItemFrom(
+        account: AccountID
+    ): (T extends CoValue ? CoID<T> : Exclude<T, CoValue>) | undefined {
+        let lastItem: CoStreamItem<T> | undefined;
 
         for (const [sessionID, items] of Object.entries(this.items)) {
             if (sessionID.startsWith(account)) {
                 if (items.length > 0) {
                     const lastItemOfSession = items[items.length - 1]!;
-                    if (!lastItem || lastItemOfSession.madeAt > lastItem.madeAt) {
+                    if (
+                        !lastItem ||
+                        lastItemOfSession.madeAt > lastItem.madeAt
+                    ) {
                         lastItem = lastItemOfSession;
                     }
                 }
@@ -130,32 +156,35 @@ export class CoStream<
         return lastItem?.item;
     }
 
-    getLastItemFromMe(): T | undefined {
+    getLastItemFromMe():
+        | (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)
+        | undefined {
         const myAccountID = this.core.node.account.id;
         if (!isAccountID(myAccountID)) return undefined;
         return this.getLastItemFrom(myAccountID);
     }
 
     toJSON(): {
-        [key: SessionID]: T[];
+        [key: SessionID]: (T extends CoValue ? CoID<T> : Exclude<T, CoValue>)[];
     } {
-        return Object.fromEntries(Object.entries(this.items).map(([sessionID, items]) =>
-            [sessionID, items.map(item => item.item)]
-        ));
+        return Object.fromEntries(
+            Object.entries(this.items).map(([sessionID, items]) => [
+                sessionID,
+                items.map((item) => item.item),
+            ])
+        );
     }
 
-    subscribe(listener: (coMap: CoStream<T, Meta>) => void): () => void {
+    subscribe(listener: (coStream: this) => void): () => void {
         return this.core.subscribe((content) => {
-            listener(content as CoStream<T, Meta>);
+            listener(content as this);
         });
     }
 
-    edit(
-        changer: (editable: WriteableCoStream<T, Meta>) => void
-    ): CoStream<T, Meta> {
+    edit(changer: (editable: WriteableCoStream<T, Meta>) => void): this {
         const editable = new WriteableCoStream<T, Meta>(this.core);
         changer(editable);
-        return new CoStream(this.core);
+        return new CoStream(this.core) as this;
     }
 }
 
@@ -165,14 +194,16 @@ export class BinaryCoStream<
         Meta extends BinaryCoStreamMeta = { type: "binary" }
     >
     extends CoStream<BinaryStreamItem, Meta>
-    implements ReadableCoValue
+    implements CoValue
 {
-    id!: CoID<BinaryCoStream<Meta>>;
+    id!: CoID<this>;
 
-    getBinaryChunks():
-        | (BinaryChunkInfo & { chunks: Uint8Array[]; finished: boolean })
+    getBinaryChunks(
+        allowUnfinished?: boolean
+    ):
+        | (BinaryStreamInfo & { chunks: Uint8Array[]; finished: boolean })
         | undefined {
-        const before = performance.now();
+        // const before = performance.now();
         const items = this.getSingleStream();
 
         if (!items) return;
@@ -184,10 +215,14 @@ export class BinaryCoStream<
             return;
         }
 
+        const end = items[items.length - 1];
+
+        if (end?.type !== "end" && !allowUnfinished) return;
+
         const chunks: Uint8Array[] = [];
 
         let finished = false;
-        let totalLength = 0;
+        // let totalLength = 0;
 
         for (const item of items.slice(1)) {
             if (item.type === "end") {
@@ -203,15 +238,15 @@ export class BinaryCoStream<
             const chunk = base64URLtoBytes(
                 item.chunk.slice(binary_U_prefixLength)
             );
-            totalLength += chunk.length;
+            // totalLength += chunk.length;
             chunks.push(chunk);
         }
 
-        const after = performance.now();
-        console.log(
-            "getBinaryChunks bandwidth in MB/s",
-            (1000 * totalLength) / (after - before) / (1024 * 1024)
-        );
+        // const after = performance.now();
+        // console.log(
+        //     "getBinaryChunks bandwidth in MB/s",
+        //     (1000 * totalLength) / (after - before) / (1024 * 1024)
+        // );
 
         return {
             mimeType: start.mimeType,
@@ -222,31 +257,33 @@ export class BinaryCoStream<
         };
     }
 
-    edit(
-        changer: (editable: WriteableBinaryCoStream<Meta>) => void
-    ): BinaryCoStream<Meta> {
+    edit(changer: (editable: WriteableBinaryCoStream<Meta>) => void): this {
         const editable = new WriteableBinaryCoStream<Meta>(this.core);
         changer(editable);
-        return new BinaryCoStream(this.core);
+        return new BinaryCoStream(this.core) as this;
     }
 }
 
 export class WriteableCoStream<
-        T extends JsonValue,
+        T extends JsonValue | CoValue,
         Meta extends JsonObject | null = null
     >
     extends CoStream<T, Meta>
-    implements WriteableCoValue
+    implements CoValue
 {
     /** @internal */
-    edit(
-        _changer: (editable: WriteableCoStream<T, Meta>) => void
-    ): CoStream<T, Meta> {
+    edit(_changer: (editable: WriteableCoStream<T, Meta>) => void): this {
         throw new Error("Already editing.");
     }
 
-    push(item: T, privacy: "private" | "trusting" = "private") {
-        this.core.makeTransaction([item], privacy);
+    push(
+        item: T extends CoValue ? T | CoID<T> : T,
+        privacy: "private" | "trusting" = "private"
+    ) {
+        this.core.makeTransaction(
+            [isCoValue(item) ? item.id : item],
+            privacy
+        );
         this.fillFromCoValue();
     }
 }
@@ -255,12 +292,10 @@ export class WriteableBinaryCoStream<
         Meta extends BinaryCoStreamMeta = { type: "binary" }
     >
     extends BinaryCoStream<Meta>
-    implements WriteableCoValue
+    implements CoValue
 {
     /** @internal */
-    edit(
-        _changer: (editable: WriteableBinaryCoStream<Meta>) => void
-    ): BinaryCoStream<Meta> {
+    edit(_changer: (editable: WriteableBinaryCoStream<Meta>) => void): this {
         throw new Error("Already editing.");
     }
 
@@ -270,7 +305,7 @@ export class WriteableBinaryCoStream<
     }
 
     startBinaryStream(
-        settings: BinaryChunkInfo,
+        settings: BinaryStreamInfo,
         privacy: "private" | "trusting" = "private"
     ) {
         this.push(
@@ -286,7 +321,7 @@ export class WriteableBinaryCoStream<
         chunk: Uint8Array,
         privacy: "private" | "trusting" = "private"
     ) {
-        const before = performance.now();
+        // const before = performance.now();
         this.push(
             {
                 type: "chunk",
@@ -294,11 +329,11 @@ export class WriteableBinaryCoStream<
             } satisfies BinaryStreamChunk,
             privacy
         );
-        const after = performance.now();
-        console.log(
-            "pushBinaryStreamChunk bandwidth in MB/s",
-            (1000 * chunk.length) / (after - before) / (1024 * 1024)
-        );
+        // const after = performance.now();
+        // console.log(
+        //     "pushBinaryStreamChunk bandwidth in MB/s",
+        //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        // );
     }
 
     endBinaryStream(privacy: "private" | "trusting" = "private") {

packages/cojson/src/coValues/static.ts

@@ -1,15 +1,15 @@
 import { JsonObject } from '../jsonValue.js';
-import { CoID, ReadableCoValue } from '../coValue.js';
+import { CoID, CoValue } from '../coValue.js';
 import { CoValueCore } from '../coValueCore.js';
 import { Group } from '../index.js';
 
-export class Static<T extends JsonObject> implements ReadableCoValue{
-    id: CoID<Static<T>>;
+export class Static<T extends JsonObject> implements CoValue{
+    id: CoID<this>;
     type = "static" as const;
     core: CoValueCore;
 
     constructor(core: CoValueCore) {
-        this.id = core.id as CoID<Static<T>>;
+        this.id = core.id as CoID<this>;
         this.core = core;
     }
 
@@ -25,7 +25,7 @@ export class Static<T extends JsonObject> implements ReadableCoValue{
         throw new Error("Method not implemented.");
     }
 
-    subscribe(_listener: (coMap: Static<T>) => void): () => void {
+    subscribe(_listener: (st: this) => void): () => void {
         throw new Error("Method not implemented.");
     }
 }

packages/cojson/src/group.ts

@@ -1,4 +1,4 @@
-import { CoID, CoValueImpl } from "./coValue.js";
+import { CoID, CoValue, AnyCoValue } from "./coValue.js";
 import { CoMap } from "./coValues/coMap.js";
 import { JsonObject, JsonValue } from "./jsonValue.js";
 import {
@@ -21,7 +21,11 @@ import { AccountID, GeneralizedControlledAccount, Profile } from "./account.js";
 import { Role } from "./permissions.js";
 import { base58 } from "@scure/base";
 import { CoList } from "./coValues/coList.js";
-import { BinaryCoStream, BinaryCoStreamMeta, CoStream } from "./coValues/coStream.js";
+import {
+    BinaryCoStream,
+    BinaryCoStreamMeta,
+    CoStream,
+} from "./coValues/coStream.js";
 
 export type GroupContent = {
     profile: CoID<Profile> | null;
@@ -35,7 +39,7 @@ export type GroupContent = {
 };
 
 export function expectGroupContent(
-    content: CoValueImpl
+    content: CoValue
 ): CoMap<GroupContent, JsonObject | null> {
     if (content.type !== "comap") {
         throw new Error("Expected map");
@@ -238,10 +242,22 @@ export class Group {
 
     /** Creates a new `CoMap` within this group, with the specified specialized
      *  `CoMap` type `M` and optional static metadata. */
-    createMap<M extends CoMap<{ [key: string]: JsonValue | undefined; }, JsonObject | null>>(
+    createMap<
+        M extends CoMap<
+            { [key: string]: JsonValue | AnyCoValue | undefined },
+            JsonObject | null
+        >
+    >(
+        init?: M extends CoMap<infer M, infer _Meta>
+            ? {
+                  [K in keyof M]: M[K] extends AnyCoValue
+                      ? M[K] | CoID<M[K]>
+                      : M[K];
+              }
+            : never,
         meta?: M["meta"]
     ): M {
-        return this.node
+        let map = this.node
             .createCoValue({
                 type: "comap",
                 ruleset: {
@@ -252,14 +268,27 @@ export class Group {
                 ...createdNowUnique(),
             })
             .getCurrentContent() as M;
+
+        if (init) {
+            map = map.edit((editable) => {
+                for (const [key, value] of Object.entries(init)) {
+                    editable.set(key, value);
+                }
+            });
+        }
+
+        return map;
     }
 
     /** Creates a new `CoList` within this group, with the specified specialized
      * `CoList` type `L` and optional static metadata. */
-    createList<L extends CoList<JsonValue, JsonObject | null>>(
+    createList<L extends CoList<JsonValue | CoValue, JsonObject | null>>(
+        init?: L extends CoList<infer I, infer _Meta>
+            ? (I extends CoValue ? CoID<I> | I : I)[]
+            : never,
         meta?: L["meta"]
     ): L {
-        return this.node
+        let list = this.node
             .createCoValue({
                 type: "colist",
                 ruleset: {
@@ -270,9 +299,19 @@ export class Group {
                 ...createdNowUnique(),
             })
             .getCurrentContent() as L;
+
+        if (init) {
+            list = list.edit((editable) => {
+                for (const item of init) {
+                    editable.push(item);
+                }
+            });
+        }
+
+        return list;
     }
 
-    createStream<C extends CoStream<JsonValue, JsonObject | null>>(
+    createStream<C extends CoStream<JsonValue | CoValue, JsonObject | null>>(
         meta?: C["meta"]
     ): C {
         return this.node
@@ -288,9 +327,9 @@ export class Group {
             .getCurrentContent() as C;
     }
 
-    createBinaryStream<
-        C extends BinaryCoStream<BinaryCoStreamMeta>
-    >(meta: C["meta"] = { type: "binary" }): C {
+    createBinaryStream<C extends BinaryCoStream<BinaryCoStreamMeta>>(
+        meta: C["meta"] = { type: "binary" }
+    ): C {
         return this.node
             .createCoValue({
                 type: "costream",

packages/cojson/src/index.ts

@@ -1,6 +1,6 @@
-import { CoValueCore, newRandomSessionID } from "./coValueCore.js";
+import { CoValueCore, newRandomSessionID, MAX_RECOMMENDED_TX_SIZE } from "./coValueCore.js";
 import { LocalNode } from "./node.js";
-import type { CoValue, ReadableCoValue } from "./coValue.js";
+import type { CoValue } from "./coValue.js";
 import { CoMap, WriteableCoMap } from "./coValues/coMap.js";
 import { CoList, WriteableCoList } from "./coValues/coList.js";
 import {
@@ -28,16 +28,17 @@ import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 import { parseJSON } from "./jsonStringify.js";
 
 import type { SessionID, AgentID } from "./ids.js";
-import type { CoID, CoValueImpl } from "./coValue.js";
-import type { BinaryChunkInfo, BinaryCoStreamMeta } from "./coValues/coStream.js";
+import type { CoID, AnyCoValue } from "./coValue.js";
+import type { Queried } from "./queries.js";
+import type { BinaryStreamInfo, BinaryCoStreamMeta } from "./coValues/coStream.js";
 import type { JsonValue } from "./jsonValue.js";
 import type { SyncMessage, Peer } from "./sync.js";
 import type { AgentSecret } from "./crypto.js";
-import type { AccountID, Profile } from "./account.js";
+import type { AccountID, Account, Profile } from "./account.js";
 import type { InviteSecret } from "./group.js";
 import type * as Media from "./media.js";
 
-type Value = JsonValue | CoValueImpl;
+type Value = JsonValue | AnyCoValue;
 
 /** @hidden */
 export const cojsonInternals = {
@@ -74,20 +75,22 @@ export {
     AnonymousControlledAccount,
     ControlledAccount,
     cryptoReady as cojsonReady,
+    MAX_RECOMMENDED_TX_SIZE
 };
 
 export type {
     Value,
     JsonValue,
     CoValue,
-    ReadableCoValue,
-    CoValueImpl,
+    AnyCoValue,
     CoID,
+    Queried,
     AccountID,
+    Account,
     Profile,
     SessionID,
     Peer,
-    BinaryChunkInfo,
+    BinaryStreamInfo,
     BinaryCoStreamMeta,
     AgentID,
     AgentSecret,

packages/cojson/src/media.ts

@@ -1,9 +1,8 @@
 import { CoMap } from './coValues/coMap.js'
-import { CoID } from './coValue.js'
 import { BinaryCoStream } from './coValues/coStream.js'
 
 export type ImageDefinition = CoMap<{
     originalSize: [number, number];
     placeholderDataURL?: string;
-    [res: `${number}x${number}`]: CoID<BinaryCoStream>;
+    [res: `${number}x${number}`]: BinaryCoStream;
 }>;

packages/cojson/src/node.ts

@@ -9,7 +9,11 @@ import {
     newRandomKeySecret,
     seal,
 } from "./crypto.js";
-import { CoValueCore, CoValueHeader, newRandomSessionID } from "./coValueCore.js";
+import {
+    CoValueCore,
+    CoValueHeader,
+    newRandomSessionID,
+} from "./coValueCore.js";
 import {
     InviteSecret,
     Group,
@@ -19,9 +23,10 @@ import {
 } from "./group.js";
 import { Peer, SyncManager } from "./sync.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
-import { CoID, CoValueImpl } from "./coValue.js";
+import { CoID } from "./coValue.js";
+import { Queried, query } from "./queries.js";
 import {
-    Account,
+    AccountGroup,
     AccountMeta,
     accountHeaderForInitialAgentSecret,
     GeneralizedControlledAccount,
@@ -30,9 +35,10 @@ import {
     AccountID,
     Profile,
     AccountContent,
-    AccountMap,
+    Account,
 } from "./account.js";
 import { CoMap } from "./coValues/coMap.js";
+import { CoValue } from "./index.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
@@ -152,16 +158,45 @@ export class LocalNode {
      * promise once a first version has been loaded. See `coValue.subscribe()` and `node.useTelepathicData()`
      * for listening to subsequent updates to the CoValue.
      */
-    async load<T extends CoValueImpl>(id: CoID<T>): Promise<T> {
+    async load<T extends CoValue>(id: CoID<T>): Promise<T> {
         return (await this.loadCoValue(id)).getCurrentContent() as T;
     }
 
+    subscribe<T extends CoValue>(id: CoID<T>, callback: (update: T) => void): () => void {
+        let stopped = false;
+        let unsubscribe!: () => void;
+
+        console.log("Subscribing to " + id);
+
+        this.load(id).then((coValue) => {
+            if (stopped) {
+                return;
+            }
+            unsubscribe = coValue.subscribe(callback);
+        }).catch((e) => {
+            console.error("Error subscribing to ", id, e);
+        });
+
+        return () => {
+            console.log("Unsubscribing from " + id);
+            stopped = true;
+            unsubscribe?.();
+        }
+    }
+
+    query<T extends CoValue>(
+        id: CoID<T>,
+        callback: (update: Queried<T> | undefined) => void
+    ): () => void {
+        return query(id, this, callback);
+    }
+
     /**
      * Loads a profile associated with an account. `Profile` is at least a `CoMap<{string: name}>`,
      * but might contain other, app-specific properties.
      */
     async loadProfile(id: AccountID): Promise<Profile> {
-        const account = await this.load<AccountMap>(id);
+        const account = await this.load<Account>(id);
         const profileID = account.get("profile");
 
         if (!profileID) {
@@ -172,7 +207,7 @@ export class LocalNode {
         ).getCurrentContent() as Profile;
     }
 
-    async acceptInvite<T extends CoValueImpl>(
+    async acceptInvite<T extends CoValue>(
         groupOrOwnedValueID: CoID<T>,
         inviteSecret: InviteSecret
     ): Promise<void> {
@@ -204,10 +239,7 @@ export class LocalNode {
                 }
             });
             setTimeout(
-                () =>
-                    reject(
-                        new Error("Couldn't find invite before timeout")
-                    ),
+                () => reject(new Error("Couldn't find invite before timeout")),
                 2000
             );
         });
@@ -224,7 +256,9 @@ export class LocalNode {
             (existingRole === "writer" && inviteRole === "reader") ||
             (existingRole === "reader" && inviteRole === "readerInvite")
         ) {
-            console.debug("Not accepting invite that would replace or downgrade role");
+            console.debug(
+                "Not accepting invite that would replace or downgrade role"
+            );
             return;
         }
 
@@ -242,7 +276,8 @@ export class LocalNode {
                 : "reader"
         );
 
-        group.underlyingMap.core._sessions = groupAsInvite.underlyingMap.core.sessions;
+        group.underlyingMap.core._sessions =
+            groupAsInvite.underlyingMap.core.sessions;
         group.underlyingMap.core._cachedContent = undefined;
 
         for (const groupListener of group.underlyingMap.core.listeners) {
@@ -332,11 +367,11 @@ export class LocalNode {
             account.node
         );
 
-        const profile = accountAsGroup.createMap<Profile>({
+        let profile = accountAsGroup.createMap<Profile>(undefined, {
             type: "profile",
         });
 
-        profile.edit((editable) => {
+        profile = profile.edit((editable) => {
             editable.set("name", name, "trusting");
         });
 
@@ -346,14 +381,26 @@ export class LocalNode {
 
         const accountOnThisNode = this.expectCoValueLoaded(account.id);
 
-        accountOnThisNode._sessions = {...accountAsGroup.underlyingMap.core.sessions};
+        accountOnThisNode._sessions = {
+            ...accountAsGroup.underlyingMap.core.sessions,
+        };
         accountOnThisNode._cachedContent = undefined;
 
+        const profileOnThisNode = this.createCoValue(profile.core.header);
+
+        profileOnThisNode._sessions = {
+            ...profile.core.sessions,
+        };
+        profileOnThisNode._cachedContent = undefined;
+
         return controlledAccount;
     }
 
     /** @internal */
-    resolveAccountAgent(id: AccountID | AgentID, expectation?: string): AgentID {
+    resolveAccountAgent(
+        id: AccountID | AgentID,
+        expectation?: string
+    ): AgentID {
         if (isAgentID(id)) {
             return id;
         }
@@ -374,7 +421,7 @@ export class LocalNode {
             );
         }
 
-        return new Account(
+        return new AccountGroup(
             coValue.getCurrentContent() as CoMap<GroupContent, AccountMeta>,
             this
         ).getCurrentAgentID();
@@ -443,7 +490,11 @@ export class LocalNode {
                     continue;
                 }
 
-                const newCoValue = new CoValueCore(entry.coValue.header, newNode, {...entry.coValue.sessions});
+                const newCoValue = new CoValueCore(
+                    entry.coValue.header,
+                    newNode,
+                    { ...entry.coValue.sessions }
+                );
 
                 newNode.coValues[coValueID as RawCoID] = {
                     state: "loaded",

packages/cojson/src/queries.ts

@@ -0,0 +1,519 @@
+import { JsonValue } from "./jsonValue.js";
+import { CoMap, WriteableCoMap } from "./coValues/coMap.js";
+import {
+    BinaryCoStream,
+    BinaryStreamInfo,
+    CoStream,
+    WriteableBinaryCoStream,
+    WriteableCoStream,
+} from "./coValues/coStream.js";
+import { Static } from "./coValues/static.js";
+import { CoList, WriteableCoList } from "./coValues/coList.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "./coValueCore.js";
+import { Group } from "./group.js";
+import { AccountID, Profile, isAccountID } from "./account.js";
+import {
+    AnyBinaryCoStream,
+    AnyCoList,
+    AnyCoMap,
+    AnyCoStream,
+    AnyCoValue,
+    AnyStatic,
+    CoID,
+    CoValue,
+} from "./coValue.js";
+import { SessionID } from "./ids.js";
+import { LocalNode } from "./node.js";
+
+export const AllReservedQueryProps = [
+    "id",
+    "type",
+    "meta",
+    "core",
+    "group",
+    "shadowed",
+    "edit",
+    "edits",
+] as const;
+
+export type ReservedQueryProps = (typeof AllReservedQueryProps)[number];
+
+export type QueriedCoMap<T extends AnyCoMap> = T extends CoMap<
+    infer M,
+    infer Meta
+>
+    ? Readonly<{
+          [K in keyof M as Exclude<K, ReservedQueryProps>]: ValueOrSubQueried<
+              M[K]
+          >;
+      }> &
+          (keyof M & ReservedQueryProps extends never
+              ? // eslint-disable-next-line @typescript-eslint/ban-types
+                {}
+              : Readonly<{
+                    shadowed: Readonly<{
+                        [K in keyof M as Extract<
+                            K,
+                            ReservedQueryProps
+                        >]: ValueOrSubQueried<M[K]>;
+                    }>;
+                }>) &
+          Readonly<{
+              id: CoID<T>;
+              type: "comap";
+              edits: Readonly<{
+                  [K in keyof M & string]: Readonly<{
+                      by?: QueriedAccountAndProfile;
+                      at: Date;
+                      // all: TODO;
+                  }>;
+              }>;
+              meta: Meta;
+              group: Group;
+              core: CoValueCore;
+              edit: (changer: (editable: WriteableCoMap<M, Meta>) => void) => T;
+          }>
+    : never;
+
+export type QueriedAccountAndProfile = Readonly<{
+    id: AccountID;
+    profile?: Readonly<{ name?: string; id: CoID<Profile> }>;
+    isMe?: boolean;
+}>;
+
+export type QueriedCoList<T extends AnyCoList> = T extends CoList<
+    infer I,
+    infer Meta
+>
+    ? readonly ValueOrSubQueried<I>[] &
+          Readonly<{
+              id: CoID<T>;
+              type: "colist";
+              meta: Meta;
+              group: Group;
+              core: CoValueCore;
+              edit: (
+                  changer: (editable: WriteableCoList<I, Meta>) => void
+              ) => T;
+              edits: readonly Readonly<{
+                  by?: QueriedAccountAndProfile;
+                  at: Date;
+              }>[] & {
+                  // deletions: TODO;
+              };
+          }>
+    : never;
+
+export type QueriedCoStreamItems<I extends JsonValue | CoValue> = Readonly<{
+    last: ValueOrSubQueried<I> | undefined;
+    by?: QueriedAccountAndProfile;
+    at?: Date;
+    all: { value: ValueOrSubQueried<I>; at: Date }[];
+}>;
+
+export type QueriedCoStream<T extends AnyCoStream> = T extends CoStream<
+    infer I,
+    infer Meta
+>
+    ? Readonly<{
+          id: CoID<T>;
+          type: "costream";
+          me?: QueriedCoStreamItems<I>;
+          perAccount: Readonly<{
+              [account: AccountID]: QueriedCoStreamItems<I>;
+          }>;
+          perSession: Readonly<{
+              [session: SessionID]: QueriedCoStreamItems<I>;
+          }>;
+          meta: Meta;
+          group: Group;
+          core: CoValueCore;
+          edit: (changer: (editable: WriteableCoStream<I, Meta>) => void) => T;
+      }>
+    : never;
+
+export type QueriedBinaryCoStreamItems = Readonly<{
+    last: Uint8Array | undefined;
+    by: QueriedAccountAndProfile;
+    at: Date;
+    all: { value: Uint8Array; at: Date }[];
+}>;
+
+export type QueriedBinaryCoStream<T extends AnyBinaryCoStream> =
+    T extends BinaryCoStream<infer Meta>
+        ? Readonly<
+              {
+                  id: CoID<T>;
+                  type: "costream";
+                  me?: QueriedBinaryCoStreamItems;
+                  perAccount: Readonly<{
+                      [account: AccountID]: QueriedBinaryCoStreamItems;
+                  }>;
+                  perSession: Readonly<{
+                      [session: SessionID]: QueriedBinaryCoStreamItems;
+                  }>;
+                  meta: Meta;
+                  group: Group;
+                  core: CoValueCore;
+                  edit: (
+                      changer: (editable: WriteableBinaryCoStream<Meta>) => void
+                  ) => T;
+              }
+          > & Readonly<BinaryStreamInfo>
+        : never;
+
+export type QueriedStatic<T extends AnyStatic> = T extends Static<infer Meta>
+    ? Readonly<{
+          id: CoID<T>;
+          type: "colist";
+          meta: Meta;
+          group: Group;
+          core: CoValueCore;
+      }>
+    : never;
+
+export type Queried<T extends CoValue> = T extends AnyCoMap
+    ? QueriedCoMap<T>
+    : T extends AnyCoList
+    ? QueriedCoList<T>
+    // : T extends BinaryCoStream<infer _>
+    // ? QueriedBinaryCoStream<T>
+    : T extends AnyCoStream
+    ? QueriedCoStream<T>
+    : T extends AnyStatic
+    ? QueriedStatic<T>
+    : never;
+
+export type ValueOrSubQueried<
+    V extends JsonValue | CoValue | CoID<CoValue> | undefined
+> = V extends CoID<infer C>
+    ? Queried<C> | undefined
+    : V extends CoValue
+    ? Queried<V> | undefined
+    : V;
+
+export type QueryInclude<T extends CoValue> = T extends CoMap<
+    infer M,
+    infer _Meta
+>
+    ? {
+          [K in keyof M as M[K] extends AnyCoValue | CoID<AnyCoValue>
+              ? K
+              : never]?: M[K] extends AnyCoValue
+              ? true | QueryInclude<M[K]>
+              : M[K] extends CoID<infer S>
+              ? true | QueryInclude<S>
+              : never;
+      }
+    : T extends CoList<infer I, infer _>
+    ? I extends AnyCoValue
+        ? [true] | [QueryInclude<I>]
+        : I extends CoID<infer S>
+        ? [true] | [QueryInclude<S>]
+        : never
+    : never; // TODO add CoStream;
+
+export function query<T extends CoValue>(
+    id: CoID<T>,
+    node: LocalNode,
+    callback: (queried: Queried<T> | undefined) => void
+): () => void {
+    console.log("querying", id);
+
+    const children: {
+        [id: CoID<CoValue>]: {
+            lastQueried: { [key: string]: any } | undefined;
+            unsubscribe: () => void;
+        };
+    } = {};
+
+    const unsubscribe = node.subscribe(id, (update) => {
+        lastRootValue = update;
+        onUpdate();
+    });
+
+    function getChildLastQueriedOrSubscribe<T extends CoValue>(
+        childID: CoID<T>
+    ) {
+        let child = children[childID];
+        if (!child) {
+            child = {
+                lastQueried: undefined,
+                unsubscribe: query(childID, node, (childQueried) => {
+                    child!.lastQueried = childQueried;
+                    onUpdate();
+                }),
+            };
+            children[childID] = child;
+        }
+        return child.lastQueried as Queried<T> | undefined;
+    }
+
+    function resolveValue<T extends JsonValue>(
+        value: T
+    ): T extends CoID<CoValue> ? Queried<CoValue> | undefined : T {
+        return (
+            typeof value === "string" && value.startsWith("co_")
+                ? getChildLastQueriedOrSubscribe(value as CoID<CoValue>)
+                : value
+        ) as T extends CoID<CoValue> ? Queried<CoValue> | undefined : T;
+    }
+
+    let lastRootValue: T | undefined;
+
+    function onUpdate() {
+        const rootValue = lastRootValue;
+
+        if (rootValue === undefined) {
+            return undefined;
+        }
+
+        if (rootValue instanceof CoMap) {
+            callback(queryMap(rootValue) as Queried<T>);
+        } else if (rootValue instanceof CoList) {
+            callback(queryList(rootValue) as unknown as Queried<T>);
+        } else if (rootValue instanceof CoStream) {
+            if (rootValue.meta?.type === "binary") {
+                // Querying binary string not yet implemented
+                return {}
+            } else {
+                callback(queryStream(rootValue) as unknown as Queried<T>);
+            }
+        }
+    }
+
+    return function cleanup() {
+        for (const child of Object.values(children)) {
+            child.unsubscribe();
+        }
+        unsubscribe();
+    };
+
+    function queryMap(rootValue: T & CoMap<any, any>) {
+        const mapResult: {
+            [key: string]: any;
+        } = {};
+        // let allChildrenAvailable = true;
+        for (const key of rootValue.keys()) {
+            const value = rootValue.get(key);
+
+            if (value === undefined) continue;
+
+            if (AllReservedQueryProps.includes(key as ReservedQueryProps)) {
+                mapResult.shadowed = mapResult.shadowed || {};
+                mapResult.shadowed[key] = resolveValue(value);
+            } else {
+                mapResult[key] = resolveValue(value);
+            }
+        }
+
+        Object.defineProperties(mapResult, {
+            id: { value: rootValue.id },
+            type: { value: "comap" },
+        });
+
+        if (
+            rootValue.meta?.type !== "account" &&
+            rootValue.meta?.type !== "profile"
+        ) {
+            Object.defineProperties(mapResult, {
+                edit: {
+                    value: (
+                        changer: (editable: WriteableCoMap<any, any>) => void
+                    ) => {
+                        rootValue.edit(changer);
+                        return rootValue;
+                    },
+                },
+                edits: {
+                    value: {},
+                },
+            });
+
+            for (const key of rootValue.keys()) {
+                const editorID = rootValue.whoEdited(key);
+                const editor =
+                    editorID && getChildLastQueriedOrSubscribe(editorID);
+                mapResult.edits[key] = {
+                    by: editor && {
+                        id: editorID,
+                        isMe: editorID === node.account.id ? true : undefined,
+                        profile: editor.profile && {
+                            id: editor.profile.id,
+                            name: editor.profile.name,
+                        },
+                    },
+                    at: new Date(rootValue.getLastEntry(key)!.at),
+                };
+            }
+        }
+
+        Object.defineProperties(mapResult, {
+            meta: { value: rootValue.meta },
+            group: {
+                get() {
+                    return rootValue.group;
+                },
+            },
+            core: {
+                get() {
+                    return rootValue.core;
+                },
+            },
+        });
+        return mapResult;
+    }
+
+    function queryList(rootValue: T & CoList<any, any>) {
+        const arr: any[] & { [key: string]: any } = rootValue
+            .asArray()
+            .map(resolveValue);
+
+        Object.defineProperties(arr, {
+            type: { value: "colist" },
+            id: { value: rootValue.id },
+            edit: {
+                value: (
+                    changer: (editable: WriteableCoList<any, any>) => void
+                ) => {
+                    rootValue.edit(changer);
+                    return rootValue;
+                },
+            },
+            edits: {
+                value: [],
+            },
+            meta: { value: rootValue.meta },
+            group: {
+                get() {
+                    return rootValue.group;
+                },
+            },
+            core: {
+                get() {
+                    return rootValue.core;
+                },
+            },
+        });
+
+        for (let i = 0; i < arr.length; i++) {
+            const editorID = rootValue.whoInserted(i);
+            const editor = editorID && getChildLastQueriedOrSubscribe(editorID);
+            arr.edits[i] = {
+                by: editor && {
+                    id: editorID,
+                    isMe: editorID === node.account.id ? true : undefined,
+                    profile: editor.profile && {
+                        id: editor.profile.id,
+                        name: editor.profile.name,
+                    },
+                },
+                at: new Date(rootValue.entries()[i]!.madeAt),
+            };
+        }
+        return arr;
+    }
+
+    function queryStream(rootValue: T & CoStream<any, any>) {
+        const seenAccounts = new Set<AccountID>();
+
+        const perSession = Object.fromEntries(
+            Object.entries(rootValue.items).map(([sessionID, items]) => {
+                const editorID = accountOrAgentIDfromSessionID(
+                    sessionID as SessionID
+                );
+                if (isAccountID(editorID)) seenAccounts.add(editorID);
+                const editor =
+                    editorID &&
+                    (isAccountID(editorID)
+                        ? getChildLastQueriedOrSubscribe(editorID)
+                        : undefined);
+                const lastItem = items[items.length - 1];
+                return [
+                    sessionID as SessionID,
+                    {
+                        last: lastItem && resolveValue(lastItem.item),
+                        by: editor && {
+                            id: editorID as AccountID,
+                            isMe:
+                                editorID === node.account.id ? true : undefined,
+                            profile: editor.profile && {
+                                id: editor.profile.id,
+                                name: editor.profile.name,
+                            },
+                        },
+                        at: lastItem && new Date(lastItem.madeAt),
+                        all: items.map((item) => ({
+                            value: item.item && resolveValue(item.item),
+                            at: new Date(item.madeAt),
+                        })),
+                    } satisfies QueriedCoStreamItems<JsonValue>,
+                ];
+            })
+        );
+
+        const perAccount = Object.fromEntries(
+            [...seenAccounts.values()].map((accountID) => {
+                const itemsFromAllMatchingSessions = Object.entries(perSession)
+                    .flatMap(([sessionID, sessionItems]) =>
+                        sessionID.startsWith(accountID) ? sessionItems.all : []
+                    )
+                    .sort((a, b) => {
+                        return a.at.getTime() - b.at.getTime();
+                    });
+                const editor = getChildLastQueriedOrSubscribe(accountID);
+                const lastItem =
+                    itemsFromAllMatchingSessions[
+                        itemsFromAllMatchingSessions.length - 1
+                    ];
+
+                return [
+                    accountID,
+                    {
+                        last: lastItem?.value,
+                        by: editor && {
+                            id: accountID,
+                            isMe:
+                                accountID === node.account.id
+                                    ? true
+                                    : undefined,
+                            profile: editor.profile && {
+                                id: editor.profile.id,
+                                name: editor.profile.name,
+                            },
+                        },
+                        at: lastItem && new Date(lastItem.at),
+                        all: itemsFromAllMatchingSessions,
+                    } satisfies QueriedCoStreamItems<JsonValue>,
+                ];
+            })
+        );
+
+        const me = isAccountID(node.account.id)
+            ? perAccount[node.account.id]
+            : undefined;
+
+        const streamResult: QueriedCoStream<AnyCoStream> = {
+            type: "costream",
+            id: rootValue.id,
+            perSession,
+            perAccount,
+            me,
+            meta: rootValue.meta,
+            get group() {
+                return rootValue.group;
+            },
+            get core() {
+                return rootValue.core;
+            },
+            edit: (
+                changer: (editable: WriteableCoStream<any, any>) => void
+            ) => {
+                rootValue.edit(changer);
+                return rootValue;
+            },
+        };
+
+        return streamResult;
+    }
+}

packages/cojson/src/streamUtils.ts

@@ -34,7 +34,14 @@ export function connectedPeers(
                     trace &&
                         console.debug(
                             `${peer2id} -> ${peer1id}`,
-                            JSON.stringify(chunk, null, 2)
+                            JSON.stringify(
+                                chunk,
+                                (k, v) =>
+                                    (k === "changes" || k === "encryptedChanges")
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                                2
+                            )
                         );
                     controller.enqueue(chunk);
                 },
@@ -52,7 +59,14 @@ export function connectedPeers(
                     trace &&
                         console.debug(
                             `${peer1id} -> ${peer2id}`,
-                            JSON.stringify(chunk, null, 2)
+                            JSON.stringify(
+                                chunk,
+                                (k, v) =>
+                                    (k === "changes" || k === "encryptedChanges")
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                                2
+                            )
                         );
                     controller.enqueue(chunk);
                 },
@@ -102,16 +116,22 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
         },
     });
 
+    let lastWritePromise = Promise.resolve();
+
     const writable = new WritableStream<T>({
         async write(chunk) {
             const enqueue = await enqueuePromise;
             if (readerClosed) {
                 throw new Error("Reader closed");
             } else {
-                // make sure write resolves before corresponding read
-                setTimeout(() => {
-                    enqueue(chunk);
-                })
+                // make sure write resolves before corresponding read, but make sure writes are still in order
+                await lastWritePromise;
+                lastWritePromise = new Promise((resolve) => {
+                    setTimeout(() => {
+                        enqueue(chunk);
+                        resolve();
+                    });
+                });
             }
         },
         async abort(reason) {

packages/cojson/src/sync.ts

@@ -215,14 +215,32 @@ export class SyncManager {
             await this.sendNewContentIncludingDependencies(id, peer);
         }
 
-        const newContent = coValue.newContentSince(
+        const newContentPieces = coValue.newContentSince(
             peer.optimisticKnownStates[id]
         );
 
-        if (newContent) {
-            await this.trySendToPeer(peer, newContent);
+        if (newContentPieces) {
+            const optimisticKnownStateBefore =
+                peer.optimisticKnownStates[id] || emptyKnownState(id);
+
+            const sendPieces = async () => {
+                for (const [i, piece] of newContentPieces.entries()) {
+                    // console.log(
+                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
+                    //     // Object.values(piece.new).map((s) => s.newTransactions)
+                    // );
+                    await this.trySendToPeer(peer, piece);
+                }
+            };
+
+            sendPieces().catch((e) => {
+                console.error("Error sending new content piece, retrying", e);
+                peer.optimisticKnownStates[id] = optimisticKnownStateBefore;
+                return this.sendNewContentIncludingDependencies(id, peer);
+            });
+
             peer.optimisticKnownStates[id] = combinedKnownStates(
-                peer.optimisticKnownStates[id] || emptyKnownState(id),
+                optimisticKnownStateBefore,
                 coValue.knownState()
             );
         }
@@ -261,10 +279,17 @@ export class SyncManager {
                 for await (const msg of peerState.incoming) {
                     try {
                         await this.handleSyncMessage(msg, peerState);
+                        await new Promise<void>((resolve) => {
+                            setTimeout(resolve, 0);
+                        });
                     } catch (e) {
                         console.error(
                             `Error reading from peer ${peer.id}, handling msg`,
-                            JSON.stringify(msg),
+                            JSON.stringify(msg, (k, v) =>
+                                k === "changes" || k === "encryptedChanges"
+                                    ? v.slice(0, 20) + "..."
+                                    : v
+                            ),
                             e
                         );
                     }
@@ -445,6 +470,10 @@ export class SyncManager {
             const newTransactions =
                 newContentForSession.newTransactions.slice(alreadyKnownOffset);
 
+            if (newTransactions.length === 0) {
+                continue;
+            }
+
             const before = performance.now();
             const success = await coValue.tryAddTransactionsAsync(
                 sessionID,
@@ -454,20 +483,30 @@ export class SyncManager {
             );
             const after = performance.now();
             if (after - before > 10) {
-                const totalTxLength = newTransactions.map(t => stableStringify(t)!.length).reduce((a, b) => a + b, 0);
+                const totalTxLength = newTransactions
+                    .map((t) =>
+                        t.privacy === "private"
+                            ? t.encryptedChanges.length
+                            : t.changes.length
+                    )
+                    .reduce((a, b) => a + b, 0);
                 console.log(
-                    "Adding incoming transactions took",
-                    after - before,
-                    "ms",
-                    totalTxLength,
-                    "bytes = ",
-                    "bandwidth: MB/s",
-                    (1000 * totalTxLength / (after - before)) / (1024 * 1024)
+                    `Adding incoming transactions took ${(
+                        after - before
+                    ).toFixed(2)}ms for ${totalTxLength} bytes = bandwidth: ${(
+                        (1000 * totalTxLength) /
+                        (after - before) /
+                        (1024 * 1024)
+                    ).toFixed(2)} MB/s`
                 );
             }
 
             if (!success) {
-                console.error("Failed to add transactions", newTransactions);
+                console.error(
+                    "Failed to add transactions",
+                    msg.id,
+                    newTransactions
+                );
                 continue;
             }
 
@@ -492,18 +531,9 @@ export class SyncManager {
     }
 
     async handleCorrection(msg: KnownStateMessage, peer: PeerState) {
-        const coValue = this.local.expectCoValueLoaded(msg.id);
+        peer.optimisticKnownStates[msg.id] = msg;
 
-        peer.optimisticKnownStates[msg.id] = combinedKnownStates(
-            msg,
-            coValue.knownState()
-        );
-
-        const newContent = coValue.newContentSince(msg);
-
-        if (newContent) {
-            await this.trySendToPeer(peer, newContent);
-        }
+        return this.sendNewContentIncludingDependencies(msg.id, peer);
     }
 
     handleUnsubscribe(_msg: DoneMessage) {

packages/cojson/src/tests/account.test.ts

@@ -1,7 +1,7 @@
-import { newRandomSessionID } from "./coValueCore.js";
-import { cojsonReady } from "./index.js";
-import { LocalNode } from "./node.js";
-import { connectedPeers } from "./streamUtils.js";
+import { newRandomSessionID } from "../coValueCore.js";
+import { cojsonReady } from "../index.js";
+import { LocalNode } from "../node.js";
+import { connectedPeers } from "../streamUtils.js";
 
 beforeEach(async () => {
     await cojsonReady;

packages/cojson/src/tests/coValue.test.ts

@@ -1,8 +1,8 @@
-import { accountOrAgentIDfromSessionID } from "./coValueCore.js";
-import { BinaryCoStream } from "./coValues/coStream.js";
-import { createdNowUnique } from "./crypto.js";
-import { cojsonReady } from "./index.js";
-import { LocalNode } from "./node.js";
+import { accountOrAgentIDfromSessionID } from "../coValueCore.js";
+import { BinaryCoStream } from "../coValues/coStream.js";
+import { createdNowUnique } from "../crypto.js";
+import { MAX_RECOMMENDED_TX_SIZE, cojsonReady } from "../index.js";
+import { LocalNode } from "../node.js";
 import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
 
 beforeEach(async () => {
@@ -382,14 +382,14 @@ test("Can push into BinaryCoStream", () => {
 
     content.edit((editable) => {
         editable.startBinaryStream({mimeType: "text/plain", fileName: "test.txt"}, "trusting");
-        expect(editable.getBinaryChunks()).toEqual({
+        expect(editable.getBinaryChunks(true)).toEqual({
             mimeType: "text/plain",
             fileName: "test.txt",
             chunks: [],
             finished: false,
         });
         editable.pushBinaryStreamChunk(new Uint8Array([1, 2, 3]), "trusting");
-        expect(editable.getBinaryChunks()).toEqual({
+        expect(editable.getBinaryChunks(true)).toEqual({
             mimeType: "text/plain",
             fileName: "test.txt",
             chunks: [new Uint8Array([1, 2, 3])],
@@ -397,7 +397,7 @@ test("Can push into BinaryCoStream", () => {
         });
         editable.pushBinaryStreamChunk(new Uint8Array([4, 5, 6]), "trusting");
 
-        expect(editable.getBinaryChunks()).toEqual({
+        expect(editable.getBinaryChunks(true)).toEqual({
             mimeType: "text/plain",
             fileName: "test.txt",
             chunks: [new Uint8Array([1, 2, 3]), new Uint8Array([4, 5, 6])],
@@ -413,3 +413,112 @@ test("Can push into BinaryCoStream", () => {
         });
     });
 });
+
+test("When adding large transactions (small fraction of MAX_RECOMMENDED_TX_SIZE), we store an inbetween signature every time we reach MAX_RECOMMENDED_TX_SIZE and split up newContentSince accordingly", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "costream",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: { type: "binary" },
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "costream" || content.meta?.type !== "binary" || !(content instanceof BinaryCoStream)) {
+        throw new Error("Expected binary stream");
+    }
+
+    content.edit((editable) => {
+        editable.startBinaryStream({mimeType: "text/plain", fileName: "test.txt"}, "trusting");
+    });
+
+    for (let i = 0; i < 10; i++) {
+        const chunk = new Uint8Array(MAX_RECOMMENDED_TX_SIZE/3 + 100);
+
+        content.edit((editable) => {
+            editable.pushBinaryStreamChunk(chunk, "trusting");
+        });
+    }
+
+    content.edit((editable) => {
+        editable.endBinaryStream("trusting");
+    });
+
+    const sessionEntry = coValue._sessions[node.currentSessionID]!;
+    expect(sessionEntry.transactions.length).toEqual(12);
+    expect(sessionEntry.signatureAfter[0]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[1]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[2]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[3]).toBeDefined();
+    expect(sessionEntry.signatureAfter[4]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[5]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[6]).toBeDefined();
+    expect(sessionEntry.signatureAfter[7]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[8]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[9]).toBeDefined();
+    expect(sessionEntry.signatureAfter[10]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[11]).not.toBeDefined();
+
+    const newContent = coValue.newContentSince({id: coValue.id, header: false, sessions: {}})!;
+
+    expect(newContent.length).toEqual(5)
+    expect(newContent[0]!.header).toBeDefined();
+    expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[3]);
+    expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[6]);
+    expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[9]);
+    expect(newContent[4]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.lastSignature);
+});
+
+test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we store an inbetween signature after every large transaction and split up newContentSince accordingly", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "costream",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: { type: "binary" },
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "costream" || content.meta?.type !== "binary" || !(content instanceof BinaryCoStream)) {
+        throw new Error("Expected binary stream");
+    }
+
+    content.edit((editable) => {
+        editable.startBinaryStream({mimeType: "text/plain", fileName: "test.txt"}, "trusting");
+    });
+
+    const chunk = new Uint8Array(MAX_RECOMMENDED_TX_SIZE + 100);
+
+    for (let i = 0; i < 3; i++) {
+        content.edit((editable) => {
+            editable.pushBinaryStreamChunk(chunk, "trusting");
+        });
+    }
+
+    content.edit((editable) => {
+        editable.endBinaryStream("trusting");
+    });
+
+    const sessionEntry = coValue._sessions[node.currentSessionID]!;
+    expect(sessionEntry.transactions.length).toEqual(5);
+    expect(sessionEntry.signatureAfter[0]).not.toBeDefined();
+    expect(sessionEntry.signatureAfter[1]).toBeDefined();
+    expect(sessionEntry.signatureAfter[2]).toBeDefined();
+    expect(sessionEntry.signatureAfter[3]).toBeDefined();
+    expect(sessionEntry.signatureAfter[4]).not.toBeDefined();
+
+    const newContent = coValue.newContentSince({id: coValue.id, header: false, sessions: {}})!;
+
+    expect(newContent.length).toEqual(5)
+    expect(newContent[0]!.header).toBeDefined();
+    expect(newContent[1]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[1]);
+    expect(newContent[2]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[2]);
+    expect(newContent[3]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.signatureAfter[3]);
+    expect(newContent[4]!.new[node.currentSessionID]!.lastSignature).toEqual(sessionEntry.lastSignature);
+});
+
+

packages/cojson/src/tests/coValueCore.test.ts

@@ -1,11 +1,11 @@
-import { Transaction } from "./coValueCore.js";
-import { LocalNode } from "./node.js";
-import { createdNowUnique, getAgentSignerSecret, newRandomAgentSecret, sign } from "./crypto.js";
+import { Transaction } from "../coValueCore.js";
+import { LocalNode } from "../node.js";
+import { createdNowUnique, getAgentSignerSecret, newRandomAgentSecret, sign } from "../crypto.js";
 import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
-import { MapOpPayload } from "./coValues/coMap.js";
-import { Role } from "./permissions.js";
-import { cojsonReady } from "./index.js";
-import { stableStringify } from "./jsonStringify.js";
+import { MapOpPayload } from "../coValues/coMap.js";
+import { Role } from "../permissions.js";
+import { cojsonReady } from "../index.js";
+import { stableStringify } from "../jsonStringify.js";
 
 beforeEach(async () => {
     await cojsonReady;

packages/cojson/src/tests/crypto.test.ts

@@ -14,14 +14,14 @@ import {
     decryptForTransaction,
     encryptKeySecret,
     decryptKeySecret,
-} from './crypto.js';
+} from '../crypto.js';
 import { base58, base64url } from "@scure/base";
 import { x25519 } from "@noble/curves/ed25519";
 import { xsalsa20_poly1305 } from "@noble/ciphers/salsa";
 import { blake3 } from "@noble/hashes/blake3";
 import stableStringify from "fast-json-stable-stringify";
-import { SessionID } from './ids.js';
-import { cojsonReady } from './index.js';
+import { SessionID } from '../ids.js';
+import { cojsonReady } from '../index.js';
 
 beforeEach(async () => {
     await cojsonReady;

packages/cojson/src/tests/group.test.ts

@@ -1,5 +1,5 @@
-import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream, cojsonReady } from "./index";
-import { randomAnonymousAccountAndSessionID } from "./testUtils";
+import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream, cojsonReady } from "../index";
+import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
 
 beforeEach(async () => {
     await cojsonReady;

packages/cojson/src/tests/permissions.test.ts

@@ -1,6 +1,6 @@
-import { newRandomSessionID } from "./coValueCore.js";
-import { expectMap } from "./coValue.js";
-import { Group, expectGroupContent } from "./group.js";
+import { newRandomSessionID } from "../coValueCore.js";
+import { expectMap } from "../coValue.js";
+import { Group, expectGroupContent } from "../group.js";
 import {
     createdNowUnique,
     newRandomKeySecret,
@@ -10,14 +10,14 @@ import {
     getAgentID,
     getAgentSealerSecret,
     getAgentSealerID,
-} from "./crypto.js";
+} from "../crypto.js";
 import {
     newGroup,
     newGroupHighLevel,
     groupWithTwoAdmins,
     groupWithTwoAdminsHighLevel,
 } from "./testUtils.js";
-import { AnonymousControlledAccount, cojsonReady } from "./index.js";
+import { AnonymousControlledAccount, cojsonReady } from "../index.js";
 
 beforeEach(async () => {
     await cojsonReady;

packages/cojson/src/tests/queries.test.ts

@@ -0,0 +1,301 @@
+import { BinaryCoStream, CoList, CoMap, CoStream, Group, LocalNode, cojsonReady } from "..";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
+
+test("Queries with maps work", async () => {
+    const { node, accountID } =
+        LocalNode.withNewlyCreatedAccount("Hermes Puggington");
+
+    const group = node.createGroup();
+
+    let map = group.createMap<
+        CoMap<{
+            hello: "world";
+            subMap: CoMap<{
+                hello: "world" | "moon" | "sun";
+                id: string;
+            }>;
+        }>
+    >();
+
+    const done = new Promise<void>((resolve) => {
+        const unsubQuery = node.query(map.id, (queriedMap) => {
+            // console.log("update", update);
+            if (queriedMap) {
+                expect(queriedMap.type).toBe("comap");
+                expect(queriedMap.id).toEqual(map.id);
+                expect(queriedMap.core).toEqual(map.core);
+                expect(queriedMap.group).toBeInstanceOf(Group);
+                expect(queriedMap.group.id).toBe(group.id);
+                expect(queriedMap.meta).toBe(null);
+                expect(queriedMap.hello).toBe("world");
+                expect(Object.keys(queriedMap)).toEqual(["hello", "subMap"]);
+                if (queriedMap.edits.hello?.by?.profile?.name) {
+                    expect(queriedMap.edits.hello).toMatchObject({
+                        by: {
+                            id: accountID,
+                            profile: {
+                                id: node.expectProfileLoaded(accountID).id,
+                                name: "Hermes Puggington",
+                            },
+                            isMe: true,
+                        },
+                        at: new Date(map.getLastEntry("hello")!.at),
+                    });
+                    if (queriedMap.subMap) {
+                        expect(queriedMap.subMap.type).toBe("comap");
+                        expect(queriedMap.subMap.id).toEqual(subMap.id);
+                        expect(queriedMap.subMap.core).toEqual(subMap.core);
+                        expect(queriedMap.subMap.group).toBeInstanceOf(Group);
+                        expect(queriedMap.subMap.group.id).toBe(group.id);
+                        expect(queriedMap.subMap.meta).toBe(null);
+                        expect(queriedMap.subMap.shadowed.id).toBe("foreignID");
+                        if (queriedMap.subMap.hello === "moon") {
+                            // console.log("got to 'moon'");
+                            queriedMap.subMap.edit((subMap) => {
+                                subMap.set("hello", "sun");
+                            });
+                        } else if (
+                            queriedMap.subMap.hello === "sun" &&
+                            queriedMap.subMap.edits.hello?.by?.profile?.name ===
+                                "Hermes Puggington"
+                        ) {
+                            // console.log("final update", queriedMap);
+                            resolve();
+                            unsubQuery();
+                        }
+                    }
+                }
+            }
+        });
+    });
+
+    map = map.edit((map) => {
+        map.set("hello", "world");
+    });
+
+    let subMap = group.createMap<
+        CoMap<{
+            hello: "world" | "moon" | "sun";
+            id: string;
+        }>
+    >();
+
+    map = map.edit((map) => {
+        map.set("subMap", subMap);
+    });
+
+    subMap = subMap.edit((subMap) => {
+        subMap.set("hello", "world");
+        subMap.set("id", "foreignID");
+    });
+
+    subMap = subMap.edit((subMap) => {
+        subMap.set("hello", "moon");
+    });
+
+    await done;
+});
+
+test("Queries with lists work", () => {
+    const { node, accountID } =
+        LocalNode.withNewlyCreatedAccount("Hermes Puggington");
+
+    const group = node.createGroup();
+
+    let list = group.createList<CoList<string>>();
+
+    const done = new Promise<void>((resolve) => {
+        const unsubQuery = node.query(list.id, (queriedList) => {
+            if (queriedList) {
+                // console.log("update", queriedList, queriedList.edits);
+                expect(queriedList.type).toBe("colist");
+                expect(queriedList.id).toEqual(list.id);
+                expect(queriedList.core).toEqual(list.core);
+                expect(queriedList.group).toBeInstanceOf(Group);
+                expect(queriedList.group.id).toBe(group.id);
+                expect(queriedList.meta).toBe(null);
+                expect(queriedList[0]).toBe("hello");
+                expect(queriedList[1]).toBe("world");
+                expect(queriedList[2]).toBe("moon");
+                if (queriedList.edits[2]?.by?.profile?.name) {
+                    expect(queriedList.edits[2]).toMatchObject({
+                        by: {
+                            id: accountID,
+                            profile: {
+                                id: node.expectProfileLoaded(accountID).id,
+                                name: "Hermes Puggington",
+                            },
+                            isMe: true,
+                        },
+                        at: expect.any(Date),
+                    });
+                    if (queriedList.length === 3) {
+                        queriedList.edit((list) => {
+                            list.push("sun");
+                        });
+                    } else if (
+                        queriedList.length === 4 &&
+                        queriedList.edits[3]?.by?.profile?.name ===
+                            "Hermes Puggington"
+                    ) {
+                        expect(queriedList[3]).toBe("sun");
+                        // console.log("final update", queriedList);
+                        resolve();
+                        unsubQuery();
+                    }
+                }
+            }
+        });
+    });
+
+    list = list.edit((list) => {
+        list.push("hello");
+        list.push("world");
+        list.push("moon");
+    });
+
+    return done;
+});
+
+test("List of nested maps works", () => {
+    const { node } = LocalNode.withNewlyCreatedAccount("Hermes Puggington");
+
+    const group = node.createGroup();
+
+    let list = group.createList<CoList<CoMap<{ hello: "world" }>>>();
+
+    const done = new Promise<void>((resolve) => {
+        const unsubQuery = node.query(list.id, (queriedList) => {
+            if (queriedList && queriedList[0]) {
+                // console.log("update", queriedList);
+                expect(queriedList[0]).toMatchObject({
+                    hello: "world",
+                    id: list.get(0)!,
+                });
+                // console.log("final update", queriedList);
+                resolve();
+                unsubQuery();
+            }
+        });
+    });
+
+    list = list.edit((list) => {
+        list.push(
+            group.createMap<CoMap<{ hello: "world" }>>({
+                hello: "world",
+            })
+        );
+    });
+
+    return done;
+});
+
+test("Queries with streams work", () => {
+    const { node, accountID } =
+        LocalNode.withNewlyCreatedAccount("Hermes Puggington");
+
+    const group = node.createGroup();
+
+    let stream = group.createStream<CoStream<string>>();
+
+    const done = new Promise<void>((resolve) => {
+        const unsubQuery = node.query(stream.id, (queriedStream) => {
+            if (queriedStream) {
+                console.log("update", queriedStream);
+                if (queriedStream.me?.by?.profile?.name) {
+                    expect(queriedStream.type).toBe("costream");
+                    expect(queriedStream.id).toEqual(stream.id);
+                    expect(queriedStream.core).toEqual(stream.core);
+                    expect(queriedStream.group).toBeInstanceOf(Group);
+                    expect(queriedStream.group.id).toBe(group.id);
+                    expect(queriedStream.meta).toBe(null);
+                    const expectedEntry = {
+                        last: "world",
+                        by: {
+                            id: accountID,
+                            isMe: true,
+                            profile: {
+                                id: node.expectProfileLoaded(accountID).id,
+                                name: "Hermes Puggington",
+                            },
+                        },
+                        at: new Date(
+                            stream.items[node.currentSessionID][1].madeAt
+                        ),
+                        all: [
+                            {
+                                value: "hello",
+                                at: new Date(
+                                    stream.items[
+                                        node.currentSessionID
+                                    ][0].madeAt
+                                ),
+                            },
+                            {
+                                value: "world",
+                                at: new Date(
+                                    stream.items[
+                                        node.currentSessionID
+                                    ][1].madeAt
+                                ),
+                            },
+                        ],
+                    };
+                    expect(queriedStream.perSession).toMatchObject({
+                        [node.currentSessionID]: expectedEntry,
+                    });
+                    expect(queriedStream.perAccount).toMatchObject({
+                        [accountID]: expectedEntry,
+                    });
+                    expect(queriedStream.me).toMatchObject(expectedEntry);
+                    console.log("final update", queriedStream);
+                    resolve();
+                    unsubQuery();
+                }
+            }
+        });
+    });
+
+    stream = stream.edit((stream) => {
+        stream.push("hello");
+        stream.push("world");
+    });
+
+    return done;
+});
+
+test("Streams of nested maps work", () => {
+    const { node } = LocalNode.withNewlyCreatedAccount("Hermes Puggington");
+
+    const group = node.createGroup();
+
+    let stream = group.createStream<CoStream<CoMap<{ hello: "world" }>>>();
+
+    const done = new Promise<void>((resolve) => {
+        const unsubQuery = node.query(stream.id, (queriedStream) => {
+            if (queriedStream && queriedStream.me?.last) {
+                // console.log("update", queriedList);
+                expect(queriedStream.me.last).toMatchObject({
+                    hello: "world",
+                    id: map.id,
+                });
+                // console.log("final update", queriedList);
+                resolve();
+                unsubQuery();
+            }
+        });
+    });
+
+    const map = group.createMap<CoMap<{ hello: "world" }>>({
+        hello: "world",
+    });
+
+    stream = stream.edit((list) => {
+        list.push(map);
+    });
+
+    return done;
+});

packages/cojson/src/tests/sync.test.ts

@@ -1,17 +1,17 @@
-import { newRandomSessionID } from "./coValueCore.js";
-import { LocalNode } from "./node.js";
-import { SyncMessage } from "./sync.js";
-import { expectMap } from "./coValue.js";
-import { MapOpPayload } from "./coValues/coMap.js";
-import { Group } from "./group.js";
+import { newRandomSessionID } from "../coValueCore.js";
+import { LocalNode } from "../node.js";
+import { SyncMessage } from "../sync.js";
+import { expectMap } from "../coValue.js";
+import { MapOpPayload } from "../coValues/coMap.js";
+import { Group } from "../group.js";
 import {
     randomAnonymousAccountAndSessionID,
     shouldNotResolve,
 } from "./testUtils.js";
-import { connectedPeers, newStreamPair } from "./streamUtils.js";
-import { AccountID } from "./account.js";
-import { cojsonReady } from "./index.js";
-import { stableStringify } from "./jsonStringify.js";
+import { connectedPeers, newStreamPair } from "../streamUtils.js";
+import { AccountID } from "../account.js";
+import { cojsonReady } from "../index.js";
+import { stableStringify } from "../jsonStringify.js";
 
 beforeEach(async () => {
     await cojsonReady;
@@ -436,8 +436,9 @@ test("No matter the optimistic known state, node respects invalid known state me
         editable.set("goodbye", "world", "trusting");
     });
 
-    const _mapEditMsg1 = await reader.read();
-    const _mapEditMsg2 = await reader.read();
+    const _mapEditMsgs = await reader.read();
+
+    console.log("Sending correction");
 
     await writer.write({
         action: "known",

packages/cojson/src/tests/testUtils.ts

@@ -1,9 +1,11 @@
-import { AgentSecret, createdNowUnique, getAgentID, newRandomAgentSecret  } from "./crypto.js";
-import { newRandomSessionID } from "./coValueCore.js";
-import { LocalNode } from "./node.js";
-import { expectGroupContent } from "./group.js";
-import { AnonymousControlledAccount } from "./account.js";
-import { SessionID } from "./ids.js";
+import { AgentSecret, createdNowUnique, getAgentID, newRandomAgentSecret  } from "../crypto.js";
+import { newRandomSessionID } from "../coValueCore.js";
+import { LocalNode } from "../node.js";
+import { expectGroupContent } from "../group.js";
+import { AnonymousControlledAccount } from "../account.js";
+import { SessionID } from "../ids.js";
+// @ts-ignore
+import { expect } from "bun:test";
 
 export function randomAnonymousAccountAndSessionID(): [AnonymousControlledAccount, SessionID] {
     const agentSecret = newRandomAgentSecret();

packages/jazz-browser-auth-local/package.json

@@ -1,11 +1,11 @@
 {
     "name": "jazz-browser-auth-local",
-    "version": "0.2.0",
+    "version": "0.2.5",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
     "license": "MIT",
     "dependencies": {
-        "jazz-browser": "^0.2.0",
+        "jazz-browser": "^0.2.5",
         "typescript": "^5.1.6"
     },
     "scripts": {

packages/jazz-browser-media-images/package.json

@@ -1,13 +1,13 @@
 {
   "name": "jazz-browser-media-images",
-  "version": "0.2.0",
+  "version": "0.2.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.2.0",
+    "cojson": "^0.2.3",
     "image-blob-reduce": "^4.1.0",
-    "jazz-browser": "^0.2.0",
+    "jazz-browser": "^0.2.5",
     "typescript": "^5.1.6"
   },
   "scripts": {

packages/jazz-browser-media-images/src/index.ts

@@ -1,4 +1,4 @@
-import { CoID, Group, LocalNode, Media } from "cojson";
+import { CoID, Group, LocalNode, Media, Queried } from "cojson";
 
 import ImageBlobReduce from "image-blob-reduce";
 import Pica from "pica";
@@ -131,7 +131,7 @@ export type LoadingImageInfo = {
 };
 
 export function loadImage(
-    imageID: CoID<Media.ImageDefinition>,
+    image: CoID<Media.ImageDefinition> | Media.ImageDefinition | Queried<Media.ImageDefinition>,
     localNode: LocalNode,
     progressiveCallback: (update: LoadingImageInfo) => void
 ): () => void {
@@ -141,7 +141,8 @@ export function loadImage(
     const resState: {
         [res: `${number}x${number}`]:
             | { state: "queued" }
-            | { state: "loading" }
+            | { state: "waiting" }
+            | { state: "loading"; doneOrFailed: Promise<void> }
             | { state: "loaded"; blobURL: string }
             | { state: "revoked" }
             | { state: "failed" }
@@ -152,15 +153,16 @@ export function loadImage(
         stopped = true;
         for (const [res, entry] of Object.entries(resState)) {
             if (entry?.state === "loaded") {
-                URL.revokeObjectURL(entry.blobURL);
                 resState[res as `${number}x${number}`] = { state: "revoked" };
+                // prevent flashing from immediate revocation
+                setTimeout(() => {URL.revokeObjectURL(entry.blobURL)}, 3000);
             }
         }
         unsubscribe?.();
     };
 
     localNode
-        .load(imageID)
+        .load(typeof image === "string" ? image : image.id)
         .then((imageDefinition) => {
             if (stopped) return;
             unsubscribe = imageDefinition.subscribe(async (imageDefinition) => {
@@ -170,7 +172,8 @@ export function loadImage(
                 const placeholderDataURL =
                     imageDefinition.get("placeholderDataURL");
 
-                const resolutions = imageDefinition.keys()
+                const resolutions = imageDefinition
+                    .keys()
                     .filter(
                         (key): key is `${number}x${number}` =>
                             !!key.match(/\d+x\d+/)
@@ -182,48 +185,126 @@ export function loadImage(
                     });
 
                 const startLoading = async () => {
-
                     const notYetQueuedOrLoading = resolutions.filter(
                         (res) => !resState[res]
-                        );
+                    );
 
-                    console.log("Loading iteration", resolutions, resState, notYetQueuedOrLoading);
+                    // console.log(
+                    //     "Loading iteration",
+                    //     resolutions,
+                    //     resState,
+                    //     notYetQueuedOrLoading
+                    // );
 
                     for (const res of notYetQueuedOrLoading) {
-                            resState[res] = { state: "queued" };
+                        resState[res] = { state: "queued" };
                     }
 
                     for (const res of notYetQueuedOrLoading) {
                         if (stopped) return;
-                        resState[res] = { state: "loading" };
+                        resState[res] = { state: "waiting" };
 
                         const binaryStreamId = imageDefinition.get(res)!;
-                        console.log("Loading image res", imageID, res, binaryStreamId);
+                        // console.log(
+                        //     "Loading image res",
+                        //     imageID,
+                        //     res,
+                        //     binaryStreamId
+                        // );
 
-                        const blob = await readBlobFromBinaryStream(
-                            binaryStreamId,
-                            localNode
+                        const binaryStream = await localNode.load(
+                            binaryStreamId
                         );
 
                         if (stopped) return;
-                        if (!blob) {
+                        if (!binaryStream) {
                             resState[res] = { state: "failed" };
-                            console.log("Loading image res failed", imageID, res, binaryStreamId);
-                            continue;
+                            console.error(
+                                "Loading image res failed",
+                                image,
+                                res,
+                                binaryStreamId
+                            );
+                            return;
                         }
 
-                        const blobURL = URL.createObjectURL(blob);
-                        resState[res] = { state: "loaded", blobURL };
+                        await new Promise<void>((resolveFullyLoaded) => {
+                            const unsubFromStream = binaryStream.subscribe(
+                                async (_) => {
+                                    if (stopped) return;
+                                    const currentState = resState[res];
+                                    if (currentState?.state === "loading") {
+                                        await currentState.doneOrFailed;
+                                        // console.log(
+                                        //     "Retrying image res after previous attempt",
+                                        //     imageID,
+                                        //     res,
+                                        //     binaryStreamId
+                                        // );
+                                    }
+                                    if (resState[res]?.state === "loaded") {
+                                        return;
+                                    }
 
-                        console.log("Loaded image res", imageID, res, binaryStreamId);
+                                    const doneOrFailed = new Promise<void>(
+                                        // eslint-disable-next-line no-async-promise-executor
+                                        async (resolveDoneOrFailed) => {
+                                            const blob =
+                                                await readBlobFromBinaryStream(
+                                                    binaryStreamId,
+                                                    localNode
+                                                );
 
-                        progressiveCallback({
-                            originalSize,
-                            placeholderDataURL,
-                            highestResSrc: blobURL,
+                                            if (stopped) return;
+                                            if (!blob) {
+                                                // console.log(
+                                                //     "Image res not available yet",
+                                                //     imageID,
+                                                //     res,
+                                                //     binaryStreamId
+                                                // );
+                                                resolveDoneOrFailed();
+                                                return;
+                                            }
+
+                                            const blobURL =
+                                                URL.createObjectURL(blob);
+                                            resState[res] = {
+                                                state: "loaded",
+                                                blobURL,
+                                            };
+
+                                            // console.log(
+                                            //     "Loaded image res",
+                                            //     imageID,
+                                            //     res,
+                                            //     binaryStreamId
+                                            // );
+
+                                            progressiveCallback({
+                                                originalSize,
+                                                placeholderDataURL,
+                                                highestResSrc: blobURL,
+                                            });
+
+                                            unsubFromStream();
+                                            resolveDoneOrFailed();
+
+                                            await new Promise((resolve) =>
+                                                setTimeout(resolve, 0)
+                                            );
+
+                                            resolveFullyLoaded();
+                                        }
+                                    );
+
+                                    resState[res] = {
+                                        state: "loading",
+                                        doneOrFailed,
+                                    };
+                                }
+                            );
                         });
-
-                        await new Promise((resolve) => setTimeout(resolve, 0));
                     }
                 };
 
@@ -239,13 +320,13 @@ export function loadImage(
                 }
 
                 startLoading().catch((err) => {
-                    console.error("Error loading image", imageID, err);
+                    console.error("Error loading image", image, err);
                     cleanUp();
                 });
             });
         })
         .catch((err) => {
-            console.error("Error loading image", imageID, err);
+            console.error("Error loading image", image, err);
             cleanUp();
         });
 

packages/jazz-browser/package.json

@@ -1,12 +1,12 @@
 {
   "name": "jazz-browser",
-  "version": "0.2.0",
+  "version": "0.2.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.2.0",
-    "jazz-storage-indexeddb": "^0.2.0",
+    "cojson": "^0.2.3",
+    "jazz-storage-indexeddb": "^0.2.5",
     "typescript": "^5.1.6"
   },
   "scripts": {

packages/jazz-browser/src/index.ts

@@ -1,5 +1,6 @@
-import { BinaryCoStream, InviteSecret } from "cojson";
+import { BinaryCoStream, CoValue, InviteSecret, Queried } from "cojson";
 import { BinaryCoStreamMeta } from "cojson";
+import { MAX_RECOMMENDED_TX_SIZE } from "cojson";
 import { cojsonReady } from "cojson";
 import {
     LocalNode,
@@ -9,7 +10,6 @@ import {
     SessionID,
     SyncMessage,
     Peer,
-    CoValueImpl,
     Group,
     CoID,
 } from "cojson";
@@ -72,6 +72,12 @@ export async function createBrowserNode({
         node,
         done: () => {
             shouldTryToReconnect = false;
+            console.log("Cleaning up node");
+            for (const peer of Object.values(node.sync.peers)) {
+                peer.outgoing
+                    .close()
+                    .catch((e) => console.error("Error while closing peer", e));
+            }
             sessionDone?.();
         },
     };
@@ -285,8 +291,8 @@ function websocketWritableStream<T>(ws: WebSocket) {
     }
 }
 
-export function createInviteLink(
-    value: CoValueImpl,
+export function createInviteLink<T extends CoValue | Queried<CoValue>>(
+    value: T | Queried<T>,
     role: "reader" | "writer" | "admin",
     // default to same address as window.location, but without hash
     {
@@ -315,7 +321,7 @@ export function createInviteLink(
     return `${baseURL}#invitedTo=${value.id}&${inviteSecret}`;
 }
 
-export function parseInviteLink<C extends CoValueImpl>(
+export function parseInviteLink<C extends CoValue>(
     inviteURL: string
 ):
     | {
@@ -334,7 +340,7 @@ export function parseInviteLink<C extends CoValueImpl>(
     return { valueID, inviteSecret };
 }
 
-export function consumeInviteLinkFromWindowLocation<C extends CoValueImpl>(
+export function consumeInviteLinkFromWindowLocation<C extends CoValue>(
     node: LocalNode
 ): Promise<
     | {
@@ -382,17 +388,17 @@ export async function createBinaryStreamFromBlob<
                     totalSizeBytes: blob.size,
                     fileName: blob instanceof File ? blob.name : undefined,
                 });
-            }) as C;// TODO: fix this
-                const chunkSize = 256 * 1024;
+            }) as C; // TODO: fix this
+            const chunkSize = MAX_RECOMMENDED_TX_SIZE;
 
-                for (let idx = 0; idx < data.length; idx += chunkSize) {
-                    stream = stream.edit((stream) => {
-                        stream.pushBinaryStreamChunk(
-                            data.slice(idx, idx + chunkSize)
-                        );
-                    }) as C; // TODO: fix this
-                    await new Promise((resolve) => setTimeout(resolve, 0));
-                }
+            for (let idx = 0; idx < data.length; idx += chunkSize) {
+                stream = stream.edit((stream) => {
+                    stream.pushBinaryStreamChunk(
+                        data.slice(idx, idx + chunkSize)
+                    );
+                }) as C; // TODO: fix this
+                await new Promise((resolve) => setTimeout(resolve, 0));
+            }
             stream = stream.edit((stream) => {
                 stream.endBinaryStream();
             }) as C; // TODO: fix this
@@ -419,15 +425,11 @@ export async function readBlobFromBinaryStream<
         return undefined;
     }
 
-    const chunks = stream.getBinaryChunks();
+    const chunks = stream.getBinaryChunks(allowUnfinished);
 
     if (!chunks) {
         return undefined;
     }
 
-    if (!allowUnfinished && !chunks.finished) {
-        return undefined;
-    }
-
     return new Blob(chunks.chunks, { type: chunks.mimeType });
 }

packages/jazz-react-auth-local/package.json

@@ -1,12 +1,12 @@
 {
     "name": "jazz-react-auth-local",
-    "version": "0.2.0",
+    "version": "0.2.5",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
     "license": "MIT",
     "dependencies": {
-        "jazz-browser-auth-local": "^0.2.0",
-        "jazz-react": "^0.2.0",
+        "jazz-browser-auth-local": "^0.2.5",
+        "jazz-react": "^0.2.5",
         "typescript": "^5.1.6"
     },
     "devDependencies": {

packages/jazz-react-media-images/package.json

@@ -1,14 +1,14 @@
 {
   "name": "jazz-react-media-images",
-  "version": "0.2.0",
+  "version": "0.2.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.2.0",
-    "jazz-browser": "^0.2.0",
-    "jazz-browser-media-images": "^0.2.0",
-    "jazz-react": "^0.2.0",
+    "cojson": "^0.2.3",
+    "jazz-browser": "^0.2.5",
+    "jazz-browser-media-images": "^0.2.5",
+    "jazz-react": "^0.2.5",
     "typescript": "^5.1.6"
   },
   "devDependencies": {

packages/jazz-react-media-images/src/index.tsx

@@ -1,11 +1,11 @@
-import { CoID, Media } from "cojson";
+import { CoID, Media, Queried } from "cojson";
 import { loadImage, LoadingImageInfo } from "jazz-browser-media-images";
 import { useJazz } from "jazz-react";
 import { useEffect, useState } from "react";
-export { createImage } from "jazz-browser-media-images";
+export { createImage, LoadingImageInfo } from "jazz-browser-media-images";
 
 export function useLoadImage(
-    imageID?: CoID<Media.ImageDefinition>
+    imageID?: CoID<Media.ImageDefinition> | Media.ImageDefinition | Queried<Media.ImageDefinition>
 ): LoadingImageInfo | undefined {
     const { localNode } = useJazz();
 

packages/jazz-react/package.json

@@ -1,12 +1,12 @@
 {
   "name": "jazz-react",
-  "version": "0.2.0",
+  "version": "0.2.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.2.0",
-    "jazz-browser": "^0.2.0",
+    "cojson": "^0.2.3",
+    "jazz-browser": "^0.2.5",
     "typescript": "^5.1.6"
   },
   "devDependencies": {

packages/jazz-react/src/index.tsx

@@ -1,6 +1,5 @@
 import {
     LocalNode,
-    CoValueImpl,
     CoID,
     CoMap,
     AccountID,
@@ -8,6 +7,8 @@ import {
     CojsonInternalTypes,
     BinaryCoStream,
     BinaryCoStreamMeta,
+    Queried,
+    CoValue
 } from "cojson";
 import React, { useEffect, useState } from "react";
 import { AuthProvider, createBrowserNode } from "jazz-browser";
@@ -47,6 +48,7 @@ export function WithJazz({
 
     useEffect(() => {
         let done: (() => void) | undefined = undefined;
+        let stop = false;
 
         (async () => {
             const nodeHandle = await createBrowserNode({
@@ -57,6 +59,11 @@ export function WithJazz({
                     undefined,
             });
 
+            if (stop) {
+                nodeHandle.done();
+                return;
+            }
+
             setNode(nodeHandle.node);
 
             done = nodeHandle.done;
@@ -65,6 +72,7 @@ export function WithJazz({
         });
 
         return () => {
+            stop = true;
             done && done();
         };
     }, [auth, syncAddress]);
@@ -92,7 +100,7 @@ export function useJazz() {
     return context;
 }
 
-export function useTelepathicState<T extends CoValueImpl>(id?: CoID<T>) {
+export function useTelepathicState<T extends CoValue>(id?: CoID<T>) {
     const [state, setState] = useState<T>();
 
     const { localNode } = useJazz();
@@ -129,6 +137,22 @@ export function useTelepathicState<T extends CoValueImpl>(id?: CoID<T>) {
     return state;
 }
 
+export function useTelepathicQuery<T extends CoValue>(
+    id?: CoID<T>
+): Queried<T> | undefined {
+    const { localNode } = useJazz();
+
+    const [result, setResult] = useState<Queried<T> | undefined>();
+
+    useEffect(() => {
+        if (!id) return;
+        const unsubscribe = localNode.query(id, setResult);
+        return unsubscribe;
+    }, [id, localNode]);
+
+    return result;
+}
+
 export function useProfile<
     P extends {
         [key: string]: JsonValue;

packages/jazz-storage-indexeddb/package.json

@@ -1,11 +1,11 @@
 {
   "name": "jazz-storage-indexeddb",
-  "version": "0.2.0",
+  "version": "0.2.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.2.0",
+    "cojson": "^0.2.3",
     "typescript": "^5.1.6"
   },
   "devDependencies": {

packages/jazz-storage-indexeddb/src/index.ts

@@ -4,6 +4,7 @@ import {
     SyncMessage,
     Peer,
     CojsonInternalTypes,
+    MAX_RECOMMENDED_TX_SIZE,
 } from "cojson";
 import {
     ReadableStream,
@@ -24,6 +25,7 @@ type SessionRow = {
     sessionID: SessionID;
     lastIdx: number;
     lastSignature: CojsonInternalTypes.Signature;
+    bytesSinceLastSignature?: number;
 };
 
 type StoredSessionRow = SessionRow & { rowID: number };
@@ -34,6 +36,12 @@ type TransactionRow = {
     tx: CojsonInternalTypes.Transaction;
 };
 
+type SignatureAfterRow = {
+    ses: number;
+    idx: number;
+    signature: CojsonInternalTypes.Signature;
+};
+
 export class IDBStorage {
     db: IDBDatabase;
     fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
@@ -55,7 +63,7 @@ export class IDBStorage {
                 done = result.done;
 
                 if (result.value) {
-                    this.handleSyncMessage(result.value);
+                    await this.handleSyncMessage(result.value);
                 }
             }
         })();
@@ -88,42 +96,63 @@ export class IDBStorage {
         toLocalNode: WritableStream<SyncMessage>
     ) {
         const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
-            const request = indexedDB.open("jazz-storage", 1);
+            const request = indexedDB.open("jazz-storage", 4);
             request.onerror = () => {
                 reject(request.error);
             };
             request.onsuccess = () => {
                 resolve(request.result);
             };
-            request.onupgradeneeded = () => {
+            request.onupgradeneeded = async (ev) => {
                 const db = request.result;
+                if (ev.oldVersion === 0) {
+                    const coValues = db.createObjectStore("coValues", {
+                        autoIncrement: true,
+                        keyPath: "rowID",
+                    });
 
-                const coValues = db.createObjectStore("coValues", {
-                    autoIncrement: true,
-                    keyPath: "rowID",
-                });
-
-                coValues.createIndex("coValuesById", "id", {
-                    unique: true,
-                });
-
-                const sessions = db.createObjectStore("sessions", {
-                    autoIncrement: true,
-                    keyPath: "rowID",
-                });
-
-                sessions.createIndex("sessionsByCoValue", "coValue");
-                sessions.createIndex(
-                    "uniqueSessions",
-                    ["coValue", "sessionID"],
-                    {
+                    coValues.createIndex("coValuesById", "id", {
                         unique: true,
-                    }
-                );
+                    });
 
-                db.createObjectStore("transactions", {
-                    keyPath: ["ses", "idx"],
-                });
+                    const sessions = db.createObjectStore("sessions", {
+                        autoIncrement: true,
+                        keyPath: "rowID",
+                    });
+
+                    sessions.createIndex("sessionsByCoValue", "coValue");
+                    sessions.createIndex(
+                        "uniqueSessions",
+                        ["coValue", "sessionID"],
+                        {
+                            unique: true,
+                        }
+                    );
+
+                    db.createObjectStore("transactions", {
+                        keyPath: ["ses", "idx"],
+                    });
+                }
+                if (ev.oldVersion <= 1) {
+                    db.createObjectStore("signatureAfter", {
+                        keyPath: ["ses", "idx"],
+                    });
+                }
+                if (ev.oldVersion !== 0 && ev.oldVersion <= 3) {
+                    // fix embarrassing off-by-one error for transaction indices
+                    console.log("Migration: fixing off-by-one error");
+                    const transaction = (ev.target as unknown as {transaction: IDBTransaction}).transaction;
+
+                    const txsStore = transaction.objectStore("transactions");
+                    const txs = await promised(txsStore.getAll());
+
+                    for (const tx of txs) {
+                        await promised(txsStore.delete([tx.ses, tx.idx]));
+                        tx.idx -= 1;
+                        await promised(txsStore.add(tx));
+                    }
+                    console.log("Migration: fixing off-by-one error - done");
+                }
             };
         });
 
@@ -153,10 +182,12 @@ export class IDBStorage {
             coValues,
             sessions,
             transactions,
+            signatureAfter,
         }: {
             coValues: IDBObjectStore;
             sessions: IDBObjectStore;
             transactions: IDBObjectStore;
+            signatureAfter: IDBObjectStore;
         },
         asDependencyOf?: CojsonInternalTypes.RawCoID
     ) {
@@ -176,12 +207,14 @@ export class IDBStorage {
             sessions: {},
         };
 
-        const newContent: CojsonInternalTypes.NewContentMessage = {
-            action: "content",
-            id: theirKnown.id,
-            header: theirKnown.header ? undefined : coValueRow?.header,
-            new: {},
-        };
+        const newContentPieces: CojsonInternalTypes.NewContentMessage[] = [
+            {
+                action: "content",
+                id: theirKnown.id,
+                header: theirKnown.header ? undefined : coValueRow?.header,
+                new: {},
+            },
+        ];
 
         for (const sessionRow of allOurSessions) {
             ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -193,6 +226,21 @@ export class IDBStorage {
                 const firstNewTxIdx =
                     theirKnown.sessions[sessionRow.sessionID] || 0;
 
+                const signaturesAndIdxs = await promised<SignatureAfterRow[]>(
+                    signatureAfter.getAll(
+                        IDBKeyRange.bound(
+                            [sessionRow.rowID, firstNewTxIdx],
+                            [sessionRow.rowID, Infinity]
+                        )
+                    )
+                );
+
+                // console.log(
+                //     theirKnown.id,
+                //     "signaturesAndIdxs",
+                //     JSON.stringify(signaturesAndIdxs)
+                // );
+
                 const newTxInSession = await promised<TransactionRow[]>(
                     transactions.getAll(
                         IDBKeyRange.bound(
@@ -202,38 +250,83 @@ export class IDBStorage {
                     )
                 );
 
-                newContent.new[sessionRow.sessionID] = {
-                    after: firstNewTxIdx,
-                    lastSignature: sessionRow.lastSignature,
-                    newTransactions: newTxInSession.map((row) => row.tx),
-                };
+                let idx = firstNewTxIdx;
+
+                // console.log(
+                //     theirKnown.id,
+                //     "newTxInSession",
+                //     newTxInSession.length
+                // );
+
+                for (const tx of newTxInSession) {
+                    let sessionEntry =
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ];
+                    if (!sessionEntry) {
+                        sessionEntry = {
+                            after: idx,
+                            lastSignature: "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
+                            newTransactions: [],
+                        };
+                        newContentPieces[newContentPieces.length - 1]!.new[
+                            sessionRow.sessionID
+                        ] = sessionEntry;
+                    }
+
+                    sessionEntry.newTransactions.push(tx.tx);
+
+                    if (
+                        signaturesAndIdxs[0] &&
+                        idx === signaturesAndIdxs[0].idx
+                    ) {
+                        sessionEntry.lastSignature =
+                            signaturesAndIdxs[0].signature;
+                        signaturesAndIdxs.shift();
+                        newContentPieces.push({
+                            action: "content",
+                            id: theirKnown.id,
+                            new: {},
+                        });
+                    } else if (
+                        idx ===
+                        firstNewTxIdx + newTxInSession.length - 1
+                    ) {
+                        sessionEntry.lastSignature = sessionRow.lastSignature;
+                    }
+                    idx += 1;
+                }
             }
         }
 
         const dependedOnCoValues =
             coValueRow?.header.ruleset.type === "group"
-                ? Object.values(newContent.new).flatMap((sessionEntry) =>
-                      sessionEntry.newTransactions.flatMap((tx) => {
-                          if (tx.privacy !== "trusting") return [];
-                          // TODO: avoid parse here?
-                          return cojsonInternals
-                              .parseJSON(tx.changes)
-                              .map(
-                                  (change) =>
-                                      change &&
-                                      typeof change === "object" &&
-                                      "op" in change &&
-                                      change.op === "set" &&
-                                      "key" in change &&
-                                      change.key
-                              )
-                              .filter(
-                                  (key): key is CojsonInternalTypes.RawCoID =>
-                                      typeof key === "string" &&
-                                      key.startsWith("co_")
-                              );
-                      })
-                  )
+                ? newContentPieces
+                      .flatMap((piece) => Object.values(piece.new))
+                      .flatMap((sessionEntry) =>
+                          sessionEntry.newTransactions.flatMap((tx) => {
+                              if (tx.privacy !== "trusting") return [];
+                              // TODO: avoid parse here?
+                              return cojsonInternals
+                                  .parseJSON(tx.changes)
+                                  .map(
+                                      (change) =>
+                                          change &&
+                                          typeof change === "object" &&
+                                          "op" in change &&
+                                          change.op === "set" &&
+                                          "key" in change &&
+                                          change.key
+                                  )
+                                  .filter(
+                                      (
+                                          key
+                                      ): key is CojsonInternalTypes.RawCoID =>
+                                          typeof key === "string" &&
+                                          key.startsWith("co_")
+                                  );
+                          })
+                      )
                 : coValueRow?.header.ruleset.type === "ownedByGroup"
                 ? [coValueRow?.header.ruleset.group]
                 : [];
@@ -241,7 +334,7 @@ export class IDBStorage {
         for (const dependedOnCoValue of dependedOnCoValues) {
             await this.sendNewContentAfter(
                 { id: dependedOnCoValue, header: false, sessions: {} },
-                { coValues, sessions, transactions },
+                { coValues, sessions, transactions, signatureAfter },
                 asDependencyOf || theirKnown.id
             );
         }
@@ -252,8 +345,15 @@ export class IDBStorage {
             asDependencyOf,
         });
 
-        if (newContent.header || Object.keys(newContent.new).length > 0) {
-            await this.toLocalNode.write(newContent);
+        const nonEmptyNewContentPieces = newContentPieces.filter(
+            (piece) => piece.header || Object.keys(piece.new).length > 0
+        );
+
+        // console.log(theirKnown.id, nonEmptyNewContentPieces);
+
+        for (const piece of nonEmptyNewContentPieces) {
+            await this.toLocalNode.write(piece);
+            await new Promise((resolve) => setTimeout(resolve, 0));
         }
     }
 
@@ -262,7 +362,7 @@ export class IDBStorage {
     }
 
     async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
-        const { coValues, sessions, transactions } =
+        const { coValues, sessions, transactions, signatureAfter } =
             this.inTransaction("readwrite");
 
         let storedCoValueRowID = (
@@ -333,18 +433,39 @@ export class IDBStorage {
                 const actuallyNewOffset =
                     (sessionRow?.lastIdx || 0) -
                     (msg.new[sessionID]?.after || 0);
+
                 const actuallyNewTransactions =
                     newTransactions.slice(actuallyNewOffset);
 
+                let newBytesSinceLastSignature =
+                    (sessionRow?.bytesSinceLastSignature || 0) +
+                    actuallyNewTransactions.reduce(
+                        (sum, tx) =>
+                            sum +
+                            (tx.privacy === "private"
+                                ? tx.encryptedChanges.length
+                                : tx.changes.length),
+                        0
+                    );
+
+                const newLastIdx =
+                    (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+
+                let shouldWriteSignature = false;
+
+                if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+                    shouldWriteSignature = true;
+                    newBytesSinceLastSignature = 0;
+                }
+
                 let nextIdx = sessionRow?.lastIdx || 0;
 
                 const sessionUpdate = {
                     coValue: storedCoValueRowID,
                     sessionID: sessionID,
-                    lastIdx:
-                        (sessionRow?.lastIdx || 0) +
-                        actuallyNewTransactions.length,
+                    lastIdx: newLastIdx,
                     lastSignature: msg.new[sessionID]!.lastSignature,
+                    bytesSinceLastSignature: newBytesSinceLastSignature,
                 };
 
                 const sessionRowID = (await promised(
@@ -358,8 +479,18 @@ export class IDBStorage {
                     )
                 )) as number;
 
+                if (shouldWriteSignature) {
+                    await promised(
+                        signatureAfter.put({
+                            ses: sessionRowID,
+                            // TODO: newLastIdx is a misnomer, it's actually more like nextIdx or length
+                            idx: newLastIdx - 1,
+                            signature: msg.new[sessionID]!.lastSignature,
+                        } satisfies SignatureAfterRow)
+                    );
+                }
+
                 for (const newTransaction of actuallyNewTransactions) {
-                    nextIdx++;
                     await promised(
                         transactions.add({
                             ses: sessionRowID,
@@ -367,6 +498,7 @@ export class IDBStorage {
                             tx: newTransaction,
                         } satisfies TransactionRow)
                     );
+                    nextIdx++;
                 }
             }
         }
@@ -390,9 +522,10 @@ export class IDBStorage {
         coValues: IDBObjectStore;
         sessions: IDBObjectStore;
         transactions: IDBObjectStore;
+        signatureAfter: IDBObjectStore;
     } {
         const tx = this.db.transaction(
-            ["coValues", "sessions", "transactions"],
+            ["coValues", "sessions", "transactions", "signatureAfter"],
             mode
         );
 
@@ -409,8 +542,9 @@ export class IDBStorage {
         const coValues = tx.objectStore("coValues");
         const sessions = tx.objectStore("sessions");
         const transactions = tx.objectStore("transactions");
+        const signatureAfter = tx.objectStore("signatureAfter");
 
-        return { coValues, sessions, transactions };
+        return { coValues, sessions, transactions, signatureAfter };
     }
 }