Publish

- jazz-example-todo@0.0.21
 - cojson@0.1.6
 - cojson-simple-sync@0.1.7
 - cojson-storage-sqlite@0.1.4
 - jazz-browser@0.1.6
 - jazz-browser-auth-local@0.1.6
 - jazz-react@0.1.7
 - jazz-react-auth-local@0.1.7
 - jazz-storage-indexeddb@0.1.6

.github/workflows/build-and-deploy.yaml

@@ -73,5 +73,5 @@ jobs:
 
         envsubst '${DOCKER_USER} ${DOCKER_PASSWORD} ${DOCKER_TAG} ${BRANCH_SUFFIX} ${BRANCH_SUBDOMAIN}' < job-template.nomad > job-instance.nomad;
         cat job-instance.nomad;
-        NOMAD_ADDR='http://control1-london:4646' nomad job run job-instance.nomad;
+        NOMAD_ADDR='http://control1v2-london:4646' nomad job run job-instance.nomad;
       working-directory: ./examples/todo

examples/todo/job-template.nomad

@@ -22,6 +22,10 @@ job "example-todo$BRANCH_SUFFIX" {
       weight    = 100
     }
 
+    constraint {
+      distinct_hosts = true
+    }
+
     task "server" {
       driver = "docker"
 

examples/todo/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-example-todo",
   "private": true,
-  "version": "0.0.18",
+  "version": "0.0.21",
   "type": "module",
   "scripts": {
     "dev": "vite",
@@ -16,8 +16,8 @@
     "@types/qrcode": "^1.5.1",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
-    "jazz-react": "^0.1.5",
-    "jazz-react-auth-local": "^0.1.5",
+    "jazz-react": "^0.1.7",
+    "jazz-react-auth-local": "^0.1.7",
     "lucide-react": "^0.265.0",
     "qrcode": "^1.5.3",
     "react": "^18.2.0",

examples/todo/src/App.tsx

@@ -6,29 +6,34 @@ import {
     useJazz,
     useProfile,
     useTelepathicState,
-    createInviteLink
+    createInviteLink,
 } from "jazz-react";
 
 import { SubmittableInput } from "./components/SubmittableInput";
 import { useToast } from "./components/ui/use-toast";
 import { Skeleton } from "./components/ui/skeleton";
 import {
-    Table, TableBody, TableCell, TableHead, TableHeader, TableRow,
+    Table,
+    TableBody,
+    TableCell,
+    TableHead,
+    TableHeader,
+    TableRow,
 } from "@/components/ui/table";
 import { Checkbox } from "@/components/ui/checkbox";
 import { Button } from "@/components/ui/button";
 import uniqolor from "uniqolor";
 import QRCode from "qrcode";
+import { CoList } from "cojson/dist/contentTypes/coList";
 
-type TaskContent = { done: boolean; text: string };
-type Task = CoMap<TaskContent>;
+type Task = CoMap<{ done: boolean; text: string }>;
 
-type TodoListContent = {
+type ListOfTasks = CoList<CoID<Task>>;
+
+type TodoList = CoMap<{
     title: string;
-    // other keys form a set of task IDs
-    [taskId: CoID<Task>]: true;
-};
-type TodoList = CoMap<TodoListContent>;
+    tasks: CoID<ListOfTasks>;
+}>;
 
 export default function App() {
     const [listId, setListId] = useState<CoID<TodoList>>();
@@ -58,11 +63,14 @@ export default function App() {
 
     const createList = useCallback(
         (title: string) => {
+            if (!title) return;
             const listGroup = localNode.createGroup();
-            const list = listGroup.createMap<TodoListContent>();
+            const list = listGroup.createMap<TodoList>();
+            const tasks = listGroup.createList<ListOfTasks>();
 
             list.edit((list) => {
                 list.set("title", title);
+                list.set("tasks", tasks.id);
             });
 
             window.location.hash = list.id;
@@ -96,18 +104,19 @@ export default function App() {
 
 export function TodoListComponent({ listId }: { listId: CoID<TodoList> }) {
     const list = useTelepathicState(listId);
+    const tasks = useTelepathicState(list?.get("tasks"));
 
     const createTask = (text: string) => {
-        if (!list) return;
-        const task = list.coValue.getGroup().createMap<TaskContent>();
+        if (!tasks || !text) return;
+        const task = tasks.coValue.getGroup().createMap<Task>();
 
         task.edit((task) => {
             task.set("text", text);
             task.set("done", false);
         });
 
-        list.edit((list) => {
-            list.set(task.id, true);
+        tasks.edit((tasks) => {
+            tasks.push(task.id);
         });
     };
 
@@ -134,12 +143,9 @@ export function TodoListComponent({ listId }: { listId: CoID<TodoList> }) {
                     </TableRow>
                 </TableHeader>
                 <TableBody>
-                    {list &&
-                        list
-                            .keys()
-                            .filter((key): key is CoID<Task> =>
-                                key.startsWith("co_")
-                            )
+                    {tasks &&
+                        tasks
+                            .asArray()
                             .map((taskId) => (
                                 <TaskRow key={taskId} taskId={taskId} />
                             ))}
@@ -181,7 +187,9 @@ function TaskRow({ taskId }: { taskId: CoID<Task> }) {
             <TableCell>
                 <div className="flex flex-row justify-between items-center gap-2">
                     <span className={task?.get("done") ? "line-through" : ""}>
-                        {task?.get("text") || <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />}
+                        {task?.get("text") || (
+                            <Skeleton className="mt-1 w-[200px] h-[1em] rounded-full" />
+                        )}
                     </span>
                     <NameBadge accountID={task?.getLastEditor("text")} />
                 </div>
@@ -201,15 +209,19 @@ function NameBadge({ accountID }: { accountID?: AccountID }) {
     const darkColor = uniqolor(accountID || "", { lightness: 20 }).color;
 
     return (
-        profile?.get("name") && <span
-            className="rounded-full py-0.5 px-2 text-xs"
-            style={{
-                color: theme == "light" ? darkColor : brightColor,
-                background: theme == "light" ? brightColor : darkColor,
-            }}
-        >
-            {profile.get("name")}
-        </span>
+        profile?.get("name") ? (
+            <span
+                className="rounded-full py-0.5 px-2 text-xs"
+                style={{
+                    color: theme == "light" ? darkColor : brightColor,
+                    background: theme == "light" ? brightColor : darkColor,
+                }}
+            >
+                {profile.get("name")}
+            </span>
+        ) : (
+            <Skeleton className="mt-1 w-[50px] h-[1em] rounded-full" />
+        )
     );
 }
 
@@ -231,11 +243,15 @@ function InviteButton({ list }: { list: TodoList }) {
                         setExistingInviteLink(inviteLink);
                     }
                     if (inviteLink) {
-                        const qr = await QRCode.toDataURL(inviteLink, { errorCorrectionLevel: 'L' });
+                        const qr = await QRCode.toDataURL(inviteLink, {
+                            errorCorrectionLevel: "L",
+                        });
                         navigator.clipboard.writeText(inviteLink).then(() =>
                             toast({
                                 title: "Copied invite link to clipboard!",
-                                description: <img src={qr} className="w-20 h-20"/>,
+                                description: (
+                                    <img src={qr} className="w-20 h-20" />
+                                ),
                             })
                         );
                     }
@@ -245,4 +261,4 @@ function InviteButton({ list }: { list: TodoList }) {
             </Button>
         )
     );
-}
+}

examples/todo/src/main.tsx

@@ -11,7 +11,7 @@ import App from "./App.tsx";
 import "./index.css";
 
 ReactDOM.createRoot(document.getElementById("root")!).render(
-    <React.StrictMode>
+    // <React.StrictMode>
         <ThemeProvider>
             <div className="flex items-center gap-2 justify-center mt-5">
                 <img src="jazz-logo.png" className="h-5" /> Jazz Todo List
@@ -31,5 +31,5 @@ ReactDOM.createRoot(document.getElementById("root")!).render(
                 <Toaster />
             </WithJazz>
         </ThemeProvider>
-    </React.StrictMode>
+    // </React.StrictMode>
 );

packages/cojson-simple-sync/package.json

@@ -4,7 +4,7 @@
     "types": "src/index.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.1.4",
+    "version": "0.1.7",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@types/ws": "^8.5.5",
@@ -16,8 +16,8 @@
         "typescript": "5.0.2"
     },
     "dependencies": {
-        "cojson": "^0.1.4",
-        "cojson-storage-sqlite": "^0.1.1",
+        "cojson": "^0.1.6",
+        "cojson-storage-sqlite": "^0.1.4",
         "ws": "^8.13.0"
     },
     "scripts": {

packages/cojson-simple-sync/src/index.ts

@@ -1,8 +1,7 @@
 import { AnonymousControlledAccount, LocalNode, cojsonInternals } from "cojson";
-import { WebSocketServer, createWebSocketStream } from "ws";
-import { Duplex } from "node:stream";
-import { TransformStream } from "node:stream/web";
+import { WebSocketServer } from "ws";
 import { SQLiteStorage } from "cojson-storage-sqlite";
+import { websocketReadableStream, websocketWritableStream } from "./websocketStreams.js";
 
 const wss = new WebSocketServer({ port: 4200 });
 
@@ -35,28 +34,7 @@ wss.on("connection", function connection(ws, req) {
         clearInterval(pinging);
     });
 
-    const duplexStream = createWebSocketStream(ws, {
-        decodeStrings: false,
-        readableObjectMode: true,
-        writableObjectMode: true,
-        encoding: "utf-8",
-        defaultEncoding: "utf-8",
-    });
 
-    const { readable: incomingStrings, writable: outgoingStrings } =
-        Duplex.toWeb(duplexStream);
-
-    const toJSON = new TransformStream({
-        transform: (chunk, controller) => {
-            controller.enqueue(JSON.parse(chunk));
-        },
-    });
-
-    const fromJSON = new TransformStream({
-        transform: (chunk, controller) => {
-            controller.enqueue(JSON.stringify(chunk));
-        },
-    });
 
     const clientAddress =
         (req.headers["x-forwarded-for"] as string | undefined)
@@ -68,11 +46,9 @@ wss.on("connection", function connection(ws, req) {
     localNode.sync.addPeer({
         id: clientId,
         role: "client",
-        incoming: incomingStrings.pipeThrough(toJSON),
-        outgoing: fromJSON.writable,
+        incoming: websocketReadableStream(ws),
+        outgoing: websocketWritableStream(ws),
     });
 
-    void fromJSON.readable.pipeTo(outgoingStrings);
-
     ws.on("error", (e) => console.error(`Error on connection ${clientId}:`, e));
 });

packages/cojson-simple-sync/src/websocketStreams.ts

@@ -0,0 +1,86 @@
+import { WebSocket } from "ws";
+import { WritableStream, ReadableStream } from "isomorphic-streams";
+
+export function websocketReadableStream<T>(ws: WebSocket) {
+    ws.binaryType = "arraybuffer";
+
+    return new ReadableStream<T>({
+        start(controller) {
+            ws.addEventListener("message", (event) => {
+                if (typeof event.data !== "string")
+                    return console.warn(
+                        "Got non-string message from client",
+                        event.data
+                    );
+                const msg = JSON.parse(event.data);
+                if (msg.type === "ping") {
+                    // console.debug(
+                    //     "Got ping from",
+                    //     msg.dc,
+                    //     "latency",
+                    //     Date.now() - msg.time,
+                    //     "ms"
+                    // );
+                    return;
+                }
+                controller.enqueue(msg);
+            });
+            ws.addEventListener("close", () => controller.close());
+            ws.addEventListener("error", () =>
+                controller.error(new Error("The WebSocket errored!"))
+            );
+        },
+
+        cancel() {
+            ws.close();
+        },
+    });
+}
+
+export function websocketWritableStream<T>(ws: WebSocket) {
+    return new WritableStream<T>({
+        start(controller) {
+            ws.addEventListener("close", () =>
+                controller.error(
+                    new Error("The WebSocket closed unexpectedly!")
+                )
+            );
+            ws.addEventListener("error", () =>
+                controller.error(new Error("The WebSocket errored!"))
+            );
+
+            if (ws.readyState === WebSocket.OPEN) {
+                return;
+            }
+
+            return new Promise((resolve) => ws.once("open", resolve));
+        },
+
+        write(chunk) {
+            ws.send(JSON.stringify(chunk));
+            // Return immediately, since the web socket gives us no easy way to tell
+            // when the write completes.
+        },
+
+        close() {
+            return closeWS(1000);
+        },
+
+        abort(reason) {
+            return closeWS(4000, reason && reason.message);
+        },
+    });
+
+    function closeWS(code: number, reasonString?: string) {
+        return new Promise<void>((resolve, reject) => {
+            ws.onclose = (e) => {
+                if (e.wasClean) {
+                    resolve();
+                } else {
+                    reject(new Error("The connection was not closed cleanly"));
+                }
+            };
+            ws.close(code, reasonString);
+        });
+    }
+}

packages/cojson-storage-sqlite/package.json

@@ -1,13 +1,13 @@
 {
   "name": "cojson-storage-sqlite",
   "type": "module",
-  "version": "0.1.1",
+  "version": "0.1.4",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
     "better-sqlite3": "^8.5.2",
-    "cojson": "^0.1.4",
+    "cojson": "^0.1.6",
     "typescript": "^5.1.6"
   },
   "scripts": {

packages/cojson-storage-sqlite/src/index.ts

@@ -338,17 +338,20 @@ export class SQLiteStorage {
                         lastSignature: msg.new[sessionID]!.lastSignature,
                     };
 
-                    const sessionRowID = this.db
+                    const upsertedSession = (this.db
                         .prepare<[number, string, number, string]>(
                             `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature) VALUES (?, ?, ?, ?)
-                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature`
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature
+                            RETURNING rowID`
                         )
-                        .run(
+                        .get(
                             sessionUpdate.coValue,
                             sessionUpdate.sessionID,
                             sessionUpdate.lastIdx,
                             sessionUpdate.lastSignature
-                        ).lastInsertRowid as number;
+                        ) as {rowID: number});
+
+                    const sessionRowID = upsertedSession.rowID;
 
                     for (const newTransaction of actuallyNewTransactions) {
                         nextIdx++;

packages/cojson/package.json

@@ -5,7 +5,7 @@
     "types": "dist/index.d.ts",
     "type": "module",
     "license": "MIT",
-    "version": "0.1.4",
+    "version": "0.1.6",
     "devDependencies": {
         "@types/jest": "^29.5.3",
         "@typescript-eslint/eslint-plugin": "^6.2.1",

packages/cojson/src/account.ts

@@ -135,7 +135,8 @@ export class AnonymousControlledAccount
 
 export type AccountContent = GroupContent & { profile: CoID<Profile> };
 export type AccountMeta = { type: "account" };
-export type AccountID = CoID<CoMap<AccountContent, AccountMeta>>;
+export type AccountMap = CoMap<AccountContent, AccountMeta>;
+export type AccountID = CoID<AccountMap>;
 
 export type AccountIDOrAgentID = AgentID | AccountID;
 export type AccountOrAgentID = AgentID | Account;

packages/cojson/src/contentType.test.ts

@@ -3,7 +3,7 @@ import { createdNowUnique } from "./crypto.js";
 import { LocalNode } from "./node.js";
 import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
 
-test("Empty COJSON Map works", () => {
+test("Empty CoMap works", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 
     const coValue = node.createCoValue({
@@ -24,7 +24,7 @@ test("Empty COJSON Map works", () => {
     expect(content.toJSON()).toEqual({});
 });
 
-test("Can insert and delete Map entries in edit()", () => {
+test("Can insert and delete CoMap entries in edit()", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 
     const coValue = node.createCoValue({
@@ -53,7 +53,7 @@ test("Can insert and delete Map entries in edit()", () => {
     });
 });
 
-test("Can get map entry values at different points in time", () => {
+test("Can get CoMap entry values at different points in time", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 
     const coValue = node.createCoValue({
@@ -89,7 +89,7 @@ test("Can get map entry values at different points in time", () => {
     });
 });
 
-test("Can get all historic values of key", () => {
+test("Can get all historic values of key in CoMap", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 
     const coValue = node.createCoValue({
@@ -141,7 +141,7 @@ test("Can get all historic values of key", () => {
     });
 });
 
-test("Can get last tx ID for a key", () => {
+test("Can get last tx ID for a key in CoMap", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 
     const coValue = node.createCoValue({
@@ -173,3 +173,112 @@ test("Can get last tx ID for a key", () => {
         expect(editable.getLastTxID("hello")?.txIndex).toEqual(2);
     });
 });
+
+test("Empty CoList works", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "colist") {
+        throw new Error("Expected list");
+    }
+
+    expect(content.type).toEqual("colist");
+    expect(content.toJSON()).toEqual([]);
+});
+
+test("Can append, prepend and delete items to CoList", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "colist") {
+        throw new Error("Expected list");
+    }
+
+    expect(content.type).toEqual("colist");
+
+    content.edit((editable) => {
+        editable.append(0, "hello", "trusting");
+        expect(editable.toJSON()).toEqual(["hello"]);
+        editable.append(0, "world", "trusting");
+        expect(editable.toJSON()).toEqual(["hello", "world"]);
+        editable.prepend(1, "beautiful", "trusting");
+        expect(editable.toJSON()).toEqual(["hello", "beautiful", "world"]);
+        editable.prepend(3, "hooray", "trusting");
+        expect(editable.toJSON()).toEqual([
+            "hello",
+            "beautiful",
+            "world",
+            "hooray",
+        ]);
+        editable.delete(2, "trusting");
+        expect(editable.toJSON()).toEqual(["hello", "beautiful", "hooray"]);
+    });
+});
+
+test("Push is equivalent to append after last item", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "colist") {
+        throw new Error("Expected list");
+    }
+
+    expect(content.type).toEqual("colist");
+
+    content.edit((editable) => {
+        editable.append(0, "hello", "trusting");
+        expect(editable.toJSON()).toEqual(["hello"]);
+        editable.push("world", "trusting");
+        expect(editable.toJSON()).toEqual(["hello", "world"]);
+        editable.push("hooray", "trusting");
+        expect(editable.toJSON()).toEqual(["hello", "world", "hooray"]);
+    });
+});
+
+test("Can push into empty list", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = coValue.getCurrentContent();
+
+    if (content.type !== "colist") {
+        throw new Error("Expected list");
+    }
+
+    expect(content.type).toEqual("colist");
+
+    content.edit((editable) => {
+        editable.push("hello", "trusting");
+        expect(editable.toJSON()).toEqual(["hello"]);
+    });
+})

packages/cojson/src/contentTypes/coList.ts

@@ -1,19 +1,262 @@
-import { JsonObject, JsonValue } from '../jsonValue.js';
-import { CoID } from '../contentType.js';
-import { CoValue } from '../coValue.js';
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { CoID } from "../contentType.js";
+import { CoValue, accountOrAgentIDfromSessionID } from "../coValue.js";
+import { SessionID, TransactionID } from "../ids.js";
+import { AccountID } from "../index.js";
+import { isAccountID } from "../account.js";
 
-export class CoList<T extends JsonValue, Meta extends JsonObject | null = null> {
+type OpID = TransactionID & { changeIdx: number };
+
+type InsertionOpPayload<T extends JsonValue> =
+    | {
+          op: "pre";
+          value: T;
+          before: OpID | "end";
+      }
+    | {
+          op: "app";
+          value: T;
+          after: OpID | "start";
+      };
+
+type DeletionOpPayload = {
+    op: "del";
+    insertion: OpID;
+};
+
+export type ListOpPayload<T extends JsonValue> =
+    | InsertionOpPayload<T>
+    | DeletionOpPayload;
+
+type InsertionEntry<T extends JsonValue> = {
+    madeAt: number;
+    predecessors: OpID[];
+    successors: OpID[];
+} & InsertionOpPayload<T>;
+
+type DeletionEntry = {
+    madeAt: number;
+    deletionID: OpID;
+} & DeletionOpPayload;
+
+export class CoList<
+    T extends JsonValue,
+    Meta extends JsonObject | null = null
+> {
     id: CoID<CoList<T, Meta>>;
     type = "colist" as const;
     coValue: CoValue;
+    afterStart: OpID[];
+    beforeEnd: OpID[];
+    insertions: {
+        [sessionID: SessionID]: {
+            [txIdx: number]: {
+                [changeIdx: number]: InsertionEntry<T>;
+            };
+        };
+    };
+    deletionsByInsertion: {
+        [deletedSessionID: SessionID]: {
+            [deletedTxIdx: number]: {
+                [deletedChangeIdx: number]: DeletionEntry[];
+            };
+        };
+    };
 
     constructor(coValue: CoValue) {
         this.id = coValue.id as CoID<CoList<T, Meta>>;
         this.coValue = coValue;
+        this.afterStart = [];
+        this.beforeEnd = [];
+        this.insertions = {};
+        this.deletionsByInsertion = {};
+
+        this.fillOpsFromCoValue();
     }
 
-    toJSON(): JsonObject {
-        throw new Error("Method not implemented.");
+
+    get meta(): Meta {
+        return this.coValue.header.meta as Meta;
+    }
+
+    protected fillOpsFromCoValue() {
+        this.insertions = {};
+        this.deletionsByInsertion = {};
+        this.afterStart = [];
+        this.beforeEnd = [];
+
+        for (const {
+            txID,
+            changes,
+            madeAt,
+        } of this.coValue.getValidSortedTransactions()) {
+            for (const [changeIdx, changeUntyped] of changes.entries()) {
+                const change = changeUntyped as ListOpPayload<T>;
+
+                if (change.op === "pre" || change.op === "app") {
+                    let sessionEntry = this.insertions[txID.sessionID];
+                    if (!sessionEntry) {
+                        sessionEntry = {};
+                        this.insertions[txID.sessionID] = sessionEntry;
+                    }
+                    let txEntry = sessionEntry[txID.txIndex];
+                    if (!txEntry) {
+                        txEntry = {};
+                        sessionEntry[txID.txIndex] = txEntry;
+                    }
+                    txEntry[changeIdx] = {
+                        madeAt,
+                        predecessors: [],
+                        successors: [],
+                        ...change,
+                    };
+                    if (change.op === "pre") {
+                        if (change.before === "end") {
+                            this.beforeEnd.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        } else {
+                            const beforeEntry =
+                                this.insertions[change.before.sessionID]?.[
+                                    change.before.txIndex
+                                ]?.[change.before.changeIdx];
+                            if (!beforeEntry) {
+                                throw new Error(
+                                    "Not yet implemented: insertion before missing op " +
+                                        change.before
+                                );
+                            }
+                            beforeEntry.predecessors.splice(0, 0, {
+                                ...txID,
+                                changeIdx,
+                            });
+                        }
+                    } else {
+                        if (change.after === "start") {
+                            this.afterStart.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        } else {
+                            const afterEntry =
+                                this.insertions[change.after.sessionID]?.[
+                                    change.after.txIndex
+                                ]?.[change.after.changeIdx];
+                            if (!afterEntry) {
+                                throw new Error(
+                                    "Not yet implemented: insertion after missing op " +
+                                        change.after
+                                );
+                            }
+                            afterEntry.successors.push({
+                                ...txID,
+                                changeIdx,
+                            });
+                        }
+                    }
+                } else if (change.op === "del") {
+                    let sessionEntry =
+                        this.deletionsByInsertion[change.insertion.sessionID];
+                    if (!sessionEntry) {
+                        sessionEntry = {};
+                        this.deletionsByInsertion[change.insertion.sessionID] =
+                            sessionEntry;
+                    }
+                    let txEntry = sessionEntry[change.insertion.txIndex];
+                    if (!txEntry) {
+                        txEntry = {};
+                        sessionEntry[change.insertion.txIndex] = txEntry;
+                    }
+                    let changeEntry = txEntry[change.insertion.changeIdx];
+                    if (!changeEntry) {
+                        changeEntry = [];
+                        txEntry[change.insertion.changeIdx] = changeEntry;
+                    }
+                    changeEntry.push({
+                        madeAt,
+                        deletionID: {
+                            ...txID,
+                            changeIdx,
+                        },
+                        ...change,
+                    });
+                } else {
+                    throw new Error(
+                        "Unknown list operation " +
+                            (change as { op: unknown }).op
+                    );
+                }
+            }
+        }
+    }
+
+    entries(): { value: T; madeAt: number; opID: OpID }[] {
+        const arr: { value: T; madeAt: number; opID: OpID }[] = [];
+        for (const opID of this.afterStart) {
+            this.fillArrayFromOpID(opID, arr);
+        }
+        for (const opID of this.beforeEnd) {
+            this.fillArrayFromOpID(opID, arr);
+        }
+        return arr;
+    }
+
+    private fillArrayFromOpID(
+        opID: OpID,
+        arr: { value: T; madeAt: number; opID: OpID }[]
+    ) {
+        const entry =
+            this.insertions[opID.sessionID]?.[opID.txIndex]?.[opID.changeIdx];
+        if (!entry) {
+            throw new Error("Missing op " + opID);
+        }
+        for (const predecessor of entry.predecessors) {
+            this.fillArrayFromOpID(predecessor, arr);
+        }
+        const deleted =
+            (this.deletionsByInsertion[opID.sessionID]?.[opID.txIndex]?.[
+                opID.changeIdx
+            ]?.length || 0) > 0;
+        if (!deleted) {
+            arr.push({
+                value: entry.value,
+                madeAt: entry.madeAt,
+                opID,
+            });
+        }
+        for (const successor of entry.successors) {
+            this.fillArrayFromOpID(successor, arr);
+        }
+    }
+
+    getLastEditor(idx: number): AccountID | undefined {
+        const entry = this.entries()[idx];
+        if (!entry) {
+            return undefined;
+        }
+        const accountID = accountOrAgentIDfromSessionID(entry.opID.sessionID);
+        if (isAccountID(accountID)) {
+            return accountID;
+        } else {
+            return undefined;
+        }
+    }
+
+    toJSON(): T[] {
+        return this.asArray();
+    }
+
+    asArray(): T[] {
+        return this.entries().map((entry) => entry.value);
+    }
+
+    edit(
+        changer: (editable: WriteableCoList<T, Meta>) => void
+    ): CoList<T, Meta> {
+        const editable = new WriteableCoList<T, Meta>(this.coValue);
+        changer(editable);
+        return new CoList(this.coValue);
     }
 
     subscribe(listener: (coMap: CoList<T, Meta>) => void): () => void {
@@ -22,3 +265,106 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
         });
     }
 }
+
+export class WriteableCoList<
+    T extends JsonValue,
+    Meta extends JsonObject | null = null
+> extends CoList<T, Meta> {
+    append(
+        after: number,
+        value: T,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const entries = this.entries();
+        let opIDBefore;
+        if (entries.length > 0) {
+            const entryBefore = entries[after];
+            if (!entryBefore) {
+                throw new Error("Invalid index " + after);
+            }
+            opIDBefore = entryBefore.opID;
+        } else {
+            if (after !== 0) {
+                throw new Error("Invalid index " + after);
+            }
+            opIDBefore = "start";
+        }
+        this.coValue.makeTransaction(
+            [
+                {
+                    op: "app",
+                    value,
+                    after: opIDBefore,
+                },
+            ],
+            privacy
+        );
+
+        this.fillOpsFromCoValue();
+    }
+
+    push(value: T, privacy: "private" | "trusting" = "private"): void {
+        // TODO: optimize
+        const entries = this.entries();
+        this.append(entries.length > 0 ? entries.length - 1 : 0, value, privacy);
+    }
+
+    prepend(
+        before: number,
+        value: T,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const entries = this.entries();
+        let opIDAfter;
+        if (entries.length > 0) {
+            const entryAfter = entries[before];
+            if (entryAfter) {
+                opIDAfter = entryAfter.opID;
+            } else {
+                if (before !== entries.length) {
+                    throw new Error("Invalid index " + before);
+                }
+                opIDAfter = "end";
+            }
+        } else {
+            if (before !== 0) {
+                throw new Error("Invalid index " + before);
+            }
+            opIDAfter = "end";
+        }
+        this.coValue.makeTransaction(
+            [
+                {
+                    op: "pre",
+                    value,
+                    before: opIDAfter,
+                },
+            ],
+            privacy
+        );
+
+        this.fillOpsFromCoValue();
+    }
+
+    delete(
+        at: number,
+        privacy: "private" | "trusting" = "private"
+    ): void {
+        const entries = this.entries();
+        const entry = entries[at];
+        if (!entry) {
+            throw new Error("Invalid index " + at);
+        }
+        this.coValue.makeTransaction(
+            [
+                {
+                    op: "del",
+                    insertion: entry.opID,
+                },
+            ],
+            privacy
+        );
+
+        this.fillOpsFromCoValue();
+    }
+}

packages/cojson/src/contentTypes/coMap.ts

@@ -46,6 +46,10 @@ export class CoMap<
         this.fillOpsFromCoValue();
     }
 
+    get meta(): Meta {
+        return this.coValue.header.meta as Meta;
+    }
+
     protected fillOpsFromCoValue() {
         this.ops = {};
 

packages/cojson/src/group.ts

@@ -24,6 +24,7 @@ import {
 } from "./account.js";
 import { Role } from "./permissions.js";
 import { base58 } from "@scure/base";
+import { CoList } from "./contentTypes/coList.js";
 
 export type GroupContent = {
     profile: CoID<Profile> | null;
@@ -186,10 +187,9 @@ export class Group {
         this.rotateReadKey();
     }
 
-    createMap<
-        M extends { [key: string]: JsonValue },
-        Meta extends JsonObject | null = null
-    >(meta?: Meta): CoMap<M, Meta> {
+    createMap<M extends CoMap<{ [key: string]: JsonValue }, JsonObject | null>>(
+        meta?: M["meta"]
+    ): M {
         return this.node
             .createCoValue({
                 type: "comap",
@@ -200,7 +200,23 @@ export class Group {
                 meta: meta || null,
                 ...createdNowUnique(),
             })
-            .getCurrentContent() as CoMap<M, Meta>;
+            .getCurrentContent() as M;
+    }
+
+    createList<L extends CoList<JsonValue, JsonObject | null>>(
+        meta?: L["meta"]
+    ): L {
+        return this.node
+            .createCoValue({
+                type: "colist",
+                ruleset: {
+                    type: "ownedByGroup",
+                    group: this.groupMap.id,
+                },
+                meta: meta || null,
+                ...createdNowUnique(),
+            })
+            .getCurrentContent() as L;
     }
 
     testWithDifferentAccount(
@@ -230,4 +246,4 @@ export function secretSeedFromInviteSecret(inviteSecret: InviteSecret) {
     }
 
     return base58.decode(inviteSecret.slice("inviteSecret_z".length));
-}
+}

packages/cojson/src/index.ts

@@ -25,6 +25,7 @@ import type {
     AccountID,
     AccountContent,
     ProfileContent,
+    ProfileMeta,
     Profile,
 } from "./account.js";
 import type { InviteSecret } from "./group.js";
@@ -70,6 +71,7 @@ export type {
     AccountContent,
     Profile,
     ProfileContent,
+    ProfileMeta,
     InviteSecret
 };
 

packages/cojson/src/node.ts

@@ -31,8 +31,7 @@ import {
     AccountID,
     Profile,
     AccountContent,
-    ProfileContent,
-    ProfileMeta,
+    AccountMap,
 } from "./account.js";
 import { CoMap } from "./index.js";
 
@@ -139,7 +138,7 @@ export class LocalNode {
     }
 
     async loadProfile(id: AccountID): Promise<Profile> {
-        const account = await this.load<CoMap<AccountContent>>(id);
+        const account = await this.load<AccountMap>(id);
         const profileID = account.get("profile");
 
         if (!profileID) {
@@ -307,7 +306,7 @@ export class LocalNode {
             account.node
         );
 
-        const profile = accountAsGroup.createMap<ProfileContent, ProfileMeta>({
+        const profile = accountAsGroup.createMap<Profile>({
             type: "profile",
         });
 

packages/cojson/src/streamUtils.ts

@@ -1,12 +1,17 @@
-import { ReadableStream, TransformStream, WritableStream } from "isomorphic-streams";
+import {
+    ReadableStream,
+    TransformStream,
+    WritableStream,
+} from "isomorphic-streams";
 import { Peer, PeerID, SyncMessage } from "./sync.js";
 
-
 export function connectedPeers(
     peer1id: PeerID,
     peer2id: PeerID,
     {
-        trace = false, peer1role = "peer", peer2role = "peer",
+        trace = false,
+        peer1role = "peer",
+        peer2role = "peer",
     }: {
         trace?: boolean;
         peer1role?: Peer["role"];
@@ -24,9 +29,13 @@ export function connectedPeers(
             new TransformStream({
                 transform(
                     chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void; }
+                    controller: { enqueue: (msg: SyncMessage) => void }
                 ) {
-                    trace && console.debug(`${peer2id} -> ${peer1id}`, JSON.stringify(chunk, null, 2));
+                    trace &&
+                        console.debug(
+                            `${peer2id} -> ${peer1id}`,
+                            JSON.stringify(chunk, null, 2)
+                        );
                     controller.enqueue(chunk);
                 },
             })
@@ -38,9 +47,13 @@ export function connectedPeers(
             new TransformStream({
                 transform(
                     chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void; }
+                    controller: { enqueue: (msg: SyncMessage) => void }
                 ) {
-                    trace && console.debug(`${peer1id} -> ${peer2id}`, JSON.stringify(chunk, null, 2));
+                    trace &&
+                        console.debug(
+                            `${peer1id} -> ${peer2id}`,
+                            JSON.stringify(chunk, null, 2)
+                        );
                     controller.enqueue(chunk);
                 },
             })
@@ -65,39 +78,22 @@ export function connectedPeers(
 }
 
 export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
-    const queue: T[] = [];
-    let resolveNextItemReady: () => void = () => { };
-    let nextItemReady: Promise<void> = new Promise((resolve) => {
-        resolveNextItemReady = resolve;
-    });
-
-    let writerClosed = false;
     let readerClosed = false;
 
+    let resolveEnqueue: (enqueue: (item: T) => void) => void;
+    const enqueuePromise = new Promise<(item: T) => void>((resolve) => {
+        resolveEnqueue = resolve;
+    });
+
+    let resolveClose: (close: () => void) => void;
+    const closePromise = new Promise<() => void>((resolve) => {
+        resolveClose = resolve;
+    });
+
     const readable = new ReadableStream<T>({
-        async pull(controller) {
-            let retriesLeft = 3;
-            while (retriesLeft > 0) {
-                if (writerClosed) {
-                    controller.close();
-                    return;
-                }
-                retriesLeft--;
-                if (queue.length > 0) {
-                    controller.enqueue(queue.shift()!);
-                    if (queue.length === 0) {
-                        nextItemReady = new Promise((resolve) => {
-                            resolveNextItemReady = resolve;
-                        });
-                    }
-                    return;
-                } else {
-                    await nextItemReady;
-                }
-            }
-            throw new Error(
-                "Should only use one retry to get next item in queue."
-            );
+        async start(controller) {
+            resolveEnqueue(controller.enqueue.bind(controller));
+            resolveClose(controller.close.bind(controller));
         },
 
         cancel(_reason) {
@@ -107,22 +103,21 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
     });
 
     const writable = new WritableStream<T>({
-        write(chunk) {
+        async write(chunk) {
+            const enqueue = await enqueuePromise;
             if (readerClosed) {
-                console.log("Reader closed, not writing chunk", chunk);
-                throw new Error("Reader closed, not writing chunk");
-            }
-            queue.push(chunk);
-            if (queue.length === 1) {
-                // make sure that await write resolves before corresponding read
-                setTimeout(() => resolveNextItemReady());
+                throw new Error("Reader closed");
+            } else {
+                // make sure write resolves before corresponding read
+                setTimeout(() => {
+                    enqueue(chunk);
+                })
             }
         },
-        abort(_reason) {
-            console.log("Manually closing writer");
-            writerClosed = true;
-            resolveNextItemReady();
-            return Promise.resolve();
+        async abort(reason) {
+            console.debug("Manually closing writer", reason);
+            const close = await closePromise;
+            close();
         },
     });
 

packages/cojson/src/sync.ts

@@ -268,6 +268,7 @@ export class SyncManager {
                         );
                     }
                 }
+                console.log("DONE!!!");
             } catch (e) {
                 console.error(`Error reading from peer ${peer.id}`, e);
             }
@@ -280,13 +281,32 @@ export class SyncManager {
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        return peer.outgoing.write(msg).catch((e) => {
-            console.error(
-                new Error(`Error writing to peer ${peer.id}, disconnecting`, {
-                    cause: e,
+        return new Promise<void>((resolve) => {
+            const timeout = setTimeout(() => {
+                console.error(
+                    new Error(
+                        `Writing to peer ${peer.id} took >1s - this should never happen as write should resolve quickly or error`
+                    )
+                );
+                resolve();
+            }, 1000);
+            peer.outgoing
+                .write(msg)
+                .then(() => {
+                    clearTimeout(timeout);
+                    resolve();
                 })
-            );
-            delete this.peers[peer.id];
+                .catch((e) => {
+                    console.error(
+                        new Error(
+                            `Error writing to peer ${peer.id}, disconnecting`,
+                            {
+                                cause: e,
+                            }
+                        )
+                    );
+                    delete this.peers[peer.id];
+                });
         });
     }
 

packages/jazz-browser-auth-local/package.json

@@ -1,11 +1,11 @@
 {
     "name": "jazz-browser-auth-local",
-    "version": "0.1.4",
+    "version": "0.1.6",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
     "license": "MIT",
     "dependencies": {
-        "jazz-browser": "^0.1.4",
+        "jazz-browser": "^0.1.6",
         "typescript": "^5.1.6"
     },
     "scripts": {

packages/jazz-browser/package.json

@@ -1,12 +1,12 @@
 {
   "name": "jazz-browser",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.1.4",
-    "jazz-storage-indexeddb": "^0.1.4",
+    "cojson": "^0.1.6",
+    "jazz-storage-indexeddb": "^0.1.6",
     "typescript": "^5.1.6"
   },
   "scripts": {

packages/jazz-browser/src/index.ts

@@ -182,10 +182,12 @@ function websocketReadableStream<T>(ws: WebSocket) {
                 }
                 controller.enqueue(msg);
             };
-            ws.addEventListener("close", () => controller.close());
-            ws.addEventListener("error", () =>
-                controller.error(new Error("The WebSocket errored!"))
-            );
+            const closeListener = () => controller.close();
+            ws.addEventListener("close", closeListener);
+            ws.addEventListener("error", () => {
+                controller.error(new Error("The WebSocket errored!"));
+                ws.removeEventListener("close", closeListener);
+            });
         },
 
         cancel() {
@@ -209,23 +211,37 @@ function createWebSocketPeer(syncAddress: string): Peer {
 }
 
 function websocketWritableStream<T>(ws: WebSocket) {
+    const initialQueue = [] as T[];
+    let isOpen = false;
+
     return new WritableStream<T>({
         start(controller) {
-            ws.addEventListener("error", () => {
-                controller.error(new Error("The WebSocket errored!"));
+            ws.addEventListener("error", (event) => {
+                controller.error(
+                    new Error("The WebSocket errored!" + JSON.stringify(event))
+                );
             });
             ws.addEventListener("close", () => {
                 controller.error(
                     new Error("The server closed the connection unexpectedly!")
                 );
             });
-            return new Promise((resolve) => (ws.addEventListener("open", resolve)));
+            ws.addEventListener("open", () => {
+                for (const item of initialQueue) {
+                    ws.send(JSON.stringify(item));
+                }
+                isOpen = true;
+            });
         },
 
-        write(chunk) {
-            ws.send(JSON.stringify(chunk));
-            // Return immediately, since the web socket gives us no easy way to tell
-            // when the write completes.
+        async write(chunk) {
+            if (isOpen) {
+                ws.send(JSON.stringify(chunk));
+                // Return immediately, since the web socket gives us no easy way to tell
+                // when the write completes.
+            } else {
+                initialQueue.push(chunk);
+            }
         },
 
         close() {

packages/jazz-react-auth-local/package.json

@@ -1,12 +1,12 @@
 {
     "name": "jazz-react-auth-local",
-    "version": "0.1.5",
+    "version": "0.1.7",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
     "license": "MIT",
     "dependencies": {
-        "jazz-browser-auth-local": "^0.1.4",
-        "jazz-react": "^0.1.5",
+        "jazz-browser-auth-local": "^0.1.6",
+        "jazz-react": "^0.1.7",
         "typescript": "^5.1.6"
     },
     "devDependencies": {

packages/jazz-react/package.json

@@ -1,12 +1,12 @@
 {
   "name": "jazz-react",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.1.4",
-    "jazz-browser": "^0.1.4",
+    "cojson": "^0.1.6",
+    "jazz-browser": "^0.1.6",
     "typescript": "^5.1.6"
   },
   "devDependencies": {

packages/jazz-react/src/index.tsx

@@ -3,9 +3,10 @@ import {
     ContentType,
     CoID,
     ProfileContent,
+    ProfileMeta,
     CoMap,
     AccountID,
-    Profile,
+    JsonValue,
 } from "cojson";
 import React, { useEffect, useState } from "react";
 import { AuthProvider, createBrowserNode } from "jazz-browser";
@@ -123,10 +124,10 @@ export function useTelepathicState<T extends ContentType>(id?: CoID<T>) {
     return state;
 }
 
-export function useProfile<P extends ProfileContent = ProfileContent>(
-    accountID?: AccountID
-): (Profile & CoMap<P>) | undefined {
-    const [profileID, setProfileID] = useState<CoID<Profile & CoMap<P>>>();
+export function useProfile<
+    P extends { [key: string]: JsonValue } & ProfileContent = ProfileContent
+>(accountID?: AccountID): CoMap<P, ProfileMeta> | undefined {
+    const [profileID, setProfileID] = useState<CoID<CoMap<P, ProfileMeta>>>();
 
     const { localNode } = useJazz();
 

packages/jazz-storage-indexeddb/package.json

@@ -1,11 +1,11 @@
 {
   "name": "jazz-storage-indexeddb",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
-    "cojson": "^0.1.4",
+    "cojson": "^0.1.6",
     "typescript": "^5.1.6"
   },
   "devDependencies": {