Merge pull request #2719 from garden-co/changeset-release/main

Version Packages

.github/workflows/code-quality.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Biome
         uses: biomejs/setup-biome@v2
         with:
-          version: 1.9.4
+          version: 2.1.3
       - name: Run Biome
         run: biome ci .
 

biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.1.3/schema.json",
   "vcs": {
     "enabled": true,
     "clientKind": "git",
@@ -7,39 +7,35 @@
   },
   "files": {
     "ignoreUnknown": false,
-    "ignore": [
-      "jazz-tools.json",
-      "**/ios/**",
-      "**/android/**",
-      "tests/jazz-svelte/src/**",
-      "examples/*svelte*/**",
-      "starters/*svelte*/**",
-      "examples/server-worker-inbox/src/routeTree.gen.ts",
-      "homepage/homepage/**",
-      "**/package.json"
+    "includes": [
+      "**",
+      "!**/jazz-tools.json",
+      "!**/ios/**",
+      "!**/android/**",
+      "!**/tests/jazz-svelte/src/**",
+      "!**/examples/**/*svelte*/**",
+      "!**/starters/**/*svelte*/**",
+      "!**/examples/server-worker-inbox/src/routeTree.gen.ts",
+      "!**/homepage/homepage/**",
+      "!**/package.json",
+      "!**/*svelte*/**"
     ]
   },
   "formatter": {
     "enabled": true,
     "indentStyle": "space"
   },
-  "organizeImports": {
-    "enabled": true
-  },
+  "assist": { "actions": { "source": { "organizeImports": "off" } } },
   "linter": {
     "enabled": false,
     "rules": {
       "recommended": true,
       "correctness": {
+        "useExhaustiveDependencies": "off",
         "useImportExtensions": {
           "level": "error",
           "options": {
-            "suggestedExtensions": {
-              "ts": {
-                "module": "js",
-                "component": "jsx"
-              }
-            }
+            "forceJsExtensions": true
           }
         }
       }
@@ -47,7 +43,7 @@
   },
   "overrides": [
     {
-      "include": ["packages/**/src/**"],
+      "includes": ["**/packages/**/src/**"],
       "linter": {
         "enabled": true,
         "rules": {
@@ -56,7 +52,10 @@
       }
     },
     {
-      "include": ["packages/cojson/src/storage/*/**", "cojson-transport-ws/**"],
+      "includes": [
+        "**/packages/cojson/src/storage/**/*/**",
+        "**/cojson-transport-ws/**"
+      ],
       "linter": {
         "enabled": true,
         "rules": {
@@ -65,7 +64,7 @@
       }
     },
     {
-      "include": ["**/tests/**"],
+      "includes": ["**/tests/**"],
       "linter": {
         "rules": {
           "correctness": {
@@ -75,7 +74,7 @@
             "noNonNullAssertion": "off"
           },
           "suspicious": {
-            "noExplicitAny": "info"
+            "noExplicitAny": "off"
           }
         }
       }

examples/chat-rn/src/chat.tsx

@@ -76,7 +76,9 @@ export function ChatScreen({ navigation }: { navigation: any }) {
 
   const renderMessageItem = ({
     item,
-  }: { item: Loaded<typeof Message, { text: true }> }) => {
+  }: {
+    item: Loaded<typeof Message, { text: true }>;
+  }) => {
     const isMe = item._edits?.text?.by?.isMe;
     return (
       <View

examples/chat-rn/src/invite.tsx

@@ -3,11 +3,7 @@ import React from "react";
 import { Text } from "react-native";
 import { Chat } from "./schema";
 
-export function HandleInviteScreen({
-  navigation,
-}: {
-  navigation: any;
-}) {
+export function HandleInviteScreen({ navigation }: { navigation: any }) {
   useAcceptInviteNative({
     invitedObjectSchema: Chat,
     onAccept: async (chatId) => {

examples/chat-svelte/CHANGELOG.md

@@ -1,5 +1,35 @@
 # passkey-svelte
 
+## 0.0.112
+
+### Patch Changes
+
+- Updated dependencies [67e0968]
+- Updated dependencies [2c8120d]
+  - jazz-tools@0.16.6
+
+## 0.0.111
+
+### Patch Changes
+
+- Updated dependencies [3cd1586]
+- Updated dependencies [33ebbf0]
+  - jazz-tools@0.16.5
+
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [16764f6]
+  - jazz-tools@0.16.4
+
+## 0.0.109
+
+### Patch Changes
+
+- Updated dependencies [43d3511]
+  - jazz-tools@0.16.3
+
 ## 0.0.108
 
 ### Patch Changes

examples/chat-svelte/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chat-svelte",
-  "version": "0.0.108",
+  "version": "0.0.112",
   "type": "module",
   "private": true,
   "scripts": {

examples/chat/package.json

@@ -15,7 +15,7 @@
     "clsx": "^2.0.0",
     "hash-slash": "workspace:*",
     "jazz-tools": "workspace:*",
-    "lucide-react": "^0.274.0",
+    "lucide-react": "^0.536.0",
     "react": "19.1.0",
     "react-dom": "19.1.0",
     "zod": "3.25.76"

examples/chat/src/chatScreen.tsx

@@ -1,6 +1,6 @@
-import { Account, co } from "jazz-tools";
+import { Account } from "jazz-tools";
 import { createImage, useAccount, useCoState } from "jazz-tools/react";
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { Chat, Message } from "./schema.ts";
 import {
   BubbleBody,
@@ -15,14 +15,17 @@ import {
   TextInput,
 } from "./ui.tsx";
 
-export function ChatScreen(props: { chatID: string }) {
-  const chat = useCoState(Chat, props.chatID, {
-    resolve: { $each: { text: true } },
-  });
-  const { me } = useAccount();
-  const [showNLastMessages, setShowNLastMessages] = useState(30);
+const INITIAL_MESSAGES_TO_SHOW = 30;
 
-  if (!chat)
+export function ChatScreen(props: { chatID: string }) {
+  const chat = useCoState(Chat, props.chatID);
+  const { me } = useAccount();
+  const [showNLastMessages, setShowNLastMessages] = useState(
+    INITIAL_MESSAGES_TO_SHOW,
+  );
+  const isLoading = useMessagesPreload(props.chatID);
+
+  if (!chat || isLoading)
     return (
       <div className="flex-1 flex justify-center items-center">Loading...</div>
     );
@@ -41,7 +44,7 @@ export function ChatScreen(props: { chatID: string }) {
       chat.push(
         Message.create(
           {
-            text: co.plainText().create(file.name, chat._owner),
+            text: file.name,
             image: image,
           },
           chat._owner,
@@ -59,9 +62,14 @@ export function ChatScreen(props: { chatID: string }) {
       <ChatBody>
         {chat.length > 0 ? (
           chat
+            // We call slice before reverse to avoid mutating the original array
             .slice(-showNLastMessages)
-            .reverse() // this plus flex-col-reverse on ChatBody gives us scroll-to-bottom behavior
-            .map((msg) => <ChatBubble me={me} msg={msg} key={msg.id} />)
+            // Reverse plus flex-col-reverse on ChatBody gives us scroll-to-bottom behavior
+            .reverse()
+            .map(
+              (msg) =>
+                msg?.text && <ChatBubble me={me} msg={msg} key={msg.id} />,
+            )
         ) : (
           <EmptyChatMessage />
         )}
@@ -80,12 +88,7 @@ export function ChatScreen(props: { chatID: string }) {
 
         <TextInput
           onSubmit={(text) => {
-            chat.push(
-              Message.create(
-                { text: co.plainText().create(text, chat._owner) },
-                chat._owner,
-              ),
-            );
+            chat.push(Message.create({ text }, chat._owner));
           }}
         />
       </InputBar>
@@ -93,10 +96,7 @@ export function ChatScreen(props: { chatID: string }) {
   );
 }
 
-function ChatBubble(props: {
-  me: Account;
-  msg: co.loaded<typeof Message, { text: true }>;
-}) {
+function ChatBubble(props: { me: Account; msg: Message }) {
   if (!props.me.canRead(props.msg) || !props.msg.text?.toString()) {
     return (
       <BubbleContainer fromMe={false}>
@@ -126,3 +126,35 @@ function ChatBubble(props: {
     </BubbleContainer>
   );
 }
+
+/**
+ * Warms the local cache with the initial messages to load only the initial messages
+ * and avoid flickering
+ */
+function useMessagesPreload(chatID: string) {
+  const [isLoading, setIsLoading] = useState(true);
+
+  useEffect(() => {
+    preloadChatMessages(chatID).finally(() => {
+      setIsLoading(false);
+    });
+  }, [chatID]);
+
+  return isLoading;
+}
+
+async function preloadChatMessages(chatID: string) {
+  const chat = await Chat.load(chatID);
+
+  if (!chat?._refs) return;
+
+  const promises = [];
+
+  for (const msg of Array.from(chat._refs)
+    .reverse()
+    .slice(0, INITIAL_MESSAGES_TO_SHOW)) {
+    promises.push(Message.load(msg.id, { resolve: { text: true } }));
+  }
+
+  await Promise.all(promises);
+}

examples/chat/src/ui.tsx

@@ -112,7 +112,9 @@ export function InputBar(props: { children: React.ReactNode }) {
 
 export function ImageInput({
   onImageChange,
-}: { onImageChange?: (event: React.ChangeEvent<HTMLInputElement>) => void }) {
+}: {
+  onImageChange?: (event: React.ChangeEvent<HTMLInputElement>) => void;
+}) {
   const inputRef = useRef<HTMLInputElement>(null);
 
   const onUploadClick = () => {

examples/clerk-expo/src/auth/SignInScreen.tsx

@@ -10,7 +10,9 @@ import {
 
 export function SignInScreen({
   setPage,
-}: { setPage: (page: "sign-in" | "sign-up") => void }) {
+}: {
+  setPage: (page: "sign-in" | "sign-up") => void;
+}) {
   const { signIn, setActive, isLoaded } = useSignIn();
 
   const [emailAddress, setEmailAddress] = useState("");

examples/clerk-expo/src/auth/SignUpScreen.tsx

@@ -10,7 +10,9 @@ import {
 
 export function SignUpScreen({
   setPage,
-}: { setPage: (page: "sign-in" | "sign-up") => void }) {
+}: {
+  setPage: (page: "sign-in" | "sign-up") => void;
+}) {
   const { isLoaded, signUp, setActive } = useSignUp();
 
   const [emailAddress, setEmailAddress] = React.useState("");

examples/form/src/CreateOrder.tsx

@@ -1,5 +1,4 @@
 import { useIframeHashRouter } from "hash-slash";
-import { Loaded } from "jazz-tools";
 import { useAccount, useCoState } from "jazz-tools/react";
 import { useState } from "react";
 import { Errors } from "./Errors.tsx";
@@ -9,7 +8,6 @@ import {
   BubbleTeaOrder,
   DraftBubbleTeaOrder,
   JazzAccount,
-  ListOfBubbleTeaAddOns,
   validateDraftOrder,
 } from "./schema.ts";
 
@@ -22,7 +20,7 @@ export function CreateOrder() {
 
   if (!me?.root) return;
 
-  const onSave = (draft: Loaded<typeof DraftBubbleTeaOrder>) => {
+  const onSave = (draft: DraftBubbleTeaOrder) => {
     const validation = validateDraftOrder(draft);
     setErrors(validation.errors);
     if (validation.errors.length > 0) {
@@ -30,11 +28,11 @@ export function CreateOrder() {
     }
 
     // turn the draft into a real order
-    me.root.orders.push(draft as Loaded<typeof BubbleTeaOrder>);
+    me.root.orders.push(draft as BubbleTeaOrder);
 
     // reset the draft
     me.root.draft = DraftBubbleTeaOrder.create({
-      addOns: ListOfBubbleTeaAddOns.create([]),
+      addOns: [],
     });
 
     router.navigate("/");
@@ -60,7 +58,7 @@ function CreateOrderForm({
   onSave,
 }: {
   id: string;
-  onSave: (draft: Loaded<typeof DraftBubbleTeaOrder>) => void;
+  onSave: (draft: DraftBubbleTeaOrder) => void;
 }) {
   const draft = useCoState(DraftBubbleTeaOrder, id, {
     resolve: { addOns: true, instructions: true },

examples/form/src/OrderForm.tsx

@@ -1,4 +1,4 @@
-import { CoPlainText, Loaded } from "jazz-tools";
+import { CoPlainText } from "jazz-tools";
 import {
   BubbleTeaAddOnTypes,
   BubbleTeaBaseTeaTypes,
@@ -10,7 +10,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: Loaded<typeof BubbleTeaOrder> | Loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   // Handles updates to the instructions field of the order.

examples/form/src/OrderThumbnail.tsx

@@ -1,11 +1,6 @@
-import { Loaded } from "jazz-tools";
 import { BubbleTeaOrder } from "./schema.ts";
 
-export function OrderThumbnail({
-  order,
-}: {
-  order: Loaded<typeof BubbleTeaOrder>;
-}) {
+export function OrderThumbnail({ order }: { order: BubbleTeaOrder }) {
   const { id, baseTea, addOns, instructions, deliveryDate, withMilk } = order;
   const date = deliveryDate.toLocaleDateString();
 

examples/form/src/schema.ts

@@ -1,4 +1,4 @@
-import { Loaded, co, z } from "jazz-tools";
+import { co, z } from "jazz-tools";
 
 export const BubbleTeaAddOnTypes = [
   "Pearl",
@@ -18,8 +18,9 @@ export const BubbleTeaBaseTeaTypes = [
 export const ListOfBubbleTeaAddOns = co.list(
   z.literal([...BubbleTeaAddOnTypes]),
 );
+export type ListOfBubbleTeaAddOns = co.loaded<typeof ListOfBubbleTeaAddOns>;
 
-function hasAddOnsChanges(list?: Loaded<typeof ListOfBubbleTeaAddOns> | null) {
+function hasAddOnsChanges(list?: ListOfBubbleTeaAddOns | null) {
   return list && Object.entries(list._raw.insertions).length > 0;
 }
 
@@ -30,16 +31,12 @@ export const BubbleTeaOrder = co.map({
   withMilk: z.boolean(),
   instructions: co.optional(co.plainText()),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  baseTea: z.optional(z.literal([...BubbleTeaBaseTeaTypes])),
-  addOns: co.optional(ListOfBubbleTeaAddOns),
-  deliveryDate: z.optional(z.date()),
-  withMilk: z.optional(z.boolean()),
-  instructions: co.optional(co.plainText()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(order: Loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(order: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!order.baseTea) {
@@ -52,7 +49,7 @@ export function validateDraftOrder(order: Loaded<typeof DraftBubbleTeaOrder>) {
   return { errors };
 }
 
-export function hasChanges(order?: Loaded<typeof DraftBubbleTeaOrder> | null) {
+export function hasChanges(order?: DraftBubbleTeaOrder | null) {
   return (
     !!order &&
     (Object.keys(order._edits).length > 1 || hasAddOnsChanges(order.addOns))
@@ -73,15 +70,9 @@ export const JazzAccount = co
   })
   .withMigration((account) => {
     if (!account.root) {
-      const orders = co.list(BubbleTeaOrder).create([], account);
-      const draft = DraftBubbleTeaOrder.create(
-        {
-          addOns: ListOfBubbleTeaAddOns.create([], account),
-          instructions: co.plainText().create("", account),
-        },
+      account.root = AccountRoot.create(
+        { draft: { addOns: [], instructions: "" }, orders: [] },
         account,
       );
-
-      account.root = AccountRoot.create({ draft, orders }, account);
     }
   });

examples/multi-cursors/src/components/Container.tsx

@@ -12,7 +12,11 @@ function Avatar({
   name,
   color,
   active,
-}: { name: string; color: string; active: boolean }) {
+}: {
+  name: string;
+  color: string;
+  active: boolean;
+}) {
   return (
     <span
       title={name}

examples/multiauth/src/index.css

@@ -47,7 +47,9 @@ button {
   font-family: inherit;
   background-color: transparent;
   cursor: pointer;
-  transition: all 0.05s ease, border-color 0.1s ease;
+  transition:
+    all 0.05s ease,
+    border-color 0.1s ease;
   box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
 }
 button:hover {
@@ -93,8 +95,9 @@ button:active {
   background-color: white;
   padding: 2rem;
   border-radius: 0.5rem;
-  box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px
-    rgba(0, 0, 0, 0.06);
+  box-shadow:
+    0 4px 6px -1px rgba(0, 0, 0, 0.1),
+    0 2px 4px -1px rgba(0, 0, 0, 0.06);
   width: 28rem;
 }
 

examples/music-player/package.json

@@ -22,7 +22,7 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "jazz-tools": "workspace:*",
-    "lucide-react": "^0.274.0",
+    "lucide-react": "^0.536.0",
     "react": "19.1.0",
     "react-dom": "19.1.0",
     "react-router": "^6.16.0",
@@ -38,6 +38,7 @@
     "postcss": "^8.4.27",
     "tailwindcss": "^4.1.10",
     "typescript": "5.6.2",
-    "vite": "^6.3.5"
+    "vite": "^6.3.5",
+    "vite-plugin-pwa": "^1.0.2"
   }
 }

examples/music-player/src/1_schema.ts

@@ -84,15 +84,14 @@ export const MusicaAccount = co
      *  You can use it to set up the account root and any other initial CoValues you need.
      */
     if (account.root === undefined) {
-      const tracks = co.list(MusicTrack).create([]);
       const rootPlaylist = Playlist.create({
-        tracks,
+        tracks: [],
         title: "",
       });
 
       account.root = MusicaAccountRoot.create({
         rootPlaylist,
-        playlists: co.list(Playlist).create([]),
+        playlists: [],
         activeTrack: undefined,
         activePlaylist: rootPlaylist,
         exampleDataLoaded: false,

examples/music-player/src/2_main.tsx

@@ -15,7 +15,7 @@ import { SidebarProvider } from "@/components/ui/sidebar";
 import { JazzReactProvider } from "jazz-tools/react";
 import { onAnonymousAccountDiscarded } from "./4_actions";
 import { KeyboardListener } from "./components/PlayerControls";
-import { useUploadExampleData } from "./lib/useUploadExampleData";
+import { usePrepareAppState } from "./lib/usePrepareAppState";
 
 /**
  * Walkthrough: The top-level provider `<JazzReactProvider/>`
@@ -31,7 +31,7 @@ import { useUploadExampleData } from "./lib/useUploadExampleData";
 function Main() {
   const mediaPlayer = useMediaPlayer();
 
-  useUploadExampleData();
+  const isReady = usePrepareAppState(mediaPlayer);
 
   const router = createHashRouter([
     {
@@ -48,6 +48,8 @@ function Main() {
     },
   ]);
 
+  if (!isReady) return null;
+
   return (
     <>
       <RouterProvider router={router} />

examples/music-player/src/3_HomePage.tsx

@@ -11,6 +11,7 @@ import { uploadMusicTracks } from "./4_actions";
 import { MediaPlayer } from "./5_useMediaPlayer";
 import { FileUploadButton } from "./components/FileUploadButton";
 import { MusicTrackRow } from "./components/MusicTrackRow";
+import { PlayerControls } from "./components/PlayerControls";
 import { PlaylistTitleInput } from "./components/PlaylistTitleInput";
 import { SidePanel } from "./components/SidePanel";
 import { Button } from "./components/ui/button";
@@ -42,7 +43,11 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   const playlistId = params.playlistId ?? me?.root._refs.rootPlaylist.id;
 
   const playlist = useCoState(Playlist, playlistId, {
-    resolve: { tracks: true },
+    resolve: {
+      tracks: {
+        $each: true,
+      },
+    },
   });
 
   const isRootPlaylist = !params.playlistId;
@@ -64,16 +69,16 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   const isAuthenticated = useIsAuthenticated();
 
   return (
-    <SidebarInset className="flex flex-col h-screen text-gray-800 bg-blue-50">
+    <SidebarInset className="flex flex-col h-screen text-gray-800">
       <div className="flex flex-1 overflow-hidden">
-        <SidePanel mediaPlayer={mediaPlayer} />
-        <main className="flex-1 p-6 overflow-y-auto overflow-x-hidden">
-          <SidebarTrigger />
-          <div className="flex items-center justify-between mb-6">
+        <SidePanel />
+        <main className="flex-1 px-2 py-4 md:px-6 overflow-y-auto overflow-x-hidden relative sm:h-[calc(100vh-80px)] bg-white h-[calc(100vh-165px)]">
+          <SidebarTrigger className="md:hidden" />
+          <div className="flex flex-row items-center justify-between mb-4 pl-1 md:pl-10 pr-2 md:pr-0 mt-2 md:mt-0 w-full">
             {isRootPlaylist ? (
               <h1 className="text-2xl font-bold text-blue-800">All tracks</h1>
             ) : (
-              <PlaylistTitleInput playlistId={playlistId} />
+              <PlaylistTitleInput className="w-full" playlistId={playlistId} />
             )}
             <div className="flex items-center space-x-4">
               {isRootPlaylist && (
@@ -90,14 +95,14 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
               )}
             </div>
           </div>
-          <ul className="flex flex-col max-w-full">
+          <ul className="flex flex-col max-w-full sm:gap-1">
             {playlist?.tracks?.map(
-              (track) =>
+              (track, index) =>
                 track && (
                   <MusicTrackRow
                     trackId={track.id}
                     key={track.id}
-                    isLoading={mediaPlayer.loading === track.id}
+                    index={index}
                     isPlaying={
                       mediaPlayer.activeTrackId === track.id &&
                       isActivePlaylist &&
@@ -106,12 +111,12 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
                     onClick={() => {
                       mediaPlayer.setActiveTrack(track, playlist);
                     }}
-                    showAddToPlaylist={isRootPlaylist}
                   />
                 ),
             )}
           </ul>
         </main>
+        <PlayerControls mediaPlayer={mediaPlayer} />
       </div>
     </SidebarInset>
   );

examples/music-player/src/4_actions.ts

@@ -1,11 +1,6 @@
 import { getAudioFileData } from "@/lib/audio/getAudioFileData";
-import { FileStream, Group, co } from "jazz-tools";
-import {
-  MusicTrack,
-  MusicTrackWaveform,
-  MusicaAccount,
-  Playlist,
-} from "./1_schema";
+import { FileStream, Group } from "jazz-tools";
+import { MusicTrack, MusicaAccount, Playlist } from "./1_schema";
 
 /**
  * Walkthrough: Actions
@@ -51,7 +46,7 @@ export async function uploadMusicTracks(
       {
         file: fileStream,
         duration: data.duration,
-        waveform: MusicTrackWaveform.create({ data: data.waveform }, group),
+        waveform: { data: data.waveform },
         title: file.name,
         isExampleTrack,
       },
@@ -73,18 +68,10 @@ export async function createNewPlaylist() {
     },
   });
 
-  // Since playlists are meant to be shared we associate them
-  // to a group which will contain the keys required to get
-  // access to the "owned" values
-  const playlistGroup = Group.create();
-
-  const playlist = Playlist.create(
-    {
-      title: "New Playlist",
-      tracks: co.list(MusicTrack).create([], playlistGroup),
-    },
-    playlistGroup,
-  );
+  const playlist = Playlist.create({
+    title: "New Playlist",
+    tracks: [],
+  });
 
   // Again, we associate the new playlist to the
   // user by pushing it into the playlists CoList
@@ -129,7 +116,7 @@ export async function removeTrackFromPlaylist(
 
   if (track._owner._type === "Group" && playlist._owner._type === "Group") {
     const trackGroup = track._owner;
-    await trackGroup.removeMember(playlist._owner);
+    trackGroup.removeMember(playlist._owner);
 
     const index =
       playlist.tracks?.findIndex(

examples/music-player/src/5_useMediaPlayer.ts

@@ -5,6 +5,7 @@ import { FileStream } from "jazz-tools";
 import { useAccount } from "jazz-tools/react";
 import { useRef, useState } from "react";
 import { updateActivePlaylist, updateActiveTrack } from "./4_actions";
+import { useAudioManager } from "./lib/audio/AudioManager";
 import { getNextTrack, getPrevTrack } from "./lib/getters";
 
 export function useMediaPlayer() {
@@ -12,6 +13,7 @@ export function useMediaPlayer() {
     resolve: { root: true },
   });
 
+  const audioManager = useAudioManager();
   const playState = usePlayState();
   const playMedia = usePlayMedia();
 
@@ -24,8 +26,10 @@ export function useMediaPlayer() {
 
   async function loadTrack(track: MusicTrack) {
     lastLoadedTrackId.current = track.id;
+    audioManager.unloadCurrentAudio();
 
     setLoading(track.id);
+    updateActiveTrack(track);
 
     const file = await FileStream.loadAsBlob(track._refs.file!.id); // TODO: see if we can avoid !
 
@@ -40,8 +44,6 @@ export function useMediaPlayer() {
       return;
     }
 
-    updateActiveTrack(track);
-
     await playMedia(file);
 
     setLoading(null);
@@ -85,6 +87,7 @@ export function useMediaPlayer() {
     playNextTrack,
     playPrevTrack,
     loading,
+    loadTrack,
   };
 }
 

examples/music-player/src/components/ConfirmDialog.tsx

@@ -0,0 +1,59 @@
+import { Button } from "@/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+
+interface ConfirmDialogProps {
+  isOpen: boolean;
+  onOpenChange: (open: boolean) => void;
+  title: string;
+  description: string;
+  confirmText?: string;
+  cancelText?: string;
+  onConfirm: () => void;
+  variant?: "default" | "destructive";
+}
+
+export function ConfirmDialog({
+  isOpen,
+  onOpenChange,
+  title,
+  description,
+  confirmText = "Confirm",
+  cancelText = "Cancel",
+  onConfirm,
+  variant = "destructive",
+}: ConfirmDialogProps) {
+  function handleConfirm() {
+    onConfirm();
+    onOpenChange(false);
+  }
+
+  function handleCancel() {
+    onOpenChange(false);
+  }
+
+  return (
+    <Dialog open={isOpen} onOpenChange={onOpenChange}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>{title}</DialogTitle>
+          <DialogDescription>{description}</DialogDescription>
+        </DialogHeader>
+        <DialogFooter>
+          <Button variant="outline" onClick={handleCancel}>
+            {cancelText}
+          </Button>
+          <Button variant={variant} onClick={handleConfirm}>
+            {confirmText}
+          </Button>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}

examples/music-player/src/components/MusicTrackRow.tsx

@@ -9,31 +9,41 @@ import {
 import { cn } from "@/lib/utils";
 import { Loaded } from "jazz-tools";
 import { useAccount, useCoState } from "jazz-tools/react";
-import { MoreHorizontal } from "lucide-react";
-import { Fragment } from "react/jsx-runtime";
-import { MusicTrackTitleInput } from "./MusicTrackTitleInput";
+import { MoreHorizontal, Pause, Play } from "lucide-react";
+import { Fragment, useCallback, useState } from "react";
+import { EditTrackDialog } from "./RenameTrackDialog";
+import { Waveform } from "./Waveform";
 import { Button } from "./ui/button";
 
+function isPartOfThePlaylist(
+  trackId: string,
+  playlist: Loaded<typeof Playlist, { tracks: true }>,
+) {
+  return Array.from(playlist.tracks._refs).some((t) => t.id === trackId);
+}
+
 export function MusicTrackRow({
   trackId,
-  isLoading,
   isPlaying,
   onClick,
-  showAddToPlaylist,
+  index,
 }: {
   trackId: string;
-  isLoading: boolean;
   isPlaying: boolean;
   onClick: (track: Loaded<typeof MusicTrack>) => void;
-  showAddToPlaylist: boolean;
+  index: number;
 }) {
   const track = useCoState(MusicTrack, trackId);
+  const [isEditDialogOpen, setIsEditDialogOpen] = useState(false);
+  const [isDropdownOpen, setIsDropdownOpen] = useState(false);
+  const [isHovered, setIsHovered] = useState(false);
 
   const { me } = useAccount(MusicaAccount, {
-    resolve: { root: { playlists: { $each: true } } },
+    resolve: { root: { playlists: { $each: { tracks: true } } } },
   });
 
   const playlists = me?.root.playlists ?? [];
+  const isActiveTrack = trackId === me?.root._refs.activeTrack?.id;
 
   function handleTrackClick() {
     if (!track) return;
@@ -60,71 +70,118 @@ export function MusicTrackRow({
     }
   }
 
+  function handleEdit() {
+    setIsEditDialogOpen(true);
+  }
+
+  const handleContextMenu = useCallback((e: React.MouseEvent) => {
+    e.preventDefault();
+    setIsDropdownOpen(true);
+  }, []);
+
+  const showWaveform = isHovered || isActiveTrack;
+
   return (
     <li
-      className={"flex gap-1 hover:bg-slate-200 group py-2 px-2 cursor-pointer"}
-      onClick={handleTrackClick}
+      className={cn(
+        "flex gap-1 hover:bg-slate-200 group py-2 cursor-pointer rounded-lg",
+        isActiveTrack && "bg-slate-200",
+      )}
+      onMouseEnter={() => {
+        setIsHovered(true);
+      }}
+      onMouseLeave={() => {
+        setIsHovered(false);
+      }}
     >
       <button
         className={cn(
-          "flex items-center justify-center bg-transparent w-8 h-8 ",
-          !isPlaying && "group-hover:bg-slate-300 rounded-full",
+          "flex items-center justify-center bg-transparent w-8 h-8 transition-opacity cursor-pointer",
+          // Show play button on hover or when active, hide otherwise
+          "md:opacity-0 opacity-50 group-hover:opacity-100",
+          isActiveTrack && "md:opacity-100 opacity-100",
         )}
         onClick={handleTrackClick}
         aria-label={`${isPlaying ? "Pause" : "Play"} ${track?.title}`}
       >
-        {isLoading ? (
-          <div className="animate-spin">߷</div>
-        ) : isPlaying ? (
-          "⏸️"
+        {isPlaying ? (
+          <Pause height={16} width={16} fill="currentColor" />
         ) : (
-          "▶️"
+          <Play height={16} width={16} fill="currentColor" />
         )}
       </button>
-      <MusicTrackTitleInput trackId={trackId} />
+      {/* Show track index when play button is hidden - hidden on mobile */}
+      <div
+        className={cn(
+          "hidden md:flex items-center justify-center w-8 h-8 text-sm text-gray-500 font-mono transition-opacity",
+        )}
+      >
+        {index + 1}
+      </div>
+      <button
+        onContextMenu={handleContextMenu}
+        onClick={handleTrackClick}
+        className="flex items-center overflow-hidden text-ellipsis whitespace-nowrap cursor-pointer flex-1 min-w-0"
+      >
+        {track?.title}
+      </button>
+
+      {/* Waveform that appears on hover */}
+      {track && showWaveform && (
+        <div className="flex-1 min-w-0 px-2 items-center hidden md:flex">
+          <Waveform
+            track={track}
+            height={20}
+            className="opacity-70 w-full"
+            showProgress={isActiveTrack}
+          />
+        </div>
+      )}
+
       <div onClick={(evt) => evt.stopPropagation()}>
-        {showAddToPlaylist && (
-          <DropdownMenu>
-            <DropdownMenuTrigger asChild>
-              <Button
-                variant="ghost"
-                className="h-8 w-8 p-0"
-                aria-label={`Open ${track?.title} menu`}
-              >
-                <span className="sr-only">Open menu</span>
-                <MoreHorizontal className="h-4 w-4" />
-              </Button>
-            </DropdownMenuTrigger>
-            <DropdownMenuContent align="end">
-              <DropdownMenuItem
-                key={`delete`}
-                onSelect={async () => {
-                  if (!track) return;
-                  deleteTrack();
-                }}
-              >
-                Delete
-              </DropdownMenuItem>
-              {playlists.map((playlist, index) => (
-                <Fragment key={index}>
+        <DropdownMenu open={isDropdownOpen} onOpenChange={setIsDropdownOpen}>
+          <DropdownMenuTrigger asChild>
+            <Button
+              variant="ghost"
+              className="h-8 w-8 p-0"
+              aria-label={`Open ${track?.title} menu`}
+            >
+              <span className="sr-only">Open menu</span>
+              <MoreHorizontal className="h-4 w-4" />
+            </Button>
+          </DropdownMenuTrigger>
+          <DropdownMenuContent align="end">
+            <DropdownMenuItem onSelect={handleEdit}>Edit</DropdownMenuItem>
+            {playlists.map((playlist, playlistIndex) => (
+              <Fragment key={playlistIndex}>
+                {isPartOfThePlaylist(trackId, playlist) ? (
                   <DropdownMenuItem
-                    key={`add-${index}`}
-                    onSelect={() => handleAddToPlaylist(playlist)}
-                  >
-                    Add to {playlist.title}
-                  </DropdownMenuItem>
-                  <DropdownMenuItem
-                    key={`remove-${index}`}
+                    key={`remove-${playlistIndex}`}
                     onSelect={() => handleRemoveFromPlaylist(playlist)}
                   >
                     Remove from {playlist.title}
                   </DropdownMenuItem>
-                </Fragment>
-              ))}
-            </DropdownMenuContent>
-          </DropdownMenu>
-        )}
+                ) : (
+                  <DropdownMenuItem
+                    key={`add-${playlistIndex}`}
+                    onSelect={() => handleAddToPlaylist(playlist)}
+                  >
+                    Add to {playlist.title}
+                  </DropdownMenuItem>
+                )}
+              </Fragment>
+            ))}
+          </DropdownMenuContent>
+        </DropdownMenu>
       </div>
+      {track && isEditDialogOpen && (
+        <EditTrackDialog
+          track={track}
+          isOpen={isEditDialogOpen}
+          onOpenChange={setIsEditDialogOpen}
+          onDelete={deleteTrack}
+        />
+      )}
     </li>
   );
 }

examples/music-player/src/components/PlayerControls.tsx

@@ -5,7 +5,8 @@ import { usePlayState } from "@/lib/audio/usePlayState";
 import { useKeyboardListener } from "@/lib/useKeyboardListener";
 import { useAccount, useCoState } from "jazz-tools/react";
 import { Pause, Play, SkipBack, SkipForward } from "lucide-react";
-import { Waveform } from "./Waveform";
+import WaveformCanvas from "./WaveformCanvas";
+import { Button } from "./ui/button";
 
 export function PlayerControls({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   const playState = usePlayState();
@@ -15,51 +16,61 @@ export function PlayerControls({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
     resolve: { root: { activePlaylist: true } },
   }).me?.root.activePlaylist;
 
-  const activeTrack = useCoState(MusicTrack, mediaPlayer.activeTrackId, {
-    resolve: { waveform: true },
-  });
+  const activeTrack = useCoState(MusicTrack, mediaPlayer.activeTrackId);
 
   if (!activeTrack) return null;
 
   const activeTrackTitle = activeTrack.title;
 
   return (
-    <footer className="flex items-center justify-between p-4 gap-4 bg-white border-t border-gray-200 fixed bottom-0 left-0 right-0 w-full">
-      <div className="flex justify-center items-center space-x-2">
-        <div className="flex items-center space-x-4">
-          <button
+    <footer className="flex flex-wrap sm:flex-nowrap items-center justify-between pt-4 p-2 sm:p-4 gap-4 sm:gap-4 bg-white border-t border-gray-200 absolute bottom-0 left-0 right-0 w-full z-50">
+      {/* Player Controls - Always on top */}
+      <div className="flex justify-center items-center space-x-1 sm:space-x-2 flex-shrink-0 w-full sm:w-auto order-1 sm:order-none">
+        <div className="flex items-center space-x-2">
+          <Button
+            variant="ghost"
+            size="icon"
             onClick={mediaPlayer.playPrevTrack}
-            className="text-blue-600 hover:text-blue-800"
             aria-label="Previous track"
           >
-            <SkipBack size={20} />
-          </button>
-          <button
+            <SkipBack className="h-5 w-5" fill="currentColor" />
+          </Button>
+          <Button
+            size="icon"
             onClick={playState.toggle}
-            className="w-[42px] h-[42px] flex items-center justify-center bg-blue-600 rounded-full text-white hover:bg-blue-700"
+            className="bg-blue-600 text-white hover:bg-blue-700"
             aria-label={isPlaying ? "Pause active track" : "Play active track"}
           >
             {isPlaying ? (
-              <Pause size={24} fill="currentColor" />
+              <Pause className="h-5 w-5" fill="currentColor" />
             ) : (
-              <Play size={24} fill="currentColor" />
+              <Play className="h-5 w-5" fill="currentColor" />
             )}
-          </button>
-          <button
+          </Button>
+          <Button
+            variant="ghost"
+            size="icon"
             onClick={mediaPlayer.playNextTrack}
-            className="text-blue-600 hover:text-blue-800"
             aria-label="Next track"
           >
-            <SkipForward size={20} />
-          </button>
+            <SkipForward className="h-5 w-5" fill="currentColor" />
+          </Button>
         </div>
       </div>
-      <div className=" sm:hidden md:flex flex-col flex-shrink-1 items-center w-[75%]">
-        <Waveform track={activeTrack} height={30} />
-      </div>
-      <div className="flex flex-col items-end  gap-1 text-right min-w-fit w-[25%]">
-        <h4 className="font-medium text-blue-800">{activeTrackTitle}</h4>
-        <p className="text-sm text-gray-600">
+
+      {/* Waveform - Below controls on mobile, between controls and info on desktop */}
+      <WaveformCanvas
+        className="order-1 sm:order-none"
+        track={activeTrack}
+        height={50}
+      />
+
+      {/* Track Info - Below waveform on mobile, on the right on desktop */}
+      <div className="flex flex-col gap-1 min-w-fit sm:flex-shrink-0 text-center w-full sm:text-right items-center sm:items-end sm:w-auto order-0 sm:order-none">
+        <h4 className="font-medium text-blue-800 text-base sm:text-base truncate max-w-80 sm:max-w-80">
+          {activeTrackTitle}
+        </h4>
+        <p className="hidden sm:block text-xs sm:text-sm text-gray-600 truncate sm:max-w-80">
           {activePlaylist?.title || "All tracks"}
         </p>
       </div>
@@ -69,7 +80,9 @@ export function PlayerControls({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
 
 export function KeyboardListener({
   mediaPlayer,
-}: { mediaPlayer: MediaPlayer }) {
+}: {
+  mediaPlayer: MediaPlayer;
+}) {
   const playState = usePlayState();
 
   useMediaEndListener(mediaPlayer.playNextTrack);

examples/music-player/src/components/PlaylistTitleInput.tsx

@@ -1,12 +1,15 @@
 import { Playlist } from "@/1_schema";
 import { updatePlaylistTitle } from "@/4_actions";
+import { cn } from "@/lib/utils";
 import { useCoState } from "jazz-tools/react";
 import { ChangeEvent, useState } from "react";
 
 export function PlaylistTitleInput({
   playlistId,
+  className,
 }: {
   playlistId: string | undefined;
+  className?: string;
 }) {
   const playlist = useCoState(Playlist, playlistId);
   const [isEditing, setIsEditing] = useState(false);
@@ -33,7 +36,10 @@ export function PlaylistTitleInput({
     <input
       value={inputValue}
       onChange={handleTitleChange}
-      className="text-2xl font-bold text-blue-800 bg-transparent"
+      className={cn(
+        "text-2xl font-bold text-blue-800 bg-transparent",
+        className,
+      )}
       onFocus={handleFoucsIn}
       onBlur={handleFocusOut}
       aria-label={`Playlist title`}

examples/music-player/src/components/RenameTrackDialog.tsx

@@ -0,0 +1,108 @@
+import { MusicTrack } from "@/1_schema";
+import { updateMusicTrackTitle } from "@/4_actions";
+import { Button } from "@/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import { Input } from "@/components/ui/input";
+import { Loaded } from "jazz-tools";
+import { useState } from "react";
+import { ConfirmDialog } from "./ConfirmDialog";
+
+interface EditTrackDialogProps {
+  track: Loaded<typeof MusicTrack>;
+  isOpen: boolean;
+  onOpenChange: (open: boolean) => void;
+  onDelete: () => void;
+}
+
+export function EditTrackDialog({
+  track,
+  isOpen,
+  onOpenChange,
+  onDelete,
+}: EditTrackDialogProps) {
+  const [newTitle, setNewTitle] = useState(track.title);
+  const [isDeleteConfirmOpen, setIsDeleteConfirmOpen] = useState(false);
+
+  function handleSave() {
+    if (track && newTitle.trim()) {
+      updateMusicTrackTitle(track, newTitle.trim());
+      onOpenChange(false);
+    }
+  }
+
+  function handleCancel() {
+    setNewTitle(track?.title || "");
+    onOpenChange(false);
+  }
+
+  function handleDeleteClick() {
+    setIsDeleteConfirmOpen(true);
+  }
+
+  function handleDeleteConfirm() {
+    onDelete();
+    onOpenChange(false);
+  }
+
+  function handleKeyDown(event: React.KeyboardEvent) {
+    if (event.key === "Enter") {
+      handleSave();
+    } else if (event.key === "Escape") {
+      handleCancel();
+    }
+  }
+
+  return (
+    <Dialog open={isOpen} onOpenChange={onOpenChange}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>Edit Track</DialogTitle>
+          <DialogDescription>Edit "{track?.title}".</DialogDescription>
+        </DialogHeader>
+        <form className="py-4" onSubmit={handleSave}>
+          <Input
+            value={newTitle}
+            onChange={(e) => setNewTitle(e.target.value)}
+            onKeyDown={handleKeyDown}
+            placeholder="Enter track name..."
+            autoFocus
+          />
+        </form>
+        <DialogFooter className="flex justify-between">
+          <Button
+            variant="destructive"
+            onClick={handleDeleteClick}
+            className="mr-auto"
+          >
+            Delete Track
+          </Button>
+          <div className="flex gap-2">
+            <Button variant="outline" onClick={handleCancel}>
+              Cancel
+            </Button>
+            <Button onClick={handleSave} disabled={!newTitle.trim()}>
+              Save
+            </Button>
+          </div>
+        </DialogFooter>
+      </DialogContent>
+      <ConfirmDialog
+        isOpen={isDeleteConfirmOpen}
+        onOpenChange={setIsDeleteConfirmOpen}
+        title="Delete Track"
+        description={`Are you sure you want to delete "${track.title}"? This action cannot be undone.`}
+        confirmText="Delete"
+        cancelText="Cancel"
+        onConfirm={handleDeleteConfirm}
+        variant="destructive"
+      />
+    </Dialog>
+  );
+}

examples/music-player/src/components/SidePanel.tsx

@@ -1,10 +1,8 @@
-import { MusicTrack, MusicaAccount } from "@/1_schema";
+import { MusicaAccount } from "@/1_schema";
 import { createNewPlaylist, deletePlaylist } from "@/4_actions";
-import { MediaPlayer } from "@/5_useMediaPlayer";
 import {
   Sidebar,
   SidebarContent,
-  SidebarFooter,
   SidebarGroup,
   SidebarGroupContent,
   SidebarGroupLabel,
@@ -14,22 +12,18 @@ import {
   SidebarMenuButton,
   SidebarMenuItem,
 } from "@/components/ui/sidebar";
-import { usePlayState } from "@/lib/audio/usePlayState";
-import { useAccount, useCoState } from "jazz-tools/react";
-import { Home, Music, Pause, Play, Plus, Trash2 } from "lucide-react";
+import { useAccount } from "jazz-tools/react";
+import { Home, Music, Plus, Trash2 } from "lucide-react";
 import { useNavigate, useParams } from "react-router";
 import { AuthButton } from "./AuthButton";
 
-export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
+export function SidePanel() {
   const { playlistId } = useParams();
   const navigate = useNavigate();
   const { me } = useAccount(MusicaAccount, {
     resolve: { root: { playlists: { $each: true } } },
   });
 
-  const playState = usePlayState();
-  const isPlaying = playState.value === "play";
-
   function handleAllTracksClick() {
     navigate(`/`);
   }
@@ -50,12 +44,6 @@ export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
     navigate(`/playlist/${playlist.id}`);
   }
 
-  const activeTrack = useCoState(MusicTrack, mediaPlayer.activeTrackId, {
-    resolve: { waveform: true },
-  });
-
-  const activeTrackTitle = activeTrack?.title;
-
   return (
     <Sidebar>
       <SidebarHeader>
@@ -137,29 +125,6 @@ export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
           </SidebarGroupContent>
         </SidebarGroup>
       </SidebarContent>
-      {activeTrack && (
-        <SidebarFooter>
-          <SidebarMenu>
-            <SidebarMenuItem className="flex justify-end">
-              <SidebarMenuButton
-                onClick={playState.toggle}
-                aria-label={
-                  isPlaying ? "Pause active track" : "Play active track"
-                }
-              >
-                <div className="w-[28px] h-[28px] flex items-center justify-center bg-blue-600 rounded-full text-white hover:bg-blue-700">
-                  {isPlaying ? (
-                    <Pause size={16} fill="currentColor" />
-                  ) : (
-                    <Play size={16} fill="currentColor" />
-                  )}
-                </div>
-                <span>{activeTrackTitle}</span>
-              </SidebarMenuButton>
-            </SidebarMenuItem>
-          </SidebarMenu>
-        </SidebarFooter>
-      )}
     </Sidebar>
   );
 }

examples/music-player/src/components/Waveform.tsx

@@ -7,6 +7,8 @@ import { useCoState } from "jazz-tools/react";
 export function Waveform(props: {
   track: Loaded<typeof MusicTrack>;
   height: number;
+  className?: string;
+  showProgress?: boolean;
 }) {
   const { track, height } = props;
   const waveformData = useCoState(
@@ -28,29 +30,24 @@ export function Waveform(props: {
   }
 
   const barCount = waveformData.length;
-  const activeBar = Math.ceil(barCount * (currentTime.value / duration));
-
-  function seek(i: number) {
-    currentTime.setValue((i / barCount) * duration);
-  }
+  const activeBar = props.showProgress
+    ? Math.ceil(barCount * (currentTime.value / duration))
+    : -1;
 
   return (
     <div
-      className="flex justify-center items-end w-full"
+      className={cn("flex justify-center items-end w-full", props.className)}
       style={{
         height,
-        gap: 1,
       }}
     >
       {waveformData.map((value, i) => (
         <button
           type="button"
           key={i}
-          onClick={() => seek(i)}
           className={cn(
             "w-1 transition-colors rounded-none rounded-t-lg min-h-1",
-            activeBar >= i ? "bg-gray-500" : "bg-gray-300",
-            "hover:bg-black hover:border hover:border-solid hover:border-black",
+            activeBar >= i ? "bg-gray-800" : "bg-gray-400",
             "focus-visible:outline-black focus:outline-hidden",
           )}
           style={{

examples/music-player/src/components/WaveformCanvas.tsx

@@ -0,0 +1,282 @@
+"use client";
+
+import { MusicTrack, MusicTrackWaveform } from "@/1_schema";
+import { AudioManager, useAudioManager } from "@/lib/audio/AudioManager";
+import {
+  getPlayerCurrentTime,
+  setPlayerCurrentTime,
+  subscribeToPlayerCurrentTime,
+  usePlayerCurrentTime,
+} from "@/lib/audio/usePlayerCurrentTime";
+import { cn } from "@/lib/utils";
+import { Loaded } from "jazz-tools";
+import type React from "react";
+
+import { useEffect, useRef } from "react";
+
+type Props = {
+  track: Loaded<typeof MusicTrack>;
+  height?: number;
+  barColor?: string;
+  progressColor?: string;
+  backgroundColor?: string;
+  className?: string;
+};
+
+const DEFAULT_HEIGHT = 96;
+
+// Downsample PCM into N peaks (abs max in window)
+function buildPeaks(channelData: number[], samples: number): Float32Array {
+  const length = channelData.length;
+  if (channelData.length < samples) {
+    // Create a peaks array that interpolates the channelData
+    const interpolatedPeaks = new Float32Array(samples);
+    for (let i = 0; i < samples; i++) {
+      const index = Math.floor(i * (length / samples));
+      interpolatedPeaks[i] = channelData[index];
+    }
+    return interpolatedPeaks;
+  }
+
+  const blockSize = Math.floor(length / samples);
+  const peaks = new Float32Array(samples);
+
+  for (let i = 0; i < samples; i++) {
+    const start = i * blockSize;
+    let end = start + blockSize;
+    if (end > length) end = length;
+    let max = 0;
+    for (let j = start; j < end; j++) {
+      const v = Math.abs(channelData[j]);
+      if (v > max) max = v;
+    }
+    peaks[i] = max;
+  }
+  return peaks;
+}
+
+type DrawWaveformCanvasProps = {
+  canvas: HTMLCanvasElement;
+  waveformData: number[] | undefined;
+  duration: number;
+  currentTime: number;
+  barColor?: string;
+  progressColor?: string;
+  backgroundColor?: string;
+  isAnimating: boolean;
+  animationProgress: number;
+  progress: number;
+};
+
+function drawWaveform(props: DrawWaveformCanvasProps) {
+  const {
+    canvas,
+    waveformData,
+    isAnimating,
+    animationProgress,
+    barColor = "hsl(215, 16%, 47%)",
+    progressColor = "hsl(142, 71%, 45%)",
+    backgroundColor = "transparent",
+    progress,
+  } = props;
+
+  const ctx = canvas.getContext("2d");
+  if (!ctx) return;
+
+  const dpr = window.devicePixelRatio || 1;
+  const cssWidth = canvas.clientWidth;
+  const cssHeight = canvas.clientHeight;
+  canvas.width = Math.floor(cssWidth * dpr);
+  canvas.height = Math.floor(cssHeight * dpr);
+  ctx.scale(dpr, dpr);
+
+  // Background
+  ctx.fillStyle = backgroundColor;
+  ctx.fillRect(0, 0, cssWidth, cssHeight);
+
+  if (!waveformData || !waveformData.length) {
+    // Draw placeholder line
+    ctx.strokeStyle = "hsl(215, 20%, 65%)";
+    ctx.lineWidth = 2;
+    ctx.beginPath();
+    ctx.moveTo(0, cssHeight / 2);
+    ctx.lineTo(cssWidth, cssHeight / 2);
+    ctx.stroke();
+    return;
+  }
+
+  const midY = cssHeight / 2;
+  const barWidth = 2; // px
+  const gap = 1;
+  const totalBars = Math.floor(cssWidth / (barWidth + gap));
+  const ds = buildPeaks(waveformData, totalBars);
+
+  const draw = (color: string, untilBar: number, start = 0) => {
+    ctx.fillStyle = color;
+    for (let i = start; i < untilBar; i++) {
+      const v = ds[i] || 0;
+      const h = Math.max(2, v * (cssHeight - 8)); // margin
+      const x = i * (barWidth + gap);
+
+      // Apply staggered animation
+      if (isAnimating) {
+        const barProgress = Math.max(0, Math.min(1, animationProgress / 0.2));
+        const animatedHeight = h * barProgress;
+
+        ctx.globalAlpha = barProgress;
+        ctx.fillRect(x, midY - animatedHeight / 2, barWidth, animatedHeight);
+      } else {
+        ctx.fillRect(x, midY - h / 2, barWidth, h);
+      }
+    }
+  };
+
+  // Progress overlay
+  const progressBars = Math.floor(
+    totalBars * Math.max(0, Math.min(1, progress || 0)),
+  );
+  draw(progressColor, progressBars);
+  // Base waveform
+  draw(barColor, totalBars, progressBars);
+}
+
+type WaveformCanvasProps = {
+  audioManager: AudioManager;
+  canvas: HTMLCanvasElement;
+  waveformId: string;
+  duration: number;
+  barColor?: string;
+  progressColor?: string;
+  backgroundColor?: string;
+};
+
+async function renderWaveform(props: WaveformCanvasProps) {
+  const { audioManager, canvas, waveformId, duration } = props;
+
+  let mounted = true;
+  let currentTime = getPlayerCurrentTime(audioManager);
+  let waveformData: undefined | number[] = undefined;
+  let isAnimating = true;
+  const startTime = performance.now();
+  let animationProgress = 0;
+  const animationDuration = 800;
+
+  function draw() {
+    const progress = currentTime / duration;
+
+    drawWaveform({
+      canvas,
+      waveformData,
+      duration,
+      currentTime,
+      isAnimating,
+      animationProgress,
+      progress,
+    });
+  }
+
+  const animate = (currentTime: number) => {
+    if (!mounted) return;
+
+    const elapsed = currentTime - startTime;
+    animationProgress = Math.min(elapsed / animationDuration, 1);
+
+    if (animationProgress < 1) {
+      requestAnimationFrame(animate);
+    } else {
+      isAnimating = false;
+    }
+
+    draw();
+  };
+
+  requestAnimationFrame(animate);
+
+  const unsubscribeFromCurrentTime = subscribeToPlayerCurrentTime(
+    audioManager,
+    (time) => {
+      currentTime = time;
+      draw();
+    },
+  );
+
+  const unsubscribeFromWaveform = MusicTrackWaveform.subscribe(
+    waveformId,
+    {},
+    (newResult) => {
+      waveformData = newResult.data;
+      draw();
+    },
+  );
+
+  return () => {
+    mounted = false;
+    unsubscribeFromCurrentTime();
+    unsubscribeFromWaveform();
+  };
+}
+
+export default function WaveformCanvas({
+  track,
+  height = DEFAULT_HEIGHT,
+  barColor, // muted-foreground-ish
+  progressColor, // green
+  backgroundColor,
+  className,
+}: Props) {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const audioManager = useAudioManager();
+
+  const duration = track.duration;
+  const waveformId = track._refs.waveform?.id;
+
+  // Animation effect
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (!canvas) return;
+    if (!waveformId) return;
+
+    renderWaveform({
+      audioManager,
+      canvas,
+      waveformId,
+      duration,
+      barColor,
+      progressColor,
+      backgroundColor,
+    });
+  }, [audioManager, canvasRef, waveformId, duration]);
+
+  const onPointer = (e: React.PointerEvent<HTMLCanvasElement>) => {
+    const rect = e.currentTarget.getBoundingClientRect();
+    const ratio = (e.clientX - rect.left) / rect.width;
+    const time = Math.max(0, Math.min(1, ratio)) * duration;
+    setPlayerCurrentTime(audioManager, time);
+  };
+
+  const currentTime = usePlayerCurrentTime();
+  const progress = currentTime.value / duration;
+
+  return (
+    <div className={cn("w-full", className)}>
+      <div
+        className="w-full rounded-md bg-background"
+        style={{ height }}
+        role="slider"
+        aria-label="Waveform scrubber"
+        aria-valuenow={Math.round((progress || 0) * 100)}
+        aria-valuemin={0}
+        aria-valuemax={100}
+      >
+        <canvas
+          ref={canvasRef}
+          className="w-full h-full rounded-md cursor-pointer"
+          onPointerDown={onPointer}
+          onPointerMove={(e) => {
+            if (e.buttons === 1) onPointer(e);
+          }}
+        />
+      </div>
+    </div>
+  );
+}

examples/music-player/src/components/ui/sidebar.tsx

@@ -532,7 +532,8 @@ const sidebarMenuButtonVariants = cva(
   {
     variants: {
       variant: {
-        default: "hover:bg-sidebar-accent hover:text-sidebar-accent-foreground",
+        default:
+          "hover:bg-sidebar-accent hover:text-sidebar-accent-foreground cursor-pointer",
         outline:
           "bg-background shadow-[0_0_0_1px_hsl(var(--sidebar-border))] hover:bg-sidebar-accent hover:text-sidebar-accent-foreground hover:shadow-[0_0_0_1px_hsl(var(--sidebar-accent))]",
       },

examples/music-player/src/index.css

@@ -2,6 +2,12 @@
 
 @custom-variant dark (&:is(.dark *));
 
+html {
+  overflow: hidden;
+  position: relative;
+  background-color: hsl(0 0% 99%);
+}
+
 :root {
   --background: hsl(0 0% 100%);
   --foreground: hsl(20 14.3% 4.1%);

examples/music-player/src/lib/audio/AudioManager.ts

@@ -15,6 +15,7 @@ export class AudioManager {
     if (this.audioObjectURL) {
       URL.revokeObjectURL(this.audioObjectURL);
       this.audioObjectURL = null;
+      this.mediaElement.src = "";
     }
   }
 

examples/music-player/src/lib/audio/usePlayerCurrentTime.ts

@@ -1,22 +1,14 @@
 import { useLayoutEffect, useState } from "react";
-import { useAudioManager } from "./AudioManager";
+import { AudioManager, useAudioManager } from "./AudioManager";
 
 export function usePlayerCurrentTime() {
   const audioManager = useAudioManager();
   const [value, setValue] = useState<number>(0);
 
   useLayoutEffect(() => {
-    setValue(audioManager.mediaElement.currentTime);
+    setValue(getPlayerCurrentTime(audioManager));
 
-    const onTimeUpdate = () => {
-      setValue(audioManager.mediaElement.currentTime);
-    };
-
-    audioManager.mediaElement.addEventListener("timeupdate", onTimeUpdate);
-
-    return () => {
-      audioManager.mediaElement.removeEventListener("timeupdate", onTimeUpdate);
-    };
+    return subscribeToPlayerCurrentTime(audioManager, setValue);
   }, [audioManager]);
 
   function setCurrentTime(time: number) {
@@ -31,3 +23,26 @@ export function usePlayerCurrentTime() {
     setValue: setCurrentTime,
   };
 }
+
+export function setPlayerCurrentTime(audioManager: AudioManager, time: number) {
+  audioManager.mediaElement.currentTime = time;
+}
+
+export function getPlayerCurrentTime(audioManager: AudioManager): number {
+  return audioManager.mediaElement.currentTime;
+}
+
+export function subscribeToPlayerCurrentTime(
+  audioManager: AudioManager,
+  callback: (time: number) => void,
+) {
+  const onTimeUpdate = () => {
+    callback(audioManager.mediaElement.currentTime);
+  };
+
+  audioManager.mediaElement.addEventListener("timeupdate", onTimeUpdate);
+
+  return () => {
+    audioManager.mediaElement.removeEventListener("timeupdate", onTimeUpdate);
+  };
+}

examples/music-player/src/lib/usePrepareAppState.ts

@@ -0,0 +1,54 @@
+import { MusicaAccount, MusicaAccountRoot } from "@/1_schema";
+import { MediaPlayer } from "@/5_useMediaPlayer";
+import { co } from "jazz-tools";
+import { useAccount } from "jazz-tools/react";
+import { useEffect, useState } from "react";
+import { uploadMusicTracks } from "../4_actions";
+
+export function usePrepareAppState(mediaPlayer: MediaPlayer) {
+  const [isReady, setIsReady] = useState(false);
+
+  const { agent } = useAccount();
+
+  useEffect(() => {
+    loadInitialData(mediaPlayer).then(() => {
+      setIsReady(true);
+    });
+  }, [agent]);
+
+  return isReady;
+}
+
+async function loadInitialData(mediaPlayer: MediaPlayer) {
+  const me = await MusicaAccount.getMe().ensureLoaded({
+    resolve: {
+      root: {
+        rootPlaylist: { tracks: { $each: true } },
+        activeTrack: true,
+        activePlaylist: true,
+      },
+    },
+  });
+
+  uploadOnboardingData(me.root);
+
+  // Load the active track in the AudioManager
+  if (me.root.activeTrack) {
+    mediaPlayer.loadTrack(me.root.activeTrack);
+  }
+}
+
+async function uploadOnboardingData(root: co.loaded<typeof MusicaAccountRoot>) {
+  if (root.exampleDataLoaded) return;
+
+  root.exampleDataLoaded = true;
+
+  try {
+    const trackFile = await (await fetch("/example.mp3")).blob();
+
+    await uploadMusicTracks([new File([trackFile], "Example song")], true);
+  } catch (error) {
+    root.exampleDataLoaded = false;
+    throw error;
+  }
+}

examples/music-player/src/lib/useUploadExampleData.ts

@@ -1,31 +0,0 @@
-import { MusicaAccount } from "@/1_schema";
-import { useAccount } from "jazz-tools/react";
-import { useEffect } from "react";
-import { uploadMusicTracks } from "../4_actions";
-
-export function useUploadExampleData() {
-  const { agent } = useAccount();
-
-  useEffect(() => {
-    uploadOnboardingData();
-  }, [agent]);
-}
-
-async function uploadOnboardingData() {
-  const me = await MusicaAccount.getMe().ensureLoaded({
-    resolve: { root: true },
-  });
-
-  if (me.root.exampleDataLoaded) return;
-
-  me.root.exampleDataLoaded = true;
-
-  try {
-    const trackFile = await (await fetch("/example.mp3")).blob();
-
-    await uploadMusicTracks([new File([trackFile], "Example song")], true);
-  } catch (error) {
-    me.root.exampleDataLoaded = false;
-    throw error;
-  }
-}

examples/music-player/tests/pages/HomePage.ts

@@ -55,10 +55,20 @@ export class HomePage {
 
   async editTrackTitle(trackTitle: string, newTitle: string) {
     await this.page
-      .getByRole("textbox", {
-        name: `Edit track title: ${trackTitle}`,
+      .getByRole("button", {
+        name: `Open ${trackTitle} menu`,
       })
-      .fill(newTitle);
+      .click();
+
+    await this.page
+      .getByRole("menuitem", {
+        name: `Edit`,
+      })
+      .click();
+
+    await this.page.getByPlaceholder("Enter track name...").fill(newTitle);
+
+    await this.page.getByRole("button", { name: "Save" }).click();
   }
 
   async createPlaylist() {

examples/music-player/vite.config.ts

@@ -1,10 +1,21 @@
 import path from "path";
 import react from "@vitejs/plugin-react-swc";
 import { defineConfig } from "vite";
+import { VitePWA } from "vite-plugin-pwa";
 
 // https://vitejs.dev/config/
 export default defineConfig({
-  plugins: [react()],
+  plugins: [
+    react(),
+    VitePWA({
+      registerType: "autoUpdate",
+      strategies: "generateSW",
+      workbox: {
+        globPatterns: ["**/*.{js,css,html,ico,png,svg}"],
+        maximumFileSizeToCacheInBytes: 1024 * 1024 * 5,
+      },
+    }),
+  ],
   resolve: {
     alias: {
       "@": path.resolve(__dirname, "./src"),

examples/organization/package.json

@@ -13,7 +13,7 @@
   },
   "dependencies": {
     "jazz-tools": "workspace:*",
-    "lucide-react": "^0.274.0",
+    "lucide-react": "^0.536.0",
     "react": "19.1.0",
     "react-dom": "19.1.0",
     "react-router": "^6.16.0",

examples/organization/src/components/InviteLink.tsx

@@ -6,7 +6,9 @@ import { Organization } from "../schema.ts";
 
 export function InviteLink({
   organization,
-}: { organization: Loaded<typeof Organization> }) {
+}: {
+  organization: Loaded<typeof Organization>;
+}) {
   let [copyCount, setCopyCount] = useState(0);
   let copied = copyCount > 0;
 

examples/organization/src/components/OrganizationMembers.tsx

@@ -4,7 +4,9 @@ import { Organization } from "../schema.ts";
 
 export function OrganizationMembers({
   organization,
-}: { organization: Loaded<typeof Organization> }) {
+}: {
+  organization: Loaded<typeof Organization>;
+}) {
   const group = organization._owner.castAs(Group);
 
   return (
@@ -25,7 +27,11 @@ function MemberItem({
   account,
   role,
   group,
-}: { account: Account; role: string; group: Group }) {
+}: {
+  account: Account;
+  role: string;
+  group: Group;
+}) {
   const { me } = useAccount();
 
   const canRemoveMember = group.myRole() === "admin" && account.id !== me?.id;

examples/passphrase/src/index.css

@@ -79,8 +79,9 @@ main {
   background-color: white;
   padding: 2rem;
   border-radius: 0.5rem;
-  box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px
-    rgba(0, 0, 0, 0.06);
+  box-shadow:
+    0 4px 6px -1px rgba(0, 0, 0, 0.1),
+    0 2px 4px -1px rgba(0, 0, 0, 0.06);
   width: 28rem;
 }
 

examples/reactions/src/ReactionsScreen.tsx

@@ -35,7 +35,9 @@ export function ReactionsScreen(props: { id: string }) {
 
 const ReactionButtons = ({
   reactions,
-}: { reactions: Loaded<typeof Reactions> }) => (
+}: {
+  reactions: Loaded<typeof Reactions>;
+}) => (
   <div className="reaction-buttons">
     {ReactionTypes.map((reactionType) => (
       <button
@@ -56,7 +58,9 @@ const ReactionButtons = ({
 
 const ReactionOverview = ({
   reactions,
-}: { reactions: Loaded<typeof Reactions> }) => (
+}: {
+  reactions: Loaded<typeof Reactions>;
+}) => (
   <>
     {Object.values(reactions.perAccount).map((reaction) => (
       <div key={reaction.by?.id} className="reaction-row">

examples/richtext-prosemirror/src/main.tsx

@@ -17,7 +17,7 @@ createRoot(document.getElementById("root")!).render(
   <StrictMode>
     <JazzReactProvider
       sync={{
-        peer: `wss://cloud.jazz.tools/?key=${apiKey}`,
+        peer: `ws://localhost:4200/?key=${apiKey}`,
       }}
       AccountSchema={JazzAccount}
     >

examples/todo/package.json

@@ -16,7 +16,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "jazz-tools": "workspace:*",
-    "lucide-react": "^0.274.0",
+    "lucide-react": "^0.536.0",
     "qrcode": "^1.5.3",
     "react": "19.1.0",
     "react-dom": "19.1.0",

homepage/design-system/src/app/covalues/layout.tsx

@@ -2,11 +2,7 @@
 
 import { JazzReactProvider } from "jazz-tools/react";
 
-export default function CovaluesLayout({
-  children,
-}: {
-  children: any;
-}) {
+export default function CovaluesLayout({ children }: { children: any }) {
   return (
     <JazzReactProvider sync={{ when: "never" }}>{children}</JazzReactProvider>
   );

homepage/design-system/src/components/atoms/Card.tsx

@@ -3,7 +3,10 @@ import { clsx } from "clsx";
 export function Card({
   children,
   className,
-}: { children: React.ReactNode; className?: string }) {
+}: {
+  children: React.ReactNode;
+  className?: string;
+}) {
   return (
     <div className={clsx(className, "border rounded-xl shadow-sm")}>
       {children}

homepage/design-system/src/components/atoms/Label.tsx

@@ -4,7 +4,11 @@ export function Label({
   label,
   htmlFor,
   className,
-}: { label: string; htmlFor: string; className?: string }) {
+}: {
+  label: string;
+  htmlFor: string;
+  className?: string;
+}) {
   return (
     <LabelRadix.Root className={className} htmlFor={htmlFor}>
       {label}

homepage/design-system/src/components/atoms/icons/CoFeed.tsx

@@ -4,7 +4,11 @@ export function IconCoFeed({
   className,
   size,
   strokeWidth,
-}: { className?: string; size?: number; strokeWidth: number }) {
+}: {
+  className?: string;
+  size?: number;
+  strokeWidth: number;
+}) {
   return (
     <svg
       xmlns="http://www.w3.org/2000/svg"

homepage/design-system/src/components/atoms/icons/CoRecord.tsx

@@ -4,7 +4,12 @@ export function IconCoRecord({
   className,
   size,
   strokeWidth,
-}: { className?: string; size?: number; color?: string; strokeWidth: number }) {
+}: {
+  className?: string;
+  size?: number;
+  color?: string;
+  strokeWidth: number;
+}) {
   return (
     <svg
       xmlns="http://www.w3.org/2000/svg"

homepage/design-system/src/components/atoms/logos/JazzLogo.tsx

@@ -4,7 +4,11 @@ export function JazzLogo({
   className,
   width = undefined,
   height = undefined,
-}: { className?: string; width?: number; height?: number }) {
+}: {
+  className?: string;
+  width?: number;
+  height?: number;
+}) {
   return (
     <svg
       viewBox="0 0 386 146"

homepage/design-system/src/components/molecules/Table.tsx

@@ -93,7 +93,11 @@ const TableDataContainer = ({
   children,
   className,
   isCopyable,
-}: { children: React.ReactNode; className?: string; isCopyable?: boolean }) => {
+}: {
+  children: React.ReactNode;
+  className?: string;
+  isCopyable?: boolean;
+}) => {
   return (
     <div
       className={clsx("flex gap-2", className, isCopyable && "cursor-pointer")}

homepage/gcmp/app/shiki.css

@@ -247,7 +247,13 @@ data-lsp {
 }
 .tag-container .twoslash-annotation {
   position: absolute;
-  font-family: "JetBrains Mono", Menlo, Monaco, Consolas, Courier New, monospace;
+  font-family:
+    "JetBrains Mono",
+    Menlo,
+    Monaco,
+    Consolas,
+    Courier New,
+    monospace;
   right: -10px;
   /** Default annotation text to 200px */
   width: 200px;

homepage/gcmp/app/team/[member]/page.tsx

@@ -6,7 +6,9 @@ import Router from "next/router";
 
 export default async function TeamMemberPage({
   params,
-}: { params: Promise<{ member: string }> }) {
+}: {
+  params: Promise<{ member: string }>;
+}) {
   const { member } = await params;
 
   const memberInfo = team.find((m) => m.slug === member);

homepage/homepage/content/docs/design-patterns/form.mdx

@@ -54,10 +54,10 @@ import { co, z, CoMap } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 ```
 </CodeGroup>
 
@@ -73,17 +73,17 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 // ---cut---
 // OrderForm.tsx
 export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -118,16 +118,16 @@ import * as React from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -177,10 +177,10 @@ import { useState, useEffect } from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({
   draft: DraftBubbleTeaOrder,
@@ -218,7 +218,7 @@ export function OrderForm({
 // CreateOrder.tsx
 export function CreateOrder() {
   const { me } = useAccount();
-  const [draft, setDraft] = useState<co.loaded<typeof DraftBubbleTeaOrder>>();
+  const [draft, setDraft] = useState<DraftBubbleTeaOrder>();
 
   useEffect(() => {
     setDraft(DraftBubbleTeaOrder.create({}));
@@ -228,7 +228,7 @@ export function CreateOrder() {
     e.preventDefault();
     if (!draft || !draft.name) return;
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>; // TODO: this should narrow correctly
+    const order = draft as BubbleTeaOrder; // TODO: this should narrow correctly
 
     console.log("Order created:", order);
   };
@@ -251,11 +251,15 @@ Update the schema to include a `validateDraftOrder` helper.
 import { co, z } from "jazz-tools";
 // ---cut---
 // schema.ts
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
+export const BubbleTeaOrder = co.map({
+  name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) { // [!code ++:9]
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
+
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) { // [!code ++:9]
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -279,12 +283,12 @@ import { useState, useEffect } from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -307,7 +311,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -330,7 +334,7 @@ export function OrderForm({
 // CreateOrder.tsx
 export function CreateOrder() {
   const { me } = useAccount();
-  const [draft, setDraft] = useState<co.loaded<typeof DraftBubbleTeaOrder>>();
+  const [draft, setDraft] = useState<DraftBubbleTeaOrder>();
 
   useEffect(() => {
     setDraft(DraftBubbleTeaOrder.create({}));
@@ -346,7 +350,7 @@ export function CreateOrder() {
       return;
     }
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>;
+    const order = draft as BubbleTeaOrder;
 
     console.log("Order created:", order);
   };
@@ -372,10 +376,10 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({ // [!code ++:15]
   draft: DraftBubbleTeaOrder,
@@ -403,10 +407,10 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({
   draft: DraftBubbleTeaOrder,
@@ -452,12 +456,12 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -492,7 +496,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -533,7 +537,7 @@ export function CreateOrder() {
       return;
     }
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>;
+    const order = draft as BubbleTeaOrder;
     console.log("Order created:", order);
 
     // create a new empty draft
@@ -577,11 +581,15 @@ Simply add a `hasChanges` helper to your schema.
 import { co, z } from "jazz-tools";
 // ---cut---
 // schema.ts
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
+export const BubbleTeaOrder = co.map({
+  name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
+
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -591,7 +599,7 @@ export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>)
   return { errors };
 };
 
-export function hasChanges(draft?: co.loaded<typeof DraftBubbleTeaOrder>) { // [!code ++:3]
+export function hasChanges(draft?: DraftBubbleTeaOrder) { // [!code ++:3]
   return draft ? Object.keys(draft._edits).length : false;
 };
 ```
@@ -608,12 +616,12 @@ import * as React from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -623,7 +631,7 @@ export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>)
   return { errors };
 };
 
-export function hasChanges(draft?: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function hasChanges(draft?: DraftBubbleTeaOrder) {
   return draft ? Object.keys(draft._edits).length : false;
 };
 

homepage/homepage/content/docs/groups/inheritance.mdx

@@ -205,6 +205,101 @@ console.log(containingGroup.getParentGroups()); // [addedGroup]
 ```
 </CodeGroup>
 
+## Group hierarchy on CoValue creation
+
+When creating CoValues that contain other CoValues using plain JSON objects, Jazz not only creates
+the necessary CoValues automatically but it will also manage their group ownership.
+
+<CodeGroup>
+```ts twoslash
+import { co, z } from "jazz-tools";
+
+// ---cut---
+const Task = co.plainText();
+const Column = co.list(Task);
+const Board = co.map({
+  title: z.string(),
+  columns: co.list(Column),
+});
+
+const board = Board.create({
+  title: "My board",
+  columns: [
+    ["Task 1.1", "Task 1.2"],
+    ["Task 2.1", "Task 2.2"],
+  ],
+});
+```
+</CodeGroup>
+
+For each created column and task CoValue, Jazz also creates a new group as its owner and
+adds the referencing CoValue's owner as a member of that group. This means permissions for nested CoValues 
+are inherited from the CoValue that references them, but can also be modified independently for each CoValue
+if needed.
+
+<CodeGroup>
+```ts twoslash
+import { co, z, Group, Account } from "jazz-tools";
+
+const alice = {} as unknown as Account;
+const bob = {} as unknown as Account;
+const Task = co.plainText();
+const Column = co.list(Task);
+const Board = co.map({
+  title: z.string(),
+  columns: co.list(Column),
+});
+// ---cut---
+const writeAccess = Group.create();
+writeAccess.addMember(bob, "writer");
+
+// Give Bob write access to the board, columns and tasks
+const board = Board.create({
+  title: "My board",
+  columns: [
+    ["Task 1.1", "Task 1.2"],
+    ["Task 2.1", "Task 2.2"],
+  ],
+}, writeAccess);
+
+// Give Alice read access to one specific task
+const task = board.columns[0][0];
+const taskGroup = task._owner.castAs(Group);
+taskGroup.addMember(alice, "reader");
+```
+</CodeGroup>
+
+If you prefer to manage permissions differently, you can always create CoValues explicitly:
+
+<CodeGroup>
+```ts twoslash
+import { co, Group, z, Account } from "jazz-tools";
+
+const bob = {} as unknown as Account;
+const Task = co.plainText();
+const Column = co.list(Task);
+const Board = co.map({
+  title: z.string(),
+  columns: co.list(Column),
+});
+
+// ---cut---
+const writeAccess = Group.create();
+writeAccess.addMember(bob, "writer");
+const readAccess =  Group.create();
+readAccess.addMember(bob, "reader");
+
+// Give Bob read access to the board and write access to the columns and tasks
+const board = Board.create({
+  title: "My board",
+  columns: co.list(Column).create([
+    ["Task 1.1", "Task 1.2"],
+    ["Task 2.1", "Task 2.2"],
+  ], writeAccess),
+}, readAccess);
+```
+</CodeGroup>
+
 ## Example: Team Hierarchy
 
 Here's a practical example of using group inheritance for team permissions:

homepage/homepage/content/docs/project-setup/svelte.mdx

@@ -28,18 +28,19 @@ See the [schema docs](/docs/schemas/covalues) for more information.
 <CodeGroup>
 ```ts
 // src/lib/schema.ts
-import { Account, Profile, coField } from "jazz-tools";
+import { co, z } from "jazz-tools"
 
-export class MyProfile extends Profile {
-  name = coField.string;
-  counter = coField.number; // This will be publically visible
-}
+export const MyProfile = co.profile({
+	name: z.string(),
+	counter: z.number()
+});
 
-export class MyAccount extends Account {
-  profile = coField.ref(MyProfile);
+export const root = co.map({});
 
-  // ...
-}
+export const UserAccount = co.account({
+	root,
+	profile: MyProfile
+});
 ```
 </CodeGroup>
 
@@ -48,17 +49,17 @@ export class MyAccount extends Account {
 <CodeGroup>
 ```svelte
 <!-- src/routes/+layout.svelte -->
-
 <script lang="ts">
-  import { JazzSvelteProvider } from 'jazz-tools/svelte';
+  import { JazzSvelteProvider } from "jazz-tools/svelte";
+
+  let { children } = $props();
 
   // Example configuration for authentication and peer connection
   let sync = { peer: "wss://cloud.jazz.tools/?key=you@example.com" };
-  let AccountSchema = MyAccount;
 </script>
 
-<JazzSvelteProvider {sync} {AccountSchema}>
-  <App />
+<JazzSvelteProvider {sync} AccountSchema={MyAccount}>
+  {@render children?.()}
 </JazzSvelteProvider>
 ```
 </CodeGroup>
@@ -69,12 +70,11 @@ export class MyAccount extends Account {
 ```svelte
 <!-- src/routes/+page.svelte -->
 <script lang="ts">
-  import { useCoState, useAccount } from 'jazz-tools/svelte';
-  import { MyProfile } from './schema';
+	import { CoState, AccountCoState } from "jazz-tools/svelte";
+	import { MyProfile, UserAccount } from "$lib/schema";
 
-  const { me } = useAccount();
-
-  const profile = $derived(useCoState(MyProfile, me._refs.profile.id));
+	const me = new AccountCoState(UserAccount);
+	const profile = new CoState(MyProfile, me.current?._refs.profile?.id);
 
   function increment() {
     if (!profile.current) return;
@@ -82,7 +82,7 @@ export class MyAccount extends Account {
   }
 </script>
 
-<button on:click={increment}>
+<button onclick={increment}>
   Count: {profile.current?.counter}
 </button>
 ```

homepage/homepage/content/docs/schemas/covalues.mdx

@@ -3,6 +3,7 @@ export const metadata = {
 };
 
 import { CodeGroup, ComingSoon } from "@/components/forMdx";
+import { Alert } from "@garden-co/design-system/src/components/atoms/Alert";
 
 # Defining schemas: CoValues
 
@@ -80,6 +81,40 @@ const project = TodoProject.create(
 
 </CodeGroup>
 
+When creating CoValues that contain other CoValues, you can pass in a plain JSON object.
+Jazz will automatically create the CoValues for you.
+
+<CodeGroup>
+```ts twoslash
+// @filename: schema.ts
+import { co, z, CoMap, CoList } from "jazz-tools";
+
+export const ListOfTasks = co.list(z.string());
+
+export const TodoProject = co.map({
+  title: z.string(),
+  tasks: ListOfTasks,
+});
+
+// @filename: app.ts
+// ---cut---
+// app.ts
+import { Group } from "jazz-tools";
+import { TodoProject, ListOfTasks } from "./schema";
+
+const group = Group.create().makePublic();
+const project = TodoProject.create({
+  title: "New Project",
+  tasks: [], // Permissions are inherited, so the tasks list will also be public
+}, group);
+```
+</CodeGroup>
+
+<Alert variant="info" className="flex gap-2 items-center my-4">
+To learn more about how permissions work when creating nested CoValues with plain JSON objects,
+refer to [Group hierarchy on CoValue creation](/docs/groups/inheritance#group-hierarchy-on-covalue-creation).
+</Alert>
+
 ## Types of CoValues
 
 ### `CoMap` (declaration)

homepage/homepage/content/docs/using-covalues/comaps.mdx

@@ -228,6 +228,54 @@ export type Project = co.loaded<typeof Project>;
 ```
 </CodeGroup>
 
+### Partial
+
+For convenience Jazz provies a dedicated API for making all the properties of a CoMap optional:
+
+<CodeGroup>
+```ts twoslash
+import { co, z } from "jazz-tools";
+
+const Project = co.map({
+  name: z.string(),
+  startDate: z.date(),
+  status: z.literal(["planning", "active", "completed"]),
+});
+
+const ProjectDraft = Project.partial();
+
+// The fields are all optional now
+const project = ProjectDraft.create({});
+```
+</CodeGroup>
+
+### Pick
+
+You can also pick specific fields from a CoMap:
+
+<CodeGroup>
+```ts twoslash
+import { co, z } from "jazz-tools";
+
+const Project = co.map({
+  name: z.string(),
+  startDate: z.date(),
+  status: z.literal(["planning", "active", "completed"]),
+});
+
+const ProjectStep1 = Project.pick({
+  name: true,
+  startDate: true,
+});
+
+// We don't provide the status field
+const project = ProjectStep1.create({
+  name: "My project",
+  startDate: new Date("2025-04-01"),
+});
+```
+</CodeGroup>
+
 ### Working with Record CoMaps
 
 For record-type CoMaps, you can access values using bracket notation:

homepage/homepage/generate-docs/llms-full.mjs

@@ -5,6 +5,8 @@ import { readFile, readdir } from "fs/promises";
 import { DOC_SECTIONS } from "./utils/config.mjs";
 import { writeDocsFile } from "./utils/index.mjs";
 
+const exclude = [/\/upgrade\//];
+
 async function readMdxContent(url) {
   try {
     // Special case for the introduction
@@ -31,12 +33,17 @@ async function readMdxContent(url) {
 
     // If it's a directory, try to read all framework variants
     const fullPath = path.join(baseDir, relativePath);
+
+    if (exclude.some((pattern) => pattern.test(fullPath))) {
+      return null;
+    }
+
     try {
       const stats = await fs.stat(fullPath);
       if (stats.isDirectory()) {
         // Read all MDX files in the directory
         const files = await fs.readdir(fullPath);
-        const mdxFiles = files.filter((f) => f.endsWith(".mdx"));
+        const mdxFiles = files.filter((f) => f.endsWith(".mdx")).filter((f) => !exclude.some((pattern) => pattern.test(f)));
 
         if (mdxFiles.length === 0) return null;
 

package.json

@@ -31,7 +31,7 @@
     "build:packages": "turbo build --filter='./packages/*'",
     "lint": "turbo lint && cd homepage/homepage && pnpm run lint",
     "test": "vitest",
-    "test:ci": "vitest run --watch=false --coverage.enabled=true",
+    "test:ci": "vitest run --watch=false",
     "test:coverage": "vitest --ui --coverage.enabled=true",
     "format-and-lint": "biome check .",
     "format-and-lint:fix": "biome check . --write",

packages/cojson-storage-indexeddb/CHANGELOG.md

@@ -1,5 +1,41 @@
 # cojson-storage-indexeddb
 
+## 0.16.6
+
+### Patch Changes
+
+- 103d1b4: Fix Unknown transaction error on IndexedDB showing up sometimes when using a readwrite transaction to read values
+- 67e0968: Fix content streaming chunking, now chunks should be splitted always respecting the MAX_RECOMMENDED_TX_SIZE
+- 4b99ff1: Add a multi-storage scheduler to avoid conflicting store operations when having multiple storage instances open on the same database
+- Updated dependencies [67e0968]
+- Updated dependencies [ce9ca54]
+- Updated dependencies [4b99ff1]
+- Updated dependencies [ac5d20d]
+- Updated dependencies [9bf7946]
+  - cojson@0.16.6
+
+## 0.16.5
+
+### Patch Changes
+
+- Updated dependencies [3cd1586]
+- Updated dependencies [267f689]
+  - cojson@0.16.5
+
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
+## 0.16.3
+
+### Patch Changes
+
+- cojson@0.16.3
+
 ## 0.16.2
 
 ### Patch Changes

packages/cojson-storage-indexeddb/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cojson-storage-indexeddb",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "main": "dist/index.js",
   "type": "module",
   "types": "dist/index.d.ts",

packages/cojson-storage-indexeddb/src/CoJsonIDBTransaction.ts

@@ -118,3 +118,43 @@ export class CoJsonIDBTransaction {
     }
   }
 }
+
+export function queryIndexedDbStore<T>(
+  db: IDBDatabase,
+  storeName: StoreName,
+  callback: (store: IDBObjectStore) => IDBRequest<T>,
+) {
+  return new Promise<T>((resolve, reject) => {
+    const tx = db.transaction(storeName, "readonly");
+    const request = callback(tx.objectStore(storeName));
+
+    request.onerror = () => {
+      reject(request.error);
+    };
+
+    request.onsuccess = () => {
+      resolve(request.result as T);
+      tx.commit();
+    };
+  });
+}
+
+export function putIndexedDbStore<T, O extends IDBValidKey>(
+  db: IDBDatabase,
+  storeName: StoreName,
+  value: T,
+) {
+  return new Promise<O>((resolve, reject) => {
+    const tx = db.transaction(storeName, "readwrite");
+    const request = tx.objectStore(storeName).put(value);
+
+    request.onerror = () => {
+      reject(request.error);
+    };
+
+    request.onsuccess = () => {
+      resolve(request.result as O);
+      tx.commit();
+    };
+  });
+}

packages/cojson-storage-indexeddb/src/idbClient.ts

@@ -8,7 +8,12 @@ import type {
   StoredSessionRow,
   TransactionRow,
 } from "cojson";
-import { CoJsonIDBTransaction } from "./CoJsonIDBTransaction.js";
+import {
+  CoJsonIDBTransaction,
+  putIndexedDbStore,
+  queryIndexedDbStore,
+} from "./CoJsonIDBTransaction.js";
+import { StoreName } from "./CoJsonIDBTransaction.js";
 
 export class IDBClient implements DBClientInterfaceAsync {
   private db;
@@ -39,17 +44,18 @@ export class IDBClient implements DBClientInterfaceAsync {
   }
 
   async getCoValue(coValueId: RawCoID): Promise<StoredCoValueRow | undefined> {
-    return this.makeRequest<StoredCoValueRow | undefined>((tx) =>
-      tx.getObjectStore("coValues").index("coValuesById").get(coValueId),
+    return queryIndexedDbStore(this.db, "coValues", (store) =>
+      store.index("coValuesById").get(coValueId),
     );
   }
 
+  async getCoValueRowID(coValueId: RawCoID): Promise<number | undefined> {
+    return this.getCoValue(coValueId).then((row) => row?.rowID);
+  }
+
   async getCoValueSessions(coValueRowId: number): Promise<StoredSessionRow[]> {
-    return this.makeRequest<StoredSessionRow[]>((tx) =>
-      tx
-        .getObjectStore("sessions")
-        .index("sessionsByCoValue")
-        .getAll(coValueRowId),
+    return queryIndexedDbStore(this.db, "sessions", (store) =>
+      store.index("sessionsByCoValue").getAll(coValueRowId),
     );
   }
 
@@ -57,11 +63,8 @@ export class IDBClient implements DBClientInterfaceAsync {
     coValueRowId: number,
     sessionID: SessionID,
   ): Promise<StoredSessionRow | undefined> {
-    return this.makeRequest<StoredSessionRow>((tx) =>
-      tx
-        .getObjectStore("sessions")
-        .index("uniqueSessions")
-        .get([coValueRowId, sessionID]),
+    return queryIndexedDbStore(this.db, "sessions", (store) =>
+      store.index("uniqueSessions").get([coValueRowId, sessionID]),
     );
   }
 
@@ -70,12 +73,10 @@ export class IDBClient implements DBClientInterfaceAsync {
     fromIdx: number,
     toIdx: number,
   ): Promise<TransactionRow[]> {
-    return this.makeRequest<TransactionRow[]>((tx) =>
-      tx
-        .getObjectStore("transactions")
-        .getAll(
-          IDBKeyRange.bound([sessionRowId, fromIdx], [sessionRowId, toIdx]),
-        ),
+    return queryIndexedDbStore(this.db, "transactions", (store) =>
+      store.getAll(
+        IDBKeyRange.bound([sessionRowId, fromIdx], [sessionRowId, toIdx]),
+      ),
     );
   }
 
@@ -83,32 +84,28 @@ export class IDBClient implements DBClientInterfaceAsync {
     sessionRowId: number,
     firstNewTxIdx: number,
   ): Promise<SignatureAfterRow[]> {
-    return this.makeRequest<SignatureAfterRow[]>((tx) =>
-      tx
-        .getObjectStore("signatureAfter")
-        .getAll(
-          IDBKeyRange.bound(
-            [sessionRowId, firstNewTxIdx],
-            [sessionRowId, Number.POSITIVE_INFINITY],
-          ),
+    return queryIndexedDbStore(this.db, "signatureAfter", (store) =>
+      store.getAll(
+        IDBKeyRange.bound(
+          [sessionRowId, firstNewTxIdx],
+          [sessionRowId, Number.POSITIVE_INFINITY],
         ),
+      ),
     );
   }
 
-  async addCoValue(
-    msg: CojsonInternalTypes.NewContentMessage,
-  ): Promise<number> {
-    if (!msg.header) {
-      throw new Error(`Header is required, coId: ${msg.id}`);
+  async upsertCoValue(
+    id: RawCoID,
+    header?: CojsonInternalTypes.CoValueHeader,
+  ): Promise<number | undefined> {
+    if (!header) {
+      return this.getCoValueRowID(id);
     }
 
-    return (await this.makeRequest<IDBValidKey>((tx) =>
-      tx.getObjectStore("coValues").put({
-        id: msg.id,
-        // biome-ignore lint/style/noNonNullAssertion: TODO(JAZZ-561): Review
-        header: msg.header!,
-      } satisfies CoValueRow),
-    )) as number;
+    return putIndexedDbStore<CoValueRow, number>(this.db, "coValues", {
+      id,
+      header,
+    }).catch(() => this.getCoValueRowID(id));
   }
 
   async addSessionUpdate({

packages/cojson-storage-indexeddb/src/tests/storage.indexeddb.test.ts

@@ -179,8 +179,9 @@ test("should load dependencies correctly (group inheritance)", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Group header: true new: After: 0 New: 5",
+      "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT ParentGroup header: true new: After: 0 New: 4",
+      "client -> CONTENT Group header: false new: After: 3 New: 2",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
     ]
   `);
@@ -527,9 +528,8 @@ test("large coValue upload streaming", async () => {
     [
       "client -> LOAD Map sessions: empty",
       "storage -> CONTENT Group header: true new: After: 0 New: 3",
-      "storage -> CONTENT Map header: true new: After: 0 New: 97",
-      "storage -> CONTENT Map header: true new: After: 97 New: 97",
-      "storage -> CONTENT Map header: true new: After: 194 New: 6",
+      "storage -> CONTENT Map header: true new: After: 0 New: 193",
+      "storage -> CONTENT Map header: true new: After: 193 New: 7",
     ]
   `);
 });
@@ -561,9 +561,10 @@ test("should sync and load accounts from storage", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Account header: true new: After: 0 New: 4",
+      "client -> CONTENT Account header: true new: After: 0 New: 3",
       "client -> CONTENT ProfileGroup header: true new: After: 0 New: 5",
       "client -> CONTENT Profile header: true new: After: 0 New: 1",
+      "client -> CONTENT Account header: false new: After: 3 New: 1",
     ]
   `);
 

packages/cojson-storage-indexeddb/src/tests/testUtils.ts

@@ -36,12 +36,11 @@ export function trackMessages() {
   };
 
   StorageApiAsync.prototype.store = async function (data, correctionCallback) {
-    for (const msg of data) {
-      messages.push({
-        from: "client",
-        msg,
-      });
-    }
+    messages.push({
+      from: "client",
+      msg: data,
+    });
+
     return originalStore.call(this, data, (msg) => {
       messages.push({
         from: "storage",
@@ -51,7 +50,18 @@ export function trackMessages() {
           ...msg,
         },
       });
-      correctionCallback(msg);
+      const correctionMessages = correctionCallback(msg);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          messages.push({
+            from: "client",
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

packages/cojson-storage-sqlite/CHANGELOG.md

@@ -1,5 +1,39 @@
 # cojson-storage-sqlite
 
+## 0.16.6
+
+### Patch Changes
+
+- 67e0968: Fix content streaming chunking, now chunks should be splitted always respecting the MAX_RECOMMENDED_TX_SIZE
+- Updated dependencies [67e0968]
+- Updated dependencies [ce9ca54]
+- Updated dependencies [4b99ff1]
+- Updated dependencies [ac5d20d]
+- Updated dependencies [9bf7946]
+  - cojson@0.16.6
+
+## 0.16.5
+
+### Patch Changes
+
+- Updated dependencies [3cd1586]
+- Updated dependencies [267f689]
+  - cojson@0.16.5
+
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
+## 0.16.3
+
+### Patch Changes
+
+- cojson@0.16.3
+
 ## 0.16.2
 
 ### Patch Changes

packages/cojson-storage-sqlite/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cojson-storage-sqlite",
   "type": "module",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",

packages/cojson-storage-sqlite/src/tests/storage.sqlite.test.ts

@@ -211,8 +211,9 @@ test("should load dependencies correctly (group inheritance)", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Group header: true new: After: 0 New: 5",
+      "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT ParentGroup header: true new: After: 0 New: 4",
+      "client -> CONTENT Group header: false new: After: 3 New: 2",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
     ]
   `);
@@ -374,6 +375,8 @@ test("should recover from data loss", async () => {
     [
       "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
+      "client -> CONTENT Map header: false new: After: 3 New: 1",
+      "storage -> KNOWN CORRECTION Map sessions: header/4",
       "client -> CONTENT Map header: false new: After: 1 New: 3",
     ]
   `);
@@ -455,10 +458,7 @@ test("should recover missing dependencies from storage", async () => {
       data,
       correctionCallback,
     ) {
-      if (
-        data[0]?.id &&
-        [group.core.id, account.core.id as string].includes(data[0].id)
-      ) {
+      if ([group.core.id, account.core.id as string].includes(data.id)) {
         return false;
       }
 
@@ -658,9 +658,8 @@ test("large coValue upload streaming", async () => {
     [
       "client -> LOAD Map sessions: empty",
       "storage -> CONTENT Group header: true new: After: 0 New: 3",
-      "storage -> CONTENT Map header: true new: After: 0 New: 97",
-      "storage -> CONTENT Map header: true new: After: 97 New: 97",
-      "storage -> CONTENT Map header: true new: After: 194 New: 6",
+      "storage -> CONTENT Map header: true new: After: 0 New: 193",
+      "storage -> CONTENT Map header: true new: After: 193 New: 7",
     ]
   `);
 });

packages/cojson-storage-sqlite/src/tests/testUtils.ts

@@ -36,12 +36,11 @@ export function trackMessages() {
   };
 
   StorageApiSync.prototype.store = function (data, correctionCallback) {
-    for (const msg of data) {
-      messages.push({
-        from: "client",
-        msg,
-      });
-    }
+    messages.push({
+      from: "client",
+      msg: data,
+    });
+
     return originalStore.call(this, data, (msg) => {
       messages.push({
         from: "storage",
@@ -51,7 +50,19 @@ export function trackMessages() {
           ...msg,
         },
       });
-      correctionCallback(msg);
+
+      const correctionMessages = correctionCallback(msg);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          messages.push({
+            from: "client",
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

packages/cojson-transport-ws/CHANGELOG.md

@@ -1,5 +1,39 @@
 # cojson-transport-nodejs-ws
 
+## 0.16.6
+
+### Patch Changes
+
+- ac5d20d: Add ingress and egress metering
+- Updated dependencies [67e0968]
+- Updated dependencies [ce9ca54]
+- Updated dependencies [4b99ff1]
+- Updated dependencies [ac5d20d]
+- Updated dependencies [9bf7946]
+  - cojson@0.16.6
+
+## 0.16.5
+
+### Patch Changes
+
+- Updated dependencies [3cd1586]
+- Updated dependencies [267f689]
+  - cojson@0.16.5
+
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
+## 0.16.3
+
+### Patch Changes
+
+- cojson@0.16.3
+
 ## 0.16.2
 
 ### Patch Changes

packages/cojson-transport-ws/package.json

@@ -1,11 +1,12 @@
 {
   "name": "cojson-transport-ws",
   "type": "module",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
     "cojson": "workspace:*"
   },
   "scripts": {
@@ -17,8 +18,9 @@
     "build": "rm -rf ./dist && tsc --sourceMap --outDir dist"
   },
   "devDependencies": {
-    "typescript": "catalog:",
+    "@opentelemetry/sdk-metrics": "^2.0.0",
     "@types/ws": "8.5.10",
+    "typescript": "catalog:",
     "ws": "^8.14.2"
   }
 }

packages/cojson-transport-ws/src/BatchedOutgoingMessages.ts

@@ -1,3 +1,4 @@
+import { ValueType, metrics } from "@opentelemetry/api";
 import type { DisconnectedError, SyncMessage } from "cojson";
 import type { Peer } from "cojson";
 import {
@@ -15,7 +16,7 @@ import {
   waitForWebSocketOpen,
 } from "./utils.js";
 
-const { CO_VALUE_PRIORITY } = cojsonInternals;
+const { CO_VALUE_PRIORITY, getContentMessageSize } = cojsonInternals;
 
 export const MAX_OUTGOING_MESSAGES_CHUNK_BYTES = 25_000;
 
@@ -26,11 +27,22 @@ export class BatchedOutgoingMessages
   private queue: PriorityBasedMessageQueue;
   private processing = false;
   private closed = false;
+  private counter = metrics
+    .getMeter("cojson-transport-ws")
+    .createCounter("jazz.usage.egress", {
+      description: "Total egress bytes",
+      unit: "bytes",
+      valueType: ValueType.INT,
+    });
 
   constructor(
     private websocket: AnyWebSocket,
     private batching: boolean,
     peerRole: Peer["role"],
+    /**
+     * Additional key-value pair of attributes to add to the egress metric.
+     */
+    private meta?: Record<string, string | number>,
   ) {
     this.queue = new PriorityBasedMessageQueue(
       CO_VALUE_PRIORITY.HIGH,
@@ -39,6 +51,9 @@ export class BatchedOutgoingMessages
         peerRole: peerRole,
       },
     );
+
+    // Initialize the counter by adding 0
+    this.counter.add(0, this.meta);
   }
 
   push(msg: SyncMessage | DisconnectedError) {
@@ -93,7 +108,11 @@ export class BatchedOutgoingMessages
     this.processing = false;
   }
 
-  processMessage(msg: SyncMessage) {
+  private processMessage(msg: SyncMessage) {
+    if (msg.action === "content") {
+      this.counter.add(getContentMessageSize(msg), this.meta);
+    }
+
     if (!this.batching) {
       this.websocket.send(JSON.stringify(msg));
       return;
@@ -116,7 +135,7 @@ export class BatchedOutgoingMessages
     }
   }
 
-  sendMessagesInBulk() {
+  private sendMessagesInBulk() {
     if (this.backlog.length > 0 && isWebSocketOpen(this.websocket)) {
       this.websocket.send(this.backlog);
       this.backlog = "";

packages/cojson-transport-ws/src/createWebSocketPeer.ts

@@ -1,9 +1,10 @@
+import { ValueType, metrics } from "@opentelemetry/api";
 import { type Peer, type SyncMessage, cojsonInternals, logger } from "cojson";
 import { BatchedOutgoingMessages } from "./BatchedOutgoingMessages.js";
 import { deserializeMessages } from "./serialization.js";
 import type { AnyWebSocket } from "./types.js";
 
-const { ConnectedPeerChannel } = cojsonInternals;
+const { ConnectedPeerChannel, getContentMessageSize } = cojsonInternals;
 
 export type CreateWebSocketPeerOpts = {
   id: string;
@@ -15,6 +16,10 @@ export type CreateWebSocketPeerOpts = {
   pingTimeout?: number;
   onClose?: () => void;
   onSuccess?: () => void;
+  /**
+   * Additional key-value attributes to add to the ingress metric.
+   */
+  meta?: Record<string, string | number>;
 };
 
 function createPingTimeoutListener(
@@ -64,7 +69,19 @@ export function createWebSocketPeer({
   pingTimeout = 10_000,
   onSuccess,
   onClose,
+  meta,
 }: CreateWebSocketPeerOpts): Peer {
+  const totalIngressBytesCounter = metrics
+    .getMeter("cojson-transport-ws")
+    .createCounter("jazz.usage.ingress", {
+      description: "Total ingress bytes from peer",
+      unit: "bytes",
+      valueType: ValueType.INT,
+    });
+
+  // Initialize the counter by adding 0
+  totalIngressBytesCounter.add(0, meta);
+
   const incoming = new ConnectedPeerChannel();
   const emitClosedEvent = createClosedEventEmitter(onClose);
 
@@ -101,6 +118,7 @@ export function createWebSocketPeer({
     websocket,
     batchingByDefault,
     role,
+    meta,
   );
   let isFirstMessage = true;
 
@@ -135,6 +153,10 @@ export function createWebSocketPeer({
     for (const msg of messages) {
       if (msg && "action" in msg) {
         incoming.push(msg);
+
+        if (msg.action === "content") {
+          totalIngressBytesCounter.add(getContentMessageSize(msg), meta);
+        }
       }
     }
   }

packages/cojson-transport-ws/src/tests/BatchedOutgoingMessages.test.ts

@@ -0,0 +1,83 @@
+import type { SyncMessage } from "cojson";
+import { type Mocked, afterEach, describe, expect, test, vi } from "vitest";
+import { BatchedOutgoingMessages } from "../BatchedOutgoingMessages";
+import type { AnyWebSocket } from "../types";
+import { createTestMetricReader, tearDownTestMetricReader } from "./utils.js";
+
+describe("BatchedOutgoingMessages", () => {
+  describe("telemetry", () => {
+    afterEach(() => {
+      tearDownTestMetricReader();
+    });
+
+    test("should correctly measure egress", async () => {
+      const metricReader = createTestMetricReader();
+
+      const mockWebSocket = {
+        readyState: 1,
+        addEventListener: vi.fn(),
+        removeEventListener: vi.fn(),
+        close: vi.fn(),
+        send: vi.fn(),
+      } as unknown as Mocked<AnyWebSocket>;
+
+      const outgoing = new BatchedOutgoingMessages(
+        mockWebSocket,
+        true,
+        "server",
+        { test: "test" },
+      );
+
+      const encryptedChanges = "Hello, world!";
+      vi.useFakeTimers();
+      outgoing.push({
+        action: "content",
+        new: {
+          someSessionId: {
+            newTransactions: [
+              {
+                privacy: "private",
+                encryptedChanges,
+              },
+            ],
+          },
+        },
+      } as unknown as SyncMessage);
+
+      await vi.runAllTimersAsync();
+      vi.useRealTimers();
+
+      expect(
+        await metricReader.getMetricValue("jazz.usage.egress", {
+          test: "test",
+        }),
+      ).toBe(encryptedChanges.length);
+
+      const trustingChanges = "Jazz is great!";
+      vi.useFakeTimers();
+      outgoing.push({
+        action: "content",
+        new: {
+          someSessionId: {
+            after: 0,
+            newTransactions: [
+              {
+                privacy: "trusting",
+                changes: trustingChanges,
+              },
+            ],
+          },
+        },
+      } as unknown as SyncMessage);
+
+      await vi.runAllTimersAsync();
+      vi.useRealTimers();
+
+      expect(
+        await metricReader.getMetricValue("jazz.usage.egress", {
+          test: "test",
+        }),
+      ).toBe(encryptedChanges.length + trustingChanges.length);
+    });
+  });
+});

packages/cojson-transport-ws/src/tests/createWebSocketPeer.test.ts

@@ -1,6 +1,6 @@
 import type { CojsonInternalTypes, SyncMessage } from "cojson";
 import { cojsonInternals } from "cojson";
-import { type Mocked, describe, expect, test, vi } from "vitest";
+import { type Mocked, afterEach, describe, expect, test, vi } from "vitest";
 import { MAX_OUTGOING_MESSAGES_CHUNK_BYTES } from "../BatchedOutgoingMessages.js";
 import {
   type CreateWebSocketPeerOpts,
@@ -8,6 +8,7 @@ import {
 } from "../createWebSocketPeer.js";
 import type { AnyWebSocket } from "../types.js";
 import { BUFFER_LIMIT, BUFFER_LIMIT_POLLING_INTERVAL } from "../utils.js";
+import { createTestMetricReader, tearDownTestMetricReader } from "./utils.js";
 
 const { CO_VALUE_PRIORITY } = cojsonInternals;
 
@@ -520,6 +521,89 @@ describe("createWebSocketPeer", () => {
       );
     });
   });
+
+  describe("telemetry", () => {
+    afterEach(() => {
+      tearDownTestMetricReader();
+    });
+
+    test("should initialize to 0 when creating a websocket peer", async () => {
+      const metricReader = createTestMetricReader();
+      setup({
+        meta: { test: "test" },
+      });
+
+      const measuredIngress = await metricReader.getMetricValue(
+        "jazz.usage.ingress",
+        {
+          test: "test",
+        },
+      );
+      expect(measuredIngress).toBe(0);
+    });
+
+    test("should correctly measure incoming ingress", async () => {
+      const metricReader = createTestMetricReader();
+      const { listeners } = setup({
+        meta: { label: "value" },
+      });
+
+      const messageHandler = listeners.get("message");
+
+      const encryptedChanges = "Hello, world!";
+      messageHandler?.(
+        new MessageEvent("message", {
+          data: JSON.stringify({
+            action: "content",
+            new: {
+              someSessionId: {
+                after: 0,
+                newTransactions: [
+                  {
+                    privacy: "private" as const,
+                    madeAt: 0,
+                    keyUsed: "key_zkey" as const,
+                    encryptedChanges,
+                  },
+                ],
+              },
+            },
+          }),
+        }),
+      );
+
+      expect(
+        await metricReader.getMetricValue("jazz.usage.ingress", {
+          label: "value",
+        }),
+      ).toBe(encryptedChanges.length);
+
+      const trustingChanges = "Jazz is great!";
+      messageHandler?.(
+        new MessageEvent("message", {
+          data: JSON.stringify({
+            action: "content",
+            new: {
+              someSessionId: {
+                newTransactions: [
+                  {
+                    privacy: "trusting",
+                    changes: trustingChanges,
+                  },
+                ],
+              },
+            },
+          }),
+        }),
+      );
+
+      expect(
+        await metricReader.getMetricValue("jazz.usage.ingress", {
+          label: "value",
+        }),
+      ).toBe(encryptedChanges.length + trustingChanges.length);
+    });
+  });
 });
 
 // biome-ignore lint/suspicious/noConfusingVoidType: Test helper

packages/cojson-transport-ws/src/tests/utils.ts

@@ -1,3 +1,12 @@
+import { metrics } from "@opentelemetry/api";
+import {
+  AggregationTemporality,
+  InMemoryMetricExporter,
+  MeterProvider,
+  MetricReader,
+} from "@opentelemetry/sdk-metrics";
+import { expect } from "vitest";
+
 // biome-ignore lint/suspicious/noConfusingVoidType: Test helper
 export function waitFor(callback: () => boolean | void) {
   return new Promise<void>((resolve, reject) => {
@@ -26,3 +35,71 @@ export function waitFor(callback: () => boolean | void) {
     }, 100);
   });
 }
+
+/**
+ * This is a test metric reader that uses an in-memory metric exporter and exposes a method to get the value of a metric given its name and attributes.
+ *
+ * This is useful for testing the values of metrics that are collected by the SDK.
+ *
+ * TODO: We may want to rethink how we access metrics (see `getMetricValue` method) to make it more flexible.
+ */
+class TestMetricReader extends MetricReader {
+  private _exporter = new InMemoryMetricExporter(
+    AggregationTemporality.CUMULATIVE,
+  );
+
+  protected onShutdown(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  protected onForceFlush(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+
+  async getMetricValue(
+    name: string,
+    attributes: { [key: string]: string | number } = {},
+  ) {
+    await this.collectAndExport();
+    const metric = this._exporter
+      .getMetrics()[0]
+      ?.scopeMetrics[0]?.metrics.find((m) => m.descriptor.name === name);
+
+    const dp = metric?.dataPoints.find(
+      (dp) => JSON.stringify(dp.attributes) === JSON.stringify(attributes),
+    );
+
+    this._exporter.reset();
+
+    return dp?.value;
+  }
+
+  async collectAndExport(): Promise<void> {
+    const result = await this.collect();
+    await new Promise<void>((resolve, reject) => {
+      this._exporter.export(result.resourceMetrics, (result) => {
+        if (result.error != null) {
+          reject(result.error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+}
+
+export function createTestMetricReader() {
+  const metricReader = new TestMetricReader();
+  const success = metrics.setGlobalMeterProvider(
+    new MeterProvider({
+      readers: [metricReader],
+    }),
+  );
+
+  expect(success).toBe(true);
+
+  return metricReader;
+}
+
+export function tearDownTestMetricReader() {
+  metrics.disable();
+}

packages/cojson-transport-ws/src/tests/webSocket.integration.test.ts

@@ -1,5 +1,4 @@
-import { assert } from "node:console";
-import { ControlledAgent, type CryptoProvider, LocalNode } from "cojson";
+import { type CryptoProvider, LocalNode } from "cojson";
 import { WasmCrypto } from "cojson/crypto/WasmCrypto";
 import { afterEach, beforeEach, describe, expect, test } from "vitest";
 import { WebSocket } from "ws";

packages/cojson/CHANGELOG.md

@@ -1,5 +1,37 @@
 # cojson
 
+## 0.16.6
+
+### Patch Changes
+
+- 67e0968: Fix content streaming chunking, now chunks should be splitted always respecting the MAX_RECOMMENDED_TX_SIZE
+- ce9ca54: Chunk CoPlainText content to avoid generating bg messages when the user past a megabytes of text
+- 4b99ff1: Add a multi-storage scheduler to avoid conflicting store operations when having multiple storage instances open on the same database
+- ac5d20d: Add ingress and egress metering
+- 9bf7946: Added a TTL based optional garbage collection for covalues
+
+## 0.16.5
+
+### Patch Changes
+
+- 3cd1586: Makes the key rotation not fail when child groups are unavailable or their readkey is not accessible.
+
+  Also changes the Group.removeMember method to not return a Promise, because:
+
+  - All the locally available child groups are rotated immediately
+  - All the remote child groups are rotated in background, but since they are not locally available the user won't need the new key immediately
+
+- 267f689: Groups: fix the readkey not being revealed to everyone when doing a key rotation
+
+## 0.16.4
+
+### Patch Changes
+
+- f9d538f: Fix the error raised when extending a group without having child groups loaded
+- 802b5a3: Refactor local updates sync to ensure that the changes are synced respecting the insertion order
+
+## 0.16.3
+
 ## 0.16.2
 
 ## 0.16.1

packages/cojson/package.json

@@ -25,7 +25,7 @@
   },
   "type": "module",
   "license": "MIT",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "devDependencies": {
     "@opentelemetry/sdk-metrics": "^2.0.0",
     "libsql": "^0.5.13",

packages/cojson/src/GarbageCollector.ts

@@ -0,0 +1,48 @@
+import { CoValueCore } from "./coValueCore/coValueCore.js";
+import { GARBAGE_COLLECTOR_CONFIG } from "./config.js";
+import { RawCoID } from "./ids.js";
+
+export class GarbageCollector {
+  private readonly interval: ReturnType<typeof setInterval>;
+
+  constructor(private readonly coValues: Map<RawCoID, CoValueCore>) {
+    this.interval = setInterval(() => {
+      this.collect();
+    }, GARBAGE_COLLECTOR_CONFIG.INTERVAL);
+  }
+
+  getCurrentTime() {
+    return performance.now();
+  }
+
+  trackCoValueAccess({ verified }: CoValueCore) {
+    if (verified) {
+      verified.lastAccessed = this.getCurrentTime();
+    }
+  }
+
+  collect() {
+    const currentTime = this.getCurrentTime();
+    for (const coValue of this.coValues.values()) {
+      const { verified } = coValue;
+
+      if (!verified?.lastAccessed) {
+        continue;
+      }
+
+      const timeSinceLastAccessed = currentTime - verified.lastAccessed;
+
+      if (timeSinceLastAccessed > GARBAGE_COLLECTOR_CONFIG.MAX_AGE) {
+        const unmounted = coValue.unmount();
+
+        if (unmounted) {
+          this.coValues.delete(coValue.id);
+        }
+      }
+    }
+  }
+
+  stop() {
+    clearInterval(this.interval);
+  }
+}

packages/cojson/src/coValue.ts

@@ -1,7 +1,4 @@
-import {
-  AvailableCoValueCore,
-  CoValueCore,
-} from "./coValueCore/coValueCore.js";
+import { AvailableCoValueCore } from "./coValueCore/coValueCore.js";
 import { RawProfile as Profile, RawAccount } from "./coValues/account.js";
 import { RawCoList } from "./coValues/coList.js";
 import { RawCoMap } from "./coValues/coMap.js";

packages/cojson/src/coValueContentMessage.ts

@@ -0,0 +1,86 @@
+import {
+  CoValueHeader,
+  Transaction,
+  VerifiedState,
+} from "./coValueCore/verifiedState.js";
+import { TRANSACTION_CONFIG } from "./config.js";
+import { Signature } from "./crypto/crypto.js";
+import { RawCoID, SessionID } from "./ids.js";
+import { getPriorityFromHeader } from "./priority.js";
+import { NewContentMessage, emptyKnownState } from "./sync.js";
+
+export function createContentMessage(
+  id: RawCoID,
+  header: CoValueHeader,
+  includeHeader = true,
+): NewContentMessage {
+  return {
+    action: "content",
+    id,
+    header: includeHeader ? header : undefined,
+    priority: getPriorityFromHeader(header),
+    new: {},
+  };
+}
+
+export function addTransactionToContentMessage(
+  content: NewContentMessage,
+  transaction: Transaction,
+  sessionID: SessionID,
+  signature: Signature,
+  txIdx: number,
+) {
+  const sessionContent = content.new[sessionID];
+
+  if (sessionContent) {
+    sessionContent.newTransactions.push(transaction);
+    sessionContent.lastSignature = signature;
+  } else {
+    content.new[sessionID] = {
+      after: txIdx,
+      newTransactions: [transaction],
+      lastSignature: signature,
+    };
+  }
+}
+
+export function getTransactionSize(transaction: Transaction) {
+  return transaction.privacy === "private"
+    ? transaction.encryptedChanges.length
+    : transaction.changes.length;
+}
+
+export function exceedsRecommendedSize(
+  baseSize: number,
+  transactionSize?: number,
+) {
+  if (transactionSize === undefined) {
+    return baseSize > TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE;
+  }
+
+  return (
+    baseSize + transactionSize > TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE
+  );
+}
+
+export function knownStateFromContent(content: NewContentMessage) {
+  const knownState = emptyKnownState(content.id);
+
+  for (const [sessionID, session] of Object.entries(content.new)) {
+    knownState.sessions[sessionID as SessionID] =
+      session.after + session.newTransactions.length;
+  }
+
+  return knownState;
+}
+
+export function getContentMessageSize(msg: NewContentMessage) {
+  return Object.values(msg.new).reduce((acc, sessionNewContent) => {
+    return (
+      acc +
+      sessionNewContent.newTransactions.reduce((acc, tx) => {
+        return acc + getTransactionSize(tx);
+      }, 0)
+    );
+  }, 0);
+}

packages/cojson/src/coValueCore/coValueCore.ts

@@ -1,10 +1,10 @@
 import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { Result, err } from "neverthrow";
-import { PeerState } from "../PeerState.js";
-import { RawCoValue } from "../coValue.js";
-import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
-import { RawGroup } from "../coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG, MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import type { PeerState } from "../PeerState.js";
+import type { RawCoValue } from "../coValue.js";
+import type { ControlledAccountOrAgent } from "../coValues/account.js";
+import type { RawGroup } from "../coValues/group.js";
+import { CO_VALUE_LOADING_CONFIG } from "../config.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
@@ -16,25 +16,15 @@ import {
   SignerID,
   StreamingHash,
 } from "../crypto/crypto.js";
-import {
-  RawCoID,
-  SessionID,
-  TransactionID,
-  getParentGroupId,
-  isParentGroupReference,
-} from "../ids.js";
+import { RawCoID, SessionID, TransactionID } from "../ids.js";
 import { parseJSON, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
-import {
-  determineValidTransactions,
-  isKeyForKeyField,
-} from "../permissions.js";
+import { determineValidTransactions } from "../permissions.js";
 import { CoValueKnownState, PeerID, emptyKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
-import { isAccountID } from "../typeUtils/isAccountID.js";
 import { getDependedOnCoValuesFromRawData } from "./utils.js";
 import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 
@@ -53,8 +43,6 @@ export type DecryptedTransaction = {
   trusting?: boolean;
 };
 
-const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
-
 export type AvailableCoValueCore = CoValueCore & { verified: VerifiedState };
 
 export class CoValueCore {
@@ -81,7 +69,9 @@ export class CoValueCore {
   }
   private readonly peers = new Map<
     PeerID,
-    | { type: "unknown" | "pending" | "available" | "unavailable" }
+    | {
+        type: "unknown" | "pending" | "available" | "unavailable";
+      }
     | {
         type: "errored";
         error: TryAddTransactionsError;
@@ -90,9 +80,8 @@ export class CoValueCore {
 
   // cached state and listeners
   private _cachedContent?: RawCoValue;
-  private readonly listeners: Set<
-    (core: CoValueCore, unsub: () => void) => void
-  > = new Set();
+  readonly listeners: Set<(core: CoValueCore, unsub: () => void) => void> =
+    new Set();
   private readonly _decryptionCache: {
     [key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined;
   } = {};
@@ -213,6 +202,26 @@ export class CoValueCore {
     }
   }
 
+  unmount() {
+    // For simplicity, we don't unmount groups and accounts
+    if (this.verified?.header.ruleset.type === "group") {
+      return false;
+    }
+
+    if (this.listeners.size > 0) {
+      return false; // The coValue is still in use
+    }
+
+    this.counter.add(-1, { state: this.loadingState });
+
+    if (this.groupInvalidationSubscription) {
+      this.groupInvalidationSubscription();
+      this.groupInvalidationSubscription = undefined;
+    }
+
+    return true;
+  }
+
   markNotFoundInPeer(peerId: PeerID) {
     const previousState = this.loadingState;
     this.peers.set(peerId, { type: "unavailable" });
@@ -380,7 +389,7 @@ export class CoValueCore {
   }
 
   knownStateWithStreaming(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownStateWithStreaming();
     } else {
       return emptyKnownState(this.id);
@@ -388,7 +397,7 @@ export class CoValueCore {
   }
 
   knownState(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownState();
     } else {
       return emptyKnownState(this.id);
@@ -605,16 +614,23 @@ export class CoValueCore {
     )._unsafeUnwrap({ withStackTrace: true });
 
     if (success) {
+      const session = this.verified.sessions.get(sessionID);
+      const txIdx = session ? session.transactions.length - 1 : 0;
+
       this.node.syncManager.recordTransactionsSize([transaction], "local");
-      void this.node.syncManager.requestCoValueSync(this);
+      this.node.syncManager.syncLocalTransaction(
+        this.verified,
+        transaction,
+        sessionID,
+        signature,
+        txIdx,
+      );
     }
 
     return success;
   }
 
-  getCurrentContent(options?: {
-    ignorePrivateTransactions: true;
-  }): RawCoValue {
+  getCurrentContent(options?: { ignorePrivateTransactions: true }): RawCoValue {
     if (!this.verified) {
       throw new Error(
         "CoValueCore: getCurrentContent called on coValue without verified state",
@@ -759,20 +775,7 @@ export class CoValueCore {
     }
 
     if (this.verified.header.ruleset.type === "group") {
-      const content = expectGroup(this.getCurrentContent());
-
-      const currentKeyId = content.getCurrentReadKeyId();
-
-      if (!currentKeyId) {
-        throw new Error("No readKey set");
-      }
-
-      const secret = this.getReadKey(currentKeyId);
-
-      return {
-        secret: secret,
-        id: currentKeyId,
-      };
+      return expectGroup(this.getCurrentContent()).getCurrentReadKey();
     } else if (this.verified.header.ruleset.type === "ownedByGroup") {
       return this.node
         .expectCoValueLoaded(this.verified.header.ruleset.group)
@@ -784,154 +787,36 @@ export class CoValueCore {
     }
   }
 
+  readKeyCache = new Map<KeyID, KeySecret>();
   getReadKey(keyID: KeyID): KeySecret | undefined {
-    let key = readKeyCache.get(this)?.[keyID];
-    if (!key) {
-      key = this.getUncachedReadKey(keyID);
-      if (key) {
-        let cache = readKeyCache.get(this);
-        if (!cache) {
-          cache = {};
-          readKeyCache.set(this, cache);
-        }
-        cache[keyID] = key;
-      }
-    }
-    return key;
-  }
+    // We want to check the cache here, to skip re-computing the group content
+    const cachedSecret = this.readKeyCache.get(keyID);
+
+    if (cachedSecret) {
+      return cachedSecret;
+    }
 
-  getUncachedReadKey(keyID: KeyID): KeySecret | undefined {
     if (!this.verified) {
       throw new Error(
         "CoValueCore: getUncachedReadKey called on coValue without verified state",
       );
     }
 
+    // Getting the readKey from accounts
     if (this.verified.header.ruleset.type === "group") {
       const content = expectGroup(
-        this.getCurrentContent({ ignorePrivateTransactions: true }), // to prevent recursion
-      );
-      const keyForEveryone = content.get(`${keyID}_for_everyone`);
-      if (keyForEveryone) {
-        return keyForEveryone;
-      }
-
-      // Try to find key revelation for us
-      const currentAgentOrAccountID = accountOrAgentIDfromSessionID(
-        this.node.currentSessionID,
+        // load the account without private transactions, because we are here
+        // to be able to decrypt those
+        this.getCurrentContent({ ignorePrivateTransactions: true }),
       );
 
-      // being careful here to avoid recursion
-      const lookupAccountOrAgentID = isAccountID(currentAgentOrAccountID)
-        ? this.id === currentAgentOrAccountID
-          ? this.crypto.getAgentID(this.node.agentSecret) // in accounts, the read key is revealed for the primitive agent
-          : currentAgentOrAccountID // current account ID
-        : currentAgentOrAccountID; // current agent ID
-
-      const lastReadyKeyEdit = content.lastEditAt(
-        `${keyID}_for_${lookupAccountOrAgentID}`,
-      );
-
-      if (lastReadyKeyEdit?.value) {
-        const revealer = lastReadyKeyEdit.by;
-        const revealerAgent = this.node
-          .resolveAccountAgent(revealer, "Expected to know revealer")
-          ._unsafeUnwrap({ withStackTrace: true });
-
-        const secret = this.crypto.unseal(
-          lastReadyKeyEdit.value,
-          this.crypto.getAgentSealerSecret(this.node.agentSecret), // being careful here to avoid recursion
-          this.crypto.getAgentSealerID(revealerAgent),
-          {
-            in: this.id,
-            tx: lastReadyKeyEdit.tx,
-          },
-        );
-
-        if (secret) {
-          return secret as KeySecret;
-        }
-      }
-
-      // Try to find indirect revelation through previousKeys
-
-      for (const co of content.keys()) {
-        if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-          const encryptingKeyID = co.split("_for_")[1] as KeyID;
-          const encryptingKeySecret = this.getReadKey(encryptingKeyID);
-
-          if (!encryptingKeySecret) {
-            continue;
-          }
-
-          const encryptedPreviousKey = content.get(co)!;
-
-          const secret = this.crypto.decryptKeySecret(
-            {
-              encryptedID: keyID,
-              encryptingID: encryptingKeyID,
-              encrypted: encryptedPreviousKey,
-            },
-            encryptingKeySecret,
-          );
-
-          if (secret) {
-            return secret as KeySecret;
-          } else {
-            logger.warn(
-              `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
-            );
-          }
-        }
-      }
-
-      // try to find revelation to parent group read keys
-      for (const co of content.keys()) {
-        if (isParentGroupReference(co)) {
-          const parentGroupID = getParentGroupId(co);
-          const parentGroup = this.node.expectCoValueLoaded(
-            parentGroupID,
-            "Expected parent group to be loaded",
-          );
-
-          const parentKeys = this.findValidParentKeys(
-            keyID,
-            content,
-            parentGroup,
-          );
-
-          for (const parentKey of parentKeys) {
-            const revelationForParentKey = content.get(
-              `${keyID}_for_${parentKey.id}`,
-            );
-
-            if (revelationForParentKey) {
-              const secret = parentGroup.crypto.decryptKeySecret(
-                {
-                  encryptedID: keyID,
-                  encryptingID: parentKey.id,
-                  encrypted: revelationForParentKey,
-                },
-                parentKey.secret,
-              );
-
-              if (secret) {
-                return secret as KeySecret;
-              } else {
-                logger.warn(
-                  `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
-                );
-              }
-            }
-          }
-        }
-      }
-
-      return undefined;
+      return content.getReadKey(keyID);
     } else if (this.verified.header.ruleset.type === "ownedByGroup") {
-      return this.node
-        .expectCoValueLoaded(this.verified.header.ruleset.group)
-        .getReadKey(keyID);
+      return expectGroup(
+        this.node
+          .expectCoValueLoaded(this.verified.header.ruleset.group)
+          .getCurrentContent(),
+      ).getReadKey(keyID);
     } else {
       throw new Error(
         "Only groups or values owned by groups have read secrets",
@@ -939,28 +824,6 @@ export class CoValueCore {
     }
   }
 
-  findValidParentKeys(keyID: KeyID, group: RawGroup, parentGroup: CoValueCore) {
-    const validParentKeys: { id: KeyID; secret: KeySecret }[] = [];
-
-    for (const co of group.keys()) {
-      if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-        const encryptingKeyID = co.split("_for_")[1] as KeyID;
-        const encryptingKeySecret = parentGroup.getReadKey(encryptingKeyID);
-
-        if (!encryptingKeySecret) {
-          continue;
-        }
-
-        validParentKeys.push({
-          id: encryptingKeyID,
-          secret: encryptingKeySecret,
-        });
-      }
-    }
-
-    return validParentKeys;
-  }
-
   getGroup(): RawGroup {
     if (!this.verified) {
       throw new Error(
@@ -1007,9 +870,7 @@ export class CoValueCore {
     }
   }
 
-  waitForSync(options?: {
-    timeout?: number;
-  }) {
+  waitForSync(options?: { timeout?: number }) {
     return this.node.syncManager.waitForSync(this.id, options?.timeout);
   }
 

packages/cojson/src/coValueCore/utils.ts

@@ -2,6 +2,7 @@ import { getGroupDependentKey } from "../ids.js";
 import { RawCoID, SessionID } from "../ids.js";
 import { Stringified, parseJSON } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { NewContentMessage } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "../typeUtils/isAccountID.js";
 import { CoValueHeader, Transaction } from "./verifiedState.js";

packages/cojson/src/coValueCore/verifiedState.ts

@@ -1,6 +1,10 @@
 import { Result, err, ok } from "neverthrow";
 import { AnyRawCoValue } from "../coValue.js";
-import { MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -14,7 +18,6 @@ import { RawCoID, SessionID, TransactionID } from "../ids.js";
 import { Stringified } from "../jsonStringify.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { PermissionsDef as RulesetDef } from "../permissions.js";
-import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { InvalidHashError, InvalidSignatureError } from "./coValueCore.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
@@ -62,6 +65,7 @@ export class VerifiedState {
   private _cachedKnownState?: CoValueKnownState;
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   private streamingKnownState?: CoValueKnownState["sessions"];
+  public lastAccessed: number | undefined;
 
   constructor(
     id: RawCoID,
@@ -151,6 +155,17 @@ export class VerifiedState {
     return ok(true as const);
   }
 
+  getLastSignatureCheckpoint(sessionID: SessionID): number {
+    const sessionLog = this.sessions.get(sessionID);
+
+    if (!sessionLog?.signatureAfter) return -1;
+
+    return Object.keys(sessionLog.signatureAfter).reduce(
+      (max, idx) => Math.max(max, parseInt(idx)),
+      -1,
+    );
+  }
+
   private doAddTransactions(
     sessionID: SessionID,
     newTransactions: Transaction[],
@@ -165,24 +180,14 @@ export class VerifiedState {
     }
 
     const signatureAfter = sessionLog?.signatureAfter ?? {};
-
-    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
-      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
-      -1,
-    );
+    const lastInbetweenSignatureIdx =
+      this.getLastSignatureCheckpoint(sessionID);
 
     const sizeOfTxsSinceLastInbetweenSignature = transactions
       .slice(lastInbetweenSignatureIdx + 1)
-      .reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+      .reduce((sum, tx) => sum + getTransactionSize(tx), 0);
 
-    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(sizeOfTxsSinceLastInbetweenSignature)) {
       signatureAfter[transactions.length - 1] = newSignature;
     }
 
@@ -242,13 +247,11 @@ export class VerifiedState {
       return this._cachedNewContentSinceEmpty;
     }
 
-    let currentPiece: NewContentMessage = {
-      action: "content",
-      id: this.id,
-      header: knownState?.header ? undefined : this.header,
-      priority: getPriorityFromHeader(this.header),
-      new: {},
-    };
+    let currentPiece: NewContentMessage = createContentMessage(
+      this.id,
+      this.header,
+      !knownState?.header,
+    );
 
     const pieces = [currentPiece];
 
@@ -299,25 +302,16 @@ export class VerifiedState {
         const oldPieceSize = pieceSize;
         for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
           const tx = log.transactions[txIdx]!;
-          pieceSize +=
-            tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length;
+          pieceSize += getTransactionSize(tx);
         }
 
-        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+        if (exceedsRecommendedSize(pieceSize)) {
           if (!currentPiece.expectContentUntil && pieces.length === 1) {
             currentPiece.expectContentUntil =
               this.knownStateWithStreaming().sessions;
           }
 
-          currentPiece = {
-            action: "content",
-            id: this.id,
-            header: undefined,
-            new: {},
-            priority: getPriorityFromHeader(this.header),
-          };
+          currentPiece = createContentMessage(this.id, this.header, false);
           pieces.push(currentPiece);
           pieceSize = pieceSize - oldPieceSize;
         }

packages/cojson/src/coValues/coPlainText.ts

@@ -1,5 +1,6 @@
 import { splitGraphemes } from "unicode-segmenter/grapheme";
 import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
+import { TRANSACTION_CONFIG } from "../config.js";
 import { JsonObject } from "../jsonValue.js";
 import { DeletionOpPayload, OpID, RawCoList } from "./coList.js";
 
@@ -110,16 +111,34 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
 
     if (idx === 0) {
-      // For insertions at start, prepend each character in reverse
-      for (const grapheme of graphemes.reverse()) {
-        this.prepend(grapheme, 0, privacy);
+      // For insertions at start, prepend the first char and append the rest
+      const firstChar = graphemes[0];
+
+      if (firstChar) {
+        this.prepend(firstChar, 0, privacy);
+      }
+
+      if (graphemes.length > 1) {
+        this.appendChars(graphemes.slice(1), 0, privacy);
       }
     } else {
       // For other insertions, append after the previous character
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
+    }
+  }
+
+  appendChars(
+    text: string[],
+    position: number,
+    privacy: "private" | "trusting" = "private",
+  ) {
+    const chunks = splitIntoChunks(text);
+    for (const chunk of chunks) {
+      this.appendItems(chunk, position, privacy);
+      position += chunk.length;
     }
   }
 
@@ -136,11 +155,12 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
+
     if (idx >= this.entries().length) {
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
     } else {
-      this.appendItems(graphemes, idx, privacy);
+      this.appendChars(graphemes, idx, privacy);
     }
   }
 
@@ -178,3 +198,15 @@ export class RawCoPlainText<
     return graphemes.join("");
   }
 }
+
+function splitIntoChunks(text: string[]) {
+  const chunks: string[][] = [];
+  for (
+    let i = 0;
+    i < text.length;
+    i += TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE
+  ) {
+    chunks.push(text.slice(i, i + TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE));
+  }
+  return chunks;
+}

packages/cojson/src/coValues/group.ts

@@ -1,8 +1,11 @@
 import { base58 } from "@scure/base";
-import { CoID } from "../coValue.js";
-import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
-import { CoValueUniqueness } from "../coValueCore/verifiedState.js";
-import {
+import type { CoID } from "../coValue.js";
+import type {
+  AvailableCoValueCore,
+  CoValueCore,
+} from "../coValueCore/coValueCore.js";
+import type { CoValueUniqueness } from "../coValueCore/verifiedState.js";
+import type {
   CryptoProvider,
   Encrypted,
   KeyID,
@@ -21,8 +24,10 @@ import {
 } from "../ids.js";
 import { JsonObject } from "../jsonValue.js";
 import { logger } from "../logger.js";
-import { AccountRole, Role } from "../permissions.js";
+import { AccountRole, Role, isKeyForKeyField } from "../permissions.js";
+import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
+import { isAccountID } from "../typeUtils/isAccountID.js";
 import {
   ControlledAccountOrAgent,
   RawAccount,
@@ -60,6 +65,59 @@ export type GroupShape = {
   [child: ChildGroupReference]: "revoked" | "extend";
 };
 
+// We had a bug on key rotation, where the new read key was not revealed to everyone
+// TODO: remove this when we hit the 0.18.0 release (either the groups are healed or they are not used often, it's a minor issue anyway)
+function healMissingKeyForEveryone(group: RawGroup) {
+  const readKeyId = group.get("readKey");
+
+  if (
+    !readKeyId ||
+    !canRead(group, EVERYONE) ||
+    group.get(`${readKeyId}_for_${EVERYONE}`)
+  ) {
+    return;
+  }
+
+  const hasAccessToReadKey = canRead(
+    group,
+    group.core.node.getCurrentAgent().id,
+  );
+
+  // If the current account has access to the read key, we can fix the group
+  if (hasAccessToReadKey) {
+    const secret = group.getReadKey(readKeyId);
+    if (secret) {
+      group.set(`${readKeyId}_for_${EVERYONE}`, secret, "trusting");
+    }
+    return;
+  }
+
+  // Fallback to the latest readable key for everyone
+  const keys = group
+    .keys()
+    .filter((key) => key.startsWith("key_") && key.endsWith("_for_everyone"));
+
+  let latestKey = keys[0];
+
+  for (const key of keys) {
+    if (!latestKey) {
+      latestKey = key;
+      continue;
+    }
+
+    const keyEntry = group.getRaw(key);
+    const latestKeyEntry = group.getRaw(latestKey);
+
+    if (keyEntry && latestKeyEntry && keyEntry.madeAt > latestKeyEntry.madeAt) {
+      latestKey = key;
+    }
+  }
+
+  if (latestKey) {
+    group._lastReadableKeyId = latestKey.replace("_for_everyone", "") as KeyID;
+  }
+}
+
 /** A `Group` is a scope for permissions of its members (`"reader" | "writer" | "admin"`), applying to objects owned by that group.
  *
  *  A `Group` object exposes methods for permission management and allows you to create new CoValues owned by that group.
@@ -86,6 +144,8 @@ export class RawGroup<
 > extends RawCoMap<GroupShape, Meta> {
   protected readonly crypto: CryptoProvider;
 
+  _lastReadableKeyId?: KeyID;
+
   constructor(
     core: AvailableCoValueCore,
     options?: {
@@ -94,6 +154,8 @@ export class RawGroup<
   ) {
     super(core, options);
     this.crypto = core.node.crypto;
+
+    healMissingKeyForEveryone(this);
   }
 
   /**
@@ -191,43 +253,7 @@ export class RawGroup<
     return groups;
   }
 
-  loadAllChildGroups() {
-    const requests: Promise<unknown>[] = [];
-    const peers = this.core.node.syncManager.getServerPeers();
-
-    for (const key of this.keys()) {
-      if (!isChildGroupReference(key)) {
-        continue;
-      }
-
-      const id = getChildGroupId(key);
-      const child = this.core.node.getCoValue(id);
-
-      if (
-        child.loadingState === "unknown" ||
-        child.loadingState === "unavailable"
-      ) {
-        child.load(peers);
-      }
-
-      requests.push(
-        child.waitForAvailableOrUnavailable().then((coValue) => {
-          if (!coValue.isAvailable()) {
-            throw new Error(`Child group ${child.id} is unavailable`);
-          }
-
-          // Recursively load child groups
-          return expectGroup(coValue.getCurrentContent()).loadAllChildGroups();
-        }),
-      );
-    }
-
-    return Promise.all(requests);
-  }
-
-  getChildGroups() {
-    const groups: RawGroup[] = [];
-
+  forEachChildGroup(callback: (child: RawGroup) => void) {
     for (const key of this.keys()) {
       if (isChildGroupReference(key)) {
         // Check if the child group reference is revoked
@@ -235,15 +261,22 @@ export class RawGroup<
           continue;
         }
 
-        const child = this.core.node.expectCoValueLoaded(
-          getChildGroupId(key),
-          "Expected child group to be loaded",
-        );
-        groups.push(expectGroup(child.getCurrentContent()));
+        const id = getChildGroupId(key);
+        const child = this.core.node.getCoValue(id);
+
+        if (child.isAvailable()) {
+          callback(expectGroup(child.getCurrentContent()));
+        } else {
+          this.core.node.load(id).then((child) => {
+            if (child !== "unavailable") {
+              callback(expectGroup(child));
+            } else {
+              logger.warn(`Unable to load child group ${id}, skipping`);
+            }
+          });
+        }
       }
     }
-
-    return groups;
   }
 
   /**
@@ -279,7 +312,7 @@ export class RawGroup<
           "Can't make everyone something other than reader, writer or writeOnly",
         );
       }
-      const currentReadKey = this.core.getCurrentReadKey();
+      const currentReadKey = this.getCurrentReadKey();
 
       if (!currentReadKey.secret) {
         throw new Error("Can't add member without read key secret");
@@ -306,7 +339,7 @@ export class RawGroup<
 
       if (role === "writeOnly") {
         if (previousRole === "reader" || previousRole === "writer") {
-          this.rotateReadKey();
+          this.rotateReadKey("everyone");
         }
 
         this.delete(`${currentReadKey.id}_for_${EVERYONE}`);
@@ -349,7 +382,7 @@ export class RawGroup<
 
       this.internalCreateWriteOnlyKeyForMember(memberKey, agent);
     } else {
-      const currentReadKey = this.core.getCurrentReadKey();
+      const currentReadKey = this.getCurrentReadKey();
 
       if (!currentReadKey.secret) {
         throw new Error("Can't add member without read key secret");
@@ -467,6 +500,10 @@ export class RawGroup<
   }
 
   getCurrentReadKeyId() {
+    if (this._lastReadableKeyId) {
+      return this._lastReadableKeyId;
+    }
+
     const myRole = this.myRole();
 
     if (myRole === "writeOnly") {
@@ -518,23 +555,173 @@ export class RawGroup<
     return memberKeys;
   }
 
+  getReadKey(keyID: KeyID): KeySecret | undefined {
+    const cache = this.core.readKeyCache;
+
+    let key = cache.get(keyID);
+    if (!key) {
+      key = this.getUncachedReadKey(keyID);
+      if (key) {
+        cache.set(keyID, key);
+      }
+    }
+    return key;
+  }
+
+  getUncachedReadKey(keyID: KeyID) {
+    const core = this.core;
+
+    const keyForEveryone = this.get(`${keyID}_for_everyone`);
+    if (keyForEveryone) {
+      return keyForEveryone;
+    }
+
+    // Try to find key revelation for us
+    const currentAgentOrAccountID = accountOrAgentIDfromSessionID(
+      core.node.currentSessionID,
+    );
+
+    // being careful here to avoid recursion
+    const lookupAccountOrAgentID = isAccountID(currentAgentOrAccountID)
+      ? core.id === currentAgentOrAccountID
+        ? core.node.crypto.getAgentID(core.node.agentSecret) // in accounts, the read key is revealed for the primitive agent
+        : currentAgentOrAccountID // current account ID
+      : currentAgentOrAccountID; // current agent ID
+
+    const lastReadyKeyEdit = this.lastEditAt(
+      `${keyID}_for_${lookupAccountOrAgentID}`,
+    );
+
+    if (lastReadyKeyEdit?.value) {
+      const revealer = lastReadyKeyEdit.by;
+      const revealerAgent = core.node
+        .resolveAccountAgent(revealer, "Expected to know revealer")
+        ._unsafeUnwrap({ withStackTrace: true });
+
+      const secret = this.crypto.unseal(
+        lastReadyKeyEdit.value,
+        this.crypto.getAgentSealerSecret(core.node.agentSecret), // being careful here to avoid recursion
+        this.crypto.getAgentSealerID(revealerAgent),
+        {
+          in: this.id,
+          tx: lastReadyKeyEdit.tx,
+        },
+      );
+
+      if (secret) {
+        return secret as KeySecret;
+      }
+    }
+
+    // Try to find indirect revelation through previousKeys
+    for (const co of this.keys()) {
+      if (isKeyForKeyField(co) && co.startsWith(keyID)) {
+        const encryptingKeyID = co.split("_for_")[1] as KeyID;
+        const encryptingKeySecret = this.getReadKey(encryptingKeyID);
+
+        if (!encryptingKeySecret) {
+          continue;
+        }
+
+        const encryptedPreviousKey = this.get(co)!;
+
+        const secret = this.crypto.decryptKeySecret(
+          {
+            encryptedID: keyID,
+            encryptingID: encryptingKeyID,
+            encrypted: encryptedPreviousKey,
+          },
+          encryptingKeySecret,
+        );
+
+        if (secret) {
+          return secret as KeySecret;
+        } else {
+          logger.warn(
+            `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
+          );
+        }
+      }
+    }
+
+    // try to find revelation to parent group read keys
+    for (const co of this.keys()) {
+      if (isParentGroupReference(co)) {
+        const parentGroupID = getParentGroupId(co);
+        const parentGroup = core.node.expectCoValueLoaded(
+          parentGroupID,
+          "Expected parent group to be loaded",
+        );
+
+        const parentKeys = this.findValidParentKeys(keyID, parentGroup);
+
+        for (const parentKey of parentKeys) {
+          const revelationForParentKey = this.get(
+            `${keyID}_for_${parentKey.id}`,
+          );
+
+          if (revelationForParentKey) {
+            const secret = parentGroup.node.crypto.decryptKeySecret(
+              {
+                encryptedID: keyID,
+                encryptingID: parentKey.id,
+                encrypted: revelationForParentKey,
+              },
+              parentKey.secret,
+            );
+
+            if (secret) {
+              return secret as KeySecret;
+            } else {
+              logger.warn(
+                `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
+              );
+            }
+          }
+        }
+      }
+    }
+
+    return undefined;
+  }
+
+  findValidParentKeys(keyID: KeyID, parentGroup: CoValueCore) {
+    const validParentKeys: { id: KeyID; secret: KeySecret }[] = [];
+
+    for (const co of this.keys()) {
+      if (isKeyForKeyField(co) && co.startsWith(keyID)) {
+        const encryptingKeyID = co.split("_for_")[1] as KeyID;
+        const encryptingKeySecret = parentGroup.getReadKey(encryptingKeyID);
+
+        if (!encryptingKeySecret) {
+          continue;
+        }
+
+        validParentKeys.push({
+          id: encryptingKeyID,
+          secret: encryptingKeySecret,
+        });
+      }
+    }
+
+    return validParentKeys;
+  }
+
   /** @internal */
   rotateReadKey(removedMemberKey?: RawAccountID | AgentID | "everyone") {
+    if (removedMemberKey !== EVERYONE && canRead(this, EVERYONE)) {
+      // When everyone has access to the group, rotating the key is useless
+      // because it would be stored unencrypted and available to everyone
+      return;
+    }
+
     const memberKeys = this.getMemberKeys().filter(
       (key) => key !== removedMemberKey,
     );
 
-    const currentlyPermittedReaders = memberKeys.filter((key) => {
-      const role = this.get(key);
-      return (
-        role === "admin" ||
-        role === "writer" ||
-        role === "reader" ||
-        role === "adminInvite" ||
-        role === "writerInvite" ||
-        role === "readerInvite"
-      );
-    });
+    const currentlyPermittedReaders = memberKeys.filter((key) =>
+      canRead(this, key),
+    );
 
     const writeOnlyMembers = memberKeys.filter((key) => {
       const role = this.get(key);
@@ -543,12 +730,12 @@ export class RawGroup<
 
     // Get these early, so we fail fast if they are unavailable
     const parentGroups = this.getParentGroups();
-    const childGroups = this.getChildGroups();
-
-    const maybeCurrentReadKey = this.core.getCurrentReadKey();
+    const maybeCurrentReadKey = this.getCurrentReadKey();
 
     if (!maybeCurrentReadKey.secret) {
-      throw new Error("Can't rotate read key secret we don't have access to");
+      throw new NoReadKeyAccessError(
+        "Can't rotate read key secret we don't have access to",
+      );
     }
 
     const currentReadKey = {
@@ -631,7 +818,7 @@ export class RawGroup<
      */
     for (const parent of parentGroups) {
       const { id: parentReadKeyID, secret: parentReadKeySecret } =
-        parent.core.getCurrentReadKey();
+        parent.getCurrentReadKey();
 
       if (!parentReadKeySecret) {
         // We can't reveal the new child key to the parent group where we don't have access to the parent read key
@@ -655,33 +842,67 @@ export class RawGroup<
       );
     }
 
-    for (const child of childGroups) {
+    this.forEachChildGroup((child) => {
       // Since child references are mantained only for the key rotation,
       // circular references are skipped here because it's more performant
       // than always checking for circular references in childs inside the permission checks
       if (child.isSelfExtension(this)) {
-        continue;
+        return;
       }
 
-      child.rotateReadKey(removedMemberKey);
-    }
+      try {
+        child.rotateReadKey(removedMemberKey);
+      } catch (error) {
+        if (error instanceof NoReadKeyAccessError) {
+          logger.warn(
+            `Can't rotate read key on child ${child.id} because we don't have access to the read key`,
+          );
+        } else {
+          throw error;
+        }
+      }
+    });
   }
 
   /** Detect circular references in group inheritance */
   isSelfExtension(parent: RawGroup) {
-    if (parent.id === this.id) {
-      return true;
-    }
+    const checkedGroups = new Set<string>();
+    const queue = [parent];
 
-    const childGroups = this.getChildGroups();
+    while (true) {
+      const current = queue.pop();
 
-    for (const child of childGroups) {
-      if (child.isSelfExtension(parent)) {
+      if (!current) {
+        return false;
+      }
+
+      if (current.id === this.id) {
         return true;
       }
+
+      checkedGroups.add(current.id);
+
+      const parentGroups = current.getParentGroups();
+
+      for (const parent of parentGroups) {
+        if (!checkedGroups.has(parent.id)) {
+          queue.push(parent);
+        }
+      }
+    }
+  }
+
+  getCurrentReadKey() {
+    const keyId = this.getCurrentReadKeyId();
+
+    if (!keyId) {
+      throw new Error("No readKey set");
     }
 
-    return false;
+    return {
+      secret: this.getReadKey(keyId),
+      id: keyId,
+    };
   }
 
   extend(
@@ -700,8 +921,8 @@ export class RawGroup<
 
     const value = role === "inherit" ? "extend" : role;
 
-    this.set(`parent_${parent.id}`, value, "trusting");
     parent.set(`child_${this.id}`, "extend", "trusting");
+    this.set(`parent_${parent.id}`, value, "trusting");
 
     if (
       parent.myRole() !== "admin" &&
@@ -716,14 +937,15 @@ export class RawGroup<
       );
     }
 
-    const { id: parentReadKeyID, secret: parentReadKeySecret } =
-      parent.core.getCurrentReadKey();
+    let { id: parentReadKeyID, secret: parentReadKeySecret } =
+      parent.getCurrentReadKey();
+
     if (!parentReadKeySecret) {
       throw new Error("Can't extend group without parent read key secret");
     }
 
     const { id: childReadKeyID, secret: childReadKeySecret } =
-      this.core.getCurrentReadKey();
+      this.getCurrentReadKey();
     if (!childReadKeySecret) {
       throw new Error("Can't extend group without child read key secret");
     }
@@ -744,7 +966,7 @@ export class RawGroup<
     );
   }
 
-  async revokeExtend(parent: RawGroup) {
+  revokeExtend(parent: RawGroup) {
     if (this.myRole() !== "admin") {
       throw new Error(
         "To unextend a group, the current account must be an admin in the child group",
@@ -775,8 +997,6 @@ export class RawGroup<
     // Set the child key on the parent group to `revoked`
     parent.set(`child_${this.id}`, "revoked", "trusting");
 
-    await this.loadAllChildGroups();
-
     // Rotate the keys on the child group
     this.rotateReadKey();
   }
@@ -788,19 +1008,7 @@ export class RawGroup<
    *
    * @category 2. Role changing
    */
-  async removeMember(
-    account: RawAccount | ControlledAccountOrAgent | Everyone,
-  ) {
-    // Ensure all child groups are loaded before removing a member
-    await this.loadAllChildGroups();
-
-    this.removeMemberInternal(account);
-  }
-
-  /** @internal */
-  removeMemberInternal(
-    account: RawAccount | ControlledAccountOrAgent | AgentID | Everyone,
-  ) {
+  removeMember(account: RawAccount | ControlledAccountOrAgent | Everyone) {
     const memberKey = typeof account === "string" ? account : account.id;
 
     if (this.myRole() === "admin") {
@@ -1011,3 +1219,25 @@ export function secretSeedFromInviteSecret(inviteSecret: InviteSecret) {
 
   return base58.decode(inviteSecret.slice("inviteSecret_z".length));
 }
+
+const canRead = (
+  group: RawGroup,
+  key: RawAccountID | AgentID | "everyone",
+): boolean => {
+  const role = group.get(key);
+  return (
+    role === "admin" ||
+    role === "writer" ||
+    role === "reader" ||
+    role === "adminInvite" ||
+    role === "writerInvite" ||
+    role === "readerInvite"
+  );
+};
+
+class NoReadKeyAccessError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "NoReadKeyAccessError";
+  }
+}

packages/cojson/src/config.ts

@@ -5,7 +5,13 @@
     This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
     to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
 **/
-export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
+export const TRANSACTION_CONFIG = {
+  MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+};
+
+export function setMaxRecommendedTxSize(size: number) {
+  TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
+}
 
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
@@ -32,3 +38,16 @@ export const SYNC_SCHEDULER_CONFIG = {
 export function setIncomingMessagesTimeBudget(budget: number) {
   SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET = budget;
 }
+
+export const GARBAGE_COLLECTOR_CONFIG = {
+  MAX_AGE: 1000 * 60 * 10, // 10 minutes
+  INTERVAL: 1000 * 60 * 5, // 5 minutes
+};
+
+export function setGarbageCollectorMaxAge(maxAge: number) {
+  GARBAGE_COLLECTOR_CONFIG.MAX_AGE = maxAge;
+}
+
+export function setGarbageCollectorInterval(interval: number) {
+  GARBAGE_COLLECTOR_CONFIG.INTERVAL = interval;
+}

packages/cojson/src/exports.ts

@@ -61,20 +61,25 @@ import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
 import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
 
-type Value = JsonValue | AnyRawCoValue;
-
-export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
+import {
+  getContentMessageSize,
+  getTransactionSize,
+} from "./coValueContentMessage.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import {
   CO_VALUE_LOADING_CONFIG,
-  MAX_RECOMMENDED_TX_SIZE,
+  TRANSACTION_CONFIG,
   setCoValueLoadingRetryDelay,
   setIncomingMessagesTimeBudget,
+  setMaxRecommendedTxSize,
 } from "./config.js";
 import { LogLevel, logger } from "./logger.js";
 import { CO_VALUE_PRIORITY, getPriorityFromHeader } from "./priority.js";
 import { getDependedOnCoValues } from "./storage/syncUtils.js";
 
+type Value = JsonValue | AnyRawCoValue;
+
+export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
 /** @hidden */
 export const cojsonInternals = {
   connectedPeers,
@@ -106,6 +111,10 @@ export const cojsonInternals = {
   ConnectedPeerChannel,
   textEncoder,
   textDecoder,
+  getTransactionSize,
+  getContentMessageSize,
+  TRANSACTION_CONFIG,
+  setMaxRecommendedTxSize,
 };
 
 export {
@@ -132,7 +141,6 @@ export {
   Media,
   CoValueCore,
   ControlledAgent,
-  MAX_RECOMMENDED_TX_SIZE,
   JsonObject,
   JsonValue,
   Peer,

packages/cojson/src/ids.ts

@@ -1,8 +1,8 @@
 import { base58 } from "@scure/base";
-import { CoID } from "./coValue.js";
-import { RawAccountID } from "./coValues/account.js";
+import type { CoID } from "./coValue.js";
+import type { RawAccountID } from "./coValues/account.js";
+import type { RawGroup } from "./coValues/group.js";
 import { shortHashLength } from "./crypto/crypto.js";
-import { RawGroup } from "./exports.js";
 
 export type RawCoID = `co_z${string}`;
 export type ParentGroupReference = `parent_${CoID<RawGroup>}`;

packages/cojson/src/localNode.ts

@@ -1,14 +1,15 @@
 import { Result, err, ok } from "neverthrow";
-import { CoID } from "./coValue.js";
-import { RawCoValue } from "./coValue.js";
+import { GarbageCollector } from "./GarbageCollector.js";
+import type { CoID } from "./coValue.js";
+import type { RawCoValue } from "./coValue.js";
 import {
-  AvailableCoValueCore,
+  type AvailableCoValueCore,
   CoValueCore,
   idforHeader,
 } from "./coValueCore/coValueCore.js";
 import {
-  CoValueHeader,
-  CoValueUniqueness,
+  type CoValueHeader,
+  type CoValueUniqueness,
   VerifiedState,
 } from "./coValueCore/verifiedState.js";
 import {
@@ -26,11 +27,11 @@ import {
   expectAccount,
 } from "./coValues/account.js";
 import {
-  InviteSecret,
-  RawGroup,
+  type InviteSecret,
+  type RawGroup,
   secretSeedFromInviteSecret,
 } from "./coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG } from "./config.js";
+import { CO_VALUE_LOADING_CONFIG, GARBAGE_COLLECTOR_CONFIG } from "./config.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { logger } from "./logger.js";
@@ -63,6 +64,7 @@ export class LocalNode {
   /** @category 3. Low-level */
   syncManager = new SyncManager(this);
 
+  garbageCollector: GarbageCollector | undefined = undefined;
   crashed: Error | undefined = undefined;
 
   storage?: StorageAPI;
@@ -78,6 +80,14 @@ export class LocalNode {
     this.crypto = crypto;
   }
 
+  enableGarbageCollector() {
+    if (this.garbageCollector) {
+      return;
+    }
+
+    this.garbageCollector = new GarbageCollector(this.coValues);
+  }
+
   setStorage(storage: StorageAPI) {
     this.storage = storage;
   }
@@ -95,6 +105,8 @@ export class LocalNode {
       this.coValues.set(id, entry);
     }
 
+    this.garbageCollector?.trackCoValueAccess(entry);
+
     return entry;
   }
 
@@ -351,7 +363,8 @@ export class LocalNode {
       new VerifiedState(id, this.crypto, header, new Map()),
     );
 
-    void this.syncManager.requestCoValueSync(coValue);
+    this.garbageCollector?.trackCoValueAccess(coValue);
+    this.syncManager.syncHeader(coValue.verified);
 
     return coValue;
   }
@@ -738,9 +751,15 @@ export class LocalNode {
     }
   }
 
-  gracefulShutdown() {
-    this.storage?.close();
+  /**
+   * Closes all the peer connections, drains all the queues and closes the storage.
+   *
+   * @returns Promise of the current pending store operation, if any.
+   */
+  gracefulShutdown(): Promise<unknown> | undefined {
     this.syncManager.gracefulShutdown();
+    this.garbageCollector?.stop();
+    return this.storage?.close();
   }
 }
 

packages/cojson/src/queue/LocalTransactionsSyncQueue.ts

@@ -0,0 +1,96 @@
+import {
+  addTransactionToContentMessage,
+  createContentMessage,
+} from "../coValueContentMessage.js";
+import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
+import { Signature } from "../crypto/crypto.js";
+import { SessionID } from "../ids.js";
+import { NewContentMessage } from "../sync.js";
+import { LinkedList } from "./LinkedList.js";
+
+/**
+ * This queue is used to batch the sync of local transactions while preserving the order of updates between CoValues.
+ *
+ * We need to preserve the order of updates between CoValues to keep the state always consistent in case of shutdown in the middle of a sync.
+ *
+ * Examples:
+ * 1. When we extend a Group we need to always ensure that the parent group is persisted before persisting the extension transaction.
+ * 2. If we do multiple updates on the same CoMap, the updates will be batched because it's safe to do so.
+ */
+export class LocalTransactionsSyncQueue {
+  private readonly queue = new LinkedList<NewContentMessage>();
+
+  constructor(private readonly sync: (content: NewContentMessage) => void) {}
+
+  syncHeader = (coValue: VerifiedState) => {
+    const lastPendingSync = this.queue.tail?.value;
+
+    if (lastPendingSync?.id === coValue.id) {
+      return;
+    }
+
+    this.enqueue(createContentMessage(coValue.id, coValue.header));
+  };
+
+  syncTransaction = (
+    coValue: VerifiedState,
+    transaction: Transaction,
+    sessionID: SessionID,
+    signature: Signature,
+    txIdx: number,
+  ) => {
+    const lastPendingSync = this.queue.tail?.value;
+    const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
+    const isSignatureCheckpoint =
+      lastSignatureIdx > -1 && lastSignatureIdx === txIdx;
+
+    if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
+      addTransactionToContentMessage(
+        lastPendingSync,
+        transaction,
+        sessionID,
+        signature,
+        txIdx,
+      );
+
+      return;
+    }
+
+    const content = createContentMessage(coValue.id, coValue.header, false);
+
+    addTransactionToContentMessage(
+      content,
+      transaction,
+      sessionID,
+      signature,
+      txIdx,
+    );
+
+    this.enqueue(content);
+  };
+
+  enqueue(content: NewContentMessage) {
+    this.queue.push(content);
+
+    this.processPendingSyncs();
+  }
+
+  private processingSyncs = false;
+  processPendingSyncs() {
+    if (this.processingSyncs) return;
+
+    this.processingSyncs = true;
+
+    queueMicrotask(() => {
+      while (this.queue.head) {
+        const content = this.queue.head.value;
+
+        this.sync(content);
+
+        this.queue.shift();
+      }
+
+      this.processingSyncs = false;
+    });
+  }
+}

packages/cojson/src/queue/StoreQueue.ts

@@ -1,19 +1,53 @@
+import { CorrectionCallback } from "../exports.js";
 import { logger } from "../logger.js";
-import { CoValueKnownState, NewContentMessage } from "../sync.js";
+import { NewContentMessage } from "../sync.js";
 import { LinkedList } from "./LinkedList.js";
 
 type StoreQueueEntry = {
-  data: NewContentMessage[];
-  correctionCallback: (data: CoValueKnownState) => void;
+  data: NewContentMessage;
+  correctionCallback: CorrectionCallback;
 };
 
-export class StoreQueue {
-  private queue = new LinkedList<StoreQueueEntry>();
+class StoreQueueManager {
+  private backlog = new LinkedList<{
+    queue: StoreQueue;
+    callback: () => Promise<unknown>;
+  }>();
+
+  private processing = false;
+
+  async schedule(queue: StoreQueue, callback: () => Promise<unknown>) {
+    this.backlog.push({ queue, callback });
+
+    if (this.processing) {
+      return;
+    }
+
+    this.processing = true;
+
+    while (this.backlog.head) {
+      const entry = this.backlog.head;
+
+      await entry.value.callback();
+
+      this.backlog.shift();
+    }
+
+    this.processing = false;
+  }
+}
+
+export class StoreQueue {
+  static manager = new StoreQueueManager();
+
+  private queue = new LinkedList<StoreQueueEntry>();
+  closed = false;
+
+  public push(data: NewContentMessage, correctionCallback: CorrectionCallback) {
+    if (this.closed) {
+      return;
+    }
 
-  public push(
-    data: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
-  ) {
     this.queue.push({ data, correctionCallback });
   }
 
@@ -22,12 +56,13 @@ export class StoreQueue {
   }
 
   processing = false;
+  lastCallback: Promise<unknown> | undefined;
 
-  async processQueue(
+  processQueue(
     callback: (
-      data: NewContentMessage[],
-      correctionCallback: (data: CoValueKnownState) => void,
-    ) => Promise<void>,
+      data: NewContentMessage,
+      correctionCallback: CorrectionCallback,
+    ) => Promise<unknown>,
   ) {
     if (this.processing) {
       return;
@@ -35,22 +70,30 @@ export class StoreQueue {
 
     this.processing = true;
 
-    let entry: StoreQueueEntry | undefined;
+    return StoreQueue.manager.schedule(this, async () => {
+      let entry: StoreQueueEntry | undefined;
 
-    while ((entry = this.pull())) {
-      const { data, correctionCallback } = entry;
+      while ((entry = this.pull())) {
+        const { data, correctionCallback } = entry;
 
-      try {
-        await callback(data, correctionCallback);
-      } catch (err) {
-        logger.error("Error processing message in store queue", { err });
+        try {
+          this.lastCallback = callback(data, correctionCallback);
+          await this.lastCallback;
+        } catch (err) {
+          logger.error("Error processing message in store queue", { err });
+        }
       }
-    }
 
-    this.processing = false;
+      this.lastCallback = undefined;
+      this.processing = false;
+    });
   }
 
-  drain() {
+  close() {
+    this.closed = true;
+
     while (this.pull()) {}
+
+    return this.lastCallback;
   }
 }

packages/cojson/src/storage/sqlite/client.ts

@@ -112,24 +112,34 @@ export class SQLiteClient implements DBClientInterfaceSync {
     ) as SignatureAfterRow[];
   }
 
-  addCoValue(msg: NewContentMessage): number {
+  getCoValueRowID(id: RawCoID): number | undefined {
+    const row = this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  upsertCoValue(id: RawCoID, header?: CoValueHeader): number | undefined {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
   }
 
-  addSessionUpdate({
-    sessionUpdate,
-  }: {
-    sessionUpdate: SessionRow;
-  }): number {
+  addSessionUpdate({ sessionUpdate }: { sessionUpdate: SessionRow }): number {
     const result = this.db.get<{ rowID: number }>(
       `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
                             ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
@@ -166,7 +176,11 @@ export class SQLiteClient implements DBClientInterfaceSync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

packages/cojson/src/storage/sqliteAsync/client.ts

@@ -112,14 +112,31 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     );
   }
 
-  async addCoValue(msg: NewContentMessage): Promise<number> {
+  async getCoValueRowID(id: RawCoID): Promise<number | undefined> {
+    const row = await this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  async upsertCoValue(
+    id: RawCoID,
+    header?: CoValueHeader,
+  ): Promise<number | undefined> {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = await this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING 
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
@@ -166,7 +183,11 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

packages/cojson/src/storage/storageAsync.ts

@@ -1,11 +1,15 @@
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   type CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
   type RawCoID,
   type SessionID,
   type StorageAPI,
+  logger,
 } from "../exports.js";
-import { getPriorityFromHeader } from "../priority.js";
 import { StoreQueue } from "../queue/StoreQueue.js";
 import {
   CoValueKnownState,
@@ -13,8 +17,13 @@ import {
   emptyKnownState,
 } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
-import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import {
+  collectNewTxs,
+  getDependedOnCoValues,
+  getNewTransactionsSize,
+} from "./syncUtils.js";
 import type {
+  CorrectionCallback,
   DBClientInterfaceAsync,
   SignatureAfterRow,
   StoredCoValueRow,
@@ -82,6 +91,7 @@ export class StorageApiAsync implements StorageAPI {
     );
 
     const knownState = this.knwonStates.getKnownState(coValueRow.id);
+    knownState.header = true;
 
     for (const sessionRow of allCoValueSessions) {
       knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -89,13 +99,7 @@ export class StorageApiAsync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = {
-      action: "content",
-      id: coValueRow.id,
-      header: coValueRow.header,
-      new: {},
-      priority: getPriorityFromHeader(coValueRow.header),
-    } as NewContentMessage;
+    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
     if (contentStreaming) {
       contentMessage.expectContentUntil = knownState["sessions"];
@@ -136,13 +140,10 @@ export class StorageApiAsync implements StorageAPI {
             contentMessage,
             callback,
           );
-          contentMessage = {
-            action: "content",
-            id: coValueRow.id,
-            header: coValueRow.header,
-            new: {},
-            priority: getPriorityFromHeader(coValueRow.header),
-          } satisfies NewContentMessage;
+          contentMessage = createContentMessage(
+            coValueRow.id,
+            coValueRow.header,
+          );
         }
       }
     }
@@ -194,50 +195,76 @@ export class StorageApiAsync implements StorageAPI {
 
   storeQueue = new StoreQueue();
 
-  async store(
-    msgs: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
-  ) {
+  async store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
     /**
      * The store operations must be done one by one, because we can't start a new transaction when there
      * is already a transaction open.
      */
-    this.storeQueue.push(msgs, correctionCallback);
+    this.storeQueue.push(msg, correctionCallback);
 
     this.storeQueue.processQueue(async (data, correctionCallback) => {
-      for (const msg of data) {
-        const success = await this.storeSingle(msg, correctionCallback);
-
-        if (!success) {
-          // Stop processing the messages for this entry, because the data is out of sync with storage
-          // and the other transactions will be rejected anyway.
-          break;
-        }
-      }
+      return this.storeSingle(data, correctionCallback);
     });
   }
 
+  /**
+   * This function is called when the storage lacks the information required to store the incoming content.
+   *
+   * It triggers a `correctionCallback` to ask the syncManager to provide the missing information.
+   *
+   * The correction is applied immediately, to ensure that, when applicable, the dependent content in the queue won't require additional corrections.
+   */
+  private async handleCorrection(
+    knownState: CoValueKnownState,
+    correctionCallback: CorrectionCallback,
+  ) {
+    const correction = correctionCallback(knownState);
+
+    if (!correction) {
+      logger.error("Correction callback returned undefined", {
+        knownState,
+        correction: correction ?? null,
+      });
+      return false;
+    }
+
+    for (const msg of correction) {
+      const success = await this.storeSingle(msg, (knownState) => {
+        logger.error("Double correction requested", {
+          msg,
+          knownState,
+        });
+        return undefined;
+      });
+
+      if (!success) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
   private async storeSingle(
     msg: NewContentMessage,
-    correctionCallback: (data: CoValueKnownState) => void,
+    correctionCallback: CorrectionCallback,
   ): Promise<boolean> {
-    const id = msg.id;
-    const coValueRow = await this.dbClient.getCoValue(id);
-
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
-
-    if (invalidAssumptionOnHeaderPresence) {
-      const knownState = emptyKnownState(id as RawCoID);
-      this.knwonStates.setKnownState(id, knownState);
-
-      correctionCallback(knownState);
+    if (this.storeQueue.closed) {
       return false;
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : await this.dbClient.addCoValue(msg);
+    const id = msg.id;
+    const storedCoValueRowID = await this.dbClient.upsertCoValue(
+      id,
+      msg.header,
+    );
+
+    if (!storedCoValueRowID) {
+      const knownState = emptyKnownState(id as RawCoID);
+      this.knwonStates.setKnownState(id, knownState);
+
+      return this.handleCorrection(knownState, correctionCallback);
+    }
 
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;
@@ -276,8 +303,7 @@ export class StorageApiAsync implements StorageAPI {
     this.knwonStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     return true;
@@ -290,38 +316,31 @@ export class StorageApiAsync implements StorageAPI {
     storedCoValueRowID: number,
   ) {
     const newTransactions = msg.new[sessionID]?.newTransactions || [];
+    const lastIdx = sessionRow?.lastIdx || 0;
 
-    const actuallyNewOffset =
-      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+    const actuallyNewOffset = lastIdx - (msg.new[sessionID]?.after || 0);
 
     const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
 
     if (actuallyNewTransactions.length === 0) {
-      return sessionRow?.lastIdx || 0;
+      return lastIdx;
     }
 
-    let newBytesSinceLastSignature =
-      (sessionRow?.bytesSinceLastSignature || 0) +
-      actuallyNewTransactions.reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+    let bytesSinceLastSignature = sessionRow?.bytesSinceLastSignature || 0;
+    const newTransactionsSize = getNewTransactionsSize(actuallyNewTransactions);
 
-    const newLastIdx =
-      (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+    const newLastIdx = lastIdx + actuallyNewTransactions.length;
 
     let shouldWriteSignature = false;
 
-    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(bytesSinceLastSignature, newTransactionsSize)) {
       shouldWriteSignature = true;
-      newBytesSinceLastSignature = 0;
+      bytesSinceLastSignature = 0;
+    } else {
+      bytesSinceLastSignature += newTransactionsSize;
     }
 
-    const nextIdx = sessionRow?.lastIdx || 0;
+    const nextIdx = lastIdx;
 
     if (!msg.new[sessionID]) throw new Error("Session ID not found");
 
@@ -330,7 +349,7 @@ export class StorageApiAsync implements StorageAPI {
       sessionID,
       lastIdx: newLastIdx,
       lastSignature: msg.new[sessionID].lastSignature,
-      bytesSinceLastSignature: newBytesSinceLastSignature,
+      bytesSinceLastSignature,
     };
 
     const sessionRowID: number = await this.dbClient.addSessionUpdate({
@@ -360,7 +379,6 @@ export class StorageApiAsync implements StorageAPI {
   }
 
   close() {
-    // Drain the store queue
-    this.storeQueue.drain();
+    return this.storeQueue.close();
   }
 }

packages/cojson/src/storage/storageSync.ts

@@ -1,20 +1,29 @@
 import { UpDownCounter, metrics } from "@opentelemetry/api";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
   RawCoID,
   type SessionID,
   type StorageAPI,
+  logger,
 } from "../exports.js";
-import { getPriorityFromHeader } from "../priority.js";
 import {
   CoValueKnownState,
   NewContentMessage,
   emptyKnownState,
 } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
-import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import {
+  collectNewTxs,
+  getDependedOnCoValues,
+  getNewTransactionsSize,
+} from "./syncUtils.js";
 import type {
+  CorrectionCallback,
   DBClientInterfaceSync,
   SignatureAfterRow,
   StoredCoValueRow,
@@ -84,6 +93,7 @@ export class StorageApiSync implements StorageAPI {
     }
 
     const knownState = this.knwonStates.getKnownState(coValueRow.id);
+    knownState.header = true;
 
     for (const sessionRow of allCoValueSessions) {
       knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -91,13 +101,7 @@ export class StorageApiSync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = {
-      action: "content",
-      id: coValueRow.id,
-      header: coValueRow.header,
-      new: {},
-      priority: getPriorityFromHeader(coValueRow.header),
-    } as NewContentMessage;
+    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
     if (contentStreaming) {
       this.streamingCounter.add(1);
@@ -137,13 +141,10 @@ export class StorageApiSync implements StorageAPI {
             contentMessage,
             callback,
           );
-          contentMessage = {
-            action: "content",
-            id: coValueRow.id,
-            header: coValueRow.header,
-            new: {},
-            priority: getPriorityFromHeader(coValueRow.header),
-          } satisfies NewContentMessage;
+          contentMessage = createContentMessage(
+            coValueRow.id,
+            coValueRow.header,
+          );
 
           // Introduce a delay to not block the main thread
           // for the entire content processing
@@ -189,42 +190,60 @@ export class StorageApiSync implements StorageAPI {
     pushCallback(contentMessage);
   }
 
-  store(
-    msgs: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
+  store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
+    return this.storeSingle(msg, correctionCallback);
+  }
+
+  /**
+   * This function is called when the storage lacks the information required to store the incoming content.
+   *
+   * It triggers a `correctionCallback` to ask the syncManager to provide the missing information.
+   */
+  private handleCorrection(
+    knownState: CoValueKnownState,
+    correctionCallback: CorrectionCallback,
   ) {
-    for (const msg of msgs) {
-      const success = this.storeSingle(msg, correctionCallback);
+    const correction = correctionCallback(knownState);
+
+    if (!correction) {
+      logger.error("Correction callback returned undefined", {
+        knownState,
+        correction: correction ?? null,
+      });
+      return false;
+    }
+
+    for (const msg of correction) {
+      const success = this.storeSingle(msg, (knownState) => {
+        logger.error("Double correction requested", {
+          msg,
+          knownState,
+        });
+        return undefined;
+      });
 
       if (!success) {
         return false;
       }
     }
+
+    return true;
   }
 
   private storeSingle(
     msg: NewContentMessage,
-    correctionCallback: (data: CoValueKnownState) => void,
+    correctionCallback: CorrectionCallback,
   ): boolean {
     const id = msg.id;
-    const coValueRow = this.dbClient.getCoValue(id);
+    const storedCoValueRowID = this.dbClient.upsertCoValue(id, msg.header);
 
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
-
-    if (invalidAssumptionOnHeaderPresence) {
+    if (!storedCoValueRowID) {
       const knownState = emptyKnownState(id as RawCoID);
-      correctionCallback(knownState);
-
       this.knwonStates.setKnownState(id, knownState);
 
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : this.dbClient.addCoValue(msg);
-
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;
 
@@ -258,8 +277,7 @@ export class StorageApiSync implements StorageAPI {
     this.knwonStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     return true;
@@ -272,35 +290,29 @@ export class StorageApiSync implements StorageAPI {
     storedCoValueRowID: number,
   ) {
     const newTransactions = msg.new[sessionID]?.newTransactions || [];
+    const lastIdx = sessionRow?.lastIdx || 0;
 
-    const actuallyNewOffset =
-      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+    const actuallyNewOffset = lastIdx - (msg.new[sessionID]?.after || 0);
 
     const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
 
     if (actuallyNewTransactions.length === 0) {
-      return sessionRow?.lastIdx || 0;
+      return lastIdx;
     }
 
-    let newBytesSinceLastSignature =
-      (sessionRow?.bytesSinceLastSignature || 0) +
-      actuallyNewTransactions.reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+    let bytesSinceLastSignature = sessionRow?.bytesSinceLastSignature || 0;
+    const newTransactionsSize = getNewTransactionsSize(actuallyNewTransactions);
 
     const newLastIdx =
       (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
 
     let shouldWriteSignature = false;
 
-    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(bytesSinceLastSignature, newTransactionsSize)) {
       shouldWriteSignature = true;
-      newBytesSinceLastSignature = 0;
+      bytesSinceLastSignature = 0;
+    } else {
+      bytesSinceLastSignature += newTransactionsSize;
     }
 
     const nextIdx = sessionRow?.lastIdx || 0;
@@ -312,7 +324,7 @@ export class StorageApiSync implements StorageAPI {
       sessionID,
       lastIdx: newLastIdx,
       lastSignature: msg.new[sessionID].lastSignature,
-      bytesSinceLastSignature: newBytesSinceLastSignature,
+      bytesSinceLastSignature,
     };
 
     const sessionRowID: number = this.dbClient.addSessionUpdate({
@@ -339,5 +351,7 @@ export class StorageApiSync implements StorageAPI {
     return this.knwonStates.waitForSync(id, coValue);
   }
 
-  close() {}
+  close() {
+    return undefined;
+  }
 }