Merge pull request #2700 from garden-co/changeset-release/main

Version Packages

examples/chat-svelte/CHANGELOG.md

@@ -1,5 +1,12 @@
 # passkey-svelte
 
+## 0.0.110
+
+### Patch Changes
+
+- Updated dependencies [16764f6]
+  - jazz-tools@0.16.4
+
 ## 0.0.109
 
 ### Patch Changes

examples/chat-svelte/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chat-svelte",
-  "version": "0.0.109",
+  "version": "0.0.110",
   "type": "module",
   "private": true,
   "scripts": {

examples/form/src/CreateOrder.tsx

@@ -1,5 +1,4 @@
 import { useIframeHashRouter } from "hash-slash";
-import { Loaded } from "jazz-tools";
 import { useAccount, useCoState } from "jazz-tools/react";
 import { useState } from "react";
 import { Errors } from "./Errors.tsx";
@@ -21,7 +20,7 @@ export function CreateOrder() {
 
   if (!me?.root) return;
 
-  const onSave = (draft: Loaded<typeof DraftBubbleTeaOrder>) => {
+  const onSave = (draft: DraftBubbleTeaOrder) => {
     const validation = validateDraftOrder(draft);
     setErrors(validation.errors);
     if (validation.errors.length > 0) {
@@ -29,7 +28,7 @@ export function CreateOrder() {
     }
 
     // turn the draft into a real order
-    me.root.orders.push(draft as Loaded<typeof BubbleTeaOrder>);
+    me.root.orders.push(draft as BubbleTeaOrder);
 
     // reset the draft
     me.root.draft = DraftBubbleTeaOrder.create({
@@ -59,7 +58,7 @@ function CreateOrderForm({
   onSave,
 }: {
   id: string;
-  onSave: (draft: Loaded<typeof DraftBubbleTeaOrder>) => void;
+  onSave: (draft: DraftBubbleTeaOrder) => void;
 }) {
   const draft = useCoState(DraftBubbleTeaOrder, id, {
     resolve: { addOns: true, instructions: true },

examples/form/src/OrderForm.tsx

@@ -1,4 +1,4 @@
-import { CoPlainText, Loaded } from "jazz-tools";
+import { CoPlainText } from "jazz-tools";
 import {
   BubbleTeaAddOnTypes,
   BubbleTeaBaseTeaTypes,
@@ -10,7 +10,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: Loaded<typeof BubbleTeaOrder> | Loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   // Handles updates to the instructions field of the order.

examples/form/src/OrderThumbnail.tsx

@@ -1,10 +1,9 @@
-import { Loaded } from "jazz-tools";
 import { BubbleTeaOrder } from "./schema.ts";
 
 export function OrderThumbnail({
   order,
 }: {
-  order: Loaded<typeof BubbleTeaOrder>;
+  order: BubbleTeaOrder;
 }) {
   const { id, baseTea, addOns, instructions, deliveryDate, withMilk } = order;
   const date = deliveryDate.toLocaleDateString();

examples/form/src/schema.ts

@@ -1,4 +1,4 @@
-import { Loaded, co, z } from "jazz-tools";
+import { co, z } from "jazz-tools";
 
 export const BubbleTeaAddOnTypes = [
   "Pearl",
@@ -18,8 +18,9 @@ export const BubbleTeaBaseTeaTypes = [
 export const ListOfBubbleTeaAddOns = co.list(
   z.literal([...BubbleTeaAddOnTypes]),
 );
+export type ListOfBubbleTeaAddOns = co.loaded<typeof ListOfBubbleTeaAddOns>;
 
-function hasAddOnsChanges(list?: Loaded<typeof ListOfBubbleTeaAddOns> | null) {
+function hasAddOnsChanges(list?: ListOfBubbleTeaAddOns | null) {
   return list && Object.entries(list._raw.insertions).length > 0;
 }
 
@@ -30,16 +31,12 @@ export const BubbleTeaOrder = co.map({
   withMilk: z.boolean(),
   instructions: co.optional(co.plainText()),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  baseTea: z.optional(z.literal([...BubbleTeaBaseTeaTypes])),
-  addOns: co.optional(ListOfBubbleTeaAddOns),
-  deliveryDate: z.optional(z.date()),
-  withMilk: z.optional(z.boolean()),
-  instructions: co.optional(co.plainText()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(order: Loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(order: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!order.baseTea) {
@@ -52,7 +49,7 @@ export function validateDraftOrder(order: Loaded<typeof DraftBubbleTeaOrder>) {
   return { errors };
 }
 
-export function hasChanges(order?: Loaded<typeof DraftBubbleTeaOrder> | null) {
+export function hasChanges(order?: DraftBubbleTeaOrder | null) {
   return (
     !!order &&
     (Object.keys(order._edits).length > 1 || hasAddOnsChanges(order.addOns))

examples/music-player/package.json

@@ -38,6 +38,7 @@
     "postcss": "^8.4.27",
     "tailwindcss": "^4.1.10",
     "typescript": "5.6.2",
-    "vite": "^6.3.5"
+    "vite": "^6.3.5",
+    "vite-plugin-pwa": "^1.0.2"
   }
 }

examples/music-player/src/1_schema.ts

@@ -84,15 +84,14 @@ export const MusicaAccount = co
      *  You can use it to set up the account root and any other initial CoValues you need.
      */
     if (account.root === undefined) {
-      const tracks = co.list(MusicTrack).create([]);
       const rootPlaylist = Playlist.create({
-        tracks,
+        tracks: [],
         title: "",
       });
 
       account.root = MusicaAccountRoot.create({
         rootPlaylist,
-        playlists: co.list(Playlist).create([]),
+        playlists: [],
         activeTrack: undefined,
         activePlaylist: rootPlaylist,
         exampleDataLoaded: false,

examples/music-player/src/3_HomePage.tsx

@@ -11,6 +11,7 @@ import { uploadMusicTracks } from "./4_actions";
 import { MediaPlayer } from "./5_useMediaPlayer";
 import { FileUploadButton } from "./components/FileUploadButton";
 import { MusicTrackRow } from "./components/MusicTrackRow";
+import { PlayerControls } from "./components/PlayerControls";
 import { PlaylistTitleInput } from "./components/PlaylistTitleInput";
 import { SidePanel } from "./components/SidePanel";
 import { Button } from "./components/ui/button";
@@ -42,7 +43,11 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   const playlistId = params.playlistId ?? me?.root._refs.rootPlaylist.id;
 
   const playlist = useCoState(Playlist, playlistId, {
-    resolve: { tracks: true },
+    resolve: {
+      tracks: {
+        $each: true,
+      },
+    },
   });
 
   const isRootPlaylist = !params.playlistId;
@@ -66,8 +71,8 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   return (
     <SidebarInset className="flex flex-col h-screen text-gray-800 bg-blue-50">
       <div className="flex flex-1 overflow-hidden">
-        <SidePanel mediaPlayer={mediaPlayer} />
-        <main className="flex-1 p-6 overflow-y-auto overflow-x-hidden">
+        <SidePanel />
+        <main className="flex-1 p-6 overflow-y-auto overflow-x-hidden relative">
           <SidebarTrigger />
           <div className="flex items-center justify-between mb-6">
             {isRootPlaylist ? (
@@ -106,12 +111,12 @@ export function HomePage({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
                     onClick={() => {
                       mediaPlayer.setActiveTrack(track, playlist);
                     }}
-                    showAddToPlaylist={isRootPlaylist}
                   />
                 ),
             )}
           </ul>
         </main>
+        <PlayerControls mediaPlayer={mediaPlayer} />
       </div>
     </SidebarInset>
   );

examples/music-player/src/4_actions.ts

@@ -1,11 +1,6 @@
 import { getAudioFileData } from "@/lib/audio/getAudioFileData";
-import { FileStream, Group, co } from "jazz-tools";
-import {
-  MusicTrack,
-  MusicTrackWaveform,
-  MusicaAccount,
-  Playlist,
-} from "./1_schema";
+import { FileStream, Group } from "jazz-tools";
+import { MusicTrack, MusicaAccount, Playlist } from "./1_schema";
 
 /**
  * Walkthrough: Actions
@@ -51,7 +46,7 @@ export async function uploadMusicTracks(
       {
         file: fileStream,
         duration: data.duration,
-        waveform: MusicTrackWaveform.create({ data: data.waveform }, group),
+        waveform: { data: data.waveform },
         title: file.name,
         isExampleTrack,
       },
@@ -73,18 +68,10 @@ export async function createNewPlaylist() {
     },
   });
 
-  // Since playlists are meant to be shared we associate them
-  // to a group which will contain the keys required to get
-  // access to the "owned" values
-  const playlistGroup = Group.create();
-
-  const playlist = Playlist.create(
-    {
-      title: "New Playlist",
-      tracks: co.list(MusicTrack).create([], playlistGroup),
-    },
-    playlistGroup,
-  );
+  const playlist = Playlist.create({
+    title: "New Playlist",
+    tracks: [],
+  });
 
   // Again, we associate the new playlist to the
   // user by pushing it into the playlists CoList

examples/music-player/src/components/ConfirmDialog.tsx

@@ -0,0 +1,59 @@
+import { Button } from "@/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+
+interface ConfirmDialogProps {
+  isOpen: boolean;
+  onOpenChange: (open: boolean) => void;
+  title: string;
+  description: string;
+  confirmText?: string;
+  cancelText?: string;
+  onConfirm: () => void;
+  variant?: "default" | "destructive";
+}
+
+export function ConfirmDialog({
+  isOpen,
+  onOpenChange,
+  title,
+  description,
+  confirmText = "Confirm",
+  cancelText = "Cancel",
+  onConfirm,
+  variant = "destructive",
+}: ConfirmDialogProps) {
+  function handleConfirm() {
+    onConfirm();
+    onOpenChange(false);
+  }
+
+  function handleCancel() {
+    onOpenChange(false);
+  }
+
+  return (
+    <Dialog open={isOpen} onOpenChange={onOpenChange}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>{title}</DialogTitle>
+          <DialogDescription>{description}</DialogDescription>
+        </DialogHeader>
+        <DialogFooter>
+          <Button variant="outline" onClick={handleCancel}>
+            {cancelText}
+          </Button>
+          <Button variant={variant} onClick={handleConfirm}>
+            {confirmText}
+          </Button>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}

examples/music-player/src/components/MusicTrackRow.tsx

@@ -10,27 +10,34 @@ import { cn } from "@/lib/utils";
 import { Loaded } from "jazz-tools";
 import { useAccount, useCoState } from "jazz-tools/react";
 import { MoreHorizontal } from "lucide-react";
-import { Fragment } from "react/jsx-runtime";
-import { MusicTrackTitleInput } from "./MusicTrackTitleInput";
+import { Fragment, useCallback, useState } from "react";
+import { EditTrackDialog } from "./RenameTrackDialog";
 import { Button } from "./ui/button";
 
+function isPartOfThePlaylist(
+  trackId: string,
+  playlist: Loaded<typeof Playlist, { tracks: true }>,
+) {
+  return Array.from(playlist.tracks._refs).some((t) => t.id === trackId);
+}
+
 export function MusicTrackRow({
   trackId,
   isLoading,
   isPlaying,
   onClick,
-  showAddToPlaylist,
 }: {
   trackId: string;
   isLoading: boolean;
   isPlaying: boolean;
   onClick: (track: Loaded<typeof MusicTrack>) => void;
-  showAddToPlaylist: boolean;
 }) {
   const track = useCoState(MusicTrack, trackId);
+  const [isEditDialogOpen, setIsEditDialogOpen] = useState(false);
+  const [isDropdownOpen, setIsDropdownOpen] = useState(false);
 
   const { me } = useAccount(MusicaAccount, {
-    resolve: { root: { playlists: { $each: true } } },
+    resolve: { root: { playlists: { $each: { tracks: true } } } },
   });
 
   const playlists = me?.root.playlists ?? [];
@@ -60,10 +67,18 @@ export function MusicTrackRow({
     }
   }
 
+  function handleEdit() {
+    setIsEditDialogOpen(true);
+  }
+
+  const handleContextMenu = useCallback((e: React.MouseEvent) => {
+    e.preventDefault();
+    setIsDropdownOpen(true);
+  }, []);
+
   return (
     <li
       className={"flex gap-1 hover:bg-slate-200 group py-2 px-2 cursor-pointer"}
-      onClick={handleTrackClick}
     >
       <button
         className={cn(
@@ -81,50 +96,57 @@ export function MusicTrackRow({
           "▶️"
         )}
       </button>
-      <MusicTrackTitleInput trackId={trackId} />
+      <button
+        onContextMenu={handleContextMenu}
+        onClick={handleTrackClick}
+        className="w-full flex items-center overflow-hidden text-ellipsis whitespace-nowrap"
+      >
+        {track?.title}
+      </button>
       <div onClick={(evt) => evt.stopPropagation()}>
-        {showAddToPlaylist && (
-          <DropdownMenu>
-            <DropdownMenuTrigger asChild>
-              <Button
-                variant="ghost"
-                className="h-8 w-8 p-0"
-                aria-label={`Open ${track?.title} menu`}
-              >
-                <span className="sr-only">Open menu</span>
-                <MoreHorizontal className="h-4 w-4" />
-              </Button>
-            </DropdownMenuTrigger>
-            <DropdownMenuContent align="end">
-              <DropdownMenuItem
-                key={`delete`}
-                onSelect={async () => {
-                  if (!track) return;
-                  deleteTrack();
-                }}
-              >
-                Delete
-              </DropdownMenuItem>
-              {playlists.map((playlist, index) => (
-                <Fragment key={index}>
-                  <DropdownMenuItem
-                    key={`add-${index}`}
-                    onSelect={() => handleAddToPlaylist(playlist)}
-                  >
-                    Add to {playlist.title}
-                  </DropdownMenuItem>
+        <DropdownMenu open={isDropdownOpen} onOpenChange={setIsDropdownOpen}>
+          <DropdownMenuTrigger asChild>
+            <Button
+              variant="ghost"
+              className="h-8 w-8 p-0"
+              aria-label={`Open ${track?.title} menu`}
+            >
+              <span className="sr-only">Open menu</span>
+              <MoreHorizontal className="h-4 w-4" />
+            </Button>
+          </DropdownMenuTrigger>
+          <DropdownMenuContent align="end">
+            <DropdownMenuItem onSelect={handleEdit}>Edit</DropdownMenuItem>
+            {playlists.map((playlist, index) => (
+              <Fragment key={index}>
+                {isPartOfThePlaylist(trackId, playlist) ? (
                   <DropdownMenuItem
                     key={`remove-${index}`}
                     onSelect={() => handleRemoveFromPlaylist(playlist)}
                   >
                     Remove from {playlist.title}
                   </DropdownMenuItem>
-                </Fragment>
-              ))}
-            </DropdownMenuContent>
-          </DropdownMenu>
-        )}
+                ) : (
+                  <DropdownMenuItem
+                    key={`add-${index}`}
+                    onSelect={() => handleAddToPlaylist(playlist)}
+                  >
+                    Add to {playlist.title}
+                  </DropdownMenuItem>
+                )}
+              </Fragment>
+            ))}
+          </DropdownMenuContent>
+        </DropdownMenu>
       </div>
+      {track && isEditDialogOpen && (
+        <EditTrackDialog
+          track={track}
+          isOpen={isEditDialogOpen}
+          onOpenChange={setIsEditDialogOpen}
+          onDelete={deleteTrack}
+        />
+      )}
     </li>
   );
 }

examples/music-player/src/components/PlayerControls.tsx

@@ -24,25 +24,25 @@ export function PlayerControls({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
   const activeTrackTitle = activeTrack.title;
 
   return (
-    <footer className="flex items-center justify-between p-4 gap-4 bg-white border-t border-gray-200 fixed bottom-0 left-0 right-0 w-full">
-      <div className="flex justify-center items-center space-x-2">
-        <div className="flex items-center space-x-4">
+    <footer className="flex items-center justify-between p-2 sm:p-4 gap-2 sm:gap-4 bg-white border-t border-gray-200 absolute bottom-0 left-0 right-0 w-full z-50">
+      <div className="flex justify-center items-center space-x-1 sm:space-x-2 flex-shrink-0">
+        <div className="flex items-center space-x-2 sm:space-x-4">
           <button
             onClick={mediaPlayer.playPrevTrack}
             className="text-blue-600 hover:text-blue-800"
             aria-label="Previous track"
           >
-            <SkipBack size={20} />
+            <SkipBack size={16} className="sm:w-5 sm:h-5" />
           </button>
           <button
             onClick={playState.toggle}
-            className="w-[42px] h-[42px] flex items-center justify-center bg-blue-600 rounded-full text-white hover:bg-blue-700"
+            className="w-8 h-8 sm:w-[42px] sm:h-[42px] flex items-center justify-center bg-blue-600 rounded-full text-white hover:bg-blue-700"
             aria-label={isPlaying ? "Pause active track" : "Play active track"}
           >
             {isPlaying ? (
-              <Pause size={24} fill="currentColor" />
+              <Pause size={16} className="sm:w-6 sm:h-6" fill="currentColor" />
             ) : (
-              <Play size={24} fill="currentColor" />
+              <Play size={16} className="sm:w-6 sm:h-6" fill="currentColor" />
             )}
           </button>
           <button
@@ -50,16 +50,22 @@ export function PlayerControls({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
             className="text-blue-600 hover:text-blue-800"
             aria-label="Next track"
           >
-            <SkipForward size={20} />
+            <SkipForward size={16} className="sm:w-5 sm:h-5" />
           </button>
         </div>
       </div>
-      <div className=" sm:hidden md:flex flex-col flex-shrink-1 items-center w-[75%]">
-        <Waveform track={activeTrack} height={30} />
+      <div className="md:hidden sm:hidden  lg:flex flex-1 justify-center items-center min-w-0 px-2">
+        <Waveform
+          track={activeTrack}
+          height={30}
+          className="h-5 sm:h-6 md:h-8 lg:h-10"
+        />
       </div>
-      <div className="flex flex-col items-end  gap-1 text-right min-w-fit w-[25%]">
-        <h4 className="font-medium text-blue-800">{activeTrackTitle}</h4>
-        <p className="text-sm text-gray-600">
+      <div className="flex flex-col items-end gap-1 text-right min-w-fit flex-shrink-0">
+        <h4 className="font-medium text-blue-800 text-sm sm:text-base truncate max-w-32 sm:max-w-80">
+          {activeTrackTitle}
+        </h4>
+        <p className="text-xs sm:text-sm text-gray-600 truncate max-w-32 sm:max-w-80">
           {activePlaylist?.title || "All tracks"}
         </p>
       </div>

examples/music-player/src/components/RenameTrackDialog.tsx

@@ -0,0 +1,108 @@
+import { MusicTrack } from "@/1_schema";
+import { updateMusicTrackTitle } from "@/4_actions";
+import { Button } from "@/components/ui/button";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import { Input } from "@/components/ui/input";
+import { Loaded } from "jazz-tools";
+import { useState } from "react";
+import { ConfirmDialog } from "./ConfirmDialog";
+
+interface EditTrackDialogProps {
+  track: Loaded<typeof MusicTrack>;
+  isOpen: boolean;
+  onOpenChange: (open: boolean) => void;
+  onDelete: () => void;
+}
+
+export function EditTrackDialog({
+  track,
+  isOpen,
+  onOpenChange,
+  onDelete,
+}: EditTrackDialogProps) {
+  const [newTitle, setNewTitle] = useState(track.title);
+  const [isDeleteConfirmOpen, setIsDeleteConfirmOpen] = useState(false);
+
+  function handleSave() {
+    if (track && newTitle.trim()) {
+      updateMusicTrackTitle(track, newTitle.trim());
+      onOpenChange(false);
+    }
+  }
+
+  function handleCancel() {
+    setNewTitle(track?.title || "");
+    onOpenChange(false);
+  }
+
+  function handleDeleteClick() {
+    setIsDeleteConfirmOpen(true);
+  }
+
+  function handleDeleteConfirm() {
+    onDelete();
+    onOpenChange(false);
+  }
+
+  function handleKeyDown(event: React.KeyboardEvent) {
+    if (event.key === "Enter") {
+      handleSave();
+    } else if (event.key === "Escape") {
+      handleCancel();
+    }
+  }
+
+  return (
+    <Dialog open={isOpen} onOpenChange={onOpenChange}>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>Edit Track</DialogTitle>
+          <DialogDescription>Edit "{track?.title}".</DialogDescription>
+        </DialogHeader>
+        <form className="py-4" onSubmit={handleSave}>
+          <Input
+            value={newTitle}
+            onChange={(e) => setNewTitle(e.target.value)}
+            onKeyDown={handleKeyDown}
+            placeholder="Enter track name..."
+            autoFocus
+          />
+        </form>
+        <DialogFooter className="flex justify-between">
+          <Button
+            variant="destructive"
+            onClick={handleDeleteClick}
+            className="mr-auto"
+          >
+            Delete Track
+          </Button>
+          <div className="flex gap-2">
+            <Button variant="outline" onClick={handleCancel}>
+              Cancel
+            </Button>
+            <Button onClick={handleSave} disabled={!newTitle.trim()}>
+              Save
+            </Button>
+          </div>
+        </DialogFooter>
+      </DialogContent>
+      <ConfirmDialog
+        isOpen={isDeleteConfirmOpen}
+        onOpenChange={setIsDeleteConfirmOpen}
+        title="Delete Track"
+        description={`Are you sure you want to delete "${track.title}"? This action cannot be undone.`}
+        confirmText="Delete"
+        cancelText="Cancel"
+        onConfirm={handleDeleteConfirm}
+        variant="destructive"
+      />
+    </Dialog>
+  );
+}

examples/music-player/src/components/SidePanel.tsx

@@ -1,10 +1,8 @@
-import { MusicTrack, MusicaAccount } from "@/1_schema";
+import { MusicaAccount } from "@/1_schema";
 import { createNewPlaylist, deletePlaylist } from "@/4_actions";
-import { MediaPlayer } from "@/5_useMediaPlayer";
 import {
   Sidebar,
   SidebarContent,
-  SidebarFooter,
   SidebarGroup,
   SidebarGroupContent,
   SidebarGroupLabel,
@@ -14,22 +12,18 @@ import {
   SidebarMenuButton,
   SidebarMenuItem,
 } from "@/components/ui/sidebar";
-import { usePlayState } from "@/lib/audio/usePlayState";
-import { useAccount, useCoState } from "jazz-tools/react";
-import { Home, Music, Pause, Play, Plus, Trash2 } from "lucide-react";
+import { useAccount } from "jazz-tools/react";
+import { Home, Music, Plus, Trash2 } from "lucide-react";
 import { useNavigate, useParams } from "react-router";
 import { AuthButton } from "./AuthButton";
 
-export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
+export function SidePanel() {
   const { playlistId } = useParams();
   const navigate = useNavigate();
   const { me } = useAccount(MusicaAccount, {
     resolve: { root: { playlists: { $each: true } } },
   });
 
-  const playState = usePlayState();
-  const isPlaying = playState.value === "play";
-
   function handleAllTracksClick() {
     navigate(`/`);
   }
@@ -50,12 +44,6 @@ export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
     navigate(`/playlist/${playlist.id}`);
   }
 
-  const activeTrack = useCoState(MusicTrack, mediaPlayer.activeTrackId, {
-    resolve: { waveform: true },
-  });
-
-  const activeTrackTitle = activeTrack?.title;
-
   return (
     <Sidebar>
       <SidebarHeader>
@@ -137,29 +125,6 @@ export function SidePanel({ mediaPlayer }: { mediaPlayer: MediaPlayer }) {
           </SidebarGroupContent>
         </SidebarGroup>
       </SidebarContent>
-      {activeTrack && (
-        <SidebarFooter>
-          <SidebarMenu>
-            <SidebarMenuItem className="flex justify-end">
-              <SidebarMenuButton
-                onClick={playState.toggle}
-                aria-label={
-                  isPlaying ? "Pause active track" : "Play active track"
-                }
-              >
-                <div className="w-[28px] h-[28px] flex items-center justify-center bg-blue-600 rounded-full text-white hover:bg-blue-700">
-                  {isPlaying ? (
-                    <Pause size={16} fill="currentColor" />
-                  ) : (
-                    <Play size={16} fill="currentColor" />
-                  )}
-                </div>
-                <span>{activeTrackTitle}</span>
-              </SidebarMenuButton>
-            </SidebarMenuItem>
-          </SidebarMenu>
-        </SidebarFooter>
-      )}
     </Sidebar>
   );
 }

examples/music-player/src/components/Waveform.tsx

@@ -7,6 +7,7 @@ import { useCoState } from "jazz-tools/react";
 export function Waveform(props: {
   track: Loaded<typeof MusicTrack>;
   height: number;
+  className?: string;
 }) {
   const { track, height } = props;
   const waveformData = useCoState(
@@ -36,7 +37,7 @@ export function Waveform(props: {
 
   return (
     <div
-      className="flex justify-center items-end w-full"
+      className={cn("flex justify-center items-end w-full", props.className)}
       style={{
         height,
         gap: 1,

examples/music-player/src/index.css

@@ -2,6 +2,12 @@
 
 @custom-variant dark (&:is(.dark *));
 
+html {
+  overflow: hidden;
+  max-width: 1200px;
+  position: relative;
+}
+
 :root {
   --background: hsl(0 0% 100%);
   --foreground: hsl(20 14.3% 4.1%);

examples/music-player/tests/pages/HomePage.ts

@@ -55,10 +55,20 @@ export class HomePage {
 
   async editTrackTitle(trackTitle: string, newTitle: string) {
     await this.page
-      .getByRole("textbox", {
-        name: `Edit track title: ${trackTitle}`,
+      .getByRole("button", {
+        name: `Open ${trackTitle} menu`,
       })
-      .fill(newTitle);
+      .click();
+
+    await this.page
+      .getByRole("menuitem", {
+        name: `Edit`,
+      })
+      .click();
+
+    await this.page.getByPlaceholder("Enter track name...").fill(newTitle);
+
+    await this.page.getByRole("button", { name: "Save" }).click();
   }
 
   async createPlaylist() {

examples/music-player/vite.config.ts

@@ -1,10 +1,21 @@
 import path from "path";
 import react from "@vitejs/plugin-react-swc";
 import { defineConfig } from "vite";
+import { VitePWA } from "vite-plugin-pwa";
 
 // https://vitejs.dev/config/
 export default defineConfig({
-  plugins: [react()],
+  plugins: [
+    react(),
+    VitePWA({
+      registerType: "autoUpdate",
+      strategies: "generateSW",
+      workbox: {
+        globPatterns: ["**/*.{js,css,html,ico,png,svg}"],
+        maximumFileSizeToCacheInBytes: 1024 * 1024 * 5,
+      },
+    }),
+  ],
   resolve: {
     alias: {
       "@": path.resolve(__dirname, "./src"),

homepage/homepage/content/docs/design-patterns/form.mdx

@@ -54,10 +54,10 @@ import { co, z, CoMap } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 ```
 </CodeGroup>
 
@@ -73,17 +73,17 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 // ---cut---
 // OrderForm.tsx
 export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -118,16 +118,16 @@ import * as React from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -177,10 +177,10 @@ import { useState, useEffect } from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({
   draft: DraftBubbleTeaOrder,
@@ -218,7 +218,7 @@ export function OrderForm({
 // CreateOrder.tsx
 export function CreateOrder() {
   const { me } = useAccount();
-  const [draft, setDraft] = useState<co.loaded<typeof DraftBubbleTeaOrder>>();
+  const [draft, setDraft] = useState<DraftBubbleTeaOrder>();
 
   useEffect(() => {
     setDraft(DraftBubbleTeaOrder.create({}));
@@ -228,7 +228,7 @@ export function CreateOrder() {
     e.preventDefault();
     if (!draft || !draft.name) return;
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>; // TODO: this should narrow correctly
+    const order = draft as BubbleTeaOrder; // TODO: this should narrow correctly
 
     console.log("Order created:", order);
   };
@@ -251,11 +251,15 @@ Update the schema to include a `validateDraftOrder` helper.
 import { co, z } from "jazz-tools";
 // ---cut---
 // schema.ts
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
+export const BubbleTeaOrder = co.map({
+  name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) { // [!code ++:9]
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
+
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) { // [!code ++:9]
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -279,12 +283,12 @@ import { useState, useEffect } from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -307,7 +311,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -330,7 +334,7 @@ export function OrderForm({
 // CreateOrder.tsx
 export function CreateOrder() {
   const { me } = useAccount();
-  const [draft, setDraft] = useState<co.loaded<typeof DraftBubbleTeaOrder>>();
+  const [draft, setDraft] = useState<DraftBubbleTeaOrder>();
 
   useEffect(() => {
     setDraft(DraftBubbleTeaOrder.create({}));
@@ -346,7 +350,7 @@ export function CreateOrder() {
       return;
     }
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>;
+    const order = draft as BubbleTeaOrder;
 
     console.log("Order created:", order);
   };
@@ -372,10 +376,10 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({ // [!code ++:15]
   draft: DraftBubbleTeaOrder,
@@ -403,10 +407,10 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
 export const AccountRoot = co.map({
   draft: DraftBubbleTeaOrder,
@@ -452,12 +456,12 @@ import { co, z } from "jazz-tools";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -492,7 +496,7 @@ export function OrderForm({
   order,
   onSave,
 }: {
-  order: co.loaded<typeof BubbleTeaOrder> | co.loaded<typeof DraftBubbleTeaOrder>;
+  order: BubbleTeaOrder | DraftBubbleTeaOrder;
   onSave?: (e: React.FormEvent<HTMLFormElement>) => void;
 }) {
   return (
@@ -533,7 +537,7 @@ export function CreateOrder() {
       return;
     }
 
-    const order = draft as co.loaded<typeof BubbleTeaOrder>;
+    const order = draft as BubbleTeaOrder;
     console.log("Order created:", order);
 
     // create a new empty draft
@@ -577,11 +581,15 @@ Simply add a `hasChanges` helper to your schema.
 import { co, z } from "jazz-tools";
 // ---cut---
 // schema.ts
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
+export const BubbleTeaOrder = co.map({
+  name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
+
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -591,7 +599,7 @@ export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>)
   return { errors };
 };
 
-export function hasChanges(draft?: co.loaded<typeof DraftBubbleTeaOrder>) { // [!code ++:3]
+export function hasChanges(draft?: DraftBubbleTeaOrder) { // [!code ++:3]
   return draft ? Object.keys(draft._edits).length : false;
 };
 ```
@@ -608,12 +616,12 @@ import * as React from "react";
 export const BubbleTeaOrder = co.map({
   name: z.string(),
 });
+export type BubbleTeaOrder = co.loaded<typeof BubbleTeaOrder>;
 
-export const DraftBubbleTeaOrder = co.map({
-  name: z.optional(z.string()),
-});
+export const DraftBubbleTeaOrder = BubbleTeaOrder.partial();
+export type DraftBubbleTeaOrder = co.loaded<typeof DraftBubbleTeaOrder>;
 
-export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function validateDraftOrder(draft: DraftBubbleTeaOrder) {
   const errors: string[] = [];
 
   if (!draft.name) {
@@ -623,7 +631,7 @@ export function validateDraftOrder(draft: co.loaded<typeof DraftBubbleTeaOrder>)
   return { errors };
 };
 
-export function hasChanges(draft?: co.loaded<typeof DraftBubbleTeaOrder>) {
+export function hasChanges(draft?: DraftBubbleTeaOrder) {
   return draft ? Object.keys(draft._edits).length : false;
 };
 

homepage/homepage/content/docs/using-covalues/comaps.mdx

@@ -228,6 +228,54 @@ export type Project = co.loaded<typeof Project>;
 ```
 </CodeGroup>
 
+### Partial
+
+For convenience Jazz provies a dedicated API for making all the properties of a CoMap optional:
+
+<CodeGroup>
+```ts twoslash
+import { co, z } from "jazz-tools";
+
+const Project = co.map({
+  name: z.string(),
+  startDate: z.date(),
+  status: z.literal(["planning", "active", "completed"]),
+});
+
+const ProjectDraft = Project.partial();
+
+// The fields are all optional now
+const project = ProjectDraft.create({});
+```
+</CodeGroup>
+
+### Pick
+
+You can also pick specific fields from a CoMap:
+
+<CodeGroup>
+```ts twoslash
+import { co, z } from "jazz-tools";
+
+const Project = co.map({
+  name: z.string(),
+  startDate: z.date(),
+  status: z.literal(["planning", "active", "completed"]),
+});
+
+const ProjectStep1 = Project.pick({
+  name: true,
+  startDate: true,
+});
+
+// We don't provide the status field
+const project = ProjectStep1.create({
+  name: "My project",
+  startDate: new Date("2025-04-01"),
+});
+```
+</CodeGroup>
+
 ### Working with Record CoMaps
 
 For record-type CoMaps, you can access values using bracket notation:

homepage/homepage/generate-docs/llms-full.mjs

@@ -5,6 +5,8 @@ import { readFile, readdir } from "fs/promises";
 import { DOC_SECTIONS } from "./utils/config.mjs";
 import { writeDocsFile } from "./utils/index.mjs";
 
+const exclude = [/\/upgrade\//];
+
 async function readMdxContent(url) {
   try {
     // Special case for the introduction
@@ -31,12 +33,17 @@ async function readMdxContent(url) {
 
     // If it's a directory, try to read all framework variants
     const fullPath = path.join(baseDir, relativePath);
+
+    if (exclude.some((pattern) => pattern.test(fullPath))) {
+      return null;
+    }
+
     try {
       const stats = await fs.stat(fullPath);
       if (stats.isDirectory()) {
         // Read all MDX files in the directory
         const files = await fs.readdir(fullPath);
-        const mdxFiles = files.filter((f) => f.endsWith(".mdx"));
+        const mdxFiles = files.filter((f) => f.endsWith(".mdx")).filter((f) => !exclude.some((pattern) => pattern.test(f)));
 
         if (mdxFiles.length === 0) return null;
 

package.json

@@ -31,7 +31,7 @@
     "build:packages": "turbo build --filter='./packages/*'",
     "lint": "turbo lint && cd homepage/homepage && pnpm run lint",
     "test": "vitest",
-    "test:ci": "vitest run --watch=false --coverage.enabled=true",
+    "test:ci": "vitest run --watch=false",
     "test:coverage": "vitest --ui --coverage.enabled=true",
     "format-and-lint": "biome check .",
     "format-and-lint:fix": "biome check . --write",

packages/cojson-storage-indexeddb/CHANGELOG.md

@@ -1,5 +1,13 @@
 # cojson-storage-indexeddb
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/cojson-storage-indexeddb/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cojson-storage-indexeddb",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "main": "dist/index.js",
   "type": "module",
   "types": "dist/index.d.ts",

packages/cojson-storage-indexeddb/src/tests/storage.indexeddb.test.ts

@@ -179,8 +179,9 @@ test("should load dependencies correctly (group inheritance)", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Group header: true new: After: 0 New: 5",
+      "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT ParentGroup header: true new: After: 0 New: 4",
+      "client -> CONTENT Group header: false new: After: 3 New: 2",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
     ]
   `);
@@ -561,9 +562,10 @@ test("should sync and load accounts from storage", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Account header: true new: After: 0 New: 4",
+      "client -> CONTENT Account header: true new: After: 0 New: 3",
       "client -> CONTENT ProfileGroup header: true new: After: 0 New: 5",
       "client -> CONTENT Profile header: true new: After: 0 New: 1",
+      "client -> CONTENT Account header: false new: After: 3 New: 1",
     ]
   `);
 

packages/cojson-storage-indexeddb/src/tests/testUtils.ts

@@ -36,12 +36,11 @@ export function trackMessages() {
   };
 
   StorageApiAsync.prototype.store = async function (data, correctionCallback) {
-    for (const msg of data) {
-      messages.push({
-        from: "client",
-        msg,
-      });
-    }
+    messages.push({
+      from: "client",
+      msg: data,
+    });
+
     return originalStore.call(this, data, (msg) => {
       messages.push({
         from: "storage",
@@ -51,7 +50,18 @@ export function trackMessages() {
           ...msg,
         },
       });
-      correctionCallback(msg);
+      const correctionMessages = correctionCallback(msg);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          messages.push({
+            from: "client",
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

packages/cojson-storage-sqlite/CHANGELOG.md

@@ -1,5 +1,13 @@
 # cojson-storage-sqlite
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/cojson-storage-sqlite/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cojson-storage-sqlite",
   "type": "module",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",

packages/cojson-storage-sqlite/src/tests/storage.sqlite.test.ts

@@ -211,8 +211,9 @@ test("should load dependencies correctly (group inheritance)", async () => {
     ),
   ).toMatchInlineSnapshot(`
     [
-      "client -> CONTENT Group header: true new: After: 0 New: 5",
+      "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT ParentGroup header: true new: After: 0 New: 4",
+      "client -> CONTENT Group header: false new: After: 3 New: 2",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
     ]
   `);
@@ -374,6 +375,8 @@ test("should recover from data loss", async () => {
     [
       "client -> CONTENT Group header: true new: After: 0 New: 3",
       "client -> CONTENT Map header: true new: After: 0 New: 1",
+      "client -> CONTENT Map header: false new: After: 3 New: 1",
+      "storage -> KNOWN CORRECTION Map sessions: header/4",
       "client -> CONTENT Map header: false new: After: 1 New: 3",
     ]
   `);
@@ -455,10 +458,7 @@ test("should recover missing dependencies from storage", async () => {
       data,
       correctionCallback,
     ) {
-      if (
-        data[0]?.id &&
-        [group.core.id, account.core.id as string].includes(data[0].id)
-      ) {
+      if ([group.core.id, account.core.id as string].includes(data.id)) {
         return false;
       }
 

packages/cojson-storage-sqlite/src/tests/testUtils.ts

@@ -36,12 +36,11 @@ export function trackMessages() {
   };
 
   StorageApiSync.prototype.store = function (data, correctionCallback) {
-    for (const msg of data) {
-      messages.push({
-        from: "client",
-        msg,
-      });
-    }
+    messages.push({
+      from: "client",
+      msg: data,
+    });
+
     return originalStore.call(this, data, (msg) => {
       messages.push({
         from: "storage",
@@ -51,7 +50,19 @@ export function trackMessages() {
           ...msg,
         },
       });
-      correctionCallback(msg);
+
+      const correctionMessages = correctionCallback(msg);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          messages.push({
+            from: "client",
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

packages/cojson-transport-ws/CHANGELOG.md

@@ -1,5 +1,13 @@
 # cojson-transport-nodejs-ws
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/cojson-transport-ws/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cojson-transport-ws",
   "type": "module",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",

packages/cojson/CHANGELOG.md

@@ -1,5 +1,12 @@
 # cojson
 
+## 0.16.4
+
+### Patch Changes
+
+- f9d538f: Fix the error raised when extending a group without having child groups loaded
+- 802b5a3: Refactor local updates sync to ensure that the changes are synced respecting the insertion order
+
 ## 0.16.3
 
 ## 0.16.2

packages/cojson/package.json

@@ -25,7 +25,7 @@
   },
   "type": "module",
   "license": "MIT",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "devDependencies": {
     "@opentelemetry/sdk-metrics": "^2.0.0",
     "libsql": "^0.5.13",

packages/cojson/src/coValueContentMessage.ts

@@ -0,0 +1,73 @@
+import {
+  CoValueHeader,
+  Transaction,
+  VerifiedState,
+} from "./coValueCore/verifiedState.js";
+import { MAX_RECOMMENDED_TX_SIZE } from "./config.js";
+import { Signature } from "./crypto/crypto.js";
+import { RawCoID, SessionID } from "./ids.js";
+import { getPriorityFromHeader } from "./priority.js";
+import { NewContentMessage, emptyKnownState } from "./sync.js";
+
+export function createContentMessage(
+  id: RawCoID,
+  header: CoValueHeader,
+  includeHeader = true,
+): NewContentMessage {
+  return {
+    action: "content",
+    id,
+    header: includeHeader ? header : undefined,
+    priority: getPriorityFromHeader(header),
+    new: {},
+  };
+}
+
+export function addTransactionToContentMessage(
+  content: NewContentMessage,
+  transaction: Transaction,
+  sessionID: SessionID,
+  signature: Signature,
+  txIdx: number,
+) {
+  const sessionContent = content.new[sessionID];
+
+  if (sessionContent) {
+    sessionContent.newTransactions.push(transaction);
+    sessionContent.lastSignature = signature;
+  } else {
+    content.new[sessionID] = {
+      after: txIdx,
+      newTransactions: [transaction],
+      lastSignature: signature,
+    };
+  }
+}
+
+export function getTransactionSize(transaction: Transaction) {
+  return transaction.privacy === "private"
+    ? transaction.encryptedChanges.length
+    : transaction.changes.length;
+}
+
+export function exceedsRecommendedSize(
+  baseSize: number,
+  transactionSize?: number,
+) {
+  if (transactionSize === undefined) {
+    return baseSize > MAX_RECOMMENDED_TX_SIZE;
+  }
+
+  return baseSize + transactionSize > MAX_RECOMMENDED_TX_SIZE;
+}
+
+export function knownStateFromContent(content: NewContentMessage) {
+  const knownState = emptyKnownState(content.id);
+
+  for (const [sessionID, session] of Object.entries(content.new)) {
+    knownState.sessions[sessionID as SessionID] =
+      session.after + session.newTransactions.length;
+  }
+
+  return knownState;
+}

packages/cojson/src/coValueCore/coValueCore.ts

@@ -2,9 +2,9 @@ import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { Result, err } from "neverthrow";
 import { PeerState } from "../PeerState.js";
 import { RawCoValue } from "../coValue.js";
-import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
+import { ControlledAccountOrAgent } from "../coValues/account.js";
 import { RawGroup } from "../coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG, MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import { CO_VALUE_LOADING_CONFIG } from "../config.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
@@ -380,7 +380,7 @@ export class CoValueCore {
   }
 
   knownStateWithStreaming(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownStateWithStreaming();
     } else {
       return emptyKnownState(this.id);
@@ -388,7 +388,7 @@ export class CoValueCore {
   }
 
   knownState(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownState();
     } else {
       return emptyKnownState(this.id);
@@ -605,8 +605,17 @@ export class CoValueCore {
     )._unsafeUnwrap({ withStackTrace: true });
 
     if (success) {
+      const session = this.verified.sessions.get(sessionID);
+      const txIdx = session ? session.transactions.length - 1 : 0;
+
       this.node.syncManager.recordTransactionsSize([transaction], "local");
-      void this.node.syncManager.requestCoValueSync(this);
+      this.node.syncManager.syncLocalTransaction(
+        this.verified,
+        transaction,
+        sessionID,
+        signature,
+        txIdx,
+      );
     }
 
     return success;

packages/cojson/src/coValueCore/verifiedState.ts

@@ -1,6 +1,10 @@
 import { Result, err, ok } from "neverthrow";
 import { AnyRawCoValue } from "../coValue.js";
-import { MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -14,7 +18,6 @@ import { RawCoID, SessionID, TransactionID } from "../ids.js";
 import { Stringified } from "../jsonStringify.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { PermissionsDef as RulesetDef } from "../permissions.js";
-import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { InvalidHashError, InvalidSignatureError } from "./coValueCore.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
@@ -151,6 +154,17 @@ export class VerifiedState {
     return ok(true as const);
   }
 
+  getLastSignatureCheckpoint(sessionID: SessionID): number {
+    const sessionLog = this.sessions.get(sessionID);
+
+    if (!sessionLog?.signatureAfter) return -1;
+
+    return Object.keys(sessionLog.signatureAfter).reduce(
+      (max, idx) => Math.max(max, parseInt(idx)),
+      -1,
+    );
+  }
+
   private doAddTransactions(
     sessionID: SessionID,
     newTransactions: Transaction[],
@@ -165,24 +179,14 @@ export class VerifiedState {
     }
 
     const signatureAfter = sessionLog?.signatureAfter ?? {};
-
-    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
-      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
-      -1,
-    );
+    const lastInbetweenSignatureIdx =
+      this.getLastSignatureCheckpoint(sessionID);
 
     const sizeOfTxsSinceLastInbetweenSignature = transactions
       .slice(lastInbetweenSignatureIdx + 1)
-      .reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+      .reduce((sum, tx) => sum + getTransactionSize(tx), 0);
 
-    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(sizeOfTxsSinceLastInbetweenSignature)) {
       signatureAfter[transactions.length - 1] = newSignature;
     }
 
@@ -242,13 +246,11 @@ export class VerifiedState {
       return this._cachedNewContentSinceEmpty;
     }
 
-    let currentPiece: NewContentMessage = {
-      action: "content",
-      id: this.id,
-      header: knownState?.header ? undefined : this.header,
-      priority: getPriorityFromHeader(this.header),
-      new: {},
-    };
+    let currentPiece: NewContentMessage = createContentMessage(
+      this.id,
+      this.header,
+      !knownState?.header,
+    );
 
     const pieces = [currentPiece];
 
@@ -299,25 +301,16 @@ export class VerifiedState {
         const oldPieceSize = pieceSize;
         for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
           const tx = log.transactions[txIdx]!;
-          pieceSize +=
-            tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length;
+          pieceSize += getTransactionSize(tx);
         }
 
-        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+        if (exceedsRecommendedSize(pieceSize)) {
           if (!currentPiece.expectContentUntil && pieces.length === 1) {
             currentPiece.expectContentUntil =
               this.knownStateWithStreaming().sessions;
           }
 
-          currentPiece = {
-            action: "content",
-            id: this.id,
-            header: undefined,
-            new: {},
-            priority: getPriorityFromHeader(this.header),
-          };
+          currentPiece = createContentMessage(this.id, this.header, false);
           pieces.push(currentPiece);
           pieceSize = pieceSize - oldPieceSize;
         }

packages/cojson/src/coValues/group.ts

@@ -669,19 +669,30 @@ export class RawGroup<
 
   /** Detect circular references in group inheritance */
   isSelfExtension(parent: RawGroup) {
-    if (parent.id === this.id) {
-      return true;
-    }
+    const checkedGroups = new Set<string>();
+    const queue = [parent];
 
-    const childGroups = this.getChildGroups();
+    while (true) {
+      const current = queue.pop();
 
-    for (const child of childGroups) {
-      if (child.isSelfExtension(parent)) {
+      if (!current) {
+        return false;
+      }
+
+      if (current.id === this.id) {
         return true;
       }
-    }
 
-    return false;
+      checkedGroups.add(current.id);
+
+      const parentGroups = current.getParentGroups();
+
+      for (const parent of parentGroups) {
+        if (!checkedGroups.has(parent.id)) {
+          queue.push(parent);
+        }
+      }
+    }
   }
 
   extend(
@@ -700,8 +711,8 @@ export class RawGroup<
 
     const value = role === "inherit" ? "extend" : role;
 
-    this.set(`parent_${parent.id}`, value, "trusting");
     parent.set(`child_${this.id}`, "extend", "trusting");
+    this.set(`parent_${parent.id}`, value, "trusting");
 
     if (
       parent.myRole() !== "admin" &&

packages/cojson/src/localNode.ts

@@ -351,7 +351,7 @@ export class LocalNode {
       new VerifiedState(id, this.crypto, header, new Map()),
     );
 
-    void this.syncManager.requestCoValueSync(coValue);
+    this.syncManager.syncHeader(coValue.verified);
 
     return coValue;
   }
@@ -738,9 +738,14 @@ export class LocalNode {
     }
   }
 
-  gracefulShutdown() {
-    this.storage?.close();
+  /**
+   * Closes all the peer connections, drains all the queues and closes the storage.
+   *
+   * @returns Promise of the current pending store operation, if any.
+   */
+  gracefulShutdown(): Promise<unknown> | undefined {
     this.syncManager.gracefulShutdown();
+    return this.storage?.close();
   }
 }
 

packages/cojson/src/queue/LocalTransactionsSyncQueue.ts

@@ -0,0 +1,96 @@
+import {
+  addTransactionToContentMessage,
+  createContentMessage,
+} from "../coValueContentMessage.js";
+import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
+import { Signature } from "../crypto/crypto.js";
+import { SessionID } from "../ids.js";
+import { NewContentMessage } from "../sync.js";
+import { LinkedList } from "./LinkedList.js";
+
+/**
+ * This queue is used to batch the sync of local transactions while preserving the order of updates between CoValues.
+ *
+ * We need to preserve the order of updates between CoValues to keep the state always consistent in case of shutdown in the middle of a sync.
+ *
+ * Examples:
+ * 1. When we extend a Group we need to always ensure that the parent group is persisted before persisting the extension transaction.
+ * 2. If we do multiple updates on the same CoMap, the updates will be batched because it's safe to do so.
+ */
+export class LocalTransactionsSyncQueue {
+  private readonly queue = new LinkedList<NewContentMessage>();
+
+  constructor(private readonly sync: (content: NewContentMessage) => void) {}
+
+  syncHeader = (coValue: VerifiedState) => {
+    const lastPendingSync = this.queue.tail?.value;
+
+    if (lastPendingSync?.id === coValue.id) {
+      return;
+    }
+
+    this.enqueue(createContentMessage(coValue.id, coValue.header));
+  };
+
+  syncTransaction = (
+    coValue: VerifiedState,
+    transaction: Transaction,
+    sessionID: SessionID,
+    signature: Signature,
+    txIdx: number,
+  ) => {
+    const lastPendingSync = this.queue.tail?.value;
+    const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
+    const isSignatureCheckpoint =
+      lastSignatureIdx > -1 && lastSignatureIdx === txIdx - 1;
+
+    if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
+      addTransactionToContentMessage(
+        lastPendingSync,
+        transaction,
+        sessionID,
+        signature,
+        txIdx,
+      );
+
+      return;
+    }
+
+    const content = createContentMessage(coValue.id, coValue.header, false);
+
+    addTransactionToContentMessage(
+      content,
+      transaction,
+      sessionID,
+      signature,
+      txIdx,
+    );
+
+    this.enqueue(content);
+  };
+
+  enqueue(content: NewContentMessage) {
+    this.queue.push(content);
+
+    this.processPendingSyncs();
+  }
+
+  private processingSyncs = false;
+  processPendingSyncs() {
+    if (this.processingSyncs) return;
+
+    this.processingSyncs = true;
+
+    queueMicrotask(() => {
+      while (this.queue.head) {
+        const content = this.queue.head.value;
+
+        this.sync(content);
+
+        this.queue.shift();
+      }
+
+      this.processingSyncs = false;
+    });
+  }
+}

packages/cojson/src/queue/StoreQueue.ts

@@ -1,19 +1,22 @@
+import { CorrectionCallback } from "../exports.js";
 import { logger } from "../logger.js";
-import { CoValueKnownState, NewContentMessage } from "../sync.js";
+import { NewContentMessage } from "../sync.js";
 import { LinkedList } from "./LinkedList.js";
 
 type StoreQueueEntry = {
-  data: NewContentMessage[];
-  correctionCallback: (data: CoValueKnownState) => void;
+  data: NewContentMessage;
+  correctionCallback: CorrectionCallback;
 };
 
 export class StoreQueue {
   private queue = new LinkedList<StoreQueueEntry>();
+  closed = false;
+
+  public push(data: NewContentMessage, correctionCallback: CorrectionCallback) {
+    if (this.closed) {
+      return;
+    }
 
-  public push(
-    data: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
-  ) {
     this.queue.push({ data, correctionCallback });
   }
 
@@ -22,12 +25,13 @@ export class StoreQueue {
   }
 
   processing = false;
+  lastCallback: Promise<unknown> | undefined;
 
   async processQueue(
     callback: (
-      data: NewContentMessage[],
-      correctionCallback: (data: CoValueKnownState) => void,
-    ) => Promise<void>,
+      data: NewContentMessage,
+      correctionCallback: CorrectionCallback,
+    ) => Promise<unknown>,
   ) {
     if (this.processing) {
       return;
@@ -41,16 +45,22 @@ export class StoreQueue {
       const { data, correctionCallback } = entry;
 
       try {
-        await callback(data, correctionCallback);
+        this.lastCallback = callback(data, correctionCallback);
+        await this.lastCallback;
       } catch (err) {
         logger.error("Error processing message in store queue", { err });
       }
     }
 
+    this.lastCallback = undefined;
     this.processing = false;
   }
 
-  drain() {
+  close() {
+    this.closed = true;
+
     while (this.pull()) {}
+
+    return this.lastCallback;
   }
 }

packages/cojson/src/storage/storageAsync.ts

@@ -1,11 +1,15 @@
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   type CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
   type RawCoID,
   type SessionID,
   type StorageAPI,
+  logger,
 } from "../exports.js";
-import { getPriorityFromHeader } from "../priority.js";
 import { StoreQueue } from "../queue/StoreQueue.js";
 import {
   CoValueKnownState,
@@ -13,8 +17,13 @@ import {
   emptyKnownState,
 } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
-import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import {
+  collectNewTxs,
+  getDependedOnCoValues,
+  getNewTransactionsSize,
+} from "./syncUtils.js";
 import type {
+  CorrectionCallback,
   DBClientInterfaceAsync,
   SignatureAfterRow,
   StoredCoValueRow,
@@ -82,6 +91,7 @@ export class StorageApiAsync implements StorageAPI {
     );
 
     const knownState = this.knwonStates.getKnownState(coValueRow.id);
+    knownState.header = true;
 
     for (const sessionRow of allCoValueSessions) {
       knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -89,13 +99,7 @@ export class StorageApiAsync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = {
-      action: "content",
-      id: coValueRow.id,
-      header: coValueRow.header,
-      new: {},
-      priority: getPriorityFromHeader(coValueRow.header),
-    } as NewContentMessage;
+    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
     if (contentStreaming) {
       contentMessage.expectContentUntil = knownState["sessions"];
@@ -136,13 +140,10 @@ export class StorageApiAsync implements StorageAPI {
             contentMessage,
             callback,
           );
-          contentMessage = {
-            action: "content",
-            id: coValueRow.id,
-            header: coValueRow.header,
-            new: {},
-            priority: getPriorityFromHeader(coValueRow.header),
-          } satisfies NewContentMessage;
+          contentMessage = createContentMessage(
+            coValueRow.id,
+            coValueRow.header,
+          );
         }
       }
     }
@@ -194,33 +195,64 @@ export class StorageApiAsync implements StorageAPI {
 
   storeQueue = new StoreQueue();
 
-  async store(
-    msgs: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
-  ) {
+  async store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
     /**
      * The store operations must be done one by one, because we can't start a new transaction when there
      * is already a transaction open.
      */
-    this.storeQueue.push(msgs, correctionCallback);
+    this.storeQueue.push(msg, correctionCallback);
 
     this.storeQueue.processQueue(async (data, correctionCallback) => {
-      for (const msg of data) {
-        const success = await this.storeSingle(msg, correctionCallback);
-
-        if (!success) {
-          // Stop processing the messages for this entry, because the data is out of sync with storage
-          // and the other transactions will be rejected anyway.
-          break;
-        }
-      }
+      return this.storeSingle(data, correctionCallback);
     });
   }
 
+  /**
+   * This function is called when the storage lacks the information required to store the incoming content.
+   *
+   * It triggers a `correctionCallback` to ask the syncManager to provide the missing information.
+   *
+   * The correction is applied immediately, to ensure that, when applicable, the dependent content in the queue won't require additional corrections.
+   */
+  private async handleCorrection(
+    knownState: CoValueKnownState,
+    correctionCallback: CorrectionCallback,
+  ) {
+    const correction = correctionCallback(knownState);
+
+    if (!correction) {
+      logger.error("Correction callback returned undefined", {
+        knownState,
+        correction: correction ?? null,
+      });
+      return false;
+    }
+
+    for (const msg of correction) {
+      const success = await this.storeSingle(msg, (knownState) => {
+        logger.error("Double correction requested", {
+          msg,
+          knownState,
+        });
+        return undefined;
+      });
+
+      if (!success) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
   private async storeSingle(
     msg: NewContentMessage,
-    correctionCallback: (data: CoValueKnownState) => void,
+    correctionCallback: CorrectionCallback,
   ): Promise<boolean> {
+    if (this.storeQueue.closed) {
+      return false;
+    }
+
     const id = msg.id;
     const coValueRow = await this.dbClient.getCoValue(id);
 
@@ -231,8 +263,7 @@ export class StorageApiAsync implements StorageAPI {
       const knownState = emptyKnownState(id as RawCoID);
       this.knwonStates.setKnownState(id, knownState);
 
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     const storedCoValueRowID: number = coValueRow
@@ -276,8 +307,7 @@ export class StorageApiAsync implements StorageAPI {
     this.knwonStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     return true;
@@ -290,38 +320,31 @@ export class StorageApiAsync implements StorageAPI {
     storedCoValueRowID: number,
   ) {
     const newTransactions = msg.new[sessionID]?.newTransactions || [];
+    const lastIdx = sessionRow?.lastIdx || 0;
 
-    const actuallyNewOffset =
-      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+    const actuallyNewOffset = lastIdx - (msg.new[sessionID]?.after || 0);
 
     const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
 
     if (actuallyNewTransactions.length === 0) {
-      return sessionRow?.lastIdx || 0;
+      return lastIdx;
     }
 
-    let newBytesSinceLastSignature =
-      (sessionRow?.bytesSinceLastSignature || 0) +
-      actuallyNewTransactions.reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+    let bytesSinceLastSignature = sessionRow?.bytesSinceLastSignature || 0;
+    const newTransactionsSize = getNewTransactionsSize(actuallyNewTransactions);
 
-    const newLastIdx =
-      (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+    const newLastIdx = lastIdx + actuallyNewTransactions.length;
 
     let shouldWriteSignature = false;
 
-    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(bytesSinceLastSignature, newTransactionsSize)) {
       shouldWriteSignature = true;
-      newBytesSinceLastSignature = 0;
+      bytesSinceLastSignature = 0;
+    } else {
+      bytesSinceLastSignature += newTransactionsSize;
     }
 
-    const nextIdx = sessionRow?.lastIdx || 0;
+    const nextIdx = lastIdx;
 
     if (!msg.new[sessionID]) throw new Error("Session ID not found");
 
@@ -330,7 +353,7 @@ export class StorageApiAsync implements StorageAPI {
       sessionID,
       lastIdx: newLastIdx,
       lastSignature: msg.new[sessionID].lastSignature,
-      bytesSinceLastSignature: newBytesSinceLastSignature,
+      bytesSinceLastSignature,
     };
 
     const sessionRowID: number = await this.dbClient.addSessionUpdate({
@@ -360,7 +383,6 @@ export class StorageApiAsync implements StorageAPI {
   }
 
   close() {
-    // Drain the store queue
-    this.storeQueue.drain();
+    return this.storeQueue.close();
   }
 }

packages/cojson/src/storage/storageSync.ts

@@ -1,20 +1,29 @@
 import { UpDownCounter, metrics } from "@opentelemetry/api";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
   RawCoID,
   type SessionID,
   type StorageAPI,
+  logger,
 } from "../exports.js";
-import { getPriorityFromHeader } from "../priority.js";
 import {
   CoValueKnownState,
   NewContentMessage,
   emptyKnownState,
 } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
-import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import {
+  collectNewTxs,
+  getDependedOnCoValues,
+  getNewTransactionsSize,
+} from "./syncUtils.js";
 import type {
+  CorrectionCallback,
   DBClientInterfaceSync,
   SignatureAfterRow,
   StoredCoValueRow,
@@ -84,6 +93,7 @@ export class StorageApiSync implements StorageAPI {
     }
 
     const knownState = this.knwonStates.getKnownState(coValueRow.id);
+    knownState.header = true;
 
     for (const sessionRow of allCoValueSessions) {
       knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -91,13 +101,7 @@ export class StorageApiSync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = {
-      action: "content",
-      id: coValueRow.id,
-      header: coValueRow.header,
-      new: {},
-      priority: getPriorityFromHeader(coValueRow.header),
-    } as NewContentMessage;
+    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
     if (contentStreaming) {
       this.streamingCounter.add(1);
@@ -137,13 +141,10 @@ export class StorageApiSync implements StorageAPI {
             contentMessage,
             callback,
           );
-          contentMessage = {
-            action: "content",
-            id: coValueRow.id,
-            header: coValueRow.header,
-            new: {},
-            priority: getPriorityFromHeader(coValueRow.header),
-          } satisfies NewContentMessage;
+          contentMessage = createContentMessage(
+            coValueRow.id,
+            coValueRow.header,
+          );
 
           // Introduce a delay to not block the main thread
           // for the entire content processing
@@ -189,22 +190,49 @@ export class StorageApiSync implements StorageAPI {
     pushCallback(contentMessage);
   }
 
-  store(
-    msgs: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
+  store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
+    return this.storeSingle(msg, correctionCallback);
+  }
+
+  /**
+   * This function is called when the storage lacks the information required to store the incoming content.
+   *
+   * It triggers a `correctionCallback` to ask the syncManager to provide the missing information.
+   */
+  private handleCorrection(
+    knownState: CoValueKnownState,
+    correctionCallback: CorrectionCallback,
   ) {
-    for (const msg of msgs) {
-      const success = this.storeSingle(msg, correctionCallback);
+    const correction = correctionCallback(knownState);
+
+    if (!correction) {
+      logger.error("Correction callback returned undefined", {
+        knownState,
+        correction: correction ?? null,
+      });
+      return false;
+    }
+
+    for (const msg of correction) {
+      const success = this.storeSingle(msg, (knownState) => {
+        logger.error("Double correction requested", {
+          msg,
+          knownState,
+        });
+        return undefined;
+      });
 
       if (!success) {
         return false;
       }
     }
+
+    return true;
   }
 
   private storeSingle(
     msg: NewContentMessage,
-    correctionCallback: (data: CoValueKnownState) => void,
+    correctionCallback: CorrectionCallback,
   ): boolean {
     const id = msg.id;
     const coValueRow = this.dbClient.getCoValue(id);
@@ -214,11 +242,9 @@ export class StorageApiSync implements StorageAPI {
 
     if (invalidAssumptionOnHeaderPresence) {
       const knownState = emptyKnownState(id as RawCoID);
-      correctionCallback(knownState);
-
       this.knwonStates.setKnownState(id, knownState);
 
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     const storedCoValueRowID: number = coValueRow
@@ -258,8 +284,7 @@ export class StorageApiSync implements StorageAPI {
     this.knwonStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     return true;
@@ -272,35 +297,29 @@ export class StorageApiSync implements StorageAPI {
     storedCoValueRowID: number,
   ) {
     const newTransactions = msg.new[sessionID]?.newTransactions || [];
+    const lastIdx = sessionRow?.lastIdx || 0;
 
-    const actuallyNewOffset =
-      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+    const actuallyNewOffset = lastIdx - (msg.new[sessionID]?.after || 0);
 
     const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
 
     if (actuallyNewTransactions.length === 0) {
-      return sessionRow?.lastIdx || 0;
+      return lastIdx;
     }
 
-    let newBytesSinceLastSignature =
-      (sessionRow?.bytesSinceLastSignature || 0) +
-      actuallyNewTransactions.reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+    let bytesSinceLastSignature = sessionRow?.bytesSinceLastSignature || 0;
+    const newTransactionsSize = getNewTransactionsSize(actuallyNewTransactions);
 
     const newLastIdx =
       (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
 
     let shouldWriteSignature = false;
 
-    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(bytesSinceLastSignature, newTransactionsSize)) {
       shouldWriteSignature = true;
-      newBytesSinceLastSignature = 0;
+      bytesSinceLastSignature = 0;
+    } else {
+      bytesSinceLastSignature += newTransactionsSize;
     }
 
     const nextIdx = sessionRow?.lastIdx || 0;
@@ -312,7 +331,7 @@ export class StorageApiSync implements StorageAPI {
       sessionID,
       lastIdx: newLastIdx,
       lastSignature: msg.new[sessionID].lastSignature,
-      bytesSinceLastSignature: newBytesSinceLastSignature,
+      bytesSinceLastSignature,
     };
 
     const sessionRowID: number = this.dbClient.addSessionUpdate({
@@ -339,5 +358,7 @@ export class StorageApiSync implements StorageAPI {
     return this.knwonStates.waitForSync(id, coValue);
   }
 
-  close() {}
+  close() {
+    return undefined;
+  }
 }

packages/cojson/src/storage/syncUtils.ts

@@ -1,5 +1,9 @@
+import { getTransactionSize } from "../coValueContentMessage.js";
 import { getDependedOnCoValuesFromRawData } from "../coValueCore/utils.js";
-import type { CoValueHeader } from "../coValueCore/verifiedState.js";
+import type {
+  CoValueHeader,
+  Transaction,
+} from "../coValueCore/verifiedState.js";
 import type { Signature } from "../crypto/crypto.js";
 import type { SessionID } from "../exports.js";
 import type { NewContentMessage } from "../sync.js";
@@ -48,3 +52,7 @@ export function getDependedOnCoValues(
 
   return getDependedOnCoValuesFromRawData(id, header, sessionIDs, transactions);
 }
+
+export function getNewTransactionsSize(newTxs: Transaction[]) {
+  return newTxs.reduce((sum, tx) => sum + getTransactionSize(tx), 0);
+}

packages/cojson/src/storage/types.ts

@@ -6,6 +6,10 @@ import { Signature } from "../crypto/crypto.js";
 import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 
+export type CorrectionCallback = (
+  correction: CoValueKnownState,
+) => NewContentMessage[] | undefined;
+
 /**
  * The StorageAPI is the interface that the StorageSync and StorageAsync classes implement.
  *
@@ -18,16 +22,13 @@ export interface StorageAPI {
     callback: (data: NewContentMessage) => void,
     done?: (found: boolean) => void,
   ): void;
-  store(
-    data: NewContentMessage[] | undefined,
-    handleCorrection: (correction: CoValueKnownState) => void,
-  ): void;
+  store(data: NewContentMessage, handleCorrection: CorrectionCallback): void;
 
   getKnownState(id: string): CoValueKnownState;
 
   waitForSync(id: string, coValue: CoValueCore): Promise<void>;
 
-  close(): void;
+  close(): Promise<unknown> | undefined;
 }
 
 export type CoValueRow = {

packages/cojson/src/sync.ts

@@ -1,15 +1,24 @@
 import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
+import {
+  getTransactionSize,
+  knownStateFromContent,
+} from "./coValueContentMessage.js";
 import { CoValueCore } from "./coValueCore/coValueCore.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
-import { CoValueHeader, Transaction } from "./coValueCore/verifiedState.js";
+import {
+  CoValueHeader,
+  Transaction,
+  VerifiedState,
+} from "./coValueCore/verifiedState.js";
 import { Signature } from "./crypto/crypto.js";
-import { RawCoID, SessionID } from "./ids.js";
+import { RawCoID, SessionID, isRawCoID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
 import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
+import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "./typeUtils/isAccountID.js";
 
@@ -57,10 +66,12 @@ export type NewContentMessage = {
 };
 
 export type SessionNewContent = {
+  // The index where to start appending the new transactions. The index counting starts from 1.
   after: number;
   newTransactions: Transaction[];
   lastSignature: Signature;
 };
+
 export type DoneMessage = {
   action: "done";
   id: RawCoID;
@@ -162,13 +173,9 @@ export class SyncManager {
   }
 
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
-    if (msg.id === undefined || msg.id === null) {
-      logger.warn("Received sync message with undefined id", {
-        msg,
-      });
-      return;
-    } else if (!msg.id.startsWith("co_z")) {
-      logger.warn("Received sync message with invalid id", {
+    if (!isRawCoID(msg.id)) {
+      const errorType = msg.id ? "invalid" : "undefined";
+      logger.warn(`Received sync message with ${errorType} id`, {
         msg,
       });
       return;
@@ -431,12 +438,9 @@ export class SyncManager {
 
   recordTransactionsSize(newTransactions: Transaction[], source: string) {
     for (const tx of newTransactions) {
-      const txLength =
-        tx.privacy === "private"
-          ? tx.encryptedChanges.length
-          : tx.changes.length;
+      const size = getTransactionSize(tx);
 
-      this.transactionsSizeHistogram.record(txLength, {
+      this.transactionsSizeHistogram.record(size, {
         source,
       });
     }
@@ -674,7 +678,7 @@ export class SyncManager {
     const syncedPeers = [];
 
     if (from !== "storage") {
-      this.storeCoValue(coValue, [msg]);
+      this.storeContent(msg);
     }
 
     for (const peer of this.peersInPriorityOrder()) {
@@ -736,60 +740,18 @@ export class SyncManager {
     };
   }
 
-  requestedSyncs = new Set<RawCoID>();
-  requestCoValueSync(coValue: CoValueCore) {
-    if (this.requestedSyncs.has(coValue.id)) {
-      return;
-    }
+  private syncQueue = new LocalTransactionsSyncQueue((content) =>
+    this.syncContent(content),
+  );
+  syncHeader = this.syncQueue.syncHeader;
+  syncLocalTransaction = this.syncQueue.syncTransaction;
 
-    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
-      trackingSet.add(coValue.id);
-    }
+  syncContent(content: NewContentMessage) {
+    const coValue = this.local.getCoValue(content.id);
 
-    queueMicrotask(() => {
-      if (this.requestedSyncs.has(coValue.id)) {
-        this.syncCoValue(coValue);
-      }
-    });
+    this.storeContent(content);
 
-    this.requestedSyncs.add(coValue.id);
-  }
-
-  storeCoValue(coValue: CoValueCore, data: NewContentMessage[] | undefined) {
-    const storage = this.local.storage;
-
-    if (!storage || !data) return;
-
-    // Try to store the content as-is for performance
-    // In case that some transactions are missing, a correction will be requested, but it's an edge case
-    storage.store(data, (correction) => {
-      if (!coValue.hasVerifiedContent()) return;
-
-      const newContentPieces = coValue.verified.newContentSince(correction);
-
-      if (!newContentPieces) return;
-
-      storage.store(newContentPieces, (response) => {
-        logger.error(
-          "Correction requested by storage after sending a correction content",
-          {
-            response,
-            knownState: coValue.knownState(),
-          },
-        );
-      });
-    });
-  }
-
-  syncCoValue(coValue: CoValueCore) {
-    this.requestedSyncs.delete(coValue.id);
-
-    if (this.local.storage && coValue.hasVerifiedContent()) {
-      const knownState = this.local.storage.getKnownState(coValue.id);
-      const newContentPieces = coValue.verified.newContentSince(knownState);
-
-      this.storeCoValue(coValue, newContentPieces);
-    }
+    const contentKnownState = knownStateFromContent(content);
 
     for (const peer of this.peersInPriorityOrder()) {
       if (peer.closed) continue;
@@ -803,7 +765,11 @@ export class SyncManager {
         continue;
       }
 
-      this.sendNewContentIncludingDependencies(coValue.id, peer);
+      // We assume that the peer already knows anything before this content
+      // Any eventual reconciliation will be handled through the known state messages exchange
+      this.trySendToPeer(peer, content);
+      peer.combineOptimisticWith(coValue.id, contentKnownState);
+      peer.trackToldKnownState(coValue.id);
     }
 
     for (const peer of this.getPeers()) {
@@ -811,6 +777,20 @@ export class SyncManager {
     }
   }
 
+  private storeContent(content: NewContentMessage) {
+    const storage = this.local.storage;
+
+    if (!storage) return;
+
+    // Try to store the content as-is for performance
+    // In case that some transactions are missing, a correction will be requested, but it's an edge case
+    storage.store(content, (correction) => {
+      return this.local
+        .getCoValue(content.id)
+        .verified?.newContentSince(correction);
+    });
+  }
+
   waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const { syncState } = this;
     const currentSyncState = syncState.getCurrentSyncState(peerId, id);

packages/cojson/src/tests/StorageApiAsync.test.ts

@@ -0,0 +1,829 @@
+import { randomUUID } from "node:crypto";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest";
+import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { CoID, LocalNode, RawCoID, RawCoMap, logger } from "../exports.js";
+import { CoValueCore } from "../exports.js";
+import {
+  CoValueKnownState,
+  NewContentMessage,
+  emptyKnownState,
+} from "../sync.js";
+import { createAsyncStorage } from "./testStorage.js";
+import {
+  SyncMessagesLog,
+  loadCoValueOrFail,
+  randomAgentAndSessionID,
+  waitFor,
+} from "./testUtils.js";
+
+const crypto = await WasmCrypto.create();
+
+/**
+ * Helper function that gets new content since a known state, throwing if:
+ * - The coValue is not verified
+ * - There is no new content
+ */
+function getNewContentSince(
+  coValue: CoValueCore,
+  knownState: CoValueKnownState,
+): NewContentMessage {
+  if (!coValue.verified) {
+    throw new Error(`CoValue ${coValue.id} is not verified`);
+  }
+
+  const contentMessage = coValue.verified.newContentSince(knownState)?.[0];
+
+  if (!contentMessage) {
+    throw new Error(`No new content available for coValue ${coValue.id}`);
+  }
+
+  return contentMessage;
+}
+
+async function createFixturesNode(customDbPath?: string) {
+  const [admin, session] = randomAgentAndSessionID();
+  const node = new LocalNode(admin.agentSecret, session, crypto);
+
+  // Create a unique database file for each test
+  const dbPath = customDbPath ?? join(tmpdir(), `test-${randomUUID()}.db`);
+  const storage = await createAsyncStorage({
+    filename: dbPath,
+    nodeName: "test",
+    storageName: "test-storage",
+  });
+
+  onTestFinished(() => {
+    try {
+      unlinkSync(dbPath);
+    } catch {}
+  });
+
+  onTestFinished(async () => {
+    await node.gracefulShutdown();
+  });
+
+  node.setStorage(storage);
+
+  return {
+    fixturesNode: node,
+    dbPath,
+  };
+}
+
+async function createTestNode(dbPath?: string) {
+  const [admin, session] = randomAgentAndSessionID();
+  const node = new LocalNode(admin.agentSecret, session, crypto);
+
+  const storage = await createAsyncStorage({
+    filename: dbPath,
+    nodeName: "test",
+    storageName: "test-storage",
+  });
+
+  onTestFinished(async () => {
+    node.gracefulShutdown();
+    await storage.close();
+  });
+
+  return {
+    node,
+    storage,
+  };
+}
+
+afterEach(() => {
+  SyncMessagesLog.clear();
+});
+
+describe("StorageApiAsync", () => {
+  describe("getKnownState", () => {
+    test("should return known state for existing coValue ID", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const id = fixturesNode.createGroup().id;
+      const knownState = storage.getKnownState(id);
+
+      expect(knownState).toEqual(emptyKnownState(id));
+      expect(storage.getKnownState(id)).toBe(knownState); // Should return same instance
+    });
+
+    test("should return different known states for different coValue IDs", async () => {
+      const { storage } = await createTestNode();
+      const id1 = "test-id-1";
+      const id2 = "test-id-2";
+
+      const knownState1 = storage.getKnownState(id1);
+      const knownState2 = storage.getKnownState(id2);
+
+      expect(knownState1).not.toBe(knownState2);
+    });
+  });
+
+  describe("load", () => {
+    test("should handle non-existent coValue gracefully", async () => {
+      const { storage } = await createTestNode();
+      const id = "non-existent-id";
+      const callback = vi.fn();
+      const done = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(id);
+      expect(initialKnownState).toEqual(emptyKnownState(id as `co_z${string}`));
+
+      await storage.load(id, callback, done);
+
+      expect(done).toHaveBeenCalledWith(false);
+      expect(callback).not.toHaveBeenCalled();
+
+      // Verify that storage known state is NOT updated when load fails
+      const afterLoadKnownState = storage.getKnownState(id);
+      expect(afterLoadKnownState).toEqual(initialKnownState);
+    });
+
+    test("should load coValue with header only successfully", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Create a real group and get its content message
+      const group = fixturesNode.createGroup();
+      await group.core.waitForSync();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.load(group.id, callback, done);
+
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: group.id,
+          header: group.core.verified.header,
+          new: expect.any(Object),
+        }),
+      );
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known state is updated after load
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.core.verified.header).toEqual(
+        group.core.verified.header,
+      );
+    });
+
+    test("should load coValue with sessions and transactions successfully", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Create a real group and add a member to create transactions
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.load(group.id, callback, done);
+
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: group.id,
+          header: group.core.verified.header,
+          new: expect.objectContaining({
+            [fixturesNode.currentSessionID]: expect.any(Object),
+          }),
+        }),
+      );
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known state is updated after load
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+  });
+
+  describe("store", () => {
+    test("should store new coValue with header successfully", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+      // Create a real group and get its content message
+      const group = fixturesNode.createGroup();
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.store(contentMessage, correctionCallback);
+      await storage.waitForSync(group.id, group.core);
+
+      // Verify that storage known state is updated after store
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.core.verified.header).toEqual(
+        group.core.verified.header,
+      );
+    });
+
+    test("should store coValue with transactions successfully", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      // Create a real group and add a member to create transactions
+      const group = fixturesNode.createGroup();
+      const knownState = group.core.verified.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.store(contentMessage, correctionCallback);
+      await storage.waitForSync(group.id, group.core);
+
+      // Verify that storage known state is updated after store
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+
+    test("should handle invalid assumption on header presence with correction", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+      const knownState = group.core.verified.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        expect(known).toEqual(emptyKnownState(group.id));
+        return group.core.verified.newContentSince(known);
+      });
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.store(contentMessage, correctionCallback);
+      await storage.waitForSync(group.id, group.core);
+
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+
+      // Verify that storage known state is updated after store with correction
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+
+    test("should handle invalid assumption on new content with correction", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const initialContent = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+
+      const initialKnownState = group.core.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const knownState = group.core.knownState();
+
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        expect(known).toEqual(initialKnownState);
+        return group.core.verified.newContentSince(known);
+      });
+
+      // Get initial storage known state
+      const initialStorageKnownState = storage.getKnownState(group.id);
+      expect(initialStorageKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.store(initialContent, correctionCallback);
+      await storage.store(contentMessage, correctionCallback);
+
+      await storage.waitForSync(group.id, group.core);
+
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+
+      // Verify that storage known state is updated after store with correction
+      const finalKnownState = storage.getKnownState(group.id);
+      expect(finalKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.get("everyone")).toEqual("writer");
+    });
+
+    test("should log an error when the correction callback returns undefined", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const knownState = group.core.knownState();
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        return undefined;
+      });
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});
+      await storage.store(contentMessage, correctionCallback);
+
+      await waitFor(() => {
+        expect(correctionCallback).toHaveBeenCalledTimes(1);
+      });
+
+      // Verify that storage known state is NOT updated when store fails
+      const afterStoreKnownState = storage.getKnownState(group.id);
+      expect(afterStoreKnownState).toEqual(initialKnownState);
+
+      expect(errorSpy).toHaveBeenCalledWith(
+        "Correction callback returned undefined",
+        {
+          knownState: expect.any(Object),
+          correction: null,
+        },
+      );
+
+      errorSpy.mockClear();
+    });
+
+    test("should log an error when the correction callback returns an invalid content message", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const knownState = group.core.knownState();
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn(() => {
+        return [contentMessage];
+      });
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});
+      await storage.store(contentMessage, correctionCallback);
+
+      await waitFor(() => {
+        expect(correctionCallback).toHaveBeenCalledTimes(1);
+      });
+
+      // Verify that storage known state is NOT updated when store fails
+      const afterStoreKnownState = storage.getKnownState(group.id);
+      expect(afterStoreKnownState).toEqual(initialKnownState);
+
+      expect(errorSpy).toHaveBeenCalledWith(
+        "Correction callback returned undefined",
+        {
+          knownState: expect.any(Object),
+          correction: null,
+        },
+      );
+
+      expect(errorSpy).toHaveBeenCalledWith("Double correction requested", {
+        knownState: expect.any(Object),
+        msg: expect.any(Object),
+      });
+
+      errorSpy.mockClear();
+    });
+
+    test("should handle invalid assumption when pushing multiple transactions with correction", async () => {
+      const { node, storage } = await createTestNode();
+
+      const core = node.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      core.makeTransaction([{ count: 1 }], "trusting");
+
+      await core.waitForSync();
+
+      // Add storage later
+      node.setStorage(storage);
+
+      core.makeTransaction([{ count: 2 }], "trusting");
+      core.makeTransaction([{ count: 3 }], "trusting");
+
+      await new Promise((resolve) => setTimeout(resolve, 10));
+
+      core.makeTransaction([{ count: 4 }], "trusting");
+      core.makeTransaction([{ count: 5 }], "trusting");
+
+      await core.waitForSync();
+
+      expect(storage.getKnownState(core.id)).toEqual(core.knownState());
+
+      expect(
+        SyncMessagesLog.getMessages({
+          Core: core,
+        }),
+      ).toMatchInlineSnapshot(`
+        [
+          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 2",
+          "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
+          "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
+          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 2",
+        ]
+      `);
+    });
+
+    test("should handle invalid assumption when pushing multiple transactions on different coValues with correction", async () => {
+      const { node, storage } = await createTestNode();
+
+      const core = node.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      const core2 = node.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      core.makeTransaction([{ count: 1 }], "trusting");
+      core2.makeTransaction([{ count: 1 }], "trusting");
+
+      await core.waitForSync();
+
+      // Add storage later
+      node.setStorage(storage);
+
+      core.makeTransaction([{ count: 2 }], "trusting");
+      core2.makeTransaction([{ count: 2 }], "trusting");
+      core.makeTransaction([{ count: 3 }], "trusting");
+      core2.makeTransaction([{ count: 3 }], "trusting");
+
+      await new Promise((resolve) => setTimeout(resolve, 10));
+
+      core.makeTransaction([{ count: 4 }], "trusting");
+      core2.makeTransaction([{ count: 4 }], "trusting");
+      core.makeTransaction([{ count: 5 }], "trusting");
+      core2.makeTransaction([{ count: 5 }], "trusting");
+
+      await core.waitForSync();
+
+      expect(storage.getKnownState(core.id)).toEqual(core.knownState());
+
+      expect(
+        SyncMessagesLog.getMessages({
+          Core: core,
+          Core2: core2,
+        }),
+      ).toMatchInlineSnapshot(`
+        [
+          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
+          "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
+          "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
+          "test-storage -> test | KNOWN CORRECTION Core2 sessions: empty",
+          "test -> test-storage | CONTENT Core2 header: true new: After: 0 New: 3",
+          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
+        ]
+      `);
+    });
+
+    test("should handle close while pushing multiple transactions on different coValues with an invalid assumption", async () => {
+      const { node, storage } = await createTestNode();
+
+      const core = node.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      const core2 = node.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      core.makeTransaction([{ count: 1 }], "trusting");
+      core2.makeTransaction([{ count: 1 }], "trusting");
+
+      await core.waitForSync();
+
+      // Add storage later
+      node.setStorage(storage);
+
+      core.makeTransaction([{ count: 2 }], "trusting");
+      core2.makeTransaction([{ count: 2 }], "trusting");
+      core.makeTransaction([{ count: 3 }], "trusting");
+      core2.makeTransaction([{ count: 3 }], "trusting");
+
+      await new Promise<void>(queueMicrotask);
+
+      await storage.close();
+      const knownState = JSON.parse(
+        JSON.stringify(storage.getKnownState(core.id)),
+      );
+
+      core.makeTransaction([{ count: 4 }], "trusting");
+      core2.makeTransaction([{ count: 4 }], "trusting");
+      core.makeTransaction([{ count: 5 }], "trusting");
+      core2.makeTransaction([{ count: 5 }], "trusting");
+
+      await new Promise<void>((resolve) => setTimeout(resolve, 10));
+
+      expect(
+        SyncMessagesLog.getMessages({
+          Core: core,
+          Core2: core2,
+        }),
+      ).toMatchInlineSnapshot(`
+        [
+          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
+          "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
+          "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
+          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
+        ]
+      `);
+
+      expect(storage.getKnownState(core.id)).toEqual(knownState);
+    });
+
+    test("should handle multiple sessions correctly", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { fixturesNode: fixtureNode2 } = await createFixturesNode(dbPath);
+      const { node, storage } = await createTestNode();
+
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      coValue.makeTransaction(
+        [
+          {
+            count: 1,
+          },
+        ],
+        "trusting",
+      );
+
+      await coValue.waitForSync();
+
+      const mapOnNode2 = await loadCoValueOrFail(
+        fixtureNode2,
+        coValue.id as CoID<RawCoMap>,
+      );
+
+      coValue.makeTransaction(
+        [
+          {
+            count: 2,
+          },
+        ],
+        "trusting",
+      );
+
+      const knownState = mapOnNode2.core.knownState();
+
+      const contentMessage = getNewContentSince(
+        mapOnNode2.core,
+        emptyKnownState(mapOnNode2.id),
+      );
+      const correctionCallback = vi.fn();
+
+      await storage.store(contentMessage, correctionCallback);
+      await storage.waitForSync(mapOnNode2.id, mapOnNode2.core);
+
+      node.setStorage(storage);
+
+      const finalMap = await loadCoValueOrFail(node, mapOnNode2.id);
+      expect(finalMap.core.knownState()).toEqual(knownState);
+    });
+  });
+
+  describe("dependencies", () => {
+    test("should push dependencies before the coValue", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group to create dependencies
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Get initial known states
+      const initialGroupKnownState = storage.getKnownState(group.id);
+      const initialMapKnownState = storage.getKnownState(map.id);
+      expect(initialGroupKnownState).toEqual(emptyKnownState(group.id));
+      expect(initialMapKnownState).toEqual(emptyKnownState(map.id));
+
+      // Load the map, which should also load the group dependency first
+      await storage.load(map.id, callback, done);
+
+      expect(callback).toHaveBeenCalledTimes(2); // Group first, then map
+      expect(callback).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          id: group.id,
+        }),
+      );
+      expect(callback).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known states are updated after load
+      const updatedGroupKnownState = storage.getKnownState(group.id);
+      const updatedMapKnownState = storage.getKnownState(map.id);
+      expect(updatedGroupKnownState).toEqual(group.core.verified.knownState());
+      expect(updatedMapKnownState).toEqual(map.core.verified.knownState());
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
+
+    test("should handle dependencies that are already loaded correctly", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Get initial known states
+      const initialGroupKnownState = storage.getKnownState(group.id);
+      const initialMapKnownState = storage.getKnownState(map.id);
+      expect(initialGroupKnownState).toEqual(emptyKnownState(group.id));
+      expect(initialMapKnownState).toEqual(emptyKnownState(map.id));
+
+      // First load the group
+      await storage.load(group.id, callback, done);
+      callback.mockClear();
+      done.mockClear();
+
+      // Verify group known state is updated after first load
+      const afterGroupLoad = storage.getKnownState(group.id);
+      expect(afterGroupLoad).toEqual(group.core.verified.knownState());
+
+      // Then load the map - the group dependency should already be loaded
+      await storage.load(map.id, callback, done);
+
+      // Should only call callback once for the map since group is already loaded
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify map known state is updated after second load
+      const finalMapKnownState = storage.getKnownState(map.id);
+      expect(finalMapKnownState).toEqual(map.core.verified.knownState());
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
+  });
+
+  describe("waitForSync", () => {
+    test("should resolve when the coValue is already synced", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and add a member
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Store the group in storage
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+      await storage.store(contentMessage, correctionCallback);
+
+      node.setStorage(storage);
+
+      // Load the group on the new node
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      // Wait for sync should resolve immediately since the coValue is already synced
+      await expect(
+        storage.waitForSync(group.id, groupOnNode.core),
+      ).resolves.toBeUndefined();
+
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+  });
+
+  describe("close", () => {
+    test("should close without throwing an error", async () => {
+      const { storage } = await createTestNode();
+
+      expect(() => storage.close()).not.toThrow();
+    });
+  });
+});

packages/cojson/src/tests/StorageApiSync.test.ts

@@ -0,0 +1,628 @@
+import { randomUUID } from "node:crypto";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, onTestFinished, test, vi } from "vitest";
+import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { CoID, LocalNode, RawCoID, RawCoMap, logger } from "../exports.js";
+import { CoValueCore } from "../exports.js";
+import {
+  CoValueKnownState,
+  NewContentMessage,
+  emptyKnownState,
+} from "../sync.js";
+import { createSyncStorage } from "./testStorage.js";
+import { loadCoValueOrFail, randomAgentAndSessionID } from "./testUtils.js";
+
+const crypto = await WasmCrypto.create();
+
+/**
+ * Helper function that gets new content since a known state, throwing if:
+ * - The coValue is not verified
+ * - There is no new content
+ */
+function getNewContentSince(
+  coValue: CoValueCore,
+  knownState: CoValueKnownState,
+): NewContentMessage {
+  if (!coValue.verified) {
+    throw new Error(`CoValue ${coValue.id} is not verified`);
+  }
+
+  const contentMessage = coValue.verified.newContentSince(knownState)?.[0];
+
+  if (!contentMessage) {
+    throw new Error(`No new content available for coValue ${coValue.id}`);
+  }
+
+  return contentMessage;
+}
+
+async function createFixturesNode(customDbPath?: string) {
+  const [admin, session] = randomAgentAndSessionID();
+  const node = new LocalNode(admin.agentSecret, session, crypto);
+
+  // Create a unique database file for each test
+  const dbPath = customDbPath ?? join(tmpdir(), `test-${randomUUID()}.db`);
+  const storage = createSyncStorage({
+    filename: dbPath,
+    nodeName: "test",
+    storageName: "test-storage",
+  });
+
+  onTestFinished(() => {
+    try {
+      unlinkSync(dbPath);
+    } catch {}
+  });
+
+  node.setStorage(storage);
+
+  return {
+    fixturesNode: node,
+    dbPath,
+  };
+}
+
+async function createTestNode(dbPath?: string) {
+  const [admin, session] = randomAgentAndSessionID();
+  const node = new LocalNode(admin.agentSecret, session, crypto);
+
+  const storage = createSyncStorage({
+    filename: dbPath,
+    nodeName: "test",
+    storageName: "test-storage",
+  });
+
+  return {
+    node,
+    storage,
+  };
+}
+
+describe("StorageApiSync", () => {
+  describe("getKnownState", () => {
+    test("should return empty known state for new coValue ID and cache the result", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const id = fixturesNode.createGroup().id;
+      const knownState = storage.getKnownState(id);
+
+      expect(knownState).toEqual(emptyKnownState(id));
+      expect(storage.getKnownState(id)).toBe(knownState); // Should return same instance
+    });
+
+    test("should return separate known state instances for different coValue IDs", async () => {
+      const { storage } = await createTestNode();
+      const id1 = "test-id-1";
+      const id2 = "test-id-2";
+
+      const knownState1 = storage.getKnownState(id1);
+      const knownState2 = storage.getKnownState(id2);
+
+      expect(knownState1).not.toBe(knownState2);
+    });
+  });
+
+  describe("load", () => {
+    test("should fail gracefully when loading non-existent coValue and preserve known state", async () => {
+      const { storage } = await createTestNode();
+      const id = "non-existent-id";
+      const callback = vi.fn();
+      const done = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(id);
+      expect(initialKnownState).toEqual(emptyKnownState(id as `co_z${string}`));
+
+      await storage.load(id, callback, done);
+
+      expect(done).toHaveBeenCalledWith(false);
+      expect(callback).not.toHaveBeenCalled();
+
+      // Verify that storage known state is NOT updated when load fails
+      const afterLoadKnownState = storage.getKnownState(id);
+      expect(afterLoadKnownState).toEqual(initialKnownState);
+    });
+
+    test("should successfully load coValue with header and update known state", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Create a real group and get its content message
+      const group = fixturesNode.createGroup();
+      await group.core.waitForSync();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.load(group.id, callback, done);
+
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: group.id,
+          header: group.core.verified.header,
+          new: expect.any(Object),
+        }),
+      );
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known state is updated after load
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.core.verified.header).toEqual(
+        group.core.verified.header,
+      );
+    });
+
+    test("should successfully load coValue with transactions and update known state", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Create a real group and add a member to create transactions
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      await storage.load(group.id, callback, done);
+
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: group.id,
+          header: group.core.verified.header,
+          new: expect.objectContaining({
+            [fixturesNode.currentSessionID]: expect.any(Object),
+          }),
+        }),
+      );
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known state is updated after load
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+  });
+
+  describe("store", () => {
+    test("should successfully store new coValue with header and update known state", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+      // Create a real group and get its content message
+      const group = fixturesNode.createGroup();
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      storage.store(contentMessage, correctionCallback);
+
+      // Verify that storage known state is updated after store
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.core.verified.header).toEqual(
+        group.core.verified.header,
+      );
+    });
+
+    test("should successfully store coValue with transactions and update known state", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      // Create a real group and add a member to create transactions
+      const group = fixturesNode.createGroup();
+      const knownState = group.core.verified.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      storage.store(contentMessage, correctionCallback);
+
+      // Verify that storage known state is updated after store
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+
+    test("should handle correction when header assumption is invalid", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+      const knownState = group.core.verified.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        expect(known).toEqual(emptyKnownState(group.id));
+        return group.core.verified.newContentSince(known);
+      });
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      const result = storage.store(contentMessage, correctionCallback);
+
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+      expect(result).toBe(true);
+
+      // Verify that storage known state is updated after store with correction
+      const updatedKnownState = storage.getKnownState(group.id);
+      expect(updatedKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+
+    test("should handle correction when new content assumption is invalid", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { node, storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const initialContent = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+
+      const initialKnownState = group.core.knownState();
+
+      group.addMember("everyone", "reader");
+
+      const knownState = group.core.knownState();
+
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        expect(known).toEqual(initialKnownState);
+        return group.core.verified.newContentSince(known);
+      });
+
+      // Get initial storage known state
+      const initialStorageKnownState = storage.getKnownState(group.id);
+      expect(initialStorageKnownState).toEqual(emptyKnownState(group.id));
+
+      storage.store(initialContent, correctionCallback);
+
+      // Verify storage known state after first store
+      const afterFirstStore = storage.getKnownState(group.id);
+      expect(afterFirstStore).toEqual(initialKnownState);
+
+      const result = storage.store(contentMessage, correctionCallback);
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+
+      expect(result).toBe(true);
+
+      // Verify that storage known state is updated after store with correction
+      const finalKnownState = storage.getKnownState(group.id);
+      expect(finalKnownState).toEqual(group.core.verified.knownState());
+
+      node.setStorage(storage);
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      expect(groupOnNode.get("everyone")).toEqual("writer");
+    });
+
+    test("should log error and fail when correction callback returns undefined", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const knownState = group.core.knownState();
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn((known) => {
+        return undefined;
+      });
+
+      // Get initial known state
+      const initialKnownState = storage.getKnownState(group.id);
+      expect(initialKnownState).toEqual(emptyKnownState(group.id));
+
+      const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});
+      const result = storage.store(contentMessage, correctionCallback);
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+
+      expect(result).toBe(false);
+
+      // Verify that storage known state is NOT updated when store fails
+      const afterStoreKnownState = storage.getKnownState(group.id);
+      expect(afterStoreKnownState).toEqual(initialKnownState);
+
+      expect(errorSpy).toHaveBeenCalledWith(
+        "Correction callback returned undefined",
+        {
+          knownState: expect.any(Object),
+          correction: null,
+        },
+      );
+
+      errorSpy.mockClear();
+    });
+
+    test("should log error and fail when correction callback returns invalid content message", async () => {
+      const { fixturesNode } = await createFixturesNode();
+      const { storage } = await createTestNode();
+
+      const group = fixturesNode.createGroup();
+
+      const knownState = group.core.knownState();
+      group.addMember("everyone", "writer");
+
+      const contentMessage = getNewContentSince(group.core, knownState);
+      const correctionCallback = vi.fn(() => {
+        return [contentMessage];
+      });
+
+      const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});
+      const result = storage.store(contentMessage, correctionCallback);
+      expect(correctionCallback).toHaveBeenCalledTimes(1);
+
+      expect(result).toBe(false);
+
+      expect(errorSpy).toHaveBeenCalledWith(
+        "Correction callback returned undefined",
+        {
+          knownState: expect.any(Object),
+          correction: null,
+        },
+      );
+
+      expect(errorSpy).toHaveBeenCalledWith("Double correction requested", {
+        knownState: expect.any(Object),
+        msg: expect.any(Object),
+      });
+
+      errorSpy.mockClear();
+    });
+
+    test("should successfully store coValue with multiple sessions", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { fixturesNode: fixtureNode2 } = await createFixturesNode(dbPath);
+      const { node, storage } = await createTestNode();
+
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      coValue.makeTransaction(
+        [
+          {
+            count: 1,
+          },
+        ],
+        "trusting",
+      );
+
+      await coValue.waitForSync();
+
+      const mapOnNode2 = await loadCoValueOrFail(
+        fixtureNode2,
+        coValue.id as CoID<RawCoMap>,
+      );
+
+      coValue.makeTransaction(
+        [
+          {
+            count: 2,
+          },
+        ],
+        "trusting",
+      );
+
+      const knownState = mapOnNode2.core.knownState();
+
+      const contentMessage = getNewContentSince(
+        mapOnNode2.core,
+        emptyKnownState(mapOnNode2.id),
+      );
+      const correctionCallback = vi.fn();
+
+      storage.store(contentMessage, correctionCallback);
+
+      node.setStorage(storage);
+
+      const finalMap = await loadCoValueOrFail(node, mapOnNode2.id);
+      expect(finalMap.core.knownState()).toEqual(knownState);
+    });
+  });
+
+  describe("dependencies", () => {
+    test("should load dependencies before dependent coValues and update all known states", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group to create dependencies
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Get initial known states
+      const initialGroupKnownState = storage.getKnownState(group.id);
+      const initialMapKnownState = storage.getKnownState(map.id);
+      expect(initialGroupKnownState).toEqual(emptyKnownState(group.id));
+      expect(initialMapKnownState).toEqual(emptyKnownState(map.id));
+
+      // Load the map, which should also load the group dependency first
+      await storage.load(map.id, callback, done);
+
+      expect(callback).toHaveBeenCalledTimes(2); // Group first, then map
+      expect(callback).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          id: group.id,
+        }),
+      );
+      expect(callback).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify that storage known states are updated after load
+      const updatedGroupKnownState = storage.getKnownState(group.id);
+      const updatedMapKnownState = storage.getKnownState(map.id);
+      expect(updatedGroupKnownState).toEqual(group.core.verified.knownState());
+      expect(updatedMapKnownState).toEqual(map.core.verified.knownState());
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
+
+    test("should skip loading already loaded dependencies", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Get initial known states
+      const initialGroupKnownState = storage.getKnownState(group.id);
+      const initialMapKnownState = storage.getKnownState(map.id);
+      expect(initialGroupKnownState).toEqual(emptyKnownState(group.id));
+      expect(initialMapKnownState).toEqual(emptyKnownState(map.id));
+
+      // First load the group
+      await storage.load(group.id, callback, done);
+      callback.mockClear();
+      done.mockClear();
+
+      // Verify group known state is updated after first load
+      const afterGroupLoad = storage.getKnownState(group.id);
+      expect(afterGroupLoad).toEqual(group.core.verified.knownState());
+
+      // Then load the map - the group dependency should already be loaded
+      await storage.load(map.id, callback, done);
+
+      // Should only call callback once for the map since group is already loaded
+      expect(callback).toHaveBeenCalledTimes(1);
+      expect(callback).toHaveBeenCalledWith(
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      // Verify map known state is updated after second load
+      const finalMapKnownState = storage.getKnownState(map.id);
+      expect(finalMapKnownState).toEqual(map.core.verified.knownState());
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
+  });
+
+  describe("waitForSync", () => {
+    test("should resolve immediately when coValue is already synced", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and add a member
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Store the group in storage
+      const contentMessage = getNewContentSince(
+        group.core,
+        emptyKnownState(group.id),
+      );
+      const correctionCallback = vi.fn();
+      storage.store(contentMessage, correctionCallback);
+
+      node.setStorage(storage);
+
+      // Load the group on the new node
+      const groupOnNode = await loadCoValueOrFail(node, group.id);
+
+      // Wait for sync should resolve immediately since the coValue is already synced
+      await expect(
+        storage.waitForSync(group.id, groupOnNode.core),
+      ).resolves.toBeUndefined();
+
+      expect(groupOnNode.get("everyone")).toEqual("reader");
+    });
+  });
+
+  describe("close", () => {
+    test("should close storage without throwing errors", async () => {
+      const { storage } = await createTestNode();
+
+      expect(() => storage.close()).not.toThrow();
+    });
+  });
+});

packages/cojson/src/tests/StoreQueue.test.ts

@@ -2,15 +2,13 @@ import { beforeEach, describe, expect, test, vi } from "vitest";
 import { StoreQueue } from "../queue/StoreQueue.js";
 import type { CoValueKnownState, NewContentMessage } from "../sync.js";
 
-function createMockNewContentMessage(id: string): NewContentMessage[] {
-  return [
-    {
-      action: "content",
-      id: id as any,
-      priority: 0,
-      new: {},
-    },
-  ];
+function createMockNewContentMessage(id: string): NewContentMessage {
+  return {
+    action: "content",
+    id: id as any,
+    priority: 0,
+    new: {},
+  };
 }
 
 function setup() {
@@ -154,14 +152,14 @@ describe("StoreQueue", () => {
       storeQueue.push(data1, mockCorrectionCallback);
       storeQueue.push(data2, mockCorrectionCallback);
 
-      storeQueue.drain();
+      storeQueue.close();
 
       expect(storeQueue.pull()).toBeUndefined();
     });
 
     test("should handle empty queue", () => {
       const { storeQueue } = setup();
-      expect(() => storeQueue.drain()).not.toThrow();
+      expect(() => storeQueue.close()).not.toThrow();
       expect(storeQueue.pull()).toBeUndefined();
     });
   });
@@ -240,23 +238,11 @@ describe("StoreQueue", () => {
   });
 
   describe("edge cases", () => {
-    test("should handle undefined data", () => {
-      const { storeQueue, mockCorrectionCallback } = setup();
-      const data: NewContentMessage[] = [];
-      storeQueue.push(data, mockCorrectionCallback);
-
-      const entry = storeQueue.pull();
-      expect(entry).toEqual({
-        data,
-        correctionCallback: mockCorrectionCallback,
-      });
-    });
-
     test("should handle null correction callback", () => {
       const { storeQueue } = setup();
       const data = createMockNewContentMessage("co1");
 
-      const nullCallback = () => {};
+      const nullCallback = () => undefined;
       storeQueue.push(data, nullCallback);
 
       const entry = storeQueue.pull();

packages/cojson/src/tests/SyncStateManager.test.ts

@@ -38,14 +38,6 @@ describe("SyncStateManager", () => {
     const updateSpy: GlobalSyncStateListenerCallback = vi.fn();
     const unsubscribe = subscriptionManager.subscribeToUpdates(updateSpy);
 
-    await client.node.syncManager.syncCoValue(map.core);
-
-    expect(updateSpy).toHaveBeenCalledWith(
-      peerState.id,
-      emptyKnownState(map.core.id),
-      { uploaded: false },
-    );
-
     await waitFor(() => {
       return subscriptionManager.getCurrentSyncState(peerState.id, map.core.id)
         .uploaded;
@@ -98,13 +90,6 @@ describe("SyncStateManager", () => {
       unsubscribe2();
     });
 
-    await client.node.syncManager.syncCoValue(map.core);
-
-    expect(updateToJazzCloudSpy).toHaveBeenCalledWith(
-      emptyKnownState(map.core.id),
-      { uploaded: false },
-    );
-
     await waitFor(() => {
       return subscriptionManager.getCurrentSyncState(peerState.id, map.core.id)
         .uploaded;
@@ -117,7 +102,7 @@ describe("SyncStateManager", () => {
       { uploaded: true },
     );
 
-    expect(updateToStorageSpy).toHaveBeenLastCalledWith(
+    expect(updateToStorageSpy).toHaveBeenCalledWith(
       emptyKnownState(group.core.id),
       { uploaded: false },
     );
@@ -133,8 +118,6 @@ describe("SyncStateManager", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     const subscriptionManager = client.node.syncManager.syncState;
 
     expect(
@@ -174,8 +157,6 @@ describe("SyncStateManager", () => {
     unsubscribe1();
     unsubscribe2();
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     anyUpdateSpy.mockClear();
 
     await waitFor(() => {
@@ -336,6 +317,26 @@ describe("SyncStateManager", () => {
 
     await map.core.waitForSync();
 
-    expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+    expect(client.node.getCoValue(map.id).hasVerifiedContent()).toBe(true);
+
+    // Since only the map is subscribed, the dependencies are pushed after the client requests them
+    await waitFor(() => {
+      expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+    });
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Map: map.core,
+        Group: group.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | LOAD Group sessions: empty",
+        "client -> server | KNOWN Map sessions: header/1",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | KNOWN Group sessions: header/3",
+      ]
+    `);
   });
 });

packages/cojson/src/tests/group.inheritance.test.ts

@@ -1,10 +1,19 @@
-import { describe, expect, test } from "vitest";
+import { beforeEach, describe, expect, test } from "vitest";
 import {
+  SyncMessagesLog,
   createThreeConnectedNodes,
   createTwoConnectedNodes,
   loadCoValueOrFail,
+  setupTestNode,
 } from "./testUtils";
 
+let jazzCloud: ReturnType<typeof setupTestNode>;
+
+beforeEach(async () => {
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
 describe("extend", () => {
   test("inherited writer roles should work correctly", async () => {
     const { node1, node2 } = await createTwoConnectedNodes("server", "server");
@@ -143,6 +152,132 @@ describe("extend", () => {
     expect(map.get("test")).toEqual("Hello!");
   });
 
+  test("should not break when checking for cycles on a loaded group", async () => {
+    const clientSession1 = setupTestNode({
+      connected: true,
+    });
+    const clientSession2 = clientSession1.spawnNewSession();
+
+    const group = clientSession1.node.createGroup();
+    const childGroup = clientSession1.node.createGroup();
+    const group2 = clientSession1.node.createGroup();
+    const group3 = clientSession1.node.createGroup();
+
+    childGroup.extend(group);
+    group.extend(group2);
+    group2.extend(group3);
+
+    await group.core.waitForSync();
+    await childGroup.core.waitForSync();
+    await group2.core.waitForSync();
+    await group3.core.waitForSync();
+
+    const groupOnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group.id,
+    );
+    const group3OnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group3.id,
+    );
+
+    expect(group3OnClientSession2.isSelfExtension(groupOnClientSession2)).toBe(
+      true,
+    );
+
+    // Child groups are not loaded as dependencies, and we want to make sure having a missing child doesn't break the extension
+    expect(clientSession2.node.getCoValue(childGroup.id).isAvailable()).toEqual(
+      false,
+    );
+
+    group3OnClientSession2.extend(groupOnClientSession2);
+
+    expect(group3OnClientSession2.getParentGroups()).toEqual([]);
+
+    const map = group3OnClientSession2.createMap();
+    map.set("test", "Hello!");
+
+    expect(map.get("test")).toEqual("Hello!");
+  });
+
+  test("should extend groups when loaded from a different session", async () => {
+    const clientSession1 = setupTestNode({
+      connected: true,
+    });
+    const clientSession2 = clientSession1.spawnNewSession();
+
+    const group = clientSession1.node.createGroup();
+    const group2 = clientSession1.node.createGroup();
+
+    await group.core.waitForSync();
+    await group2.core.waitForSync();
+
+    const groupOnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group.id,
+    );
+    const group2OnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group2.id,
+    );
+
+    group2OnClientSession2.extend(groupOnClientSession2);
+
+    expect(group2OnClientSession2.getParentGroups()).toEqual([
+      groupOnClientSession2,
+    ]);
+
+    const map = group2OnClientSession2.createMap();
+    map.set("test", "Hello!");
+
+    expect(map.get("test")).toEqual("Hello!");
+  });
+
+  test("should extend groups when there is a cycle in the parent groups", async () => {
+    const clientSession1 = setupTestNode({
+      connected: true,
+    });
+    const clientSession2 = clientSession1.spawnNewSession();
+
+    const group = clientSession1.node.createGroup();
+    const group2 = clientSession1.node.createGroup();
+
+    await group.core.waitForSync();
+    await group2.core.waitForSync();
+
+    const groupOnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group.id,
+    );
+    const group2OnClientSession2 = await loadCoValueOrFail(
+      clientSession2.node,
+      group2.id,
+    );
+
+    group.extend(group2);
+    group2OnClientSession2.extend(groupOnClientSession2);
+
+    expect(group.getParentGroups()).toEqual([group2]);
+
+    expect(group2OnClientSession2.getParentGroups()).toEqual([
+      groupOnClientSession2,
+    ]);
+
+    await group.core.waitForSync();
+    await group2OnClientSession2.core.waitForSync();
+
+    const group3 = clientSession1.node.createGroup();
+
+    group3.extend(group2);
+
+    expect(group3.getParentGroups()).toEqual([group2]);
+
+    const map = group3.createMap();
+    map.set("test", "Hello!");
+
+    expect(map.get("test")).toEqual("Hello!");
+  });
+
   test("a writerInvite role should not be inherited", async () => {
     const { node1, node2 } = await createTwoConnectedNodes("server", "server");
 

packages/cojson/src/tests/sync.auth.test.ts

@@ -53,12 +53,14 @@ describe("LocalNode auth sync", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> server | CONTENT Account header: true new: After: 0 New: 4",
+        "client -> server | CONTENT Account header: true new: After: 0 New: 3",
         "client -> server | CONTENT ProfileGroup header: true new: After: 0 New: 5",
         "client -> server | CONTENT Profile header: true new: After: 0 New: 1",
-        "server -> client | KNOWN Account sessions: header/4",
+        "client -> server | CONTENT Account header: false new: After: 3 New: 1",
+        "server -> client | KNOWN Account sessions: header/3",
         "server -> client | KNOWN ProfileGroup sessions: header/5",
         "server -> client | KNOWN Profile sessions: header/1",
+        "server -> client | KNOWN Account sessions: header/4",
       ]
     `);
   });
@@ -114,12 +116,18 @@ describe("LocalNode auth sync", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> server | CONTENT Account header: true new: After: 0 New: 5",
-        "client -> server | CONTENT Root header: true new: After: 0 New: 1",
-        "client -> server | CONTENT Profile header: true new: After: 0 New: 1",
-        "server -> client | KNOWN Account sessions: header/5",
+        "client -> server | CONTENT Account header: true new: After: 0 New: 3",
+        "client -> server | CONTENT Root header: true new: ",
+        "client -> server | CONTENT Profile header: true new: ",
+        "client -> server | CONTENT Root header: false new: After: 0 New: 1",
+        "client -> server | CONTENT Profile header: false new: After: 0 New: 1",
+        "client -> server | CONTENT Account header: false new: After: 3 New: 2",
+        "server -> client | KNOWN Account sessions: header/3",
+        "server -> client | KNOWN Root sessions: header/0",
+        "server -> client | KNOWN Profile sessions: header/0",
         "server -> client | KNOWN Root sessions: header/1",
         "server -> client | KNOWN Profile sessions: header/1",
+        "server -> client | KNOWN Account sessions: header/5",
       ]
     `);
   });
@@ -168,13 +176,15 @@ describe("LocalNode auth sync", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "creation-node -> server | CONTENT Account header: true new: After: 0 New: 4",
+        "creation-node -> server | CONTENT Account header: true new: After: 0 New: 3",
         "creation-node -> server | CONTENT ProfileGroup header: true new: After: 0 New: 5",
         "creation-node -> server | CONTENT Profile header: true new: After: 0 New: 1",
+        "creation-node -> server | CONTENT Account header: false new: After: 3 New: 1",
         "auth-node -> server | LOAD Account sessions: empty",
-        "server -> creation-node | KNOWN Account sessions: header/4",
+        "server -> creation-node | KNOWN Account sessions: header/3",
         "server -> creation-node | KNOWN ProfileGroup sessions: header/5",
         "server -> creation-node | KNOWN Profile sessions: header/1",
+        "server -> creation-node | KNOWN Account sessions: header/4",
         "server -> auth-node | CONTENT Account header: true new: After: 0 New: 4",
         "auth-node -> server | KNOWN Account sessions: header/4",
         "auth-node -> server | LOAD Profile sessions: empty",
@@ -236,12 +246,14 @@ describe("LocalNode auth sync", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "creation-node -> server | CONTENT Account header: true new: After: 0 New: 4",
+        "creation-node -> server | CONTENT Account header: true new: After: 0 New: 3",
         "creation-node -> server | CONTENT ProfileGroup header: true new: After: 0 New: 5",
         "creation-node -> server | CONTENT Profile header: true new: After: 0 New: 1",
-        "server -> creation-node | KNOWN Account sessions: header/4",
+        "creation-node -> server | CONTENT Account header: false new: After: 3 New: 1",
+        "server -> creation-node | KNOWN Account sessions: header/3",
         "server -> creation-node | KNOWN ProfileGroup sessions: header/5",
         "server -> creation-node | KNOWN Profile sessions: header/1",
+        "server -> creation-node | KNOWN Account sessions: header/4",
         "auth-node -> server | LOAD Account sessions: empty",
         "server -> auth-node | CONTENT Account header: true new: After: 0 New: 4",
         "auth-node -> server | KNOWN Account sessions: header/4",

packages/cojson/src/tests/sync.load.test.ts

@@ -15,15 +15,15 @@ import {
   waitFor,
 } from "./testUtils";
 
-// We want to simulate a real world communication that happens asynchronously
-TEST_NODE_CONFIG.withAsyncPeers = true;
-
 let jazzCloud: ReturnType<typeof setupTestNode>;
 
 // Set a short timeout to make the tests on unavailable complete faster
 setCoValueLoadingRetryDelay(100);
 
 beforeEach(async () => {
+  // We want to simulate a real world communication that happens asynchronously
+  TEST_NODE_CONFIG.withAsyncPeers = true;
+
   SyncMessagesLog.clear();
   jazzCloud = setupTestNode({ isSyncServer: true });
 });
@@ -295,10 +295,9 @@ describe("loading coValues from server", () => {
       [
         "client -> server | LOAD Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/1",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/3",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
         "client -> server | KNOWN Map sessions: header/2",
       ]
@@ -340,11 +339,10 @@ describe("loading coValues from server", () => {
       [
         "client -> server | LOAD Group sessions: header/5",
         "client -> server | LOAD Map sessions: header/2",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
-        "client -> server | CONTENT Map header: false new: After: 0 New: 1",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/5",
+        "client -> server | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/3",
         "server -> client | KNOWN Map sessions: header/3",
         "client -> server | KNOWN Map sessions: header/3",
@@ -550,6 +548,8 @@ describe("loading coValues from server", () => {
   });
 
   test("should handle reconnections in the middle of a load with a persistent peer", async () => {
+    TEST_NODE_CONFIG.withAsyncPeers = false; // To avoid flakiness
+
     const client = setupTestNode();
     const connection1 = client.connectToSyncServer({
       persistent: true,
@@ -591,9 +591,9 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | KNOWN Group sessions: header/5",
         "client -> server | LOAD Map sessions: empty",
-        "client -> server | LOAD Group sessions: header/5",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | LOAD Group sessions: header/5",
         "client -> server | KNOWN Map sessions: header/1",
       ]
     `);
@@ -634,7 +634,7 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | KNOWN ParentGroup sessions: header/6",
         "client -> server | LOAD Group sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Map sessions: header/1",
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | KNOWN Group sessions: header/5",
       ]
@@ -677,8 +677,8 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD ParentGroup sessions: empty",
-        "client -> server | KNOWN Group sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Group sessions: header/5",
+        "client -> server | KNOWN Map sessions: header/1",
         "server -> client | CONTENT ParentGroup header: true new: After: 0 New: 6",
         "client -> server | KNOWN ParentGroup sessions: header/6",
       ]
@@ -723,8 +723,8 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD Account sessions: empty",
-        "client -> server | KNOWN Group sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Group sessions: header/0",
+        "client -> server | KNOWN Map sessions: header/0",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | KNOWN Account sessions: header/4",
         "client -> server | KNOWN Group sessions: header/5",
@@ -787,7 +787,7 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Map header: true new: After: 0 New: 1 | After: 0 New: 1",
         "client -> server | KNOWN Group sessions: header/5",
         "client -> server | LOAD Account sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Map sessions: header/1",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | KNOWN Account sessions: header/4",
         "client -> server | KNOWN Map sessions: header/2",
@@ -851,8 +851,8 @@ describe("loading coValues from server", () => {
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD Account sessions: empty",
-        "client -> server | KNOWN Group sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Group sessions: header/0",
+        "client -> server | KNOWN Map sessions: header/0",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | KNOWN Account sessions: header/4",
@@ -922,7 +922,7 @@ describe("loading coValues from server", () => {
         "client -> server | LOAD Map sessions: empty",
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD Account sessions: empty",
-        "client -> server | KNOWN Map sessions: empty",
+        "client -> server | KNOWN Map sessions: header/0",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "server -> client | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | KNOWN Account sessions: header/4",
@@ -960,14 +960,17 @@ describe("loading coValues from server", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 8",
-        "client -> server | CONTENT Group header: true new: After: 0 New: 6",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "client -> server | CONTENT Group header: false new: After: 3 New: 3",
+        "client -> server | CONTENT ParentGroup header: false new: After: 6 New: 2",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
-        "server -> client | LOAD Group sessions: empty",
-        "server -> client | KNOWN ParentGroup sessions: empty",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | KNOWN ParentGroup sessions: header/6",
         "server -> client | KNOWN Group sessions: header/6",
+        "server -> client | KNOWN ParentGroup sessions: header/8",
         "server -> client | KNOWN Map sessions: header/1",
-        "client -> server | CONTENT Group header: true new: After: 0 New: 6",
+        "client -> server | CONTENT ParentGroup header: true new: ",
       ]
     `);
   });

packages/cojson/src/tests/sync.mesh.test.ts

@@ -142,19 +142,26 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "edge-france -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "edge-france -> core | CONTENT ParentGroup header: true new: After: 0 New: 6",
-        "edge-france -> core | CONTENT Group header: true new: After: 0 New: 5",
+        "edge-france -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "edge-france -> core | CONTENT Group header: true new: After: 0 New: 3",
         "edge-france -> storage | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "edge-france -> core | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "edge-france -> storage | CONTENT Group header: false new: After: 3 New: 2",
+        "edge-france -> core | CONTENT Group header: false new: After: 3 New: 2",
         "edge-france -> storage | CONTENT Map header: true new: After: 0 New: 1",
         "edge-france -> core | CONTENT Map header: true new: After: 0 New: 1",
+        "core -> edge-france | KNOWN Group sessions: header/3",
+        "core -> storage | CONTENT Group header: true new: After: 0 New: 3",
         "core -> edge-france | KNOWN ParentGroup sessions: header/6",
         "core -> storage | CONTENT ParentGroup header: true new: After: 0 New: 6",
         "core -> edge-france | KNOWN Group sessions: header/5",
-        "core -> storage | CONTENT Group header: true new: After: 0 New: 5",
+        "core -> storage | CONTENT Group header: false new: After: 3 New: 2",
         "core -> edge-france | KNOWN Map sessions: header/1",
         "core -> storage | CONTENT Map header: true new: After: 0 New: 1",
+        "edge-france -> core | CONTENT ParentGroup header: true new: ",
         "client -> edge-italy | LOAD Map sessions: empty",
+        "core -> edge-france | KNOWN ParentGroup sessions: header/6",
+        "core -> storage | CONTENT ParentGroup header: true new: ",
         "edge-italy -> storage | LOAD Map sessions: empty",
         "storage -> edge-italy | KNOWN Map sessions: empty",
         "edge-italy -> core | LOAD Map sessions: empty",
@@ -509,8 +516,7 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
     ).toMatchInlineSnapshot(`
       [
         "edge -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "edge -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "edge -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "edge -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "edge -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "edge -> storage | CONTENT Map header: false new: After: 146 New: 54",
       ]

packages/cojson/src/tests/sync.peerReconciliation.test.ts

@@ -47,6 +47,8 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN Map sessions: empty",
         "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN Map sessions: header/1",
+        "client -> server | CONTENT Group header: true new: ",
+        "client -> server | CONTENT Map header: true new: ",
       ]
     `);
   });
@@ -203,7 +205,7 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN CORRECTION Map sessions: empty",
         "client -> server | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | LOAD Group sessions: empty",
-        "server -> client | KNOWN Map sessions: empty",
+        "server -> client | KNOWN Map sessions: header/2",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "server -> client | KNOWN Group sessions: header/3",
       ]
@@ -276,8 +278,8 @@ describe("peer reconciliation", () => {
         "client -> server | CONTENT Profile header: true new: After: 0 New: 1",
         "client -> server | CONTENT Map header: false new: After: 1 New: 1",
         "server -> client | LOAD Account sessions: empty",
-        "server -> client | KNOWN ProfileGroup sessions: empty",
-        "server -> client | KNOWN Profile sessions: empty",
+        "server -> client | KNOWN ProfileGroup sessions: header/0",
+        "server -> client | KNOWN Profile sessions: header/0",
         "server -> client | KNOWN CORRECTION Map sessions: empty",
         "client -> server | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | CONTENT Map header: true new: After: 0 New: 2",
@@ -285,7 +287,7 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN ProfileGroup sessions: header/5",
         "server -> client | KNOWN Profile sessions: header/1",
         "server -> client | LOAD Group sessions: empty",
-        "server -> client | KNOWN Map sessions: empty",
+        "server -> client | KNOWN Map sessions: header/2",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "server -> client | KNOWN Group sessions: header/3",
       ]

packages/cojson/src/tests/sync.storage.test.ts

@@ -181,13 +181,11 @@ describe("client with storage syncs with server", () => {
       [
         "client -> server | LOAD Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/1",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/3",
-        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
-        "client -> storage | CONTENT Map header: true new: After: 0 New: 2",
+        "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
         "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
       ]
@@ -291,20 +289,16 @@ describe("client syncs with a server with storage", () => {
       [
         "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "client -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 73",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 73",
         "client -> storage | CONTENT Map header: false new: After: 73 New: 73",
-        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
-        "client -> server | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
         "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
         "client -> server | CONTENT Map header: false new: After: 146 New: 54",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Map sessions: header/0",
-        "server -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
         "server -> client | KNOWN Map sessions: header/73",
-        "server -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "server -> client | KNOWN Map sessions: header/146",
         "server -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "server -> client | KNOWN Map sessions: header/200",
@@ -410,7 +404,7 @@ describe("client syncs with a server with storage", () => {
 
     const correctionSpy = vi.fn();
 
-    client.node.storage?.store(newContentChunks.slice(1, 2), correctionSpy);
+    client.node.storage?.store(newContentChunks[1]!, correctionSpy);
 
     expect(correctionSpy).not.toHaveBeenCalled();
 

packages/cojson/src/tests/sync.storageAsync.test.ts

@@ -16,6 +16,7 @@ describe("client with storage syncs with server", () => {
   let jazzCloud: ReturnType<typeof setupTestNode>;
 
   beforeEach(async () => {
+    vi.resetAllMocks();
     SyncMessagesLog.clear();
     jazzCloud = setupTestNode({
       isSyncServer: true,
@@ -174,15 +175,43 @@ describe("client with storage syncs with server", () => {
       [
         "client -> server | LOAD Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/1",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/3",
-        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
-        "client -> server | KNOWN Map sessions: header/2",
-        "client -> storage | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
         "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
+        "client -> server | KNOWN Map sessions: header/2",
+        "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
+      ]
+    `);
+  });
+
+  test("the order of updates between CoValues should be preserved to ensure consistency in case of shutdown in the middle of sync", async () => {
+    const client = setupTestNode();
+
+    await client.addAsyncStorage();
+
+    const group = client.node.createGroup();
+    const initialMap = group.createMap();
+
+    const child = group.createMap();
+    child.set("parent", initialMap.id);
+    initialMap.set("child", child.id);
+
+    await initialMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        InitialMap: initialMap.core,
+        ChildMap: child.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> storage | CONTENT InitialMap header: true new: ",
+        "client -> storage | CONTENT ChildMap header: true new: After: 0 New: 1",
+        "client -> storage | CONTENT InitialMap header: false new: After: 0 New: 1",
       ]
     `);
   });
@@ -271,20 +300,16 @@ describe("client syncs with a server with storage", () => {
       [
         "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "client -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 73",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 73",
         "client -> storage | CONTENT Map header: false new: After: 73 New: 73",
-        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
-        "client -> server | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
         "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
         "client -> server | CONTENT Map header: false new: After: 146 New: 54",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Map sessions: header/0",
-        "server -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
         "server -> client | KNOWN Map sessions: header/73",
-        "server -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "server -> client | KNOWN Map sessions: header/146",
         "server -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "server -> client | KNOWN Map sessions: header/200",
@@ -369,7 +394,7 @@ describe("client syncs with a server with storage", () => {
 
     const correctionSpy = vi.fn();
 
-    client.node.storage?.store(newContentChunks.slice(1, 2), correctionSpy);
+    client.node.storage?.store(newContentChunks[1]!, correctionSpy);
 
     // Wait for the content to be stored in the storage
     // We can't use waitForSync because we are trying to store stale data

packages/cojson/src/tests/sync.test.ts

@@ -100,15 +100,18 @@ test("Can sync a coValue with private transactions through a server to another c
 
   const map = group.createMap();
   map.set("hello", "world", "private");
+
   group.addMember("everyone", "reader");
 
   const { node: client2 } = await setupTestAccount({
     connected: true,
   });
 
-  const mapOnClient2 = await loadCoValueOrFail(client2, map.id);
+  await waitFor(async () => {
+    const loadedMap = await loadCoValueOrFail(client2, map.id);
 
-  expect(mapOnClient2.get("hello")).toEqual("world");
+    expect(loadedMap.get("hello")).toEqual("world");
+  });
 });
 
 test("should keep the peer state when the peer closes if persistent is true", async () => {
@@ -563,8 +566,6 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
     const mapOnClient = group.createMap();
     mapOnClient.set("key1", "value1", "trusting");
 
-    await client.syncManager.syncCoValue(mapOnClient.core);
-
     // Wait for the full sync to complete
     await mapOnClient.core.waitForSync();
 
@@ -594,7 +595,6 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
     await map.core.waitForSync();
 
     // Block the content messages
@@ -606,7 +606,7 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
 
     map.set("key2", "value2", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
+    await new Promise<void>(queueMicrotask);
 
     expect(peerState.optimisticKnownStates.get(map.core.id)).not.toEqual(
       peerState.knownStates.get(map.core.id),
@@ -638,8 +638,6 @@ describe("SyncManager.addPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     // Wait for initial sync
     await map.core.waitForSync();
 
@@ -671,8 +669,6 @@ describe("SyncManager.addPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     // Wait for initial sync
     await map.core.waitForSync();
 
@@ -843,8 +839,6 @@ describe("waitForSyncWithPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     await expect(
       client.node.syncManager.waitForSyncWithPeer(
         peerState.id,
@@ -868,8 +862,6 @@ describe("waitForSyncWithPeer", () => {
       return Promise.resolve();
     });
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     await expect(
       client.node.syncManager.waitForSyncWithPeer(
         peerState.id,

packages/cojson/src/tests/sync.upload.test.ts

@@ -78,12 +78,15 @@ describe("client to server upload", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 6",
-        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Group header: false new: After: 3 New: 2",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN ParentGroup sessions: header/6",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> client | KNOWN Map sessions: header/1",
+        "client -> server | CONTENT ParentGroup header: true new: ",
       ]
     `);
   });
@@ -253,6 +256,40 @@ describe("client to server upload", () => {
     `);
   });
 
+  test("local updates batching", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const initialMap = group.createMap();
+
+    const child = group.createMap();
+    child.set("parent", initialMap.id);
+    initialMap.set("child", child.id);
+
+    await initialMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        InitialMap: initialMap.core,
+        ChildMap: child.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | CONTENT InitialMap header: true new: ",
+        "client -> server | CONTENT ChildMap header: true new: After: 0 New: 1",
+        "client -> server | CONTENT InitialMap header: false new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | KNOWN InitialMap sessions: header/0",
+        "server -> client | KNOWN ChildMap sessions: header/1",
+        "server -> client | KNOWN InitialMap sessions: header/1",
+      ]
+    `);
+  });
+
   test("large coValue upload streaming", async () => {
     const client = setupTestNode({
       connected: true,

packages/cojson/src/tests/testStorage.ts

@@ -5,11 +5,7 @@ import { join } from "node:path";
 import Database, { type Database as DatabaseT } from "libsql";
 import { onTestFinished } from "vitest";
 import { RawCoID, StorageAPI } from "../exports";
-import {
-  SQLiteDatabaseDriver,
-  StorageApiAsync,
-  StorageApiSync,
-} from "../storage";
+import { SQLiteDatabaseDriver } from "../storage";
 import { getSqliteStorage } from "../storage/sqlite";
 import {
   SQLiteDatabaseDriverAsync,
@@ -148,13 +144,11 @@ function trackStorageMessages(
   const originalLoad = storage.load;
 
   storage.store = function (data, correctionCallback) {
-    for (const msg of data ?? []) {
-      SyncMessagesLog.add({
-        from: nodeName,
-        to: storageName,
-        msg,
-      });
-    }
+    SyncMessagesLog.add({
+      from: nodeName,
+      to: storageName,
+      msg: data,
+    });
 
     return originalStore.call(storage, data, (correction) => {
       SyncMessagesLog.add({
@@ -167,7 +161,19 @@ function trackStorageMessages(
         },
       });
 
-      return correctionCallback(correction);
+      const correctionMessages = correctionCallback(correction);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          SyncMessagesLog.add({
+            from: nodeName,
+            to: storageName,
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

packages/cojson/src/tests/testUtils.ts

@@ -40,6 +40,14 @@ export function randomAgentAndSessionID(): [ControlledAgent, SessionID] {
   return [new ControlledAgent(agentSecret, Crypto), sessionID];
 }
 
+export function agentAndSessionIDFromSecret(
+  secret: AgentSecret,
+): [ControlledAgent, SessionID] {
+  const sessionID = Crypto.newRandomSessionID(Crypto.getAgentID(secret));
+
+  return [new ControlledAgent(secret, Crypto), sessionID];
+}
+
 export function nodeWithRandomAgentAndSessionID() {
   const [agent, session] = randomAgentAndSessionID();
   return new LocalNode(agent.agentSecret, session, Crypto);
@@ -154,8 +162,8 @@ export function connectTwoPeers(
   bRole: "client" | "server",
 ) {
   const [aAsPeer, bAsPeer] = connectedPeers(
-    "peer:" + a.getCurrentAgent().id,
-    "peer:" + b.getCurrentAgent().id,
+    "peer:" + a.currentSessionID,
+    "peer:" + b.currentSessionID,
     {
       peer1role: aRole,
       peer2role: bRole,
@@ -443,7 +451,7 @@ export function getSyncServerConnectedPeer(opts: {
 
   const { peer1, peer2 } = connectedPeersWithMessagesTracking({
     peer1: {
-      id: currentSyncServer.getCurrentAgent().id,
+      id: currentSyncServer.currentSessionID,
       role: "server",
       name: opts.syncServerName,
     },
@@ -472,9 +480,13 @@ export function setupTestNode(
   opts: {
     isSyncServer?: boolean;
     connected?: boolean;
+    secret?: AgentSecret;
   } = {},
 ) {
-  const [admin, session] = randomAgentAndSessionID();
+  const [admin, session] = opts.secret
+    ? agentAndSessionIDFromSecret(opts.secret)
+    : randomAgentAndSessionID();
+
   let node = new LocalNode(admin.agentSecret, session, Crypto);
 
   if (opts.isSyncServer) {
@@ -489,7 +501,7 @@ export function setupTestNode(
   }) {
     const { peer, peerStateOnServer, peerOnServer } =
       getSyncServerConnectedPeer({
-        peerId: node.getCurrentAgent().id,
+        peerId: session,
         syncServerName: opts?.syncServerName,
         ourName: opts?.ourName,
         syncServer: opts?.syncServer,
@@ -551,6 +563,13 @@ export function setupTestNode(
 
       return node;
     },
+    spawnNewSession: () => {
+      return setupTestNode({
+        secret: node.agentSecret,
+        connected: opts.connected,
+        isSyncServer: opts.isSyncServer,
+      });
+    },
   };
 
   return ctx;

packages/jazz-auth-betterauth/CHANGELOG.md

@@ -1,5 +1,16 @@
 # jazz-auth-betterauth
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [16764f6]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+  - jazz-tools@0.16.4
+  - jazz-betterauth-client-plugin@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-auth-betterauth/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "jazz-auth-betterauth",
-	"version": "0.16.3",
+	"version": "0.16.4",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "src/index.ts",

packages/jazz-betterauth-client-plugin/CHANGELOG.md

@@ -1,5 +1,11 @@
 # jazz-betterauth-client-plugin
 
+## 0.16.4
+
+### Patch Changes
+
+- jazz-betterauth-server-plugin@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-betterauth-client-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jazz-betterauth-client-plugin",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "type": "module",
   "main": "dist/index.js",
   "types": "src/index.ts",

packages/jazz-betterauth-server-plugin/CHANGELOG.md

@@ -1,5 +1,15 @@
 # jazz-betterauth-server-plugin
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [16764f6]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+  - jazz-tools@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-betterauth-server-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jazz-betterauth-server-plugin",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "type": "module",
   "main": "dist/index.js",
   "types": "src/index.ts",

packages/jazz-react-auth-betterauth/CHANGELOG.md

@@ -1,5 +1,17 @@
 # jazz-react-auth-betterauth
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [16764f6]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+  - jazz-tools@0.16.4
+  - jazz-auth-betterauth@0.16.4
+  - jazz-betterauth-client-plugin@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-react-auth-betterauth/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "jazz-react-auth-betterauth",
-	"version": "0.16.3",
+	"version": "0.16.4",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "src/index.tsx",

packages/jazz-run/CHANGELOG.md

@@ -1,5 +1,17 @@
 # jazz-run
 
+## 0.16.4
+
+### Patch Changes
+
+- Updated dependencies [f9d538f]
+- Updated dependencies [16764f6]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+  - jazz-tools@0.16.4
+  - cojson-storage-sqlite@0.16.4
+  - cojson-transport-ws@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-run/package.json

@@ -3,7 +3,7 @@
   "bin": "./dist/index.js",
   "type": "module",
   "license": "MIT",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "exports": {
     "./startSyncServer": {
       "types": "./dist/startSyncServer.d.ts",
@@ -28,11 +28,11 @@
     "@effect/printer-ansi": "^0.34.5",
     "@effect/schema": "^0.71.1",
     "@effect/typeclass": "^0.25.5",
-    "cojson": "workspace:0.16.3",
-    "cojson-storage-sqlite": "workspace:0.16.3",
-    "cojson-transport-ws": "workspace:0.16.3",
+    "cojson": "workspace:0.16.4",
+    "cojson-storage-sqlite": "workspace:0.16.4",
+    "cojson-transport-ws": "workspace:0.16.4",
     "effect": "^3.6.5",
-    "jazz-tools": "workspace:0.16.3",
+    "jazz-tools": "workspace:0.16.4",
     "ws": "^8.14.2"
   },
   "devDependencies": {

packages/jazz-tools/CHANGELOG.md

@@ -1,5 +1,16 @@
 # jazz-tools
 
+## 0.16.4
+
+### Patch Changes
+
+- 16764f6: Added `pick()` and `partial()` methods to CoMapSchema
+- Updated dependencies [f9d538f]
+- Updated dependencies [802b5a3]
+  - cojson@0.16.4
+  - cojson-storage-indexeddb@0.16.4
+  - cojson-transport-ws@0.16.4
+
 ## 0.16.3
 
 ### Patch Changes

packages/jazz-tools/package.json

@@ -139,7 +139,7 @@
   },
   "type": "module",
   "license": "MIT",
-  "version": "0.16.3",
+  "version": "0.16.4",
   "dependencies": {
     "@manuscripts/prosemirror-recreate-steps": "^0.1.4",
     "@scure/base": "1.2.1",

packages/jazz-tools/src/tools/coValues/interfaces.ts

@@ -2,7 +2,6 @@ import {
   type CoValueUniqueness,
   type CojsonInternalTypes,
   type RawCoValue,
-  emptyKnownState,
 } from "cojson";
 import { AvailableCoValueCore } from "cojson/dist/coValueCore/coValueCore.js";
 import {
@@ -546,9 +545,9 @@ function loadContentPiecesFromCoValue(
     }
   }
 
-  const pieces = core.verified.newContentSince(emptyKnownState(core.id));
+  const pieces = core.verified.newContentSince(undefined) ?? [];
 
-  for (const piece of pieces ?? []) {
+  for (const piece of pieces) {
     contentPieces.push(piece);
   }
 }

packages/jazz-tools/src/tools/implementation/zodSchema/schemaTypes/CoMapSchema.ts

@@ -10,6 +10,7 @@ import {
   Resolved,
   Simplify,
   SubscribeListenerOptions,
+  coMapDefiner,
   coOptionalDefiner,
   hydrateCoreCoValueSchema,
   isAnyCoValueSchema,
@@ -20,8 +21,9 @@ import { CoMapSchemaInit } from "../typeConverters/CoFieldInit.js";
 import { InstanceOrPrimitiveOfSchema } from "../typeConverters/InstanceOrPrimitiveOfSchema.js";
 import { InstanceOrPrimitiveOfSchemaCoValuesNullable } from "../typeConverters/InstanceOrPrimitiveOfSchemaCoValuesNullable.js";
 import { z } from "../zodReExport.js";
-import { AnyZodOrCoValueSchema } from "../zodSchema.js";
+import { AnyZodOrCoValueSchema, AnyZodSchema } from "../zodSchema.js";
 import { CoOptionalSchema } from "./CoOptionalSchema.js";
+import { CoreCoValueSchema } from "./CoValueSchema.js";
 
 export interface CoMapSchema<
   Shape extends z.core.$ZodLooseShape,
@@ -133,6 +135,23 @@ export interface CoMapSchema<
   getCoValueClass: () => typeof CoMap;
 
   optional(): CoOptionalSchema<this>;
+
+  /**
+   * Creates a new CoMap schema by picking the specified keys from the original schema.
+   *
+   * @param keys - The keys to pick from the original schema.
+   * @returns A new CoMap schema with the picked keys.
+   */
+  pick<Keys extends keyof Shape>(
+    keys: { [key in Keys]: true },
+  ): CoMapSchema<Simplify<Pick<Shape, Keys>>, unknown, Owner>;
+
+  /**
+   * Creates a new CoMap schema by making all fields optional.
+   *
+   * @returns A new CoMap schema with all fields optional.
+   */
+  partial(): CoMapSchema<PartialShape<Shape>, CatchAll, Owner>;
 }
 
 export function createCoreCoMapSchema<
@@ -218,10 +237,40 @@ export function enrichCoMapSchema<
     getCoValueClass: () => {
       return coValueClass;
     },
-
     optional: () => {
       return coOptionalDefiner(coValueSchema);
     },
+    pick: <Keys extends keyof Shape>(keys: { [key in Keys]: true }) => {
+      const keysSet = new Set(Object.keys(keys));
+      const pickedShape: Record<string, AnyZodOrCoValueSchema> = {};
+
+      for (const [key, value] of Object.entries(coValueSchema.shape)) {
+        if (keysSet.has(key)) {
+          pickedShape[key] = value;
+        }
+      }
+
+      return coMapDefiner(pickedShape);
+    },
+    partial: () => {
+      const partialShape: Record<string, AnyZodOrCoValueSchema> = {};
+
+      for (const [key, value] of Object.entries(coValueSchema.shape)) {
+        if (isAnyCoValueSchema(value)) {
+          partialShape[key] = coOptionalDefiner(value);
+        } else {
+          partialShape[key] = z.optional(coValueSchema.shape[key]);
+        }
+      }
+
+      const partialCoMapSchema = coMapDefiner(partialShape);
+      if (coValueSchema.catchAll) {
+        return partialCoMapSchema.catchall(
+          coValueSchema.catchAll as unknown as AnyZodOrCoValueSchema,
+        );
+      }
+      return partialCoMapSchema;
+    },
   }) as unknown as CoMapSchema<Shape, CatchAll>;
   return coValueSchema;
 }
@@ -262,3 +311,11 @@ export type CoMapInstanceCoValuesNullable<Shape extends z.core.$ZodLooseShape> =
       Shape[key]
     >;
   };
+
+export type PartialShape<Shape extends z.core.$ZodLooseShape> = Simplify<{
+  -readonly [key in keyof Shape]: Shape[key] extends AnyZodSchema
+    ? z.ZodOptional<Shape[key]>
+    : Shape[key] extends CoreCoValueSchema
+      ? CoOptionalSchema<Shape[key]>
+      : never;
+}>;

packages/jazz-tools/src/tools/implementation/zodSchema/zodSchema.ts

@@ -101,7 +101,7 @@ export type AnyCoreCoValueSchema =
   | CoreRichTextSchema
   | CoreFileStreamSchema;
 
-type AnyZodSchema = z.core.$ZodType;
+export type AnyZodSchema = z.core.$ZodType;
 
 export type AnyZodOrCoValueSchema = AnyZodSchema | CoreCoValueSchema;
 

packages/jazz-tools/src/tools/tests/coMap.test-d.ts

@@ -1,4 +1,5 @@
 import { assert, describe, expectTypeOf, test } from "vitest";
+import { ZodNumber, ZodOptional, ZodString } from "zod/v4";
 import { Group, co, z } from "../exports.js";
 import { Account } from "../index.js";
 import { CoMap, Loaded } from "../internal.js";
@@ -350,6 +351,148 @@ describe("CoMap", async () => {
       matchesNarrowed(mapWithEnum.child);
     }
   });
+
+  test("CoMap.pick()", () => {
+    const Person = co.map({
+      name: z.string(),
+      age: z.number(),
+      dog: co.map({
+        name: z.string(),
+        breed: z.string(),
+      }),
+    });
+
+    const PersonWithoutDog = Person.pick({
+      name: true,
+      age: true,
+    });
+
+    type ExpectedType = co.Map<{
+      name: ZodString;
+      age: ZodNumber;
+    }>;
+
+    function matches(value: ExpectedType) {
+      return value;
+    }
+
+    matches(PersonWithoutDog);
+  });
+
+  test("CoMap.pick() with a recursive reference", () => {
+    const Person = co.map({
+      name: z.string(),
+      age: z.number(),
+      dog: co.map({
+        name: z.string(),
+        breed: z.string(),
+      }),
+      get friend() {
+        return Person.pick({
+          name: true,
+          age: true,
+        }).optional();
+      },
+    });
+
+    type ExpectedType = co.Map<{
+      name: ZodString;
+      age: ZodNumber;
+      dog: co.Map<{
+        name: ZodString;
+        breed: ZodString;
+      }>;
+      friend: co.Optional<
+        co.Map<{
+          name: ZodString;
+          age: ZodNumber;
+        }>
+      >;
+    }>;
+
+    function matches(value: ExpectedType) {
+      return value;
+    }
+
+    matches(Person);
+  });
+
+  test("CoMap.partial()", () => {
+    const Person = co.map({
+      name: z.string(),
+      age: z.number(),
+      dog: co.map({
+        name: z.string(),
+        breed: z.string(),
+      }),
+    });
+
+    const PersonPartial = Person.partial();
+
+    type ExpectedType = co.Map<{
+      name: ZodOptional<ZodString>;
+      age: ZodOptional<ZodNumber>;
+      dog: co.Optional<
+        co.Map<{
+          name: ZodString;
+          breed: ZodString;
+        }>
+      >;
+    }>;
+
+    function matches(value: ExpectedType) {
+      return value;
+    }
+
+    matches(PersonPartial);
+  });
+
+  test("CoMap.partial() with a recursive reference", () => {
+    const Person = co.map({
+      get draft() {
+        return Person.partial()
+          .pick({
+            name: true,
+            age: true,
+            dog: true,
+          })
+          .optional();
+      },
+      name: z.string(),
+      age: z.number(),
+      dog: co.map({
+        name: z.string(),
+        breed: z.string(),
+      }),
+    });
+
+    type ExpectedType = co.Map<{
+      draft: co.Optional<
+        co.Map<{
+          name: ZodOptional<ZodString>;
+          age: ZodOptional<ZodNumber>;
+          dog: co.Optional<
+            co.Map<{
+              name: ZodString;
+              breed: ZodString;
+            }>
+          >;
+        }>
+      >;
+      name: ZodString;
+      age: ZodNumber;
+      dog: co.Map<{
+        name: ZodString;
+        breed: ZodString;
+      }>;
+    }>;
+
+    function matches(value: ExpectedType) {
+      return value;
+    }
+
+    matches(Person);
+  });
 });
 
 describe("CoMap resolution", async () => {

packages/jazz-tools/src/tools/tests/coMap.test.ts

@@ -2181,70 +2181,30 @@ describe("CoMap migration", () => {
     expect(loaded?.friend?.name).toEqual("Charlie");
     expect(loaded?.friend?.version).toEqual(2);
   });
-  describe("Time", () => {
-    test("empty map created time", () => {
-      const currentTimestampInSeconds = Math.floor(Date.now() / 1000);
-      const emptyMap = co.map({}).create({});
-      const createdAtInSeconds = Math.floor(emptyMap._createdAt / 1000);
+});
 
-      expect(createdAtInSeconds).toEqual(currentTimestampInSeconds);
-      expect(emptyMap._lastUpdatedAt).toEqual(emptyMap._createdAt);
+describe("createdAt & lastUpdatedAt", () => {
+  test("empty map created time", () => {
+    const emptyMap = co.map({}).create({});
+
+    expect(emptyMap._lastUpdatedAt).toEqual(emptyMap._createdAt);
+  });
+
+  test("created time and last updated time", async () => {
+    const Person = co.map({
+      name: z.string(),
     });
 
-    test("created time and last updated time", async () => {
-      const Person = co.map({
-        name: z.string(),
-      });
+    const person = Person.create({ name: "John" });
 
-      let currentTimestampInSeconds = Math.floor(Date.now() / 1000);
-      const person = Person.create({ name: "John" });
+    const createdAt = person._createdAt;
+    expect(person._lastUpdatedAt).toEqual(createdAt);
 
-      const createdAt = person._createdAt;
-      const createdAtInSeconds = Math.floor(createdAt / 1000);
-      expect(createdAtInSeconds).toEqual(currentTimestampInSeconds);
-      expect(person._lastUpdatedAt).toEqual(createdAt);
+    await new Promise((r) => setTimeout(r, 10));
+    person.name = "Jane";
 
-      await new Promise((r) => setTimeout(r, 1000));
-      currentTimestampInSeconds = Math.floor(Date.now() / 1000);
-      person.name = "Jane";
-
-      const lastUpdatedAtInSeconds = Math.floor(person._lastUpdatedAt / 1000);
-      expect(lastUpdatedAtInSeconds).toEqual(currentTimestampInSeconds);
-      expect(person._createdAt).toEqual(createdAt);
-      expect(person._lastUpdatedAt).not.toEqual(createdAt);
-    });
-
-    test("comap with custom uniqueness", () => {
-      const Person = co.map({
-        name: z.string(),
-      });
-
-      let currentTimestampInSeconds = Math.floor(Date.now() / 1000);
-      const person = Person.create(
-        { name: "John" },
-        { unique: "name", owner: Account.getMe() },
-      );
-
-      const createdAt = person._createdAt;
-      const createdAtInSeconds = Math.floor(createdAt / 1000);
-      expect(createdAtInSeconds).toEqual(currentTimestampInSeconds);
-    });
-
-    test("empty comap with custom uniqueness", () => {
-      const Person = co.map({
-        name: z.optional(z.string()),
-      });
-
-      let currentTimestampInSeconds = Math.floor(Date.now() / 1000);
-      const person = Person.create(
-        {},
-        { unique: "name", owner: Account.getMe() },
-      );
-
-      const createdAt = person._createdAt;
-      const createdAtInSeconds = Math.floor(createdAt / 1000);
-      expect(createdAtInSeconds).toEqual(currentTimestampInSeconds);
-    });
+    expect(person._createdAt).toEqual(createdAt);
+    expect(person._lastUpdatedAt).not.toEqual(createdAt);
   });
 });
 
@@ -2260,6 +2220,93 @@ describe("co.map schema", () => {
 
     expect(person.name.toString()).toEqual("John");
   });
+
+  describe("pick()", () => {
+    test("creates a new CoMap schema by picking fields of another CoMap schema", () => {
+      const Person = co.map({
+        name: z.string(),
+        age: z.number(),
+      });
+
+      const PersonWithName = Person.pick({
+        name: true,
+      });
+
+      const person = PersonWithName.create({
+        name: "John",
+      });
+
+      expect(person.name).toEqual("John");
+    });
+
+    test("the new schema does not include catchall properties", () => {
+      const Person = co
+        .map({
+          name: z.string(),
+          age: z.number(),
+        })
+        .catchall(z.string());
+
+      const PersonWithName = Person.pick({
+        name: true,
+      });
+
+      expect(PersonWithName.catchAll).toBeUndefined();
+
+      const person = PersonWithName.create({
+        name: "John",
+      });
+      // @ts-expect-error - property `extraField` does not exist in person
+      expect(person.extraField).toBeUndefined();
+    });
+  });
+
+  describe("partial()", () => {
+    test("creates a new CoMap schema by making all properties optional", () => {
+      const Dog = co.map({
+        name: z.string(),
+        breed: z.string(),
+      });
+      const Person = co.map({
+        name: z.string(),
+        age: z.number(),
+        pet: Dog,
+      });
+
+      const DraftPerson = Person.partial();
+
+      const draftPerson = DraftPerson.create({});
+
+      expect(draftPerson.name).toBeUndefined();
+      expect(draftPerson.age).toBeUndefined();
+      expect(draftPerson.pet).toBeUndefined();
+
+      draftPerson.name = "John";
+      draftPerson.age = 20;
+      const rex = Dog.create({ name: "Rex", breed: "Labrador" });
+      draftPerson.pet = rex;
+
+      expect(draftPerson.name).toEqual("John");
+      expect(draftPerson.age).toEqual(20);
+      expect(draftPerson.pet).toEqual(rex);
+    });
+
+    test("the new schema includes catchall properties", () => {
+      const Person = co
+        .map({
+          name: z.string(),
+          age: z.number(),
+        })
+        .catchall(z.string());
+
+      const DraftPerson = Person.partial();
+
+      const draftPerson = DraftPerson.create({});
+      draftPerson.extraField = "extra";
+
+      expect(draftPerson.extraField).toEqual("extra");
+    });
+  });
 });
 
 describe("Updating a nested reference", () => {

packages/jazz-tools/src/tools/tests/load.test.ts

@@ -184,6 +184,25 @@ test("returns null if the value is unavailable after retries", async () => {
   expect(john).toBeNull();
 });
 
+test("load works even when the coValue access is granted after the creation", async () => {
+  const alice = await createJazzTestAccount();
+  const bob = await createJazzTestAccount();
+
+  const Person = co.map({
+    name: z.string(),
+  });
+
+  const group = Group.create(alice);
+  const map = Person.create({ name: "John" }, group);
+
+  group.addMember("everyone", "reader");
+
+  const mapOnBob = await Person.load(map.id, { loadAs: bob });
+
+  expect(mapOnBob).not.toBeNull();
+  expect(mapOnBob?.name).toBe("John");
+});
+
 test("load a large coValue", async () => {
   const syncServer = await setupJazzTestSync({ asyncPeers: true });
 

starters/react-passkey-auth/CHANGELOG.md

@@ -1,5 +1,12 @@
 # jazz-react-tailwind-starter
 
+## 0.0.141
+
+### Patch Changes
+
+- Updated dependencies [16764f6]
+  - jazz-tools@0.16.4
+
 ## 0.0.140
 
 ### Patch Changes

starters/react-passkey-auth/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jazz-react-passkey-auth-starter",
   "private": true,
-  "version": "0.0.140",
+  "version": "0.0.141",
   "type": "module",
   "scripts": {
     "dev": "vite",

starters/svelte-passkey-auth/CHANGELOG.md

@@ -1,5 +1,12 @@
 # svelte-passkey-auth
 
+## 0.0.115
+
+### Patch Changes
+
+- Updated dependencies [16764f6]
+  - jazz-tools@0.16.4
+
 ## 0.0.114
 
 ### Patch Changes

starters/svelte-passkey-auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "svelte-passkey-auth",
-  "version": "0.0.114",
+  "version": "0.0.115",
   "type": "module",
   "private": true,
   "scripts": {

tests/stress-test/src/ProjectScreen.tsx

@@ -7,7 +7,11 @@ export function ProjectScreen() {
   const { projectId } = useParams();
   const project = useCoState(TodoProject, projectId, {
     resolve: {
-      tasks: true,
+      tasks: {
+        $each: {
+          text: true,
+        },
+      },
     },
   });
   const { me } = useAccount(TodoAccount, {

tests/stress-test/src/generate.ts

@@ -4,7 +4,7 @@ import { Task, TodoProject } from "./1_schema";
 
 export function generateRandomProject(numTasks: number) {
   // Create a list of tasks
-  const tasks = TodoProject.def.shape.tasks.create([]);
+  const tasks = TodoProject.shape.tasks.create([]);
 
   // Generate random tasks
   function populateTasks() {