test(dependencies): refactor the dependencies tests to use dynamic fixtures

packages/cojson/src/CoValuesStore.ts

@@ -1,38 +0,0 @@
-import { CoValueCore } from "./coValueCore.js";
-import { CoValueState } from "./coValueState.js";
-import { RawCoID } from "./ids.js";
-
-export class CoValuesStore {
-  coValues = new Map<RawCoID, CoValueState>();
-
-  get(id: RawCoID) {
-    let entry = this.coValues.get(id);
-
-    if (!entry) {
-      entry = CoValueState.Unknown(id);
-      this.coValues.set(id, entry);
-    }
-
-    return entry;
-  }
-
-  setAsAvailable(id: RawCoID, coValue: CoValueCore) {
-    const entry = this.get(id);
-    entry.dispatch({
-      type: "available",
-      coValue,
-    });
-  }
-
-  getEntries() {
-    return this.coValues.entries();
-  }
-
-  getValues() {
-    return this.coValues.values();
-  }
-
-  getKeys() {
-    return this.coValues.keys();
-  }
-}

packages/cojson/src/PeerKnownStates.ts

@@ -1,116 +0,0 @@
-import { RawCoID, SessionID } from "./ids.js";
-import {
-  CoValueKnownState,
-  combinedKnownStates,
-  emptyKnownState,
-} from "./sync.js";
-
-export type PeerKnownStateActions =
-  | {
-      type: "SET_AS_EMPTY";
-      id: RawCoID;
-    }
-  | {
-      type: "UPDATE_HEADER";
-      id: RawCoID;
-      header: boolean;
-    }
-  | {
-      type: "UPDATE_SESSION_COUNTER";
-      id: RawCoID;
-      sessionId: SessionID;
-      value: number;
-    }
-  | {
-      type: "SET";
-      id: RawCoID;
-      value: CoValueKnownState;
-    }
-  | {
-      type: "COMBINE_WITH";
-      id: RawCoID;
-      value: CoValueKnownState;
-    };
-
-export class PeerKnownStates {
-  private coValues = new Map<RawCoID, CoValueKnownState>();
-
-  private updateHeader(id: RawCoID, header: boolean) {
-    const knownState = this.coValues.get(id) ?? emptyKnownState(id);
-    knownState.header = header;
-    this.coValues.set(id, knownState);
-  }
-
-  private combineWith(id: RawCoID, value: CoValueKnownState) {
-    const knownState = this.coValues.get(id) ?? emptyKnownState(id);
-    this.coValues.set(id, combinedKnownStates(knownState, value));
-  }
-
-  private updateSessionCounter(
-    id: RawCoID,
-    sessionId: SessionID,
-    value: number,
-  ) {
-    const knownState = this.coValues.get(id) ?? emptyKnownState(id);
-    const currentValue = knownState.sessions[sessionId] || 0;
-    knownState.sessions[sessionId] = Math.max(currentValue, value);
-
-    this.coValues.set(id, knownState);
-  }
-
-  get(id: RawCoID) {
-    return this.coValues.get(id);
-  }
-
-  has(id: RawCoID) {
-    return this.coValues.has(id);
-  }
-
-  clone() {
-    const clone = new PeerKnownStates();
-    clone.coValues = new Map(this.coValues);
-    return clone;
-  }
-
-  dispatch(action: PeerKnownStateActions) {
-    switch (action.type) {
-      case "UPDATE_HEADER":
-        this.updateHeader(action.id, action.header);
-        break;
-      case "UPDATE_SESSION_COUNTER":
-        this.updateSessionCounter(action.id, action.sessionId, action.value);
-        break;
-      case "SET":
-        this.coValues.set(action.id, action.value);
-        break;
-      case "COMBINE_WITH":
-        this.combineWith(action.id, action.value);
-        break;
-      case "SET_AS_EMPTY":
-        this.coValues.set(action.id, emptyKnownState(action.id));
-        break;
-    }
-
-    this.triggerUpdate(action.id);
-  }
-
-  listeners = new Set<(id: RawCoID, knownState: CoValueKnownState) => void>();
-
-  triggerUpdate(id: RawCoID) {
-    this.trigger(id, this.coValues.get(id) ?? emptyKnownState(id));
-  }
-
-  private trigger(id: RawCoID, knownState: CoValueKnownState) {
-    for (const listener of this.listeners) {
-      listener(id, knownState);
-    }
-  }
-
-  subscribe(listener: (id: RawCoID, knownState: CoValueKnownState) => void) {
-    this.listeners.add(listener);
-
-    return () => {
-      this.listeners.delete(listener);
-    };
-  }
-}

packages/cojson/src/PeerState.ts

@@ -1,149 +0,0 @@
-import { PeerKnownStateActions, PeerKnownStates } from "./PeerKnownStates.js";
-import {
-  PriorityBasedMessageQueue,
-  QueueEntry,
-} from "./PriorityBasedMessageQueue.js";
-import { TryAddTransactionsError } from "./coValueCore.js";
-import { RawCoID } from "./ids.js";
-import { logger } from "./logger.js";
-import { CO_VALUE_PRIORITY } from "./priority.js";
-import { Peer, SyncMessage } from "./sync.js";
-
-export class PeerState {
-  constructor(
-    private peer: Peer,
-    knownStates: PeerKnownStates | undefined,
-  ) {
-    this.optimisticKnownStates = knownStates?.clone() ?? new PeerKnownStates();
-
-    // We assume that exchanges with storage peers are always successful
-    // hence we don't need to differentiate between knownStates and optimisticKnownStates
-    if (peer.role === "storage") {
-      this.knownStates = this.optimisticKnownStates;
-    } else {
-      this.knownStates = knownStates?.clone() ?? new PeerKnownStates();
-    }
-  }
-
-  /**
-   * Here we to collect all the known states that a given peer has told us about.
-   *
-   * This can be used to safely track the sync state of a coValue in a given peer.
-   */
-  readonly knownStates: PeerKnownStates;
-
-  /**
-   * This one collects the known states "optimistically".
-   * We use it to keep track of the content we have sent to a given peer.
-   *
-   * The main difference with knownState is that this is updated when the content is sent to the peer without
-   * waiting for any acknowledgement from the peer.
-   */
-  readonly optimisticKnownStates: PeerKnownStates;
-  readonly toldKnownState: Set<RawCoID> = new Set();
-
-  dispatchToKnownStates(action: PeerKnownStateActions) {
-    this.knownStates.dispatch(action);
-
-    if (this.role !== "storage") {
-      this.optimisticKnownStates.dispatch(action);
-    }
-  }
-
-  readonly erroredCoValues: Map<RawCoID, TryAddTransactionsError> = new Map();
-
-  get id() {
-    return this.peer.id;
-  }
-
-  get role() {
-    return this.peer.role;
-  }
-
-  get priority() {
-    return this.peer.priority;
-  }
-
-  get crashOnClose() {
-    return this.peer.crashOnClose;
-  }
-
-  shouldRetryUnavailableCoValues() {
-    return this.peer.role === "server";
-  }
-
-  isServerOrStoragePeer() {
-    return this.peer.role === "server" || this.peer.role === "storage";
-  }
-
-  /**
-   * We set as default priority HIGH to handle all the messages without a
-   * priority property as HIGH priority.
-   *
-   * This way we consider all the non-content messsages as HIGH priority.
-   */
-  private queue = new PriorityBasedMessageQueue(CO_VALUE_PRIORITY.HIGH);
-  private processing = false;
-  public closed = false;
-
-  async processQueue() {
-    if (this.processing) {
-      return;
-    }
-
-    this.processing = true;
-
-    let entry: QueueEntry<SyncMessage> | undefined;
-    while ((entry = this.queue.pull())) {
-      // Awaiting the push to send one message at a time
-      // This way when the peer is "under pressure" we can enqueue all
-      // the coming messages and organize them by priority
-      await this.peer.outgoing
-        .push(entry.msg)
-        .then(entry.resolve)
-        .catch(entry.reject);
-    }
-
-    this.processing = false;
-  }
-
-  pushOutgoingMessage(msg: SyncMessage) {
-    if (this.closed) {
-      return Promise.resolve();
-    }
-
-    const promise = this.queue.push(msg);
-
-    void this.processQueue();
-
-    return promise;
-  }
-
-  get incoming() {
-    if (this.closed) {
-      return (async function* () {
-        yield "Disconnected" as const;
-      })();
-    }
-
-    return this.peer.incoming;
-  }
-
-  private closeQueue() {
-    let entry: QueueEntry<SyncMessage> | undefined;
-    while ((entry = this.queue.pull())) {
-      // Using resolve here to avoid unnecessary noise in the logs
-      entry.resolve();
-    }
-  }
-
-  gracefulShutdown() {
-    logger.debug("Gracefully closing", {
-      peerId: this.id,
-      peerRole: this.role,
-    });
-    this.closeQueue();
-    this.peer.outgoing.close();
-    this.closed = true;
-  }
-}

packages/cojson/src/SyncStateManager.ts

@@ -1,154 +0,0 @@
-import { RawCoID } from "./ids.js";
-import {
-  CoValueKnownState,
-  PeerID,
-  SyncManager,
-  emptyKnownState,
-} from "./sync.js";
-
-export type SyncState = {
-  uploaded: boolean;
-};
-
-export type GlobalSyncStateListenerCallback = (
-  peerId: PeerID,
-  knownState: CoValueKnownState,
-  sync: SyncState,
-) => void;
-
-export type PeerSyncStateListenerCallback = (
-  knownState: CoValueKnownState,
-  sync: SyncState,
-) => void;
-
-export class SyncStateManager {
-  constructor(private syncManager: SyncManager) {}
-
-  private listeners = new Set<GlobalSyncStateListenerCallback>();
-  private listenersByPeers = new Map<
-    PeerID,
-    Set<PeerSyncStateListenerCallback>
-  >();
-
-  subscribeToUpdates(listener: GlobalSyncStateListenerCallback) {
-    this.listeners.add(listener);
-
-    return () => {
-      this.listeners.delete(listener);
-    };
-  }
-
-  subscribeToPeerUpdates(
-    peerId: PeerID,
-    listener: PeerSyncStateListenerCallback,
-  ) {
-    const listeners = this.listenersByPeers.get(peerId) ?? new Set();
-
-    if (listeners.size === 0) {
-      this.listenersByPeers.set(peerId, listeners);
-    }
-
-    listeners.add(listener);
-
-    return () => {
-      listeners.delete(listener);
-    };
-  }
-
-  getCurrentSyncState(peerId: PeerID, id: RawCoID) {
-    // Build a lazy sync state object to process the isUploaded info
-    // only when requested
-    const syncState = {} as SyncState;
-
-    const getIsUploaded = () =>
-      this.getIsCoValueFullyUploadedIntoPeer(peerId, id);
-
-    Object.defineProperties(syncState, {
-      uploaded: {
-        enumerable: true,
-        get: getIsUploaded,
-      },
-    });
-
-    return syncState;
-  }
-
-  triggerUpdate(peerId: PeerID, id: RawCoID) {
-    const peer = this.syncManager.peers[peerId];
-
-    if (!peer) {
-      return;
-    }
-
-    const peerListeners = this.listenersByPeers.get(peer.id);
-
-    // If we don't have any active listeners do nothing
-    if (!peerListeners?.size && !this.listeners.size) {
-      return;
-    }
-
-    const knownState = peer.knownStates.get(id) ?? emptyKnownState(id);
-    const syncState = this.getCurrentSyncState(peerId, id);
-
-    for (const listener of this.listeners) {
-      listener(peerId, knownState, syncState);
-    }
-
-    if (!peerListeners) return;
-
-    for (const listener of peerListeners) {
-      listener(knownState, syncState);
-    }
-  }
-
-  private getKnownStateSessions(peerId: PeerID, id: RawCoID) {
-    const peer = this.syncManager.peers[peerId];
-
-    if (!peer) {
-      return undefined;
-    }
-
-    const peerSessions = peer.knownStates.get(id)?.sessions;
-
-    if (!peerSessions) {
-      return undefined;
-    }
-
-    const entry = this.syncManager.local.coValuesStore.get(id);
-
-    if (entry.state.type !== "available") {
-      return undefined;
-    }
-
-    const coValue = entry.state.coValue;
-    const coValueSessions = coValue.knownState().sessions;
-
-    return {
-      peer: peerSessions,
-      coValue: coValueSessions,
-    };
-  }
-
-  private getIsCoValueFullyUploadedIntoPeer(peerId: PeerID, id: RawCoID) {
-    const sessions = this.getKnownStateSessions(peerId, id);
-
-    if (!sessions) {
-      return false;
-    }
-
-    return getIsUploaded(sessions.coValue, sessions.peer);
-  }
-}
-
-function getIsUploaded(
-  from: Record<string, number>,
-  to: Record<string, number>,
-) {
-  for (const sessionId of Object.keys(from)) {
-    if (from[sessionId] !== to[sessionId]) {
-      return false;
-    }
-  }
-
-  return true;
-}

packages/cojson/src/coValueCore.ts

@@ -1,974 +0,0 @@
-import { Result, err, ok } from "neverthrow";
-import { AnyRawCoValue, RawCoValue } from "./coValue.js";
-import { ControlledAccountOrAgent, RawAccountID } from "./coValues/account.js";
-import { RawGroup } from "./coValues/group.js";
-import { coreToCoValue } from "./coreToCoValue.js";
-import {
-  CryptoProvider,
-  Encrypted,
-  Hash,
-  KeyID,
-  KeySecret,
-  Signature,
-  SignerID,
-  StreamingHash,
-} from "./crypto/crypto.js";
-import {
-  RawCoID,
-  SessionID,
-  TransactionID,
-  getGroupDependentKeyList,
-  getParentGroupId,
-  isParentGroupReference,
-} from "./ids.js";
-import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
-import { JsonObject, JsonValue } from "./jsonValue.js";
-import { LocalNode, ResolveAccountAgentError } from "./localNode.js";
-import { logger } from "./logger.js";
-import {
-  PermissionsDef as RulesetDef,
-  determineValidTransactions,
-  isKeyForKeyField,
-} from "./permissions.js";
-import { getPriorityFromHeader } from "./priority.js";
-import { CoValueKnownState, NewContentMessage } from "./sync.js";
-import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
-import { expectGroup } from "./typeUtils/expectGroup.js";
-import { isAccountID } from "./typeUtils/isAccountID.js";
-
-/**
-    In order to not block other concurrently syncing CoValues we introduce a maximum size of transactions,
-    since they are the smallest unit of progress that can be synced within a CoValue.
-    This is particularly important for storing binary data in CoValues, since they are likely to be at least on the order of megabytes.
-    This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
-    to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
-**/
-export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
-
-export type CoValueHeader = {
-  type: AnyRawCoValue["type"];
-  ruleset: RulesetDef;
-  meta: JsonObject | null;
-} & CoValueUniqueness;
-
-export type CoValueUniqueness = {
-  uniqueness: JsonValue;
-  createdAt?: `2${string}` | null;
-};
-
-export function idforHeader(
-  header: CoValueHeader,
-  crypto: CryptoProvider,
-): RawCoID {
-  const hash = crypto.shortHash(header);
-  return `co_z${hash.slice("shortHash_z".length)}`;
-}
-
-type SessionLog = {
-  transactions: Transaction[];
-  lastHash?: Hash;
-  streamingHash: StreamingHash;
-  signatureAfter: { [txIdx: number]: Signature | undefined };
-  lastSignature: Signature;
-};
-
-export type PrivateTransaction = {
-  privacy: "private";
-  madeAt: number;
-  keyUsed: KeyID;
-  encryptedChanges: Encrypted<JsonValue[], { in: RawCoID; tx: TransactionID }>;
-};
-
-export type TrustingTransaction = {
-  privacy: "trusting";
-  madeAt: number;
-  changes: Stringified<JsonValue[]>;
-};
-
-export type Transaction = PrivateTransaction | TrustingTransaction;
-
-export type DecryptedTransaction = {
-  txID: TransactionID;
-  changes: JsonValue[];
-  madeAt: number;
-};
-
-const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
-
-export class CoValueCore {
-  id: RawCoID;
-  node: LocalNode;
-  crypto: CryptoProvider;
-  header: CoValueHeader;
-  _sessionLogs: Map<SessionID, SessionLog>;
-  _cachedContent?: RawCoValue;
-  listeners: Set<(content?: RawCoValue) => void> = new Set();
-  _decryptionCache: {
-    [key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined;
-  } = {};
-  _cachedKnownState?: CoValueKnownState;
-  _cachedDependentOn?: RawCoID[];
-  _cachedNewContentSinceEmpty?: NewContentMessage[] | undefined;
-  _currentAsyncAddTransaction?: Promise<void>;
-
-  constructor(
-    header: CoValueHeader,
-    node: LocalNode,
-    internalInitSessions: Map<SessionID, SessionLog> = new Map(),
-  ) {
-    this.crypto = node.crypto;
-    this.id = idforHeader(header, node.crypto);
-    this.header = header;
-    this._sessionLogs = internalInitSessions;
-    this.node = node;
-
-    if (header.ruleset.type == "ownedByGroup") {
-      this.node
-        .expectCoValueLoaded(header.ruleset.group)
-        .subscribe((_groupUpdate) => {
-          this._cachedContent = undefined;
-          this.notifyUpdate("immediate");
-        });
-    }
-  }
-
-  get sessionLogs(): Map<SessionID, SessionLog> {
-    return this._sessionLogs;
-  }
-
-  testWithDifferentAccount(
-    account: ControlledAccountOrAgent,
-    currentSessionID: SessionID,
-  ): CoValueCore {
-    const newNode = this.node.testWithDifferentAccount(
-      account,
-      currentSessionID,
-    );
-
-    return newNode.expectCoValueLoaded(this.id);
-  }
-
-  knownState(): CoValueKnownState {
-    if (this._cachedKnownState) {
-      return this._cachedKnownState;
-    } else {
-      const knownState = this.knownStateUncached();
-      this._cachedKnownState = knownState;
-      return knownState;
-    }
-  }
-
-  /** @internal */
-  knownStateUncached(): CoValueKnownState {
-    const sessions: CoValueKnownState["sessions"] = {};
-
-    for (const [sessionID, sessionLog] of this.sessionLogs.entries()) {
-      sessions[sessionID] = sessionLog.transactions.length;
-    }
-
-    return {
-      id: this.id,
-      header: true,
-      sessions,
-    };
-  }
-
-  get meta(): JsonValue {
-    return this.header?.meta ?? null;
-  }
-
-  nextTransactionID(): TransactionID {
-    // This is an ugly hack to get a unique but stable session ID for editing the current account
-    const sessionID =
-      this.header.meta?.type === "account"
-        ? (this.node.currentSessionID.replace(
-            this.node.account.id,
-            this.node.account
-              .currentAgentID()
-              ._unsafeUnwrap({ withStackTrace: true }),
-          ) as SessionID)
-        : this.node.currentSessionID;
-
-    return {
-      sessionID,
-      txIndex: this.sessionLogs.get(sessionID)?.transactions.length || 0,
-    };
-  }
-
-  tryAddTransactions(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-    givenExpectedNewHash: Hash | undefined,
-    newSignature: Signature,
-    skipVerify: boolean = false,
-  ): Result<true, TryAddTransactionsError> {
-    return this.node
-      .resolveAccountAgent(
-        accountOrAgentIDfromSessionID(sessionID),
-        "Expected to know signer of transaction",
-      )
-      .andThen((agent) => {
-        const signerID = this.crypto.getAgentSignerID(agent);
-
-        const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
-          sessionID,
-          newTransactions,
-        );
-
-        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
-          return err({
-            type: "InvalidHash",
-            id: this.id,
-            expectedNewHash,
-            givenExpectedNewHash,
-          } satisfies InvalidHashError);
-        }
-
-        if (
-          skipVerify !== true &&
-          !this.crypto.verify(newSignature, expectedNewHash, signerID)
-        ) {
-          return err({
-            type: "InvalidSignature",
-            id: this.id,
-            newSignature,
-            sessionID,
-            signerID,
-          } satisfies InvalidSignatureError);
-        }
-
-        this.doAddTransactions(
-          sessionID,
-          newTransactions,
-          newSignature,
-          expectedNewHash,
-          newStreamingHash,
-          "immediate",
-        );
-
-        return ok(true as const);
-      });
-  }
-
-  private doAddTransactions(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-    newSignature: Signature,
-    expectedNewHash: Hash,
-    newStreamingHash: StreamingHash,
-    notifyMode: "immediate" | "deferred",
-  ) {
-    if (this.node.crashed) {
-      throw new Error("Trying to add transactions after node is crashed");
-    }
-    const transactions = this.sessionLogs.get(sessionID)?.transactions ?? [];
-
-    for (const tx of newTransactions) {
-      transactions.push(tx);
-    }
-
-    const signatureAfter =
-      this.sessionLogs.get(sessionID)?.signatureAfter ?? {};
-
-    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
-      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
-      -1,
-    );
-
-    const sizeOfTxsSinceLastInbetweenSignature = transactions
-      .slice(lastInbetweenSignatureIdx + 1)
-      .reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
-
-    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
-      signatureAfter[transactions.length - 1] = newSignature;
-    }
-
-    this._sessionLogs.set(sessionID, {
-      transactions,
-      lastHash: expectedNewHash,
-      streamingHash: newStreamingHash,
-      lastSignature: newSignature,
-      signatureAfter: signatureAfter,
-    });
-
-    if (
-      this._cachedContent &&
-      "processNewTransactions" in this._cachedContent &&
-      typeof this._cachedContent.processNewTransactions === "function"
-    ) {
-      this._cachedContent.processNewTransactions();
-    } else {
-      this._cachedContent = undefined;
-    }
-
-    this._cachedKnownState = undefined;
-    this._cachedDependentOn = undefined;
-    this._cachedNewContentSinceEmpty = undefined;
-
-    this.notifyUpdate(notifyMode);
-  }
-
-  deferredUpdates = 0;
-  nextDeferredNotify: Promise<void> | undefined;
-
-  notifyUpdate(notifyMode: "immediate" | "deferred") {
-    if (this.listeners.size === 0) {
-      return;
-    }
-
-    if (notifyMode === "immediate") {
-      const content = this.getCurrentContent();
-      for (const listener of this.listeners) {
-        listener(content);
-      }
-    } else {
-      if (!this.nextDeferredNotify) {
-        this.nextDeferredNotify = new Promise((resolve) => {
-          setTimeout(() => {
-            this.nextDeferredNotify = undefined;
-            this.deferredUpdates = 0;
-            const content = this.getCurrentContent();
-            for (const listener of this.listeners) {
-              listener(content);
-            }
-            resolve();
-          }, 0);
-        });
-      }
-      this.deferredUpdates++;
-    }
-  }
-
-  subscribe(listener: (content?: RawCoValue) => void): () => void {
-    this.listeners.add(listener);
-    listener(this.getCurrentContent());
-
-    return () => {
-      this.listeners.delete(listener);
-    };
-  }
-
-  expectedNewHashAfter(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-  ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
-    const streamingHash =
-      this.sessionLogs.get(sessionID)?.streamingHash.clone() ??
-      new StreamingHash(this.crypto);
-    for (const transaction of newTransactions) {
-      streamingHash.update(transaction);
-    }
-
-    const newStreamingHash = streamingHash.clone();
-
-    return {
-      expectedNewHash: streamingHash.digest(),
-      newStreamingHash,
-    };
-  }
-
-  async expectedNewHashAfterAsync(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-  ): Promise<{ expectedNewHash: Hash; newStreamingHash: StreamingHash }> {
-    const streamingHash =
-      this.sessionLogs.get(sessionID)?.streamingHash.clone() ??
-      new StreamingHash(this.crypto);
-    let before = performance.now();
-    for (const transaction of newTransactions) {
-      streamingHash.update(transaction);
-      const after = performance.now();
-      if (after - before > 1) {
-        await new Promise((resolve) => setTimeout(resolve, 0));
-        before = performance.now();
-      }
-    }
-
-    const newStreamingHash = streamingHash.clone();
-
-    return {
-      expectedNewHash: streamingHash.digest(),
-      newStreamingHash,
-    };
-  }
-
-  makeTransaction(
-    changes: JsonValue[],
-    privacy: "private" | "trusting",
-  ): boolean {
-    const madeAt = Date.now();
-
-    let transaction: Transaction;
-
-    if (privacy === "private") {
-      const { secret: keySecret, id: keyID } = this.getCurrentReadKey();
-
-      if (!keySecret) {
-        throw new Error("Can't make transaction without read key secret");
-      }
-
-      const encrypted = this.crypto.encryptForTransaction(changes, keySecret, {
-        in: this.id,
-        tx: this.nextTransactionID(),
-      });
-
-      this._decryptionCache[encrypted] = changes;
-
-      transaction = {
-        privacy: "private",
-        madeAt,
-        keyUsed: keyID,
-        encryptedChanges: encrypted,
-      };
-    } else {
-      transaction = {
-        privacy: "trusting",
-        madeAt,
-        changes: stableStringify(changes),
-      };
-    }
-
-    // This is an ugly hack to get a unique but stable session ID for editing the current account
-    const sessionID =
-      this.header.meta?.type === "account"
-        ? (this.node.currentSessionID.replace(
-            this.node.account.id,
-            this.node.account
-              .currentAgentID()
-              ._unsafeUnwrap({ withStackTrace: true }),
-          ) as SessionID)
-        : this.node.currentSessionID;
-
-    const { expectedNewHash } = this.expectedNewHashAfter(sessionID, [
-      transaction,
-    ]);
-
-    const signature = this.crypto.sign(
-      this.node.account.currentSignerSecret(),
-      expectedNewHash,
-    );
-
-    const success = this.tryAddTransactions(
-      sessionID,
-      [transaction],
-      expectedNewHash,
-      signature,
-      true,
-    )._unsafeUnwrap({ withStackTrace: true });
-
-    if (success) {
-      void this.node.syncManager.syncCoValue(this);
-    }
-
-    return success;
-  }
-
-  getCurrentContent(options?: {
-    ignorePrivateTransactions: true;
-  }): RawCoValue {
-    if (!options?.ignorePrivateTransactions && this._cachedContent) {
-      return this._cachedContent;
-    }
-
-    const newContent = coreToCoValue(this, options);
-
-    if (!options?.ignorePrivateTransactions) {
-      this._cachedContent = newContent;
-    }
-
-    return newContent;
-  }
-
-  getValidTransactions(options?: {
-    ignorePrivateTransactions: boolean;
-    knownTransactions?: CoValueKnownState["sessions"];
-  }): DecryptedTransaction[] {
-    const validTransactions = determineValidTransactions(this);
-
-    const allTransactions: DecryptedTransaction[] = [];
-
-    for (const { txID, tx } of validTransactions) {
-      if (options?.knownTransactions?.[txID.sessionID]! >= txID.txIndex) {
-        continue;
-      }
-
-      if (tx.privacy === "trusting") {
-        allTransactions.push({
-          txID,
-          madeAt: tx.madeAt,
-          changes: parseJSON(tx.changes),
-        });
-        continue;
-      }
-
-      if (options?.ignorePrivateTransactions) {
-        continue;
-      }
-
-      const readKey = this.getReadKey(tx.keyUsed);
-
-      if (!readKey) {
-        continue;
-      }
-
-      let decryptedChanges = this._decryptionCache[tx.encryptedChanges];
-
-      if (!decryptedChanges) {
-        const decryptedString = this.crypto.decryptRawForTransaction(
-          tx.encryptedChanges,
-          readKey,
-          {
-            in: this.id,
-            tx: txID,
-          },
-        );
-        decryptedChanges = decryptedString && parseJSON(decryptedString);
-        this._decryptionCache[tx.encryptedChanges] = decryptedChanges;
-      }
-
-      if (!decryptedChanges) {
-        logger.error("Failed to decrypt transaction despite having key");
-        continue;
-      }
-
-      allTransactions.push({
-        txID,
-        madeAt: tx.madeAt,
-        changes: decryptedChanges,
-      });
-    }
-
-    return allTransactions;
-  }
-
-  getValidSortedTransactions(options?: {
-    ignorePrivateTransactions: boolean;
-  }): DecryptedTransaction[] {
-    const allTransactions = this.getValidTransactions(options);
-
-    allTransactions.sort(this.compareTransactions);
-
-    return allTransactions;
-  }
-
-  compareTransactions(
-    a: Pick<DecryptedTransaction, "madeAt" | "txID">,
-    b: Pick<DecryptedTransaction, "madeAt" | "txID">,
-  ) {
-    return (
-      a.madeAt - b.madeAt ||
-      (a.txID.sessionID < b.txID.sessionID ? -1 : 1) ||
-      a.txID.txIndex - b.txID.txIndex
-    );
-  }
-
-  getCurrentReadKey(): { secret: KeySecret | undefined; id: KeyID } {
-    if (this.header.ruleset.type === "group") {
-      const content = expectGroup(this.getCurrentContent());
-
-      const currentKeyId = content.getCurrentReadKeyId();
-
-      if (!currentKeyId) {
-        throw new Error("No readKey set");
-      }
-
-      const secret = this.getReadKey(currentKeyId);
-
-      return {
-        secret: secret,
-        id: currentKeyId,
-      };
-    } else if (this.header.ruleset.type === "ownedByGroup") {
-      return this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
-        .getCurrentReadKey();
-    } else {
-      throw new Error(
-        "Only groups or values owned by groups have read secrets",
-      );
-    }
-  }
-
-  getReadKey(keyID: KeyID): KeySecret | undefined {
-    let key = readKeyCache.get(this)?.[keyID];
-    if (!key) {
-      key = this.getUncachedReadKey(keyID);
-      if (key) {
-        let cache = readKeyCache.get(this);
-        if (!cache) {
-          cache = {};
-          readKeyCache.set(this, cache);
-        }
-        cache[keyID] = key;
-      }
-    }
-    return key;
-  }
-
-  getUncachedReadKey(keyID: KeyID): KeySecret | undefined {
-    if (this.header.ruleset.type === "group") {
-      const content = expectGroup(
-        this.getCurrentContent({ ignorePrivateTransactions: true }),
-      );
-
-      const keyForEveryone = content.get(`${keyID}_for_everyone`);
-      if (keyForEveryone) return keyForEveryone;
-
-      // Try to find key revelation for us
-      const lookupAccountOrAgentID =
-        this.header.meta?.type === "account"
-          ? this.node.account
-              .currentAgentID()
-              ._unsafeUnwrap({ withStackTrace: true })
-          : this.node.account.id;
-
-      const lastReadyKeyEdit = content.lastEditAt(
-        `${keyID}_for_${lookupAccountOrAgentID}`,
-      );
-
-      if (lastReadyKeyEdit?.value) {
-        const revealer = lastReadyKeyEdit.by;
-        const revealerAgent = this.node
-          .resolveAccountAgent(revealer, "Expected to know revealer")
-          ._unsafeUnwrap({ withStackTrace: true });
-
-        const secret = this.crypto.unseal(
-          lastReadyKeyEdit.value,
-          this.node.account.currentSealerSecret(),
-          this.crypto.getAgentSealerID(revealerAgent),
-          {
-            in: this.id,
-            tx: lastReadyKeyEdit.tx,
-          },
-        );
-
-        if (secret) {
-          return secret as KeySecret;
-        }
-      }
-
-      // Try to find indirect revelation through previousKeys
-
-      for (const co of content.keys()) {
-        if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-          const encryptingKeyID = co.split("_for_")[1] as KeyID;
-          const encryptingKeySecret = this.getReadKey(encryptingKeyID);
-
-          if (!encryptingKeySecret) {
-            continue;
-          }
-
-          const encryptedPreviousKey = content.get(co)!;
-
-          const secret = this.crypto.decryptKeySecret(
-            {
-              encryptedID: keyID,
-              encryptingID: encryptingKeyID,
-              encrypted: encryptedPreviousKey,
-            },
-            encryptingKeySecret,
-          );
-
-          if (secret) {
-            return secret as KeySecret;
-          } else {
-            logger.warn(
-              `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
-            );
-          }
-        }
-      }
-
-      // try to find revelation to parent group read keys
-      for (const co of content.keys()) {
-        if (isParentGroupReference(co)) {
-          const parentGroupID = getParentGroupId(co);
-          const parentGroup = this.node.expectCoValueLoaded(
-            parentGroupID,
-            "Expected parent group to be loaded",
-          );
-
-          const parentKeys = this.findValidParentKeys(
-            keyID,
-            content,
-            parentGroup,
-          );
-
-          for (const parentKey of parentKeys) {
-            const revelationForParentKey = content.get(
-              `${keyID}_for_${parentKey.id}`,
-            );
-
-            if (revelationForParentKey) {
-              const secret = parentGroup.crypto.decryptKeySecret(
-                {
-                  encryptedID: keyID,
-                  encryptingID: parentKey.id,
-                  encrypted: revelationForParentKey,
-                },
-                parentKey.secret,
-              );
-
-              if (secret) {
-                return secret as KeySecret;
-              } else {
-                logger.warn(
-                  `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
-                );
-              }
-            }
-          }
-        }
-      }
-
-      return undefined;
-    } else if (this.header.ruleset.type === "ownedByGroup") {
-      return this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
-        .getReadKey(keyID);
-    } else {
-      throw new Error(
-        "Only groups or values owned by groups have read secrets",
-      );
-    }
-  }
-
-  findValidParentKeys(keyID: KeyID, group: RawGroup, parentGroup: CoValueCore) {
-    const validParentKeys: { id: KeyID; secret: KeySecret }[] = [];
-
-    for (const co of group.keys()) {
-      if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-        const encryptingKeyID = co.split("_for_")[1] as KeyID;
-        const encryptingKeySecret = parentGroup.getReadKey(encryptingKeyID);
-
-        if (!encryptingKeySecret) {
-          continue;
-        }
-
-        validParentKeys.push({
-          id: encryptingKeyID,
-          secret: encryptingKeySecret,
-        });
-      }
-    }
-
-    return validParentKeys;
-  }
-
-  getGroup(): RawGroup {
-    if (this.header.ruleset.type !== "ownedByGroup") {
-      throw new Error("Only values owned by groups have groups");
-    }
-
-    return expectGroup(
-      this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
-        .getCurrentContent(),
-    );
-  }
-
-  getTx(txID: TransactionID): Transaction | undefined {
-    return this.sessionLogs.get(txID.sessionID)?.transactions[txID.txIndex];
-  }
-
-  newContentSince(
-    knownState: CoValueKnownState | undefined,
-  ): NewContentMessage[] | undefined {
-    const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
-
-    if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
-      return this._cachedNewContentSinceEmpty;
-    }
-
-    let currentPiece: NewContentMessage = {
-      action: "content",
-      id: this.id,
-      header: knownState?.header ? undefined : this.header,
-      priority: getPriorityFromHeader(this.header),
-      new: {},
-    };
-
-    const pieces = [currentPiece];
-
-    const sentState: CoValueKnownState["sessions"] = {};
-
-    let pieceSize = 0;
-
-    let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
-
-    while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
-      if (sessionsTodoAgain === "first") {
-        sessionsTodoAgain = undefined;
-      }
-      const sessionsTodo = sessionsTodoAgain ?? this.sessionLogs.keys();
-
-      for (const sessionIDKey of sessionsTodo) {
-        const sessionID = sessionIDKey as SessionID;
-        const log = this.sessionLogs.get(sessionID)!;
-        const knownStateForSessionID = knownState?.sessions[sessionID];
-        const sentStateForSessionID = sentState[sessionID];
-        const nextKnownSignatureIdx = getNextKnownSignatureIdx(
-          log,
-          knownStateForSessionID,
-          sentStateForSessionID,
-        );
-
-        const firstNewTxIdx =
-          sentStateForSessionID ?? knownStateForSessionID ?? 0;
-        const afterLastNewTxIdx =
-          nextKnownSignatureIdx === undefined
-            ? log.transactions.length
-            : nextKnownSignatureIdx + 1;
-
-        const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
-
-        if (nNewTx === 0) {
-          sessionsTodoAgain?.delete(sessionID);
-          continue;
-        }
-
-        if (afterLastNewTxIdx < log.transactions.length) {
-          if (!sessionsTodoAgain) {
-            sessionsTodoAgain = new Set();
-          }
-          sessionsTodoAgain.add(sessionID);
-        }
-
-        const oldPieceSize = pieceSize;
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          pieceSize +=
-            tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length;
-        }
-
-        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
-          currentPiece = {
-            action: "content",
-            id: this.id,
-            header: undefined,
-            new: {},
-            priority: getPriorityFromHeader(this.header),
-          };
-          pieces.push(currentPiece);
-          pieceSize = pieceSize - oldPieceSize;
-        }
-
-        let sessionEntry = currentPiece.new[sessionID];
-        if (!sessionEntry) {
-          sessionEntry = {
-            after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
-            newTransactions: [],
-            lastSignature: "WILL_BE_REPLACED" as Signature,
-          };
-          currentPiece.new[sessionID] = sessionEntry;
-        }
-
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          sessionEntry.newTransactions.push(tx);
-        }
-
-        sessionEntry.lastSignature =
-          nextKnownSignatureIdx === undefined
-            ? log.lastSignature!
-            : log.signatureAfter[nextKnownSignatureIdx]!;
-
-        sentState[sessionID] =
-          (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
-      }
-    }
-
-    const piecesWithContent = pieces.filter(
-      (piece) => Object.keys(piece.new).length > 0 || piece.header,
-    );
-
-    if (piecesWithContent.length === 0) {
-      return undefined;
-    }
-
-    if (isKnownStateEmpty) {
-      this._cachedNewContentSinceEmpty = piecesWithContent;
-    }
-
-    return piecesWithContent;
-  }
-
-  getDependedOnCoValues(): RawCoID[] {
-    if (this._cachedDependentOn) {
-      return this._cachedDependentOn;
-    } else {
-      const dependentOn = this.getDependedOnCoValuesUncached();
-      this._cachedDependentOn = dependentOn;
-      return dependentOn;
-    }
-  }
-
-  /** @internal */
-  getDependedOnCoValuesUncached(): RawCoID[] {
-    return this.header.ruleset.type === "group"
-      ? getGroupDependentKeyList(expectGroup(this.getCurrentContent()).keys())
-      : this.header.ruleset.type === "ownedByGroup"
-        ? [
-            this.header.ruleset.group,
-            ...new Set(
-              [...this.sessionLogs.keys()]
-                .map((sessionID) =>
-                  accountOrAgentIDfromSessionID(sessionID as SessionID),
-                )
-                .filter(
-                  (session): session is RawAccountID =>
-                    isAccountID(session) && session !== this.id,
-                ),
-            ),
-          ]
-        : [];
-  }
-
-  waitForSync(options?: {
-    timeout?: number;
-  }) {
-    return this.node.syncManager.waitForSync(this.id, options?.timeout);
-  }
-}
-
-function getNextKnownSignatureIdx(
-  log: SessionLog,
-  knownStateForSessionID?: number,
-  sentStateForSessionID?: number,
-) {
-  return Object.keys(log.signatureAfter)
-    .map(Number)
-    .sort((a, b) => a - b)
-    .find(
-      (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
-    );
-}
-
-export type InvalidHashError = {
-  type: "InvalidHash";
-  id: RawCoID;
-  expectedNewHash: Hash;
-  givenExpectedNewHash: Hash;
-};
-
-export type InvalidSignatureError = {
-  type: "InvalidSignature";
-  id: RawCoID;
-  newSignature: Signature;
-  sessionID: SessionID;
-  signerID: SignerID;
-};
-
-export type TryAddTransactionsError =
-  | ResolveAccountAgentError
-  | InvalidHashError
-  | InvalidSignatureError;

packages/cojson/src/coValueState.ts

@@ -1,374 +0,0 @@
-import { PeerState } from "./PeerState.js";
-import { CoValueCore } from "./coValueCore.js";
-import { RawCoID } from "./ids.js";
-import { logger } from "./logger.js";
-import { PeerID } from "./sync.js";
-
-export const CO_VALUE_LOADING_CONFIG = {
-  MAX_RETRIES: 2,
-  TIMEOUT: 30_000,
-};
-
-export class CoValueUnknownState {
-  type = "unknown" as const;
-}
-
-export class CoValueLoadingState {
-  type = "loading" as const;
-  private peers = new Map<
-    PeerID,
-    ReturnType<typeof createResolvablePromise<void>>
-  >();
-  private resolveResult: (value: CoValueCore | "unavailable") => void;
-
-  result: Promise<CoValueCore | "unavailable">;
-
-  constructor(peersIds: Iterable<PeerID>) {
-    this.peers = new Map();
-
-    for (const peerId of peersIds) {
-      this.peers.set(peerId, createResolvablePromise<void>());
-    }
-
-    const { resolve, promise } = createResolvablePromise<
-      CoValueCore | "unavailable"
-    >();
-
-    this.result = promise;
-    this.resolveResult = resolve;
-  }
-
-  markAsUnavailable(peerId: PeerID) {
-    const entry = this.peers.get(peerId);
-
-    if (entry) {
-      entry.resolve();
-    }
-
-    this.peers.delete(peerId);
-
-    // If none of the peers have the coValue, we resolve to unavailable
-    if (this.peers.size === 0) {
-      this.resolve("unavailable");
-    }
-  }
-
-  resolve(value: CoValueCore | "unavailable") {
-    this.resolveResult(value);
-    for (const entry of this.peers.values()) {
-      entry.resolve();
-    }
-    this.peers.clear();
-  }
-
-  // Wait for a specific peer to have a known state
-  waitForPeer(peerId: PeerID) {
-    const entry = this.peers.get(peerId);
-
-    if (!entry) {
-      return Promise.resolve();
-    }
-
-    return entry.promise;
-  }
-}
-
-export class CoValueAvailableState {
-  type = "available" as const;
-
-  constructor(public coValue: CoValueCore) {}
-}
-
-export class CoValueUnavailableState {
-  type = "unavailable" as const;
-}
-
-type CoValueStateAction =
-  | {
-      type: "load-requested";
-      peersIds: PeerID[];
-    }
-  | {
-      type: "not-found-in-peer";
-      peerId: PeerID;
-    }
-  | {
-      type: "available";
-      coValue: CoValueCore;
-    };
-
-type CoValueStateType =
-  | CoValueUnknownState
-  | CoValueLoadingState
-  | CoValueAvailableState
-  | CoValueUnavailableState;
-
-export class CoValueState {
-  promise?: Promise<CoValueCore | "unavailable">;
-  private resolve?: (value: CoValueCore | "unavailable") => void;
-
-  constructor(
-    public id: RawCoID,
-    public state: CoValueStateType,
-  ) {}
-
-  static Unknown(id: RawCoID) {
-    return new CoValueState(id, new CoValueUnknownState());
-  }
-
-  static Loading(id: RawCoID, peersIds: Iterable<PeerID>) {
-    return new CoValueState(id, new CoValueLoadingState(peersIds));
-  }
-
-  static Available(coValue: CoValueCore) {
-    return new CoValueState(coValue.id, new CoValueAvailableState(coValue));
-  }
-
-  static Unavailable(id: RawCoID) {
-    return new CoValueState(id, new CoValueUnavailableState());
-  }
-
-  async getCoValue() {
-    if (this.state.type === "available") {
-      return this.state.coValue;
-    }
-    if (this.state.type === "unavailable") {
-      return "unavailable";
-    }
-
-    // If we don't have a resolved state we return a new promise
-    // that will be resolved when the state will move to available or unavailable
-    if (!this.promise) {
-      const { promise, resolve } = createResolvablePromise<
-        CoValueCore | "unavailable"
-      >();
-
-      this.promise = promise;
-      this.resolve = resolve;
-    }
-
-    return this.promise;
-  }
-
-  private moveToState(value: CoValueStateType) {
-    this.state = value;
-
-    if (!this.resolve) {
-      return;
-    }
-
-    // If the state is available we resolve the promise
-    // and clear it to handle the possible transition from unavailable to available
-    if (value.type === "available") {
-      this.resolve(value.coValue);
-      this.clearPromise();
-    } else if (value.type === "unavailable") {
-      this.resolve("unavailable");
-      this.clearPromise();
-    }
-  }
-
-  private clearPromise() {
-    this.promise = undefined;
-    this.resolve = undefined;
-  }
-
-  async loadFromPeers(peers: PeerState[]) {
-    const state = this.state;
-
-    if (state.type !== "unknown" && state.type !== "unavailable") {
-      return;
-    }
-
-    if (peers.length === 0) {
-      return;
-    }
-
-    const doLoad = async (peersToLoadFrom: PeerState[]) => {
-      const peersWithoutErrors = getPeersWithoutErrors(
-        peersToLoadFrom,
-        this.id,
-      );
-
-      // If we are in the loading state we move to a new loading state
-      // to reset all the loading promises
-      if (this.state.type === "loading" || this.state.type === "unknown") {
-        this.moveToState(
-          new CoValueLoadingState(peersWithoutErrors.map((p) => p.id)),
-        );
-      }
-
-      // Assign the current state to a variable to not depend on the state changes
-      // that may happen while we wait for loadCoValueFromPeers to complete
-      const currentState = this.state;
-
-      // If we entered successfully the loading state, we load the coValue from the peers
-      //
-      // We may not enter the loading state if the coValue has become available in between
-      // of the retries
-      if (currentState.type === "loading") {
-        await loadCoValueFromPeers(this, peersWithoutErrors);
-
-        const result = await currentState.result;
-        return result !== "unavailable";
-      }
-
-      return currentState.type === "available";
-    };
-
-    await doLoad(peers);
-
-    // Retry loading from peers that have the retry flag enabled
-    const peersWithRetry = peers.filter((p) =>
-      p.shouldRetryUnavailableCoValues(),
-    );
-
-    if (peersWithRetry.length > 0) {
-      // We want to exit early if the coValue becomes available in between the retries
-      await Promise.race([
-        this.getCoValue(),
-        runWithRetry(
-          () => doLoad(peersWithRetry),
-          CO_VALUE_LOADING_CONFIG.MAX_RETRIES,
-        ),
-      ]);
-    }
-
-    // If after the retries the coValue is still loading, we consider the load failed
-    if (this.state.type === "loading") {
-      this.moveToState(new CoValueUnavailableState());
-    }
-  }
-
-  dispatch(action: CoValueStateAction) {
-    const currentState = this.state;
-
-    switch (action.type) {
-      case "available":
-        if (currentState.type === "loading") {
-          currentState.resolve(action.coValue);
-        }
-
-        // It should be always possible to move to the available state
-        this.moveToState(new CoValueAvailableState(action.coValue));
-
-        break;
-      case "not-found-in-peer":
-        if (currentState.type === "loading") {
-          currentState.markAsUnavailable(action.peerId);
-        }
-
-        break;
-    }
-  }
-}
-
-async function loadCoValueFromPeers(
-  coValueEntry: CoValueState,
-  peers: PeerState[],
-) {
-  for (const peer of peers) {
-    if (peer.closed) {
-      continue;
-    }
-
-    if (coValueEntry.state.type === "available") {
-      /**
-       * We don't need to wait for the message to be delivered here.
-       *
-       * This way when the coValue becomes available because it's cached we don't wait for the server
-       * peer to consume the messages queue before moving forward.
-       */
-      peer
-        .pushOutgoingMessage({
-          action: "load",
-          ...coValueEntry.state.coValue.knownState(),
-        })
-        .catch((err) => {
-          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
-        });
-    } else {
-      /**
-       * We only wait for the load state to be resolved.
-       */
-      peer
-        .pushOutgoingMessage({
-          action: "load",
-          id: coValueEntry.id,
-          header: false,
-          sessions: {},
-        })
-        .catch((err) => {
-          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
-        });
-    }
-
-    if (coValueEntry.state.type === "loading") {
-      const timeout = setTimeout(() => {
-        if (coValueEntry.state.type === "loading") {
-          logger.warn("Failed to load coValue from peer", {
-            coValueId: coValueEntry.id,
-            peerId: peer.id,
-            peerRole: peer.role,
-          });
-          coValueEntry.dispatch({
-            type: "not-found-in-peer",
-            peerId: peer.id,
-          });
-        }
-      }, CO_VALUE_LOADING_CONFIG.TIMEOUT);
-      await coValueEntry.state.waitForPeer(peer.id);
-      clearTimeout(timeout);
-    }
-  }
-}
-
-async function runWithRetry<T>(fn: () => Promise<T>, maxRetries: number) {
-  let retries = 1;
-
-  while (retries < maxRetries) {
-    /**
-     * With maxRetries of 5 we should wait:
-     * 300ms
-     * 900ms
-     * 2700ms
-     * 8100ms
-     */
-    await sleep(3 ** retries * 100);
-
-    const result = await fn();
-
-    if (result === true) {
-      return;
-    }
-
-    retries++;
-  }
-}
-
-function createResolvablePromise<T>() {
-  let resolve!: (value: T) => void;
-
-  const promise = new Promise<T>((res) => {
-    resolve = res;
-  });
-
-  return { promise, resolve };
-}
-
-function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-function getPeersWithoutErrors(peers: PeerState[], coValueId: RawCoID) {
-  return peers.filter((p) => {
-    if (p.erroredCoValues.has(coValueId)) {
-      logger.warn(
-        `Skipping load on errored coValue ${coValueId} from peer ${p.id}`,
-      );
-      return false;
-    }
-
-    return true;
-  });
-}

packages/cojson/src/coValues/coMap.ts

@@ -1,7 +1,7 @@
 import { CoID, RawCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
 import { AgentID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
+import { LocalNodeState } from "../localNode/structure.js";
 import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
@@ -39,7 +39,7 @@ export class RawCoMapView<
   /** @category 6. Meta */
   type = "comap" as const;
   /** @category 6. Meta */
-  core: CoValueCore;
+  node: LocalNodeState;
   /** @internal */
   latest: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>;

packages/cojson/src/coValues/group.ts

@@ -158,7 +158,7 @@ export class RawGroup<
         child.state.type === "unknown" ||
         child.state.type === "unavailable"
       ) {
-        child.loadFromPeers(peers).catch(() => {
+        child.loadCoValue(this.core.node.storageDriver, peers).catch(() => {
           logger.error(`Failed to load child group ${id}`);
         });
       }

packages/cojson/src/crypto/WasmCrypto.ts

@@ -104,9 +104,13 @@ export class WasmCrypto extends CryptoProvider<Uint8Array> {
   }
 
   verify(signature: Signature, message: JsonValue, id: SignerID): boolean {
-    return new Ed25519VerifyingKey(
-      new Memory(base58.decode(id.substring("signer_z".length))),
-    ).verify(
+    const idBytes = base58.decode(id.substring("signer_z".length));
+    if (idBytes.length !== 32) {
+      throw new Error(
+        `Invalid signer ID ${id} - ID bytes length is ${idBytes.length} instead of 32`,
+      );
+    }
+    return new Ed25519VerifyingKey(new Memory(idBytes)).verify(
       new Memory(textEncoder.encode(stableStringify(message))),
       new Ed25519Signature(
         new Memory(base58.decode(signature.substring("signature_z".length))),

packages/cojson/src/exports.ts

@@ -1,11 +1,5 @@
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 import { type RawCoValue } from "./coValue.js";
-import {
-  CoValueCore,
-  type CoValueUniqueness,
-  MAX_RECOMMENDED_TX_SIZE,
-  idforHeader,
-} from "./coValueCore.js";
 import {
   ControlledAgent,
   RawAccount,
@@ -39,7 +33,6 @@ import {
   rawCoIDtoBytes,
 } from "./ids.js";
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
-import { LocalNode } from "./localNode.js";
 import type { AccountRole, Role } from "./permissions.js";
 import { Channel, connectedPeers } from "./streamUtils.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
@@ -76,11 +69,8 @@ import {
 
 type Value = JsonValue | AnyRawCoValue;
 
-import { CO_VALUE_LOADING_CONFIG } from "./coValueState.js";
 import { logger } from "./logger.js";
 import { getPriorityFromHeader } from "./priority.js";
-import { FileSystem } from "./storage/FileSystem.js";
-import { BlockFilename, LSMStorage, WalFilename } from "./storage/index.js";
 
 /** @hidden */
 export const cojsonInternals = {
@@ -97,18 +87,15 @@ export const cojsonInternals = {
   accountOrAgentIDfromSessionID,
   isAccountID,
   accountHeaderForInitialAgentSecret,
-  idforHeader,
   StreamingHash,
   Channel,
   getPriorityFromHeader,
   getGroupDependentKeyList,
   getGroupDependentKey,
   disablePermissionErrors,
-  CO_VALUE_LOADING_CONFIG,
 };
 
 export {
-  LocalNode,
   RawGroup,
   Role,
   EVERYONE,
@@ -128,10 +115,8 @@ export {
   RawProfile as Profile,
   SessionID,
   Media,
-  CoValueCore,
   ControlledAgent,
   RawControlledAccount,
-  MAX_RECOMMENDED_TX_SIZE,
   JsonObject,
   JsonValue,
   Peer,
@@ -143,7 +128,6 @@ export {
   CryptoProvider,
   SyncMessage,
   isRawCoID,
-  LSMStorage,
   emptyKnownState,
   RawCoPlainText,
   stringifyOpID,
@@ -154,14 +138,10 @@ export {
 
 export type {
   Value,
-  FileSystem,
-  BlockFilename,
-  WalFilename,
   IncomingSyncStream,
   OutgoingSyncQueue,
   DisconnectedError,
   PingTimeoutError,
-  CoValueUniqueness,
   Stringified,
   CoStreamItem,
   BinaryStreamItem,
@@ -179,8 +159,8 @@ export namespace CojsonInternalTypes {
   export type LoadMessage = import("./sync.js").LoadMessage;
   export type NewContentMessage = import("./sync.js").NewContentMessage;
   export type SessionNewContent = import("./sync.js").SessionNewContent;
-  export type CoValueHeader = import("./coValueCore.js").CoValueHeader;
-  export type Transaction = import("./coValueCore.js").Transaction;
+  export type CoValueHeader = import("./localNode/structure.js").CoValueHeader;
+  export type Transaction = import("./localNode/structure.js").Transaction;
   export type TransactionID = import("./ids.js").TransactionID;
   export type Signature = import("./crypto/crypto.js").Signature;
   export type RawCoID = import("./ids.js").RawCoID;

packages/cojson/src/localNode.ts

@@ -1,703 +0,0 @@
-import { Result, ResultAsync, err, ok, okAsync } from "neverthrow";
-import { CoValuesStore } from "./CoValuesStore.js";
-import { CoID } from "./coValue.js";
-import { RawCoValue } from "./coValue.js";
-import {
-  CoValueCore,
-  CoValueHeader,
-  CoValueUniqueness,
-} from "./coValueCore.js";
-import {
-  AccountMeta,
-  ControlledAccountOrAgent,
-  ControlledAgent,
-  InvalidAccountAgentIDError,
-  RawProfile as Profile,
-  RawAccount,
-  RawAccountID,
-  RawAccountMigration,
-  RawControlledAccount,
-  RawProfile,
-  accountHeaderForInitialAgentSecret,
-} from "./coValues/account.js";
-import {
-  InviteSecret,
-  RawGroup,
-  secretSeedFromInviteSecret,
-} from "./coValues/group.js";
-import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
-import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
-import { logger } from "./logger.js";
-import { Peer, PeerID, SyncManager } from "./sync.js";
-import { expectGroup } from "./typeUtils/expectGroup.js";
-
-/** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
-
-A `LocalNode` can have peers that it syncs to, for example some form of local persistence, or a sync server, such as `cloud.jazz.tools` (Jazz Cloud).
-
-@example
-You typically get hold of a `LocalNode` using `jazz-react`'s `useJazz()`:
-
-```typescript
-const { localNode } = useJazz();
-```
-*/
-export class LocalNode {
-  /** @internal */
-  crypto: CryptoProvider;
-  /** @internal */
-  coValuesStore = new CoValuesStore();
-  /** @category 3. Low-level */
-  account: ControlledAccountOrAgent;
-  /** @category 3. Low-level */
-  currentSessionID: SessionID;
-  /** @category 3. Low-level */
-  syncManager = new SyncManager(this);
-
-  crashed: Error | undefined = undefined;
-
-  /** @category 3. Low-level */
-  constructor(
-    account: ControlledAccountOrAgent,
-    currentSessionID: SessionID,
-    crypto: CryptoProvider,
-  ) {
-    this.account = account;
-    this.currentSessionID = currentSessionID;
-    this.crypto = crypto;
-  }
-
-  /** @category 2. Node Creation */
-  static async withNewlyCreatedAccount<Meta extends AccountMeta = AccountMeta>({
-    creationProps,
-    peersToLoadFrom,
-    migration,
-    crypto,
-    initialAgentSecret = crypto.newRandomAgentSecret(),
-  }: {
-    creationProps: { name: string };
-    peersToLoadFrom?: Peer[];
-    migration?: RawAccountMigration<Meta>;
-    crypto: CryptoProvider;
-    initialAgentSecret?: AgentSecret;
-  }): Promise<{
-    node: LocalNode;
-    accountID: RawAccountID;
-    accountSecret: AgentSecret;
-    sessionID: SessionID;
-  }> {
-    const throwawayAgent = crypto.newRandomAgentSecret();
-    const setupNode = new LocalNode(
-      new ControlledAgent(throwawayAgent, crypto),
-      crypto.newRandomSessionID(crypto.getAgentID(throwawayAgent)),
-      crypto,
-    );
-
-    const account = setupNode.createAccount(initialAgentSecret);
-
-    const nodeWithAccount = account.core.node.testWithDifferentAccount(
-      account,
-      crypto.newRandomSessionID(account.id),
-    );
-
-    const accountOnNodeWithAccount =
-      nodeWithAccount.account as RawControlledAccount<Meta>;
-
-    if (peersToLoadFrom) {
-      for (const peer of peersToLoadFrom) {
-        nodeWithAccount.syncManager.addPeer(peer);
-      }
-    }
-
-    if (migration) {
-      await migration(accountOnNodeWithAccount, nodeWithAccount, creationProps);
-    } else {
-      const profileGroup = accountOnNodeWithAccount.createGroup();
-      profileGroup.addMember("everyone", "reader");
-      const profile = profileGroup.createMap<Profile>({
-        name: creationProps.name,
-      });
-      accountOnNodeWithAccount.set("profile", profile.id, "trusting");
-    }
-
-    const controlledAccount = new RawControlledAccount(
-      accountOnNodeWithAccount.core,
-      accountOnNodeWithAccount.agentSecret,
-    );
-
-    nodeWithAccount.account = controlledAccount;
-    nodeWithAccount.coValuesStore.setAsAvailable(
-      controlledAccount.id,
-      controlledAccount.core,
-    );
-    controlledAccount.core._cachedContent = undefined;
-
-    if (!controlledAccount.get("profile")) {
-      throw new Error("Must set account profile in initial migration");
-    }
-
-    // we shouldn't need this, but it fixes account data not syncing for new accounts
-    function syncAllCoValuesAfterCreateAccount() {
-      for (const coValueEntry of nodeWithAccount.coValuesStore.getValues()) {
-        if (coValueEntry.state.type === "available") {
-          void nodeWithAccount.syncManager.syncCoValue(
-            coValueEntry.state.coValue,
-          );
-        }
-      }
-    }
-
-    syncAllCoValuesAfterCreateAccount();
-
-    setTimeout(syncAllCoValuesAfterCreateAccount, 500);
-
-    return {
-      node: nodeWithAccount,
-      accountID: accountOnNodeWithAccount.id,
-      accountSecret: accountOnNodeWithAccount.agentSecret,
-      sessionID: nodeWithAccount.currentSessionID,
-    };
-  }
-
-  /** @category 2. Node Creation */
-  static async withLoadedAccount<Meta extends AccountMeta = AccountMeta>({
-    accountID,
-    accountSecret,
-    sessionID,
-    peersToLoadFrom,
-    crypto,
-    migration,
-  }: {
-    accountID: RawAccountID;
-    accountSecret: AgentSecret;
-    sessionID: SessionID | undefined;
-    peersToLoadFrom: Peer[];
-    crypto: CryptoProvider;
-    migration?: RawAccountMigration<Meta>;
-  }): Promise<LocalNode> {
-    try {
-      const loadingNode = new LocalNode(
-        new ControlledAgent(accountSecret, crypto),
-        crypto.newRandomSessionID(accountID),
-        crypto,
-      );
-
-      for (const peer of peersToLoadFrom) {
-        loadingNode.syncManager.addPeer(peer);
-      }
-
-      const accountPromise = loadingNode.load(accountID);
-
-      const account = await accountPromise;
-
-      if (account === "unavailable") {
-        throw new Error("Account unavailable from all peers");
-      }
-
-      const controlledAccount = new RawControlledAccount(
-        account.core,
-        accountSecret,
-      );
-
-      // since this is all synchronous, we can just swap out nodes for the SyncManager
-      const node = loadingNode.testWithDifferentAccount(
-        controlledAccount,
-        sessionID || crypto.newRandomSessionID(accountID),
-      );
-      node.syncManager = loadingNode.syncManager;
-      node.syncManager.local = node;
-
-      controlledAccount.core.node = node;
-      node.coValuesStore.setAsAvailable(accountID, controlledAccount.core);
-      controlledAccount.core._cachedContent = undefined;
-
-      const profileID = account.get("profile");
-      if (!profileID) {
-        throw new Error("Account has no profile");
-      }
-      const profile = await node.load(profileID);
-
-      if (profile === "unavailable") {
-        throw new Error("Profile unavailable from all peers");
-      }
-
-      if (migration) {
-        await migration(controlledAccount as RawControlledAccount<Meta>, node);
-        node.account = new RawControlledAccount(
-          controlledAccount.core,
-          controlledAccount.agentSecret,
-        );
-      }
-
-      return node;
-    } catch (e) {
-      logger.error("Error withLoadedAccount: " + (e as Error)?.message);
-      throw e;
-    }
-  }
-
-  /** @internal */
-  createCoValue(header: CoValueHeader): CoValueCore {
-    if (this.crashed) {
-      throw new Error("Trying to create CoValue after node has crashed", {
-        cause: this.crashed,
-      });
-    }
-
-    const coValue = new CoValueCore(header, this);
-    this.coValuesStore.setAsAvailable(coValue.id, coValue);
-
-    void this.syncManager.syncCoValue(coValue);
-
-    return coValue;
-  }
-
-  /** @internal */
-  async loadCoValueCore(
-    id: RawCoID,
-    skipLoadingFromPeer?: PeerID,
-  ): Promise<CoValueCore | "unavailable"> {
-    if (this.crashed) {
-      throw new Error("Trying to load CoValue after node has crashed", {
-        cause: this.crashed,
-      });
-    }
-
-    const entry = this.coValuesStore.get(id);
-
-    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
-      const peers =
-        this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
-
-      await entry.loadFromPeers(peers).catch((e) => {
-        logger.error("Error loading from peers: " + (e as Error)?.message, {
-          id,
-        });
-      });
-    }
-
-    return entry.getCoValue();
-  }
-
-  /**
-   * Loads a CoValue's content, syncing from peers as necessary and resolving the returned
-   * promise once a first version has been loaded. See `coValue.subscribe()` and `node.useTelepathicData()`
-   * for listening to subsequent updates to the CoValue.
-   *
-   * @category 3. Low-level
-   */
-  async load<T extends RawCoValue>(id: CoID<T>): Promise<T | "unavailable"> {
-    const core = await this.loadCoValueCore(id);
-
-    if (core === "unavailable") {
-      return "unavailable";
-    }
-
-    return core.getCurrentContent() as T;
-  }
-
-  getLoaded<T extends RawCoValue>(id: CoID<T>): T | undefined {
-    const entry = this.coValuesStore.get(id);
-
-    if (entry.state.type === "available") {
-      return entry.state.coValue.getCurrentContent() as T;
-    }
-
-    return undefined;
-  }
-
-  /** @category 3. Low-level */
-  subscribe<T extends RawCoValue>(
-    id: CoID<T>,
-    callback: (update: T | "unavailable") => void,
-  ): () => void {
-    let stopped = false;
-    let unsubscribe!: () => void;
-
-    this.load(id)
-      .then((coValue) => {
-        if (stopped) {
-          return;
-        }
-        if (coValue === "unavailable") {
-          callback("unavailable");
-          return;
-        }
-        unsubscribe = coValue.subscribe(callback);
-      })
-      .catch((e) => {
-        logger.error(
-          "Error subscribing to " + id + ": " + (e as Error)?.message,
-        );
-      });
-
-    return () => {
-      stopped = true;
-      unsubscribe?.();
-    };
-  }
-
-  /** @deprecated Use Account.acceptInvite instead */
-  async acceptInvite<T extends RawCoValue>(
-    groupOrOwnedValueID: CoID<T>,
-    inviteSecret: InviteSecret,
-  ): Promise<void> {
-    const groupOrOwnedValue = await this.load(groupOrOwnedValueID);
-
-    if (groupOrOwnedValue === "unavailable") {
-      throw new Error(
-        "Trying to accept invite: Group/owned value unavailable from all peers",
-      );
-    }
-
-    if (groupOrOwnedValue.core.header.ruleset.type === "ownedByGroup") {
-      return this.acceptInvite(
-        groupOrOwnedValue.core.header.ruleset.group as CoID<RawGroup>,
-        inviteSecret,
-      );
-    } else if (groupOrOwnedValue.core.header.ruleset.type !== "group") {
-      throw new Error("Can only accept invites to groups");
-    }
-
-    const group = expectGroup(groupOrOwnedValue);
-
-    const inviteAgentSecret = this.crypto.agentSecretFromSecretSeed(
-      secretSeedFromInviteSecret(inviteSecret),
-    );
-    const inviteAgentID = this.crypto.getAgentID(inviteAgentSecret);
-
-    const inviteRole = await new Promise((resolve, reject) => {
-      group.subscribe((groupUpdate) => {
-        const role = groupUpdate.get(inviteAgentID);
-        if (role) {
-          resolve(role);
-        }
-      });
-      setTimeout(
-        () => reject(new Error("Couldn't find invite before timeout")),
-        2000,
-      );
-    });
-
-    if (!inviteRole) {
-      throw new Error("No invite found");
-    }
-
-    const existingRole = group.get(this.account.id);
-
-    if (
-      existingRole === "admin" ||
-      (existingRole === "writer" && inviteRole === "writerInvite") ||
-      (existingRole === "writer" && inviteRole === "reader") ||
-      (existingRole === "reader" && inviteRole === "readerInvite") ||
-      (existingRole && inviteRole === "writeOnlyInvite")
-    ) {
-      logger.debug("Not accepting invite that would replace or downgrade role");
-      return;
-    }
-
-    const groupAsInvite = expectGroup(
-      group.core
-        .testWithDifferentAccount(
-          new ControlledAgent(inviteAgentSecret, this.crypto),
-          this.crypto.newRandomSessionID(inviteAgentID),
-        )
-        .getCurrentContent(),
-    );
-
-    groupAsInvite.addMemberInternal(
-      this.account,
-      inviteRole === "adminInvite"
-        ? "admin"
-        : inviteRole === "writerInvite"
-          ? "writer"
-          : inviteRole === "writeOnlyInvite"
-            ? "writeOnly"
-            : "reader",
-    );
-
-    group.core._sessionLogs = groupAsInvite.core.sessionLogs;
-    group.core._cachedContent = undefined;
-
-    for (const groupListener of group.core.listeners) {
-      groupListener(group.core.getCurrentContent());
-    }
-  }
-
-  /** @internal */
-  expectCoValueLoaded(id: RawCoID, expectation?: string): CoValueCore {
-    const entry = this.coValuesStore.get(id);
-
-    if (entry.state.type !== "available") {
-      throw new Error(
-        `${expectation ? expectation + ": " : ""}CoValue ${id} not yet loaded. Current state: ${entry.state.type}`,
-      );
-    }
-    return entry.state.coValue;
-  }
-
-  /** @internal */
-  expectProfileLoaded(id: RawAccountID, expectation?: string): RawProfile {
-    const account = this.expectCoValueLoaded(id, expectation);
-    const profileID = expectGroup(account.getCurrentContent()).get("profile");
-    if (!profileID) {
-      throw new Error(
-        `${expectation ? expectation + ": " : ""}Account ${id} has no profile`,
-      );
-    }
-    return this.expectCoValueLoaded(
-      profileID,
-      expectation,
-    ).getCurrentContent() as RawProfile;
-  }
-
-  /** @internal */
-  createAccount(
-    agentSecret = this.crypto.newRandomAgentSecret(),
-  ): RawControlledAccount {
-    const accountAgentID = this.crypto.getAgentID(agentSecret);
-    const account = expectGroup(
-      this.createCoValue(
-        accountHeaderForInitialAgentSecret(agentSecret, this.crypto),
-      )
-        .testWithDifferentAccount(
-          new ControlledAgent(agentSecret, this.crypto),
-          this.crypto.newRandomSessionID(accountAgentID),
-        )
-        .getCurrentContent(),
-    );
-
-    account.set(accountAgentID, "admin", "trusting");
-
-    const readKey = this.crypto.newRandomKeySecret();
-
-    const sealed = this.crypto.seal({
-      message: readKey.secret,
-      from: this.crypto.getAgentSealerSecret(agentSecret),
-      to: this.crypto.getAgentSealerID(accountAgentID),
-      nOnceMaterial: {
-        in: account.id,
-        tx: account.core.nextTransactionID(),
-      },
-    });
-
-    account.set(`${readKey.id}_for_${accountAgentID}`, sealed, "trusting");
-
-    account.set("readKey", readKey.id, "trusting");
-
-    const accountOnThisNode = this.expectCoValueLoaded(account.id);
-
-    accountOnThisNode._sessionLogs = new Map(account.core.sessionLogs);
-
-    accountOnThisNode._cachedContent = undefined;
-
-    return new RawControlledAccount(accountOnThisNode, agentSecret);
-  }
-
-  /** @internal */
-  resolveAccountAgent(
-    id: RawAccountID | AgentID,
-    expectation?: string,
-  ): Result<AgentID, ResolveAccountAgentError> {
-    if (isAgentID(id)) {
-      return ok(id);
-    }
-
-    let coValue: CoValueCore;
-
-    try {
-      coValue = this.expectCoValueLoaded(id, expectation);
-    } catch (e) {
-      return err({
-        type: "ErrorLoadingCoValueCore",
-        expectation,
-        id,
-        error: e,
-      } satisfies LoadCoValueCoreError);
-    }
-
-    if (
-      coValue.header.type !== "comap" ||
-      coValue.header.ruleset.type !== "group" ||
-      !coValue.header.meta ||
-      !("type" in coValue.header.meta) ||
-      coValue.header.meta.type !== "account"
-    ) {
-      return err({
-        type: "UnexpectedlyNotAccount",
-        expectation,
-        id,
-      } satisfies UnexpectedlyNotAccountError);
-    }
-
-    return (coValue.getCurrentContent() as RawAccount).currentAgentID();
-  }
-
-  resolveAccountAgentAsync(
-    id: RawAccountID | AgentID,
-    expectation?: string,
-  ): ResultAsync<AgentID, ResolveAccountAgentError> {
-    if (isAgentID(id)) {
-      return okAsync(id);
-    }
-
-    return ResultAsync.fromPromise(
-      this.loadCoValueCore(id),
-      (e) =>
-        ({
-          type: "ErrorLoadingCoValueCore",
-          expectation,
-          id,
-          error: e,
-        }) satisfies LoadCoValueCoreError,
-    ).andThen((coValue) => {
-      if (coValue === "unavailable") {
-        return err({
-          type: "AccountUnavailableFromAllPeers" as const,
-          expectation,
-          id,
-        } satisfies AccountUnavailableFromAllPeersError);
-      }
-
-      if (
-        coValue.header.type !== "comap" ||
-        coValue.header.ruleset.type !== "group" ||
-        !coValue.header.meta ||
-        !("type" in coValue.header.meta) ||
-        coValue.header.meta.type !== "account"
-      ) {
-        return err({
-          type: "UnexpectedlyNotAccount" as const,
-          expectation,
-          id,
-        } satisfies UnexpectedlyNotAccountError);
-      }
-
-      return (coValue.getCurrentContent() as RawAccount).currentAgentID();
-    });
-  }
-
-  /**
-   * @deprecated use Account.createGroup() instead
-   */
-  createGroup(
-    uniqueness: CoValueUniqueness = this.crypto.createdNowUnique(),
-  ): RawGroup {
-    const groupCoValue = this.createCoValue({
-      type: "comap",
-      ruleset: { type: "group", initialAdmin: this.account.id },
-      meta: null,
-      ...uniqueness,
-    });
-
-    const group = expectGroup(groupCoValue.getCurrentContent());
-
-    group.set(this.account.id, "admin", "trusting");
-
-    const readKey = this.crypto.newRandomKeySecret();
-
-    group.set(
-      `${readKey.id}_for_${this.account.id}`,
-      this.crypto.seal({
-        message: readKey.secret,
-        from: this.account.currentSealerSecret(),
-        to: this.account
-          .currentSealerID()
-          ._unsafeUnwrap({ withStackTrace: true }),
-        nOnceMaterial: {
-          in: groupCoValue.id,
-          tx: groupCoValue.nextTransactionID(),
-        },
-      }),
-      "trusting",
-    );
-
-    group.set("readKey", readKey.id, "trusting");
-
-    return group;
-  }
-
-  /** @internal */
-  testWithDifferentAccount(
-    account: ControlledAccountOrAgent,
-    currentSessionID: SessionID,
-  ): LocalNode {
-    const newNode = new LocalNode(account, currentSessionID, this.crypto);
-
-    const coValuesToCopy = Array.from(this.coValuesStore.getEntries());
-
-    while (coValuesToCopy.length > 0) {
-      const [coValueID, entry] = coValuesToCopy[coValuesToCopy.length - 1]!;
-
-      if (entry.state.type !== "available") {
-        coValuesToCopy.pop();
-        continue;
-      } else {
-        const allDepsCopied = entry.state.coValue
-          .getDependedOnCoValues()
-          .every(
-            (dep) => newNode.coValuesStore.get(dep).state.type === "available",
-          );
-
-        if (!allDepsCopied) {
-          // move to end of queue
-          coValuesToCopy.unshift(coValuesToCopy.pop()!);
-          continue;
-        }
-
-        const newCoValue = new CoValueCore(
-          entry.state.coValue.header,
-          newNode,
-          new Map(entry.state.coValue.sessionLogs),
-        );
-
-        newNode.coValuesStore.setAsAvailable(coValueID, newCoValue);
-
-        coValuesToCopy.pop();
-      }
-    }
-
-    if (account instanceof RawControlledAccount) {
-      // To make sure that when we edit the account, we're modifying the correct sessions
-      const accountInNode = new RawControlledAccount(
-        newNode.expectCoValueLoaded(account.id),
-        account.agentSecret,
-      );
-      if (accountInNode.core.node !== newNode) {
-        throw new Error("Account's node is not the new node");
-      }
-      newNode.account = accountInNode;
-    }
-
-    return newNode;
-  }
-
-  gracefulShutdown() {
-    this.syncManager.gracefulShutdown();
-  }
-}
-
-export type LoadCoValueCoreError = {
-  type: "ErrorLoadingCoValueCore";
-  error: unknown;
-  expectation?: string;
-  id: RawAccountID;
-};
-
-export type AccountUnavailableFromAllPeersError = {
-  type: "AccountUnavailableFromAllPeers";
-  expectation?: string;
-  id: RawAccountID;
-};
-
-export type UnexpectedlyNotAccountError = {
-  type: "UnexpectedlyNotAccount";
-  expectation?: string;
-  id: RawAccountID;
-};
-
-export type ResolveAccountAgentError =
-  | InvalidAccountAgentIDError
-  | LoadCoValueCoreError
-  | AccountUnavailableFromAllPeersError
-  | UnexpectedlyNotAccountError;

packages/cojson/src/localNode/actions/createCoValue.ts

@@ -0,0 +1,5 @@
+import { CoValueHeader, LocalNodeState } from "../structure.js";
+
+export function createCoValue(node: LocalNodeState, header: CoValueHeader) {
+  throw new Error("Not implemented");
+}

packages/cojson/src/localNode/actions/makeTransaction.ts

@@ -0,0 +1,13 @@
+import { RawCoID, SessionID } from "../../exports.js";
+import { LocalNodeState, Transaction } from "../structure.js";
+
+export function makeTransaction(
+  node: LocalNodeState,
+  id: RawCoID,
+  sessionID: SessionID,
+  tx: Transaction,
+): {
+  result: "success" | "failure";
+} {
+  throw new Error("Not implemented");
+}

packages/cojson/src/localNode/actions/peers.ts

@@ -0,0 +1,23 @@
+import { PeerID } from "../../sync.js";
+import { LocalNodeState } from "../structure.js";
+
+export function addPeer(node: LocalNodeState, peerID: PeerID) {
+  node.peers.push(peerID);
+  for (const coValue of Object.values(node.coValues)) {
+    coValue.peerState[peerID] = {
+      confirmed: "unknown",
+      optimistic: "unknown",
+    };
+  }
+}
+
+export function removePeer(node: LocalNodeState, peerID: PeerID) {
+  const index = node.peers.indexOf(peerID);
+  if (index === -1) {
+    throw new Error("Peer not found");
+  }
+  node.peers.splice(index, 1);
+  for (const coValue of Object.values(node.coValues)) {
+    delete coValue.peerState[peerID];
+  }
+}

packages/cojson/src/localNode/actions/subscribing.ts

@@ -0,0 +1,33 @@
+import { RawCoID } from "../../exports.js";
+import { ListenerID, LocalNodeState, emptyCoValueState } from "../structure.js";
+
+export function subscribe(
+  node: LocalNodeState,
+  id: RawCoID,
+): {
+  listenerID: ListenerID;
+} {
+  const existing = node.coValues[id];
+  if (!existing) {
+    const entry = emptyCoValueState(id);
+    entry.listeners[1] = "unknown";
+    node.coValues[id] = entry;
+    return { listenerID: 1 };
+  } else {
+    const nextListenerID = Object.keys(existing.listeners).length + 1;
+    existing.listeners[nextListenerID] = "unknown";
+    return { listenerID: nextListenerID };
+  }
+}
+
+export function unsubscribe(
+  node: LocalNodeState,
+  id: RawCoID,
+  listenerID: ListenerID,
+) {
+  const existing = node.coValues[id];
+  if (!existing) {
+    throw new Error("CoValue not found");
+  }
+  delete existing.listeners[listenerID];
+}

packages/cojson/src/localNode/effects.ts

@@ -0,0 +1,36 @@
+import { RawCoID, SessionID, SyncMessage } from "../exports.js";
+import { StoredSessionLog } from "../storage.js";
+import { PeerID } from "../sync.js";
+import { CoValueHeader, ListenerID } from "./structure.js";
+
+export type LoadMetadataFromStorageEffect = {
+  type: "loadMetadataFromStorage";
+  id: RawCoID;
+};
+
+export type LoadTransactionsFromStorageEffect = {
+  type: "loadTransactionsFromStorage";
+  id: RawCoID;
+  sessionID: SessionID;
+  from: number;
+  to: number;
+};
+
+export type SendMessageToPeerEffect = {
+  type: "sendMessageToPeer";
+  id: RawCoID;
+  peerID: PeerID;
+  message: SyncMessage;
+};
+
+export type NotifyListenerEffect = {
+  type: "notifyListener";
+  listenerID: ListenerID;
+};
+
+export type WriteToStorageEffect = {
+  type: "writeToStorage";
+  id: RawCoID;
+  header: CoValueHeader | null;
+  sessions: { [key: SessionID]: StoredSessionLog };
+};

packages/cojson/src/localNode/handlers/addTransaction.ts

@@ -0,0 +1,46 @@
+import { Signature } from "../../crypto/crypto.js";
+import { RawCoID, SessionID } from "../../exports.js";
+import { LocalNodeState, Transaction } from "../structure.js";
+
+export function addTransaction(
+  node: LocalNodeState,
+  id: RawCoID,
+  sessionID: SessionID,
+  after: number,
+  transactions: Transaction[],
+  signature: Signature,
+): {
+  result: { type: "success" } | { type: "gap"; expectedAfter: number };
+} {
+  const entry = node.coValues[id];
+  if (!entry) {
+    throw new Error("CoValue not found");
+  }
+  const session = entry.sessions[sessionID];
+  if (!session) {
+    throw new Error("Session not found");
+  }
+  if (after > session.transactions.length) {
+    return {
+      result: { type: "gap", expectedAfter: session.transactions.length },
+    };
+  }
+  for (let i = 0; i < transactions.length; i++) {
+    const sessionIdx = after + i;
+    if (
+      session.transactions[sessionIdx] &&
+      session.transactions[sessionIdx]!.state !== "availableInStorage"
+    ) {
+      throw new Error(
+        `Unexpected existing state ${session.transactions[sessionIdx]!.state} at index ${sessionIdx}`,
+      );
+    }
+    session.transactions[sessionIdx] = {
+      state: "available",
+      tx: transactions[i]!,
+      signature: i === transactions.length - 1 ? signature : null,
+    };
+    session.lastAvailable = Math.max(session.lastAvailable, sessionIdx);
+  }
+  return { result: { type: "success" } };
+}

packages/cojson/src/localNode/handlers/onMetadataLoaded.ts

@@ -0,0 +1,40 @@
+import { RawCoID, SessionID } from "../../exports.js";
+import { CoValueHeader, KnownState, LocalNodeState } from "../structure.js";
+
+export function onMetadataLoaded(
+  node: LocalNodeState,
+  id: RawCoID,
+  header: CoValueHeader | null,
+  knownState: KnownState,
+) {
+  const entry = node.coValues[id];
+  if (!entry) {
+    throw new Error("CoValue not found");
+  }
+  if (header) {
+    entry.header = header;
+  }
+  entry.storageState = knownState;
+  if (knownState !== "unknown" && knownState !== "unavailable") {
+    for (const sessionID of Object.keys(knownState.sessions) as SessionID[]) {
+      let session = entry.sessions[sessionID];
+      if (!session) {
+        session = {
+          id: sessionID,
+          transactions: [],
+          streamingHash: null,
+          lastVerified: -1,
+          lastAvailable: -1,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+        };
+        entry.sessions[sessionID] = session;
+      }
+      for (let i = 0; i < (knownState.sessions[sessionID] || 0); i++) {
+        if (!session.transactions[i]) {
+          session.transactions[i] = { state: "availableInStorage" };
+        }
+      }
+    }
+  }
+}

packages/cojson/src/localNode/handlers/onSyncMessageReceived.ts

@@ -0,0 +1,12 @@
+import { SyncMessage } from "../../exports.js";
+import { PeerID } from "../../sync.js";
+import { LocalNodeState } from "../structure.js";
+
+export function onSyncMessageReceived(
+  node: LocalNodeState,
+  message: SyncMessage,
+  fromPeer: PeerID,
+  priority: number,
+) {
+  throw new Error("Not implemented");
+}

packages/cojson/src/localNode/stages/0_load.ts

@@ -0,0 +1,62 @@
+import { SessionID } from "../../exports.js";
+import {
+  LoadMetadataFromStorageEffect,
+  LoadTransactionsFromStorageEffect,
+} from "../effects.js";
+import { LocalNodeState, SessionState } from "../structure.js";
+
+export function stageLoad(node: LocalNodeState): {
+  effects: (
+    | LoadMetadataFromStorageEffect
+    | LoadTransactionsFromStorageEffect
+  )[];
+} {
+  const effects: (
+    | LoadMetadataFromStorageEffect
+    | LoadTransactionsFromStorageEffect
+  )[] = [];
+  for (const coValue of Object.values(node.coValues)) {
+    if (coValue.storageState === "unknown") {
+      effects.push({ type: "loadMetadataFromStorage", id: coValue.id });
+      coValue.storageState = "pending";
+    } else if (coValue.storageState === "pending") {
+      continue;
+    } else if (coValue.storageState === "unavailable") {
+      continue;
+    } else {
+      if (
+        Object.keys(coValue.listeners).length == 0 &&
+        coValue.dependents.length === 0
+      )
+        continue;
+      for (const [sessionID, session] of Object.entries(coValue.sessions) as [
+        SessionID,
+        SessionState,
+      ][]) {
+        let firstToLoad = -1;
+        let lastToLoad = -1;
+
+        for (let i = 0; i < session.transactions.length; i++) {
+          if (session.transactions[i]?.state === "availableInStorage") {
+            if (firstToLoad === -1) {
+              firstToLoad = i;
+            }
+            lastToLoad = i;
+            session.transactions[i] = { state: "loadingFromStorage" };
+          }
+        }
+
+        if (firstToLoad !== -1) {
+          effects.push({
+            type: "loadTransactionsFromStorage",
+            id: coValue.id,
+            sessionID,
+            from: firstToLoad,
+            to: lastToLoad,
+          });
+        }
+      }
+    }
+  }
+  return { effects };
+}

packages/cojson/src/localNode/stages/1_loadDeps.ts

@@ -0,0 +1,58 @@
+import { getGroupDependentKey } from "../../ids.js";
+import { parseJSON } from "../../jsonStringify.js";
+import { LocalNodeState, emptyCoValueState } from "../structure.js";
+
+export function stageLoadDeps(node: LocalNodeState) {
+  for (const coValue of Object.values(node.coValues)) {
+    if (Object.keys(coValue.listeners).length === 0) {
+      continue;
+    }
+    if (coValue.storageState === "pending") {
+      continue;
+    }
+    if (coValue.header?.ruleset.type === "ownedByGroup") {
+      const existing = node.coValues[coValue.header.ruleset.group];
+      if (existing) {
+        if (!existing.dependents.includes(coValue.id)) {
+          existing.dependents.push(coValue.id);
+        }
+      } else {
+        const entry = emptyCoValueState(coValue.header.ruleset.group);
+        entry.dependents.push(coValue.id);
+        node.coValues[coValue.header.ruleset.group] = entry;
+      }
+    } else if (coValue.header?.ruleset.type === "group") {
+      for (const session of Object.values(coValue.sessions)) {
+        for (const tx of session.transactions) {
+          if (tx.state === "available" && tx.tx.privacy === "trusting") {
+            // TODO: this should read from the tx.decryptionState.changes instead
+            const changes = parseJSON(tx.tx.changes);
+            for (const change of changes) {
+              if (
+                typeof change === "object" &&
+                change !== null &&
+                "op" in change &&
+                change.op === "set" &&
+                "key" in change
+              ) {
+                const groupDependency = getGroupDependentKey(change.key);
+                if (groupDependency) {
+                  const existing = node.coValues[groupDependency];
+                  if (existing) {
+                    if (!existing.dependents.includes(coValue.id)) {
+                      existing.dependents.push(coValue.id);
+                    }
+                  } else {
+                    const entry = emptyCoValueState(groupDependency);
+                    entry.dependents.push(coValue.id);
+                    node.coValues[groupDependency] = entry;
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

packages/cojson/src/localNode/stages/2_verify.ts

@@ -0,0 +1,143 @@
+import { SignerID, StreamingHash } from "../../crypto/crypto.js";
+import { AgentID, CryptoProvider, RawCoID } from "../../exports.js";
+import { isAgentID } from "../../ids.js";
+import { parseJSON } from "../../jsonStringify.js";
+import { accountOrAgentIDfromSessionID } from "../../typeUtils/accountOrAgentIDfromSessionID.js";
+import {
+  CoValueState,
+  LocalNodeState,
+  SessionState,
+  TransactionState,
+} from "../structure.js";
+
+export function stageVerify(node: LocalNodeState, crypto: CryptoProvider) {
+  for (const coValue of Object.values(node.coValues)) {
+    if (
+      coValue.storageState === "pending" ||
+      coValue.storageState === "unknown" ||
+      (Object.keys(coValue.listeners).length === 0 &&
+        coValue.dependents.length === 0)
+    ) {
+      continue;
+    }
+    for (const session of Object.values(coValue.sessions)) {
+      if (session.lastVerified == session.lastAvailable) {
+        continue;
+      }
+
+      verifySession(node, session, coValue.id, crypto);
+    }
+  }
+}
+
+function verifySession(
+  node: LocalNodeState,
+  session: SessionState,
+  coValueID: RawCoID,
+  crypto: CryptoProvider,
+) {
+  const streamingHash =
+    session.streamingHash?.clone() ?? new StreamingHash(crypto);
+
+  for (let i = session.lastVerified + 1; i <= session.lastAvailable; i++) {
+    const txState = session.transactions[i];
+
+    if (txState?.state !== "available") {
+      throw new Error(
+        `Transaction ${i} is not available in ${coValueID} ${session.id}`,
+      );
+    }
+
+    streamingHash.update(txState.tx);
+
+    if (txState.signature) {
+      const hash = streamingHash.digest();
+      const authorID = accountOrAgentIDfromSessionID(session.id);
+      let signerID: SignerID;
+      if (isAgentID(authorID)) {
+        signerID = crypto.getAgentSignerID(authorID);
+      } else {
+        const authorAccount = node.coValues[authorID];
+        if (!authorAccount) {
+          throw new Error(
+            `Author covalue ${authorID} not present, not yet handled`,
+          );
+        }
+        const foundAgentIDs = findAgentIDsInAccount(authorAccount);
+        if (foundAgentIDs.length > 1) {
+          throw new Error(
+            `Multiple agent IDs found in ${authorID} - not yet handled`,
+          );
+        }
+        const onlyAgent = foundAgentIDs[0];
+        if (!onlyAgent) {
+          throw new Error(`No agent ID found in ${authorID} - not yet handled`);
+        }
+        signerID = crypto.getAgentSignerID(onlyAgent);
+      }
+      if (crypto.verify(txState.signature, hash, signerID)) {
+        for (let v = session.lastVerified + 1; v <= i; v++) {
+          session.transactions[v] = {
+            ...(session.transactions[v] as TransactionState & {
+              state: "available";
+            }),
+            state: "verified",
+            validity: { type: "unknown" },
+            decryptionState: { type: "notDecrypted" },
+            stored: false,
+          };
+        }
+        session.lastVerified = i;
+      } else {
+        console.log(
+          `Signature verification failed for transaction ${i} in ${coValueID} ${session.id}`,
+        );
+
+        for (
+          let iv = session.lastVerified + 1;
+          iv <= session.lastAvailable;
+          iv++
+        ) {
+          session.transactions[iv] = {
+            ...(session.transactions[iv] as TransactionState & {
+              state: "available";
+            }),
+            state: "verificationFailed",
+            reason: `Invalid signature ${iv === i ? "(here)" : `at idx ${i}`}`,
+            hash: iv === i ? hash : null,
+          };
+        }
+        session.lastVerified = session.lastAvailable;
+        return;
+      }
+    }
+  }
+}
+
+function findAgentIDsInAccount(authorAccount: CoValueState): AgentID[] {
+  return Object.values(authorAccount.sessions).flatMap((session) =>
+    session.transactions.flatMap((tx) => {
+      if (tx.state === "verified" && tx.tx.privacy === "trusting") {
+        // TODO: this should read from the tx.decryptionState.changes instead
+        const changes = parseJSON(tx.tx.changes);
+        return changes.flatMap((change) => {
+          if (
+            typeof change === "object" &&
+            change !== null &&
+            "op" in change &&
+            change.op === "set" &&
+            "key" in change &&
+            typeof change.key === "string" &&
+            isAgentID(change.key)
+          ) {
+            return [change.key as AgentID];
+          } else {
+            return [];
+          }
+        });
+      } else {
+        return [];
+      }
+    }),
+  );
+}

packages/cojson/src/localNode/stages/3_validate.ts

@@ -0,0 +1,3 @@
+import { LocalNodeState } from "../structure.js";
+
+export function stageValidate(node: LocalNodeState) {}

packages/cojson/src/localNode/stages/4_decrypt.ts

@@ -0,0 +1,3 @@
+import { LocalNodeState } from "../structure.js";
+
+export function stageDecrypt(node: LocalNodeState) {}

packages/cojson/src/localNode/stages/5_notify.ts

@@ -0,0 +1,8 @@
+import { NotifyListenerEffect } from "../effects.js";
+import { LocalNodeState } from "../structure.js";
+
+export function stageNotify(node: LocalNodeState): {
+  effects: NotifyListenerEffect[];
+} {
+  throw new Error("Not implemented");
+}

packages/cojson/src/localNode/stages/6_syncOut.ts

@@ -0,0 +1,18 @@
+import { SendMessageToPeerEffect } from "../effects.js";
+import { LocalNodeState } from "../structure.js";
+
+export function stageSyncOut(node: LocalNodeState): {
+  effects: SendMessageToPeerEffect[];
+} {
+  for (const coValue of Object.values(node.coValues)) {
+    if (
+      coValue.storageState === "pending" ||
+      coValue.storageState === "unknown"
+    ) {
+      continue;
+    } else {
+      throw new Error("CoValue is not pending or unknown");
+    }
+  }
+  return { effects: [] };
+}

packages/cojson/src/localNode/stages/7_store.ts

@@ -0,0 +1,8 @@
+import { WriteToStorageEffect } from "../effects.js";
+import { LocalNodeState } from "../structure.js";
+
+export function stageStore(node: LocalNodeState): {
+  effects: WriteToStorageEffect[];
+} {
+  throw new Error("Not implemented");
+}

packages/cojson/src/localNode/structure.ts

@@ -0,0 +1,158 @@
+import {
+  Encrypted,
+  Hash,
+  KeyID,
+  Signature,
+  StreamingHash,
+} from "../crypto/crypto.js";
+import {
+  AnyRawCoValue,
+  RawCoID,
+  SessionID,
+  Stringified,
+  SyncMessage,
+} from "../exports.js";
+import { TransactionID } from "../ids.js";
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { PermissionsDef } from "../permissions.js";
+import { PeerID } from "../sync.js";
+
+export type LocalNodeState = {
+  coValues: { [key: RawCoID]: CoValueState };
+  peers: PeerID[];
+};
+
+export interface CoValueState {
+  id: RawCoID;
+  header: CoValueHeader | null;
+  sessions: { [key: SessionID]: SessionState };
+  storageState: KnownState | "pending";
+  peerState: {
+    [key: PeerID]: {
+      confirmed: KnownState | "pending";
+      optimistic: KnownState;
+    };
+  };
+  listeners: { [key: ListenerID]: KnownState };
+  dependents: RawCoID[];
+  incomingMessages: {
+    [key: PeerID]: {
+      [priority: number]: SyncMessage[];
+    };
+  };
+}
+
+export type CoValueHeader = {
+  type: AnyRawCoValue["type"];
+  ruleset: PermissionsDef;
+  meta: JsonObject | null;
+} & CoValueUniqueness;
+
+export type CoValueUniqueness = {
+  uniqueness: JsonValue;
+  createdAt?: `2${string}` | null;
+};
+
+export type ListenerID = number;
+
+export type KnownState =
+  | {
+      header: boolean;
+      sessions: { [key: SessionID]: number };
+    }
+  | "unknown"
+  | "unavailable";
+
+export type SessionState = {
+  id: SessionID;
+  transactions: TransactionState[];
+  streamingHash: StreamingHash | null;
+  lastAvailable: number;
+  lastDepsAvailable: number;
+  lastVerified: number;
+  lastDecrypted: number;
+};
+
+export type TransactionState =
+  | {
+      state: "availableInStorage";
+    }
+  | {
+      state: "loadingFromStorage";
+    }
+  | {
+      state: "available";
+      tx: Transaction;
+      signature: Signature | null;
+    }
+  | {
+      state: "verified";
+      tx: Transaction;
+      signature: Signature | null;
+      validity:
+        | { type: "unknown" }
+        | { type: "pending" }
+        | { type: "valid" }
+        | { type: "invalid"; reason: string };
+      decryptionState:
+        | {
+            type: "notDecrypted";
+          }
+        | {
+            type: "decrypted";
+            changes: JsonValue[];
+          }
+        | {
+            type: "error";
+            error: Error;
+          };
+      stored: boolean;
+    }
+  | {
+      state: "verificationFailed";
+      tx: Transaction;
+      signature: Signature | null;
+      reason: string;
+      hash: Hash | null;
+    };
+
+export type PrivateTransaction = {
+  privacy: "private";
+  madeAt: number;
+  keyUsed: KeyID;
+  encryptedChanges: Encrypted<JsonValue[], { in: RawCoID; tx: TransactionID }>;
+};
+
+export type TrustingTransaction = {
+  privacy: "trusting";
+  madeAt: number;
+  changes: Stringified<JsonValue[]>;
+};
+
+export type Transaction = PrivateTransaction | TrustingTransaction;
+
+export type DecryptedTransaction = {
+  txID: TransactionID;
+  changes: JsonValue[];
+  madeAt: number;
+};
+
+export function emptyNode(): LocalNodeState {
+  return {
+    coValues: {},
+    peers: [],
+  };
+}
+
+export function emptyCoValueState(id: RawCoID): CoValueState {
+  return {
+    id,
+    header: null,
+    sessions: {},
+    storageState: "unknown",
+    peerState: {},
+    listeners: {},
+    dependents: [],
+    incomingMessages: {},
+  };
+}

packages/cojson/src/localNode/tick.ts

@@ -0,0 +1,42 @@
+import { CryptoProvider } from "../exports.js";
+import {
+  LoadMetadataFromStorageEffect,
+  LoadTransactionsFromStorageEffect,
+  NotifyListenerEffect,
+  SendMessageToPeerEffect,
+  WriteToStorageEffect,
+} from "./effects.js";
+import { stageLoad } from "./stages/0_load.js";
+import { stageLoadDeps } from "./stages/1_loadDeps.js";
+import { stageVerify } from "./stages/2_verify.js";
+import { stageValidate } from "./stages/3_validate.js";
+import { stageDecrypt } from "./stages/4_decrypt.js";
+import { stageNotify } from "./stages/5_notify.js";
+import { stageSyncOut } from "./stages/6_syncOut.js";
+import { stageStore } from "./stages/7_store.js";
+import { LocalNodeState } from "./structure.js";
+
+export function tick(
+  node: LocalNodeState,
+  crypto: CryptoProvider,
+): {
+  effects: (
+    | NotifyListenerEffect
+    | SendMessageToPeerEffect
+    | LoadMetadataFromStorageEffect
+    | LoadTransactionsFromStorageEffect
+    | WriteToStorageEffect
+  )[];
+} {
+  const effects = [];
+
+  effects.push(...stageLoad(node).effects);
+  stageLoadDeps(node);
+  stageVerify(node, crypto);
+  stageValidate(node);
+  stageDecrypt(node);
+  effects.push(...stageNotify(node).effects);
+  effects.push(...stageSyncOut(node).effects);
+  effects.push(...stageStore(node).effects);
+  return { effects };
+}

packages/cojson/src/storage.ts

@@ -0,0 +1,79 @@
+import { CoValueHeader, Transaction } from "./coValueCore.js";
+import { Signature } from "./crypto/crypto.js";
+import { CoValueCore, LocalNode, RawCoID, SessionID } from "./exports.js";
+import { KnownStateMessage } from "./sync.js";
+
+export type StoredSessionLog = {
+  transactions: Transaction[];
+  signatureAfter: { [txIdx: number]: Signature | undefined };
+  lastSignature: Signature;
+};
+
+export interface StorageAdapter {
+  get(id: RawCoID): Promise<{
+    header: CoValueHeader;
+    sessions: Map<SessionID, StoredSessionLog>;
+  } | null>;
+
+  writeHeader(id: RawCoID, header: CoValueHeader): Promise<void>;
+
+  appendToSession(
+    id: RawCoID,
+    sessionID: SessionID,
+    afterIdx: number,
+    tx: Transaction[],
+    lastSignature: Signature,
+  ): Promise<void>;
+}
+
+export class StorageDriver {
+  private storageAdapter: StorageAdapter;
+  private storedStates: Map<RawCoID, KnownStateMessage> = new Map();
+  private node: LocalNode;
+  constructor(storageAdapter: StorageAdapter, node: LocalNode) {
+    this.storageAdapter = storageAdapter;
+    this.node = node;
+  }
+
+  async get(id: RawCoID) {
+    const storedCoValue = await this.storageAdapter.get(id);
+
+    if (!storedCoValue) {
+      return null;
+    }
+
+    const core = new CoValueCore(storedCoValue.header, this.node);
+
+    for (const [sessionID, sessionLog] of storedCoValue.sessions) {
+      let start = 0;
+      for (const [signatureAt, signature] of Object.entries(
+        sessionLog.signatureAfter,
+      )) {
+        if (!signature) {
+          throw new Error(
+            `Expected signature at ${signatureAt} for session ${sessionID}`,
+          );
+        }
+        core
+          .tryAddTransactions(
+            sessionID,
+            sessionLog.transactions.slice(start, parseInt(signatureAt)),
+            undefined,
+            signature,
+            { skipStorage: true },
+          )
+          ._unsafeUnwrap();
+      }
+    }
+
+    return core;
+  }
+
+  async set(core: CoValueCore): Promise<void> {
+    const currentState = this.storedStates.get(core.id);
+    const knownState = core.knownState();
+
+    currentState;
+    knownState;
+  }
+}

packages/cojson/src/storage/FileSystem.ts

@@ -1,113 +0,0 @@
-import { CryptoProvider, StreamingHash } from "../crypto/crypto.js";
-import { RawCoID } from "../ids.js";
-import { CoValueChunk } from "./index.js";
-
-export type BlockFilename = `L${number}-${string}-${string}-H${number}.jsonl`;
-
-export type BlockHeader = { id: RawCoID; start: number; length: number }[];
-
-export type WalEntry = { id: RawCoID } & CoValueChunk;
-
-export type WalFilename = `wal-${number}.jsonl`;
-
-export interface FileSystem<WriteHandle, ReadHandle> {
-  crypto: CryptoProvider;
-  createFile(filename: string): Promise<WriteHandle>;
-  append(handle: WriteHandle, data: Uint8Array): Promise<void>;
-  close(handle: ReadHandle | WriteHandle): Promise<void>;
-  closeAndRename(handle: WriteHandle, filename: BlockFilename): Promise<void>;
-  openToRead(filename: string): Promise<{ handle: ReadHandle; size: number }>;
-  read(handle: ReadHandle, offset: number, length: number): Promise<Uint8Array>;
-  listFiles(): Promise<string[]>;
-  removeFile(filename: BlockFilename | WalFilename): Promise<void>;
-}
-
-export const textEncoder = new TextEncoder();
-export const textDecoder = new TextDecoder();
-
-export async function readChunk<RH, FS extends FileSystem<unknown, RH>>(
-  handle: RH,
-  header: { start: number; length: number },
-  fs: FS,
-): Promise<CoValueChunk> {
-  const chunkBytes = await fs.read(handle, header.start, header.length);
-
-  const chunk = JSON.parse(textDecoder.decode(chunkBytes));
-  return chunk;
-}
-
-export async function readHeader<RH, FS extends FileSystem<unknown, RH>>(
-  filename: string,
-  handle: RH,
-  size: number,
-  fs: FS,
-): Promise<BlockHeader> {
-  const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
-
-  const headerBytes = await fs.read(handle, size - headerLength, headerLength);
-
-  const header = JSON.parse(textDecoder.decode(headerBytes));
-  return header;
-}
-
-export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
-  chunks: Map<RawCoID, CoValueChunk>,
-  level: number,
-  blockNumber: number,
-  fs: FS,
-): Promise<BlockFilename> {
-  if (chunks.size === 0) {
-    throw new Error("No chunks to write");
-  }
-
-  const blockHeader: BlockHeader = [];
-
-  let offset = 0;
-
-  const file = await fs.createFile(
-    "wipBlock" + Math.random().toString(36).substring(7) + ".tmp.jsonl",
-  );
-  const hash = new StreamingHash(fs.crypto);
-
-  const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
-    id1.localeCompare(id2),
-  );
-
-  for (const [id, chunk] of chunksSortedById) {
-    const encodedBytes = hash.update(chunk);
-    const encodedBytesWithNewline = new Uint8Array(encodedBytes.length + 1);
-    encodedBytesWithNewline.set(encodedBytes);
-    encodedBytesWithNewline[encodedBytes.length] = 10;
-    await fs.append(file, encodedBytesWithNewline);
-    const length = encodedBytesWithNewline.length;
-    blockHeader.push({ id, start: offset, length });
-    offset += length;
-  }
-
-  const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
-  await fs.append(file, headerBytes);
-
-  const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
-    3,
-    "0",
-  )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
-    headerBytes.length
-  }.jsonl`;
-  await fs.closeAndRename(file, filename);
-
-  return filename;
-}
-
-export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
-  handle: WH,
-  fs: FS,
-  id: RawCoID,
-  chunk: CoValueChunk,
-) {
-  const walEntry: WalEntry = {
-    id,
-    ...chunk,
-  };
-  const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  return fs.append(handle, bytes);
-}

packages/cojson/src/storage/chunksAndKnownStates.ts

@@ -1,137 +0,0 @@
-import { MAX_RECOMMENDED_TX_SIZE } from "../coValueCore.js";
-import { RawCoID, SessionID } from "../ids.js";
-import { getPriorityFromHeader } from "../priority.js";
-import { CoValueKnownState, NewContentMessage } from "../sync.js";
-import { CoValueChunk } from "./index.js";
-
-export function contentSinceChunk(
-  id: RawCoID,
-  chunk: CoValueChunk,
-  known?: CoValueKnownState,
-): NewContentMessage[] {
-  const newContentPieces: NewContentMessage[] = [];
-
-  newContentPieces.push({
-    id: id,
-    action: "content",
-    header: known?.header ? undefined : chunk.header,
-    new: {},
-    priority: getPriorityFromHeader(chunk.header),
-  });
-
-  for (const [sessionID, sessionsEntry] of Object.entries(
-    chunk.sessionEntries,
-  )) {
-    for (const entry of sessionsEntry) {
-      const knownStart = known?.sessions[sessionID as SessionID] || 0;
-
-      if (entry.after + entry.transactions.length <= knownStart) {
-        continue;
-      }
-
-      const actuallyNewTransactions = entry.transactions.slice(
-        Math.max(0, knownStart - entry.after),
-      );
-
-      const newAfter =
-        entry.after +
-        (actuallyNewTransactions.length - entry.transactions.length);
-
-      let newContentEntry = newContentPieces[0]?.new[sessionID as SessionID];
-
-      if (!newContentEntry) {
-        newContentEntry = {
-          after: newAfter,
-          lastSignature: entry.lastSignature,
-          newTransactions: actuallyNewTransactions,
-        };
-        newContentPieces[0]!.new[sessionID as SessionID] = newContentEntry;
-      } else {
-        newContentEntry.newTransactions.push(...actuallyNewTransactions);
-        newContentEntry.lastSignature = entry.lastSignature;
-      }
-    }
-  }
-
-  return newContentPieces;
-}
-
-export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
-  const ourKnown: CoValueKnownState = {
-    id,
-    header: !!chunk.header,
-    sessions: {},
-  };
-
-  for (const [sessionID, sessionEntries] of Object.entries(
-    chunk.sessionEntries,
-  )) {
-    for (const entry of sessionEntries) {
-      ourKnown.sessions[sessionID as SessionID] =
-        entry.after + entry.transactions.length;
-    }
-  }
-  return ourKnown;
-}
-
-export function mergeChunks(
-  chunkA: CoValueChunk,
-  chunkB: CoValueChunk,
-): "nonContigous" | CoValueChunk {
-  const header = chunkA.header || chunkB.header;
-
-  const newSessions = { ...chunkA.sessionEntries };
-  for (const sessionID in chunkB.sessionEntries) {
-    // figure out if we can merge the chunks
-    const sessionEntriesA = chunkA.sessionEntries[sessionID];
-    const sessionEntriesB = chunkB.sessionEntries[sessionID]!;
-
-    if (!sessionEntriesA) {
-      newSessions[sessionID] = sessionEntriesB;
-      continue;
-    }
-
-    const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
-    const firstEntryOfB = sessionEntriesB[0]!;
-
-    if (
-      lastEntryOfA.after + lastEntryOfA.transactions.length ===
-      firstEntryOfB.after
-    ) {
-      const newEntries = [];
-      let bytesSinceLastSignature = 0;
-      for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
-        const entryByteLength = entry.transactions.reduce(
-          (sum, tx) =>
-            sum +
-            (tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length),
-          0,
-        );
-        if (
-          newEntries.length === 0 ||
-          bytesSinceLastSignature + entryByteLength > MAX_RECOMMENDED_TX_SIZE
-        ) {
-          newEntries.push({
-            after: entry.after,
-            lastSignature: entry.lastSignature,
-            transactions: entry.transactions,
-          });
-          bytesSinceLastSignature = 0;
-        } else {
-          const lastNewEntry = newEntries[newEntries.length - 1]!;
-          lastNewEntry.transactions.push(...entry.transactions);
-          lastNewEntry.lastSignature = entry.lastSignature;
-
-          bytesSinceLastSignature += entry.transactions.length;
-        }
-      }
-      newSessions[sessionID] = newEntries;
-    } else {
-      return "nonContigous" as const;
-    }
-  }
-
-  return { header, sessionEntries: newSessions };
-}

packages/cojson/src/storage/index.ts

@@ -1,536 +0,0 @@
-import { CoID, RawCoValue } from "../coValue.js";
-import { CoValueHeader, Transaction } from "../coValueCore.js";
-import { Signature } from "../crypto/crypto.js";
-import { RawCoID } from "../ids.js";
-import { logger } from "../logger.js";
-import { connectedPeers } from "../streamUtils.js";
-import {
-  CoValueKnownState,
-  IncomingSyncStream,
-  NewContentMessage,
-  OutgoingSyncQueue,
-  Peer,
-} from "../sync.js";
-import {
-  BlockFilename,
-  FileSystem,
-  WalEntry,
-  WalFilename,
-  readChunk,
-  readHeader,
-  textDecoder,
-  writeBlock,
-  writeToWal,
-} from "./FileSystem.js";
-import {
-  chunkToKnownState,
-  contentSinceChunk,
-  mergeChunks,
-} from "./chunksAndKnownStates.js";
-export type { BlockFilename, WalFilename } from "./FileSystem.js";
-
-const MAX_N_LEVELS = 3;
-
-export type CoValueChunk = {
-  header?: CoValueHeader;
-  sessionEntries: {
-    [sessionID: string]: {
-      after: number;
-      lastSignature: Signature;
-      transactions: Transaction[];
-    }[];
-  };
-};
-
-export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
-  currentWal: WH | undefined;
-  coValues: {
-    [id: RawCoID]: CoValueChunk | undefined;
-  };
-  fileCache: string[] | undefined;
-  headerCache = new Map<
-    BlockFilename,
-    { [id: RawCoID]: { start: number; length: number } }
-  >();
-  blockFileHandles = new Map<
-    BlockFilename,
-    Promise<{ handle: RH; size: number }>
-  >();
-
-  constructor(
-    public fs: FS,
-    public fromLocalNode: IncomingSyncStream,
-    public toLocalNode: OutgoingSyncQueue,
-  ) {
-    this.coValues = {};
-    this.currentWal = undefined;
-
-    let nMsg = 0;
-
-    const processMessages = async () => {
-      for await (const msg of fromLocalNode) {
-        try {
-          if (msg === "Disconnected" || msg === "PingTimeout") {
-            throw new Error("Unexpected Disconnected message");
-          }
-          if (msg.action === "done") {
-            return;
-          }
-
-          if (msg.action === "content") {
-            await this.handleNewContent(msg);
-          } else if (msg.action === "load" || msg.action === "known") {
-            await this.sendNewContent(msg.id, msg, undefined);
-          }
-        } catch (e) {
-          logger.error(
-            `Error reading from localNode, handling msg\n\n${JSON.stringify(
-              msg,
-              (k, v) =>
-                k === "changes" || k === "encryptedChanges"
-                  ? v.slice(0, 20) + "..."
-                  : v,
-            )}
-            Error: ${e instanceof Error ? e.message : "Unknown error"},
-            `,
-          );
-        }
-        nMsg++;
-      }
-    };
-
-    processMessages().catch((e) =>
-      logger.error("Error in processMessages in storage", e),
-    );
-
-    setTimeout(
-      () =>
-        this.compact().catch((e) => {
-          logger.error("Error while compacting", e);
-        }),
-      20000,
-    );
-  }
-
-  async sendNewContent(
-    id: RawCoID,
-    known: CoValueKnownState | undefined,
-    asDependencyOf: RawCoID | undefined,
-  ) {
-    let coValue = this.coValues[id];
-
-    if (!coValue) {
-      coValue = await this.loadCoValue(id, this.fs);
-    }
-
-    if (!coValue) {
-      this.toLocalNode
-        .push({
-          id: id,
-          action: "known",
-          header: false,
-          sessions: {},
-          asDependencyOf,
-        })
-        .catch((e) => logger.error("Error while pushing known", e));
-
-      return;
-    }
-
-    if (!known?.header && coValue.header?.ruleset.type === "ownedByGroup") {
-      await this.sendNewContent(
-        coValue.header.ruleset.group,
-        undefined,
-        asDependencyOf || id,
-      );
-    } else if (!known?.header && coValue.header?.ruleset.type === "group") {
-      const dependedOnAccountsAndGroups = new Set();
-      for (const session of Object.values(coValue.sessionEntries)) {
-        for (const entry of session) {
-          for (const tx of entry.transactions) {
-            if (tx.privacy === "trusting") {
-              const parsedChanges = JSON.parse(tx.changes);
-              for (const change of parsedChanges) {
-                if (change.op === "set" && change.key.startsWith("co_")) {
-                  dependedOnAccountsAndGroups.add(change.key);
-                }
-                if (
-                  change.op === "set" &&
-                  change.key.startsWith("parent_co_")
-                ) {
-                  dependedOnAccountsAndGroups.add(
-                    change.key.replace("parent_", ""),
-                  );
-                }
-              }
-            }
-          }
-        }
-      }
-      for (const accountOrGroup of dependedOnAccountsAndGroups) {
-        await this.sendNewContent(
-          accountOrGroup as CoID<RawCoValue>,
-          undefined,
-          asDependencyOf || id,
-        );
-      }
-    }
-
-    const newContentMessages = contentSinceChunk(id, coValue, known).map(
-      (message) => ({ ...message, asDependencyOf }),
-    );
-
-    const ourKnown: CoValueKnownState = chunkToKnownState(id, coValue);
-
-    this.toLocalNode
-      .push({
-        action: "known",
-        ...ourKnown,
-        asDependencyOf,
-      })
-      .catch((e) => logger.error("Error while pushing known", e));
-
-    for (const message of newContentMessages) {
-      if (Object.keys(message.new).length === 0) continue;
-      this.toLocalNode
-        .push(message)
-        .catch((e) => logger.error("Error while pushing new content", e));
-    }
-
-    this.coValues[id] = coValue;
-  }
-
-  async withWAL(handler: (wal: WH) => Promise<void>) {
-    if (!this.currentWal) {
-      this.currentWal = await this.fs.createFile(
-        `wal-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`,
-      );
-    }
-    await handler(this.currentWal);
-  }
-
-  async handleNewContent(newContent: NewContentMessage) {
-    const coValue = this.coValues[newContent.id];
-
-    const newContentAsChunk: CoValueChunk = {
-      header: newContent.header,
-      sessionEntries: Object.fromEntries(
-        Object.entries(newContent.new).map(([sessionID, newInSession]) => [
-          sessionID,
-          [
-            {
-              after: newInSession.after,
-              lastSignature: newInSession.lastSignature,
-              transactions: newInSession.newTransactions,
-            },
-          ],
-        ]),
-      ),
-    };
-
-    if (!coValue) {
-      if (newContent.header) {
-        await this.withWAL((wal) =>
-          writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
-        );
-
-        this.coValues[newContent.id] = newContentAsChunk;
-      } else {
-        logger.warn("Incontiguous incoming update for " + newContent.id);
-        return;
-      }
-    } else {
-      const merged = mergeChunks(coValue, newContentAsChunk);
-      if (merged === "nonContigous") {
-        console.warn(
-          "Non-contigous new content for " + newContent.id,
-          Object.entries(coValue.sessionEntries).map(([session, entries]) =>
-            entries.map((entry) => ({
-              session: session,
-              after: entry.after,
-              length: entry.transactions.length,
-            })),
-          ),
-          Object.entries(newContentAsChunk.sessionEntries).map(
-            ([session, entries]) =>
-              entries.map((entry) => ({
-                session: session,
-                after: entry.after,
-                length: entry.transactions.length,
-              })),
-          ),
-        );
-      } else {
-        await this.withWAL((wal) =>
-          writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
-        );
-
-        this.coValues[newContent.id] = merged;
-      }
-    }
-  }
-
-  async getBlockHandle(
-    blockFile: BlockFilename,
-    fs: FS,
-  ): Promise<{ handle: RH; size: number }> {
-    if (!this.blockFileHandles.has(blockFile)) {
-      this.blockFileHandles.set(blockFile, fs.openToRead(blockFile));
-    }
-
-    return this.blockFileHandles.get(blockFile)!;
-  }
-
-  async loadCoValue(id: RawCoID, fs: FS): Promise<CoValueChunk | undefined> {
-    const files = this.fileCache || (await fs.listFiles());
-    this.fileCache = files;
-    const blockFiles = (
-      files.filter((name) => name.startsWith("L")) as BlockFilename[]
-    ).sort();
-
-    let result;
-
-    for (const blockFile of blockFiles) {
-      let cachedHeader:
-        | { [id: RawCoID]: { start: number; length: number } }
-        | undefined = this.headerCache.get(blockFile);
-
-      const { handle, size } = await this.getBlockHandle(blockFile, fs);
-
-      if (!cachedHeader) {
-        cachedHeader = {};
-        const header = await readHeader(blockFile, handle, size, fs);
-        for (const entry of header) {
-          cachedHeader[entry.id] = {
-            start: entry.start,
-            length: entry.length,
-          };
-        }
-
-        this.headerCache.set(blockFile, cachedHeader);
-      }
-      const headerEntry = cachedHeader[id];
-
-      if (headerEntry) {
-        const nextChunk = await readChunk(handle, headerEntry, fs);
-        if (result) {
-          const merged = mergeChunks(result, nextChunk);
-
-          if (merged === "nonContigous") {
-            console.warn(
-              "Non-contigous chunks while loading " + id,
-              result,
-              nextChunk,
-            );
-          } else {
-            result = merged;
-          }
-        } else {
-          result = nextChunk;
-        }
-      }
-
-      // await fs.close(handle);
-    }
-
-    return result;
-  }
-
-  async compact() {
-    const fileNames = await this.fs.listFiles();
-
-    const walFiles = fileNames.filter((name) =>
-      name.startsWith("wal-"),
-    ) as WalFilename[];
-    walFiles.sort();
-
-    const coValues = new Map<RawCoID, CoValueChunk>();
-
-    if (walFiles.length === 0) return;
-
-    const oldWal = this.currentWal;
-    this.currentWal = undefined;
-
-    if (oldWal) {
-      await this.fs.close(oldWal);
-    }
-
-    for (const fileName of walFiles) {
-      const { handle, size }: { handle: RH; size: number } =
-        await this.fs.openToRead(fileName);
-      if (size === 0) {
-        await this.fs.close(handle);
-        continue;
-      }
-      const bytes = await this.fs.read(handle, 0, size);
-
-      const decoded = textDecoder.decode(bytes);
-      const lines = decoded.split("\n");
-
-      for (const line of lines) {
-        if (line.length === 0) continue;
-        const chunk = JSON.parse(line) as WalEntry;
-
-        const existingChunk = coValues.get(chunk.id);
-
-        if (existingChunk) {
-          const merged = mergeChunks(existingChunk, chunk);
-          if (merged === "nonContigous") {
-            console.log(
-              "Non-contigous chunks in " + chunk.id + ", " + fileName,
-              existingChunk,
-              chunk,
-            );
-          } else {
-            coValues.set(chunk.id, merged);
-          }
-        } else {
-          coValues.set(chunk.id, chunk);
-        }
-      }
-
-      await this.fs.close(handle);
-    }
-
-    const highestBlockNumber = fileNames.reduce((acc, name) => {
-      if (name.startsWith("L" + MAX_N_LEVELS)) {
-        const num = parseInt(name.split("-")[1]!);
-        if (num > acc) {
-          return num;
-        }
-      }
-      return acc;
-    }, 0);
-
-    await writeBlock(coValues, MAX_N_LEVELS, highestBlockNumber + 1, this.fs);
-
-    for (const walFile of walFiles) {
-      await this.fs.removeFile(walFile);
-    }
-    this.fileCache = undefined;
-
-    const fileNames2 = await this.fs.listFiles();
-
-    const blockFiles = (
-      fileNames2.filter((name) => name.startsWith("L")) as BlockFilename[]
-    ).sort();
-
-    const blockFilesByLevelInOrder: {
-      [level: number]: BlockFilename[];
-    } = {};
-
-    for (const blockFile of blockFiles) {
-      const level = parseInt(blockFile.split("-")[0]!.slice(1));
-      if (!blockFilesByLevelInOrder[level]) {
-        blockFilesByLevelInOrder[level] = [];
-      }
-      blockFilesByLevelInOrder[level]!.push(blockFile);
-    }
-
-    for (let level = MAX_N_LEVELS; level > 0; level--) {
-      const nBlocksDesired = Math.pow(2, level);
-      const blocksInLevel = blockFilesByLevelInOrder[level];
-
-      if (blocksInLevel && blocksInLevel.length > nBlocksDesired) {
-        const coValues = new Map<RawCoID, CoValueChunk>();
-
-        for (const blockFile of blocksInLevel) {
-          const { handle, size }: { handle: RH; size: number } =
-            await this.getBlockHandle(blockFile, this.fs);
-
-          if (size === 0) {
-            continue;
-          }
-          const header = await readHeader(blockFile, handle, size, this.fs);
-          for (const entry of header) {
-            const chunk = await readChunk(handle, entry, this.fs);
-
-            const existingChunk = coValues.get(entry.id);
-
-            if (existingChunk) {
-              const merged = mergeChunks(existingChunk, chunk);
-              if (merged === "nonContigous") {
-                console.log(
-                  "Non-contigous chunks in " + entry.id + ", " + blockFile,
-                  existingChunk,
-                  chunk,
-                );
-              } else {
-                coValues.set(entry.id, merged);
-              }
-            } else {
-              coValues.set(entry.id, chunk);
-            }
-          }
-        }
-
-        let levelBelow = blockFilesByLevelInOrder[level - 1];
-        if (!levelBelow) {
-          levelBelow = [];
-          blockFilesByLevelInOrder[level - 1] = levelBelow;
-        }
-
-        const highestBlockNumberInLevelBelow = levelBelow.reduce(
-          (acc, name) => {
-            const num = parseInt(name.split("-")[1]!);
-            if (num > acc) {
-              return num;
-            }
-            return acc;
-          },
-          0,
-        );
-
-        const newBlockName = await writeBlock(
-          coValues,
-          level - 1,
-          highestBlockNumberInLevelBelow + 1,
-          this.fs,
-        );
-        levelBelow.push(newBlockName);
-
-        // delete blocks that went into this one
-        for (const blockFile of blocksInLevel) {
-          const handle = await this.getBlockHandle(blockFile, this.fs);
-          await this.fs.close(handle.handle);
-          await this.fs.removeFile(blockFile);
-          this.blockFileHandles.delete(blockFile);
-        }
-      }
-    }
-
-    setTimeout(
-      () =>
-        this.compact().catch((e) => {
-          logger.error("Error while compacting", e);
-        }),
-      5000,
-    );
-  }
-
-  static asPeer<WH, RH, FS extends FileSystem<WH, RH>>({
-    fs,
-    trace,
-    localNodeName = "local",
-  }: {
-    fs: FS;
-    trace?: boolean;
-    localNodeName?: string;
-  }): Peer {
-    const [localNodeAsPeer, storageAsPeer] = connectedPeers(
-      localNodeName,
-      "storage",
-      {
-        peer1role: "client",
-        peer2role: "storage",
-        trace,
-        crashOnClose: true,
-      },
-    );
-
-    new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
-
-    // return { ...storageAsPeer, priority: 200 };
-    return storageAsPeer;
-  }
-}

packages/cojson/src/sync.ts

@@ -1,12 +1,6 @@
-import { ValueType, metrics } from "@opentelemetry/api";
-import { PeerState } from "./PeerState.js";
-import { SyncStateManager } from "./SyncStateManager.js";
-import { CoValueHeader, Transaction } from "./coValueCore.js";
-import { CoValueCore } from "./coValueCore.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
-import { LocalNode } from "./localNode.js";
-import { logger } from "./logger.js";
+import { CoValueHeader, Transaction } from "./localNode/structure.js";
 import { CoValuePriority } from "./priority.js";
 
 export type CoValueKnownState = {
@@ -112,686 +106,6 @@ export function combinedKnownStates(
   };
 }
 
-export class SyncManager {
-  peers: { [key: PeerID]: PeerState } = {};
-  local: LocalNode;
-  requestedSyncs: {
-    [id: RawCoID]:
-      | { done: Promise<void>; nRequestsThisTick: number }
-      | undefined;
-  } = {};
-
-  peersCounter = metrics.getMeter("cojson").createUpDownCounter("jazz.peers", {
-    description: "Amount of connected peers",
-    valueType: ValueType.INT,
-    unit: "peer",
-  });
-
-  constructor(local: LocalNode) {
-    this.local = local;
-    this.syncState = new SyncStateManager(this);
-  }
-
-  syncState: SyncStateManager;
-
-  peersInPriorityOrder(): PeerState[] {
-    return Object.values(this.peers).sort((a, b) => {
-      const aPriority = a.priority || 0;
-      const bPriority = b.priority || 0;
-
-      return bPriority - aPriority;
-    });
-  }
-
-  getPeers(): PeerState[] {
-    return Object.values(this.peers);
-  }
-
-  getServerAndStoragePeers(excludePeerId?: PeerID): PeerState[] {
-    return this.peersInPriorityOrder().filter(
-      (peer) => peer.isServerOrStoragePeer() && peer.id !== excludePeerId,
-    );
-  }
-
-  async handleSyncMessage(msg: SyncMessage, peer: PeerState) {
-    if (peer.erroredCoValues.has(msg.id)) {
-      logger.warn(
-        `Skipping message ${msg.action} on errored coValue ${msg.id} from peer ${peer.id}`,
-      );
-      return;
-    }
-    // TODO: validate
-    switch (msg.action) {
-      case "load":
-        return await this.handleLoad(msg, peer);
-      case "known":
-        if (msg.isCorrection) {
-          return await this.handleCorrection(msg, peer);
-        } else {
-          return await this.handleKnownState(msg, peer);
-        }
-      case "content":
-        // await new Promise<void>((resolve) => setTimeout(resolve, 0));
-        return await this.handleNewContent(msg, peer);
-      case "done":
-        return await this.handleUnsubscribe(msg);
-      default:
-        throw new Error(
-          `Unknown message type ${(msg as { action: "string" }).action}`,
-        );
-    }
-  }
-
-  async subscribeToIncludingDependencies(id: RawCoID, peer: PeerState) {
-    const entry = this.local.coValuesStore.get(id);
-
-    if (entry.state.type !== "available") {
-      entry.loadFromPeers([peer]).catch((e: unknown) => {
-        logger.error("Error sending load: " + getErrorMessage(e));
-      });
-      return;
-    }
-
-    const coValue = entry.state.coValue;
-
-    for (const id of coValue.getDependedOnCoValues()) {
-      await this.subscribeToIncludingDependencies(id, peer);
-    }
-
-    if (!peer.toldKnownState.has(id)) {
-      peer.toldKnownState.add(id);
-      this.trySendToPeer(peer, {
-        action: "load",
-        ...coValue.knownState(),
-      }).catch((e: unknown) => {
-        logger.error("Error sending load: " + getErrorMessage(e));
-      });
-    }
-  }
-
-  async tellUntoldKnownStateIncludingDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    asDependencyOf?: RawCoID,
-  ) {
-    const coValue = this.local.expectCoValueLoaded(id);
-
-    await Promise.all(
-      coValue
-        .getDependedOnCoValues()
-        .map((dependentCoID) =>
-          this.tellUntoldKnownStateIncludingDependencies(
-            dependentCoID,
-            peer,
-            asDependencyOf || id,
-          ),
-        ),
-    );
-
-    if (!peer.toldKnownState.has(id)) {
-      this.trySendToPeer(peer, {
-        action: "known",
-        asDependencyOf,
-        ...coValue.knownState(),
-      }).catch((e: unknown) => {
-        logger.error("Error sending known state: " + getErrorMessage(e));
-      });
-
-      peer.toldKnownState.add(id);
-    }
-  }
-
-  async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
-    const coValue = this.local.expectCoValueLoaded(id);
-
-    await Promise.all(
-      coValue
-        .getDependedOnCoValues()
-        .map((id) => this.sendNewContentIncludingDependencies(id, peer)),
-    );
-
-    const newContentPieces = coValue.newContentSince(
-      peer.optimisticKnownStates.get(id),
-    );
-
-    if (newContentPieces) {
-      const optimisticKnownStateBefore =
-        peer.optimisticKnownStates.get(id) || emptyKnownState(id);
-
-      const sendPieces = async () => {
-        let lastYield = performance.now();
-        for (const [_i, piece] of newContentPieces.entries()) {
-          this.trySendToPeer(peer, piece).catch((e: unknown) => {
-            logger.error("Error sending content piece: " + getErrorMessage(e));
-          });
-
-          if (performance.now() - lastYield > 10) {
-            await new Promise<void>((resolve) => {
-              setTimeout(resolve, 0);
-            });
-            lastYield = performance.now();
-          }
-        }
-      };
-
-      sendPieces().catch((e) => {
-        logger.error("Error sending new content piece, retrying", e);
-        peer.optimisticKnownStates.dispatch({
-          type: "SET",
-          id,
-          value: optimisticKnownStateBefore ?? emptyKnownState(id),
-        });
-        return this.sendNewContentIncludingDependencies(id, peer);
-      });
-
-      peer.optimisticKnownStates.dispatch({
-        type: "COMBINE_WITH",
-        id,
-        value: coValue.knownState(),
-      });
-    }
-  }
-
-  addPeer(peer: Peer) {
-    const prevPeer = this.peers[peer.id];
-    const peerState = new PeerState(peer, prevPeer?.knownStates);
-    this.peers[peer.id] = peerState;
-
-    if (prevPeer && !prevPeer.closed) {
-      prevPeer.gracefulShutdown();
-    }
-
-    this.peersCounter.add(1, { role: peer.role });
-
-    const unsubscribeFromKnownStatesUpdates = peerState.knownStates.subscribe(
-      (id) => {
-        this.syncState.triggerUpdate(peer.id, id);
-      },
-    );
-
-    if (peerState.isServerOrStoragePeer()) {
-      const initialSync = async () => {
-        for (const entry of this.local.coValuesStore.getValues()) {
-          await this.subscribeToIncludingDependencies(entry.id, peerState);
-
-          if (entry.state.type === "available") {
-            await this.sendNewContentIncludingDependencies(entry.id, peerState);
-          }
-
-          if (!peerState.optimisticKnownStates.has(entry.id)) {
-            peerState.optimisticKnownStates.dispatch({
-              type: "SET_AS_EMPTY",
-              id: entry.id,
-            });
-          }
-        }
-      };
-      void initialSync();
-    }
-
-    const processMessages = async () => {
-      for await (const msg of peerState.incoming) {
-        if (msg === "Disconnected") {
-          return;
-        }
-        if (msg === "PingTimeout") {
-          logger.error("Ping timeout from peer", {
-            peerId: peer.id,
-            peerRole: peer.role,
-          });
-          return;
-        }
-        try {
-          await this.handleSyncMessage(msg, peerState);
-        } catch (e) {
-          throw new Error(
-            `Error reading from peer ${
-              peer.id
-            }, handling msg\n\n${JSON.stringify(msg, (k, v) =>
-              k === "changes" || k === "encryptedChanges"
-                ? v.slice(0, 20) + "..."
-                : v,
-            )}`,
-            { cause: e },
-          );
-        }
-      }
-    };
-
-    processMessages()
-      .then(() => {
-        if (peer.crashOnClose) {
-          logger.error("Unexepcted close from peer", {
-            peerId: peer.id,
-            peerRole: peer.role,
-          });
-          this.local.crashed = new Error("Unexpected close from peer");
-          throw new Error("Unexpected close from peer");
-        }
-      })
-      .catch((e) => {
-        logger.error(
-          "Error processing messages from peer: " + getErrorMessage(e),
-          {
-            peerId: peer.id,
-            peerRole: peer.role,
-          },
-        );
-        if (peer.crashOnClose) {
-          this.local.crashed = e;
-          throw new Error(e);
-        }
-      })
-      .finally(() => {
-        const state = this.peers[peer.id];
-        state?.gracefulShutdown();
-        unsubscribeFromKnownStatesUpdates();
-        this.peersCounter.add(-1, { role: peer.role });
-
-        if (peer.deletePeerStateOnClose) {
-          delete this.peers[peer.id];
-        }
-      });
-  }
-
-  trySendToPeer(peer: PeerState, msg: SyncMessage) {
-    return peer.pushOutgoingMessage(msg);
-  }
-
-  async handleLoad(msg: LoadMessage, peer: PeerState) {
-    peer.dispatchToKnownStates({
-      type: "SET",
-      id: msg.id,
-      value: knownStateIn(msg),
-    });
-    const entry = this.local.coValuesStore.get(msg.id);
-
-    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
-      const eligiblePeers = this.getServerAndStoragePeers(peer.id);
-
-      if (eligiblePeers.length === 0) {
-        // If the load request contains a header or any session data
-        // and we don't have any eligible peers to load the coValue from
-        // we try to load it from the sender because it is the only place
-        // where we can get informations about the coValue
-        if (msg.header || Object.keys(msg.sessions).length > 0) {
-          entry.loadFromPeers([peer]).catch((e) => {
-            logger.error("Error loading coValue in handleLoad", e);
-          });
-        }
-        return;
-      } else {
-        this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
-          logger.error("Error loading coValue in handleLoad", e);
-        });
-      }
-    }
-
-    if (entry.state.type === "loading") {
-      // We need to return from handleLoad immediately and wait for the CoValue to be loaded
-      // in a new task, otherwise we might block further incoming content messages that would
-      // resolve the CoValue as available. This can happen when we receive fresh
-      // content from a client, but we are a server with our own upstream server(s)
-      entry
-        .getCoValue()
-        .then(async (value) => {
-          if (value === "unavailable") {
-            peer.dispatchToKnownStates({
-              type: "SET",
-              id: msg.id,
-              value: knownStateIn(msg),
-            });
-            peer.toldKnownState.add(msg.id);
-
-            this.trySendToPeer(peer, {
-              action: "known",
-              id: msg.id,
-              header: false,
-              sessions: {},
-            }).catch((e) => {
-              logger.error("Error sending known state back", e);
-            });
-
-            return;
-          }
-
-          await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-          await this.sendNewContentIncludingDependencies(msg.id, peer);
-        })
-        .catch((e) => {
-          logger.error("Error loading coValue in handleLoad loading state", e);
-        });
-    }
-
-    if (entry.state.type === "available") {
-      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-      await this.sendNewContentIncludingDependencies(msg.id, peer);
-    }
-  }
-
-  async handleKnownState(msg: KnownStateMessage, peer: PeerState) {
-    const entry = this.local.coValuesStore.get(msg.id);
-
-    peer.dispatchToKnownStates({
-      type: "COMBINE_WITH",
-      id: msg.id,
-      value: knownStateIn(msg),
-    });
-
-    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
-      if (msg.asDependencyOf) {
-        const dependencyEntry = this.local.coValuesStore.get(
-          msg.asDependencyOf,
-        );
-
-        if (
-          dependencyEntry.state.type === "available" ||
-          dependencyEntry.state.type === "loading"
-        ) {
-          this.local
-            .loadCoValueCore(
-              msg.id,
-              peer.role === "storage" ? undefined : peer.id,
-            )
-            .catch((e) => {
-              logger.error(
-                `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
-                e,
-              );
-            });
-        }
-      }
-    }
-
-    // The header is a boolean value that tells us if the other peer do have information about the header.
-    // If it's false in this point it means that the coValue is unavailable on the other peer.
-    if (entry.state.type !== "available") {
-      const availableOnPeer = peer.optimisticKnownStates.get(msg.id)?.header;
-
-      if (!availableOnPeer) {
-        entry.dispatch({
-          type: "not-found-in-peer",
-          peerId: peer.id,
-        });
-      }
-
-      return;
-    }
-
-    if (entry.state.type === "available") {
-      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-      await this.sendNewContentIncludingDependencies(msg.id, peer);
-    }
-  }
-
-  async handleNewContent(msg: NewContentMessage, peer: PeerState) {
-    const entry = this.local.coValuesStore.get(msg.id);
-
-    let coValue: CoValueCore;
-
-    if (entry.state.type !== "available") {
-      if (!msg.header) {
-        logger.error("Expected header to be sent in first message", {
-          coValueId: msg.id,
-          peerId: peer.id,
-          peerRole: peer.role,
-        });
-        return;
-      }
-
-      peer.dispatchToKnownStates({
-        type: "UPDATE_HEADER",
-        id: msg.id,
-        header: true,
-      });
-
-      coValue = new CoValueCore(msg.header, this.local);
-
-      entry.dispatch({
-        type: "available",
-        coValue,
-      });
-    } else {
-      coValue = entry.state.coValue;
-    }
-
-    let invalidStateAssumed = false;
-
-    for (const [sessionID, newContentForSession] of Object.entries(msg.new) as [
-      SessionID,
-      SessionNewContent,
-    ][]) {
-      const ourKnownTxIdx =
-        coValue.sessionLogs.get(sessionID)?.transactions.length;
-      const theirFirstNewTxIdx = newContentForSession.after;
-
-      if ((ourKnownTxIdx || 0) < theirFirstNewTxIdx) {
-        invalidStateAssumed = true;
-        continue;
-      }
-
-      const alreadyKnownOffset = ourKnownTxIdx
-        ? ourKnownTxIdx - theirFirstNewTxIdx
-        : 0;
-
-      const newTransactions =
-        newContentForSession.newTransactions.slice(alreadyKnownOffset);
-
-      if (newTransactions.length === 0) {
-        continue;
-      }
-
-      const before = performance.now();
-      // eslint-disable-next-line neverthrow/must-use-result
-      const result = coValue.tryAddTransactions(
-        sessionID,
-        newTransactions,
-        undefined,
-        newContentForSession.lastSignature,
-      );
-      const after = performance.now();
-      if (after - before > 80) {
-        const totalTxLength = newTransactions
-          .map((t) =>
-            t.privacy === "private"
-              ? t.encryptedChanges.length
-              : t.changes.length,
-          )
-          .reduce((a, b) => a + b, 0);
-        logger.debug(
-          `Adding incoming transactions took ${(after - before).toFixed(
-            2,
-          )}ms for ${totalTxLength} bytes = bandwidth: ${(
-            (1000 * totalTxLength) / (after - before) / (1024 * 1024)
-          ).toFixed(2)} MB/s`,
-        );
-      }
-
-      // const theirTotalnTxs = Object.values(
-      //     peer.optimisticKnownStates[msg.id]?.sessions || {},
-      // ).reduce((sum, nTxs) => sum + nTxs, 0);
-      // const ourTotalnTxs = [...coValue.sessionLogs.values()].reduce(
-      //     (sum, session) => sum + session.transactions.length,
-      //     0,
-      // );
-
-      if (result.isErr()) {
-        logger.error("Failed to add transactions: " + result.error.type, {
-          peerId: peer.id,
-          peerRole: peer.role,
-          id: msg.id,
-        });
-        peer.erroredCoValues.set(msg.id, result.error);
-        continue;
-      }
-
-      peer.dispatchToKnownStates({
-        type: "UPDATE_SESSION_COUNTER",
-        id: msg.id,
-        sessionId: sessionID,
-        value:
-          newContentForSession.after +
-          newContentForSession.newTransactions.length,
-      });
-    }
-
-    if (invalidStateAssumed) {
-      this.trySendToPeer(peer, {
-        action: "known",
-        isCorrection: true,
-        ...coValue.knownState(),
-      }).catch((e) => {
-        logger.error(
-          "Error sending known state correction: " + getErrorMessage(e),
-          {
-            peerId: peer.id,
-            peerRole: peer.role,
-          },
-        );
-      });
-    } else {
-      /**
-       * We are sending a known state message to the peer to acknowledge the
-       * receipt of the new content.
-       *
-       * This way the sender knows that the content has been received and applied
-       * and can update their peer's knownState accordingly.
-       */
-      this.trySendToPeer(peer, {
-        action: "known",
-        ...coValue.knownState(),
-      }).catch((e: unknown) => {
-        logger.error("Error sending known state: " + getErrorMessage(e), {
-          peerId: peer.id,
-          peerRole: peer.role,
-        });
-      });
-    }
-
-    /**
-     * We do send a correction/ack message before syncing to give an immediate
-     * response to the peers that are waiting for confirmation that a coValue is
-     * fully synced
-     */
-    await this.syncCoValue(coValue);
-  }
-
-  async handleCorrection(msg: KnownStateMessage, peer: PeerState) {
-    peer.dispatchToKnownStates({
-      type: "SET",
-      id: msg.id,
-      value: knownStateIn(msg),
-    });
-
-    return this.sendNewContentIncludingDependencies(msg.id, peer);
-  }
-
-  handleUnsubscribe(_msg: DoneMessage) {
-    throw new Error("Method not implemented.");
-  }
-
-  async syncCoValue(coValue: CoValueCore) {
-    if (this.requestedSyncs[coValue.id]) {
-      this.requestedSyncs[coValue.id]!.nRequestsThisTick++;
-      return this.requestedSyncs[coValue.id]!.done;
-    } else {
-      const done = new Promise<void>((resolve) => {
-        queueMicrotask(async () => {
-          delete this.requestedSyncs[coValue.id];
-          await this.actuallySyncCoValue(coValue);
-          resolve();
-        });
-      });
-      const entry = {
-        done,
-        nRequestsThisTick: 1,
-      };
-      this.requestedSyncs[coValue.id] = entry;
-      return done;
-    }
-  }
-
-  async actuallySyncCoValue(coValue: CoValueCore) {
-    // let blockingSince = performance.now();
-    for (const peer of this.peersInPriorityOrder()) {
-      if (peer.closed) continue;
-      if (peer.erroredCoValues.has(coValue.id)) continue;
-      // if (performance.now() - blockingSince > 5) {
-      //     await new Promise<void>((resolve) => {
-      //         setTimeout(resolve, 0);
-      //     });
-      //     blockingSince = performance.now();
-      // }
-      if (peer.optimisticKnownStates.has(coValue.id)) {
-        await this.tellUntoldKnownStateIncludingDependencies(coValue.id, peer);
-        await this.sendNewContentIncludingDependencies(coValue.id, peer);
-      } else if (peer.isServerOrStoragePeer()) {
-        await this.subscribeToIncludingDependencies(coValue.id, peer);
-        await this.sendNewContentIncludingDependencies(coValue.id, peer);
-      }
-    }
-
-    for (const peer of this.getPeers()) {
-      this.syncState.triggerUpdate(peer.id, coValue.id);
-    }
-  }
-
-  async waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
-    const { syncState } = this;
-    const currentSyncState = syncState.getCurrentSyncState(peerId, id);
-
-    const isTheConditionAlreadyMet = currentSyncState.uploaded;
-
-    if (isTheConditionAlreadyMet) {
-      return true;
-    }
-
-    return new Promise((resolve, reject) => {
-      const unsubscribe = this.syncState.subscribeToPeerUpdates(
-        peerId,
-        (knownState, syncState) => {
-          if (syncState.uploaded && knownState.id === id) {
-            resolve(true);
-            unsubscribe?.();
-            clearTimeout(timeoutId);
-          }
-        },
-      );
-
-      const timeoutId = setTimeout(() => {
-        reject(new Error(`Timeout waiting for sync on ${peerId}/${id}`));
-        unsubscribe?.();
-      }, timeout);
-    });
-  }
-
-  async waitForSync(id: RawCoID, timeout = 30_000) {
-    const peers = this.getPeers();
-
-    return Promise.all(
-      peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
-    );
-  }
-
-  async waitForAllCoValuesSync(timeout = 60_000) {
-    const coValues = this.local.coValuesStore.getValues();
-    const validCoValues = Array.from(coValues).filter(
-      (coValue) =>
-        coValue.state.type === "available" || coValue.state.type === "loading",
-    );
-
-    return Promise.all(
-      validCoValues.map((coValue) => this.waitForSync(coValue.id, timeout)),
-    );
-  }
-
-  gracefulShutdown() {
-    for (const peer of Object.values(this.peers)) {
-      peer.gracefulShutdown();
-    }
-  }
-}
-
 function knownStateIn(msg: LoadMessage | KnownStateMessage) {
   return {
     id: msg.id,

packages/cojson/src/tests/MockCrypto.ts

@@ -0,0 +1,241 @@
+import { WasmCrypto } from "../crypto/WasmCrypto";
+import {
+  Encrypted,
+  Hash,
+  KeyID,
+  KeySecret,
+  Sealed,
+  SealerID,
+  SealerSecret,
+  ShortHash,
+  Signature,
+  SignerID,
+  SignerSecret,
+} from "../crypto/crypto";
+import {
+  AgentID,
+  AgentSecret,
+  CryptoProvider,
+  RawAccountID,
+  RawCoID,
+  SessionID,
+} from "../exports";
+import { TransactionID } from "../ids";
+import { Stringified } from "../jsonStringify";
+import { JsonValue } from "../jsonValue";
+
+export class MockCrypto implements CryptoProvider<Uint8Array> {
+  inner: WasmCrypto;
+
+  constructor(wasmCrypto: WasmCrypto) {
+    this.inner = wasmCrypto;
+  }
+
+  randomBytes(length: number): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  newEd25519SigningKey(): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  newRandomSigner(): SignerSecret {
+    throw new Error("Method not implemented.");
+  }
+  signerSecretToBytes(secret: SignerSecret): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  signerSecretFromBytes(bytes: Uint8Array): SignerSecret {
+    throw new Error("Method not implemented.");
+  }
+  getSignerID(secret: SignerSecret): SignerID {
+    throw new Error("Method not implemented.");
+  }
+  sign(secret: SignerSecret, message: JsonValue): Signature {
+    const signerID = this.getSignerID(secret);
+
+    return `signature_z[${signerID}/${this.shortHash(message)}]` as Signature;
+  }
+  verify(signature: Signature, message: JsonValue, id: SignerID): boolean {
+    const expected =
+      `signature_z[${id}/${this.shortHash(message)}]` as Signature;
+    if (signature !== expected) {
+      console.error(
+        `Signature ${signature} does not match expected ${expected}`,
+      );
+      return false;
+    }
+    return true;
+  }
+  newX25519StaticSecret(): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  newRandomSealer(): SealerSecret {
+    throw new Error("Method not implemented.");
+  }
+  sealerSecretToBytes(secret: SealerSecret): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  sealerSecretFromBytes(bytes: Uint8Array): SealerSecret {
+    throw new Error("Method not implemented.");
+  }
+  getSealerID(secret: SealerSecret): SealerID {
+    throw new Error("Method not implemented.");
+  }
+  newRandomAgentSecret(): AgentSecret {
+    throw new Error("Method not implemented.");
+  }
+  agentSecretToBytes(secret: AgentSecret): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  agentSecretFromBytes(bytes: Uint8Array): AgentSecret {
+    throw new Error("Method not implemented.");
+  }
+  getAgentID(secret: AgentSecret): AgentID {
+    throw new Error("Method not implemented.");
+  }
+  getAgentSignerID(agentId: AgentID): SignerID {
+    return this.inner.getAgentSignerID(agentId);
+  }
+  getAgentSignerSecret(agentSecret: AgentSecret): SignerSecret {
+    throw new Error("Method not implemented.");
+  }
+  getAgentSealerID(agentId: AgentID): SealerID {
+    throw new Error("Method not implemented.");
+  }
+  getAgentSealerSecret(agentSecret: AgentSecret): SealerSecret {
+    throw new Error("Method not implemented.");
+  }
+  emptyBlake3State() {
+    return this.inner.emptyBlake3State();
+  }
+  cloneBlake3State(state: Uint8Array) {
+    return this.inner.cloneBlake3State(state);
+  }
+  blake3HashOnce(data: Uint8Array): Uint8Array {
+    return this.inner.blake3HashOnce(data);
+  }
+  blake3HashOnceWithContext(
+    data: Uint8Array,
+    { context }: { context: Uint8Array },
+  ): Uint8Array {
+    return this.inner.blake3HashOnceWithContext(data, { context });
+  }
+  blake3IncrementalUpdate(state: any, data: Uint8Array) {
+    return this.inner.blake3IncrementalUpdate(state, data);
+  }
+  blake3DigestForState(state: any): Uint8Array {
+    return this.inner.blake3DigestForState(state);
+  }
+  secureHash(value: JsonValue): Hash {
+    return this.inner.secureHash(value);
+  }
+  shortHash(value: JsonValue): ShortHash {
+    return this.inner.shortHash(value);
+  }
+  encrypt<T extends JsonValue, N extends JsonValue>(
+    value: T,
+    keySecret: KeySecret,
+    nOnceMaterial: N,
+  ): Encrypted<T, N> {
+    throw new Error("Method not implemented.");
+  }
+  encryptForTransaction<T extends JsonValue>(
+    value: T,
+    keySecret: KeySecret,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID },
+  ): Encrypted<T, { in: RawCoID; tx: TransactionID }> {
+    throw new Error("Method not implemented.");
+  }
+  decryptRaw<T extends JsonValue, N extends JsonValue>(
+    encrypted: Encrypted<T, N>,
+    keySecret: KeySecret,
+    nOnceMaterial: N,
+  ): Stringified<T> {
+    throw new Error("Method not implemented.");
+  }
+  decrypt<T extends JsonValue, N extends JsonValue>(
+    encrypted: Encrypted<T, N>,
+    keySecret: KeySecret,
+    nOnceMaterial: N,
+  ): T | undefined {
+    throw new Error("Method not implemented.");
+  }
+  newRandomKeySecret(): { secret: KeySecret; id: KeyID } {
+    throw new Error("Method not implemented.");
+  }
+  decryptRawForTransaction<T extends JsonValue>(
+    encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
+    keySecret: KeySecret,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID },
+  ): Stringified<T> | undefined {
+    throw new Error("Method not implemented.");
+  }
+  decryptForTransaction<T extends JsonValue>(
+    encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
+    keySecret: KeySecret,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID },
+  ): T | undefined {
+    throw new Error("Method not implemented.");
+  }
+  encryptKeySecret(keys: {
+    toEncrypt: { id: KeyID; secret: KeySecret };
+    encrypting: { id: KeyID; secret: KeySecret };
+  }): {
+    encryptedID: KeyID;
+    encryptingID: KeyID;
+    encrypted: Encrypted<
+      KeySecret,
+      { encryptedID: KeyID; encryptingID: KeyID }
+    >;
+  } {
+    throw new Error("Method not implemented.");
+  }
+  decryptKeySecret(
+    encryptedInfo: {
+      encryptedID: KeyID;
+      encryptingID: KeyID;
+      encrypted: Encrypted<
+        KeySecret,
+        { encryptedID: KeyID; encryptingID: KeyID }
+      >;
+    },
+    sealingSecret: KeySecret,
+  ): KeySecret | undefined {
+    throw new Error("Method not implemented.");
+  }
+  seal<T extends JsonValue>({
+    message,
+    from,
+    to,
+    nOnceMaterial,
+  }: {
+    message: T;
+    from: SealerSecret;
+    to: SealerID;
+    nOnceMaterial: { in: RawCoID; tx: TransactionID };
+  }): Sealed<T> {
+    throw new Error("Method not implemented.");
+  }
+  unseal<T extends JsonValue>(
+    sealed: Sealed<T>,
+    sealer: SealerSecret,
+    from: SealerID,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID },
+  ): T | undefined {
+    throw new Error("Method not implemented.");
+  }
+  uniquenessForHeader(): `z${string}` {
+    throw new Error("Method not implemented.");
+  }
+  createdNowUnique(): { createdAt: `2${string}`; uniqueness: `z${string}` } {
+    throw new Error("Method not implemented.");
+  }
+  newRandomSecretSeed(): Uint8Array {
+    throw new Error("Method not implemented.");
+  }
+  agentSecretFromSecretSeed(secretSeed: Uint8Array): AgentSecret {
+    throw new Error("Method not implemented.");
+  }
+  newRandomSessionID(accountID: RawAccountID | AgentID): SessionID {
+    throw new Error("Method not implemented.");
+  }
+}

packages/cojson/src/tests/coValueState.oldtest.ts

@@ -3,6 +3,7 @@ import { PeerState } from "../PeerState";
 import { CoValueCore } from "../coValueCore";
 import { CO_VALUE_LOADING_CONFIG, CoValueState } from "../coValueState";
 import { RawCoID } from "../ids";
+import { StorageDriver } from "../storage";
 import { Peer } from "../sync";
 
 const initialMaxRetries = CO_VALUE_LOADING_CONFIG.MAX_RETRIES;
@@ -17,8 +18,11 @@ function mockMaxRetries(maxRetries: number) {
 
 beforeEach(() => {
   mockMaxRetries(5);
+  vi.clearAllMocks();
 });
 
+const mockStorageDriver = null;
+
 describe("CoValueState", () => {
   const mockCoValueId = "co_test123" as RawCoID;
 
@@ -29,14 +33,6 @@ describe("CoValueState", () => {
     expect(state.state.type).toBe("unknown");
   });
 
-  test("should create loading state", () => {
-    const peerIds = ["peer1", "peer2"];
-    const state = CoValueState.Loading(mockCoValueId, peerIds);
-
-    expect(state.id).toBe(mockCoValueId);
-    expect(state.state.type).toBe("loading");
-  });
-
   test("should create available state", async () => {
     const mockCoValue = createMockCoValueCore(mockCoValueId);
     const state = CoValueState.Available(mockCoValue);
@@ -47,22 +43,6 @@ describe("CoValueState", () => {
     await expect(state.getCoValue()).resolves.toEqual(mockCoValue);
   });
 
-  test("should handle found action", async () => {
-    const mockCoValue = createMockCoValueCore(mockCoValueId);
-    const state = CoValueState.Loading(mockCoValueId, ["peer1", "peer2"]);
-
-    const stateValuePromise = state.getCoValue();
-
-    state.dispatch({
-      type: "available",
-      coValue: mockCoValue,
-    });
-
-    const result = await state.getCoValue();
-    expect(result).toBe(mockCoValue);
-    await expect(stateValuePromise).resolves.toBe(mockCoValue);
-  });
-
   test("should ignore actions when not in loading state", () => {
     const state = CoValueState.Unknown(mockCoValueId);
 
@@ -104,7 +84,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1, peer2] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     // Should attempt CO_VALUE_LOADING_CONFIG.MAX_RETRIES retries
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
@@ -157,7 +137,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1, peer2] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     // Should attempt CO_VALUE_LOADING_CONFIG.MAX_RETRIES retries
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
@@ -206,7 +186,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1, peer2] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     // Should attempt CO_VALUE_LOADING_CONFIG.MAX_RETRIES retries
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
@@ -258,7 +238,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     // Should attempt CO_VALUE_LOADING_CONFIG.MAX_RETRIES retries
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES + 1; i++) {
@@ -292,7 +272,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     // Should attempt CO_VALUE_LOADING_CONFIG.MAX_RETRIES retries
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
@@ -345,7 +325,7 @@ describe("CoValueState", () => {
     const mockPeers = [peer1] as unknown as PeerState[];
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers(mockPeers);
+    const loadPromise = state.loadCoValue(mockStorageDriver, mockPeers);
 
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
       await vi.runAllTimersAsync();
@@ -390,7 +370,7 @@ describe("CoValueState", () => {
     );
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers([peer1, peer2]);
+    const loadPromise = state.loadCoValue(mockStorageDriver, [peer1, peer2]);
 
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
       await vi.runAllTimersAsync();
@@ -439,7 +419,7 @@ describe("CoValueState", () => {
     peer1.closed = true;
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers([peer1, peer2]);
+    const loadPromise = state.loadCoValue(mockStorageDriver, [peer1, peer2]);
 
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES; i++) {
       await vi.runAllTimersAsync();
@@ -467,7 +447,7 @@ describe("CoValueState", () => {
     );
 
     const state = CoValueState.Unknown(mockCoValueId);
-    const loadPromise = state.loadFromPeers([peer1]);
+    const loadPromise = state.loadCoValue(mockStorageDriver, [peer1]);
 
     for (let i = 0; i < CO_VALUE_LOADING_CONFIG.MAX_RETRIES * 2; i++) {
       await vi.runAllTimersAsync();

packages/cojson/src/tests/localNode/dependencies.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, test } from "vitest";
+import { SessionID } from "../../exports.js";
+import { subscribe } from "../../localNode/actions/subscribing.js";
+import { stageLoadDeps } from "../../localNode/stages/1_loadDeps.js";
+import { emptyNode } from "../../localNode/structure.js";
+import {
+  addMemberTestTransaction,
+  addParentGroupTestTransaction,
+  createTestCoMap,
+  createTestGroup,
+} from "./testUtils.js";
+
+describe("Loading dependencies", () => {
+  test("stageLoadDeps does nothing for CoValues without listeners or dependents", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const coValue = createTestCoMap(group.id, "coMap1");
+
+    node.coValues = {
+      [coValue.id]: coValue,
+      [group.id]: group,
+    };
+    node.peers = [];
+
+    const coValuesBefore = structuredClone(node.coValues);
+
+    stageLoadDeps(node);
+    expect(node.coValues).toEqual(coValuesBefore);
+  });
+
+  test("stageLoadDeps adds dependent covalues to an existing coValue's dependencies if the dependent has listeners (ownedByGroup)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const coValue = createTestCoMap(group.id, "coMap1");
+
+    node.coValues = {
+      [coValue.id]: coValue,
+      [group.id]: group,
+    };
+
+    node.peers = [];
+
+    const _ = subscribe(node, coValue.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[group.id].dependents).toEqual([coValue.id]);
+
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[group.id].dependents).toEqual([coValue.id]);
+  });
+
+  test("stageLoadDeps adds dependents and adds a new entry on missing dependency if the dependent has listeners (ownedByGroup)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const coValue = createTestCoMap(group.id, "coMap1");
+
+    node.coValues = {
+      [coValue.id]: coValue,
+    };
+    node.peers = [];
+
+    const _ = subscribe(node, coValue.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[group.id].dependents).toEqual([coValue.id]);
+
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[group.id].dependents).toEqual([coValue.id]);
+  });
+
+  test("stageLoadDeps adds dependent covalues to an existing coValue's dependencies if the dependent has listeners (group member)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const member = createTestCoMap(null, "member");
+
+    addMemberTestTransaction(group, member.id, "session1" as SessionID);
+
+    node.coValues = {
+      [group.id]: group,
+      [member.id]: member,
+    };
+    node.peers = [];
+
+    const _ = subscribe(node, group.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[member.id].dependents).toEqual([group.id]);
+
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[member.id].dependents).toEqual([group.id]);
+  });
+
+  test("stageLoadDeps adds dependents and adds a new entry on missing dependency if the dependent has listeners (group member)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const member = createTestCoMap(null, "member");
+
+    addMemberTestTransaction(group, member.id, "session1" as SessionID);
+
+    node.coValues = {
+      [group.id]: group,
+    };
+    node.peers = [];
+
+    const _ = subscribe(node, group.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[member.id].dependents).toEqual([group.id]);
+
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[member.id].dependents).toEqual([group.id]);
+  });
+
+  test("stageLoadDeps adds dependent covalues to an existing coValue's dependencies if the dependent has listeners (extended group)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const parent = createTestGroup("sealer_z1/signer_z1_session1", "parent");
+
+    addParentGroupTestTransaction(group, parent.id, "session1" as SessionID);
+
+    node.coValues = {
+      [group.id]: group,
+      [parent.id]: parent,
+    };
+    node.peers = [];
+
+    const _ = subscribe(node, group.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[parent.id].dependents).toEqual([group.id]);
+
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[parent.id].dependents).toEqual([group.id]);
+  });
+
+  test("stageLoadDeps adds dependents and adds a new entry on missing dependency if the dependent has listeners (extended group)", () => {
+    const node = emptyNode();
+
+    const group = createTestGroup("sealer_z1/signer_z1_session1", "group1");
+    const parent = createTestGroup("sealer_z1/signer_z1_session1", "parent");
+
+    addParentGroupTestTransaction(group, parent.id, "session1" as SessionID);
+
+    node.coValues = {
+      [group.id]: group,
+    };
+    node.peers = [];
+
+    const _ = subscribe(node, group.id);
+
+    stageLoadDeps(node);
+
+    expect(node.coValues[parent.id].dependents).toEqual([group.id]);
+    stageLoadDeps(node);
+
+    // idempotency
+    expect(node.coValues[parent.id].dependents).toEqual([group.id]);
+  });
+});

packages/cojson/src/tests/localNode/peers.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "vitest";
+import { addPeer, removePeer } from "../../localNode/actions/peers.js";
+import { subscribe } from "../../localNode/actions/subscribing.js";
+
+import { emptyNode } from "../../localNode/structure.js";
+import { PeerID } from "../../sync.js";
+import { coValueID1, coValueID2 } from "./setup.js";
+
+describe("Modifying peers", () => {
+  const node = emptyNode();
+  const _1 = subscribe(node, coValueID1);
+  const _2 = subscribe(node, coValueID2);
+
+  test("Adding a peer adds it to the node and to every CoValue with an unknown peer state", () => {
+    const peerID = "peer1" as PeerID;
+    addPeer(node, peerID);
+    expect(node.peers).toEqual([peerID]);
+    expect(node.coValues[coValueID1].peerState[peerID]).toEqual({
+      confirmed: "unknown",
+      optimistic: "unknown",
+    });
+    expect(node.coValues[coValueID2].peerState[peerID]).toEqual({
+      confirmed: "unknown",
+      optimistic: "unknown",
+    });
+  });
+
+  test("Removing a peer removes it from the node and from every CoValue", () => {
+    const peerID = "peer1" as PeerID;
+    removePeer(node, peerID);
+    expect(node.peers).toEqual([]);
+    expect(node.coValues[coValueID1].peerState[peerID]).toBeUndefined();
+    expect(node.coValues[coValueID2].peerState[peerID]).toBeUndefined();
+  });
+});

packages/cojson/src/tests/localNode/setup.ts

@@ -0,0 +1,377 @@
+import { WasmCrypto } from "../../crypto/WasmCrypto.js";
+import { Signature, StreamingHash } from "../../crypto/crypto.js";
+import { JsonValue, RawCoID, SessionID, Stringified } from "../../exports.js";
+
+import { LocalNodeState, Transaction } from "../../localNode/structure.js";
+
+export const crypto = await WasmCrypto.create();
+
+export const coValueID1 = "co_zCoValueID1" as RawCoID;
+export const coValueID2 = "co_zCoValueID2" as RawCoID;
+
+export const sessionID1 = "sealer_z1/signer_z1_session1" as SessionID;
+
+export const tx1 = {
+  privacy: "trusting",
+  changes: '["ch1"]' as Stringified<string[]>,
+  madeAt: 1,
+} satisfies Transaction;
+
+export const tx2 = {
+  privacy: "trusting",
+  changes: '["ch2"]' as Stringified<string[]>,
+  madeAt: 2,
+} satisfies Transaction;
+
+export const tx3 = {
+  privacy: "trusting",
+  changes: '["ch3"]' as Stringified<string[]>,
+  madeAt: 3,
+} satisfies Transaction;
+
+export const tx4 = {
+  privacy: "trusting",
+  changes: '["ch4"]' as Stringified<string[]>,
+  madeAt: 4,
+} satisfies Transaction;
+
+export const tx5 = {
+  privacy: "trusting",
+  changes: '["ch5"]' as Stringified<string[]>,
+  madeAt: 5,
+} satisfies Transaction;
+
+export const streamingHash = new StreamingHash(crypto);
+streamingHash.update(tx1);
+streamingHash.update(tx2);
+export const signatureAfter2 =
+  `signature_z[signer_z1/${crypto.shortHash(streamingHash.digest())}]` as Signature;
+
+streamingHash.update(tx3);
+streamingHash.update(tx4);
+streamingHash.update(tx5);
+export const signatureAfter5 =
+  `signature_z[signer_z1/${crypto.shortHash(streamingHash.digest())}]` as Signature;
+
+export const scenarios = {
+  coValuesWithAvailableInStorageTxs: {
+    [coValueID1]: {
+      id: coValueID1,
+      header: {
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        uniqueness: 1,
+      },
+      sessions: {
+        [sessionID1]: {
+          transactions: [
+            { state: "availableInStorage" as const },
+            { state: "availableInStorage" as const },
+          ],
+          id: sessionID1,
+          lastVerified: -1,
+          lastAvailable: -1,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: {
+        header: true,
+        sessions: {
+          [sessionID1]: 2,
+        },
+      },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+    [coValueID2]: {
+      id: coValueID2,
+      header: {
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        uniqueness: 2,
+      },
+      sessions: {
+        [sessionID1]: {
+          transactions: [
+            { state: "availableInStorage" as const },
+            { state: "availableInStorage" as const },
+          ],
+          id: sessionID1,
+          lastVerified: -1,
+          lastAvailable: -1,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: {
+        header: true,
+        sessions: {
+          [sessionID1]: 2,
+        },
+      },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+  } satisfies LocalNodeState["coValues"],
+  coValue2IsGroupOfCoValue1: {
+    [coValueID1]: {
+      id: coValueID1,
+      header: {
+        type: "comap",
+        ruleset: { type: "ownedByGroup", group: coValueID2 },
+        meta: null,
+        uniqueness: 0,
+      },
+      sessions: {
+        [sessionID1]: {
+          id: sessionID1,
+          transactions: [
+            { state: "available" as const, tx: tx1, signature: null },
+            {
+              state: "available" as const,
+              tx: tx2,
+              signature: signatureAfter2,
+            },
+            { state: "available" as const, tx: tx3, signature: null },
+            { state: "available" as const, tx: tx4, signature: null },
+            {
+              state: "available" as const,
+              tx: tx5,
+              signature: signatureAfter5,
+            },
+          ],
+          lastVerified: -1,
+          lastAvailable: 4,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: { header: true, sessions: { [sessionID1]: 5 } },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+    [coValueID2]: {
+      id: coValueID2,
+      header: null,
+      sessions: {},
+      storageState: "unknown",
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+  } satisfies LocalNodeState["coValues"],
+  coValue2IsMemberInCoValue1WhichIsAGroup: {
+    [coValueID1]: {
+      id: coValueID1,
+      header: {
+        type: "comap",
+        ruleset: { type: "group", initialAdmin: "sealer_z1/signer_z1" },
+        meta: null,
+        uniqueness: 0,
+      },
+      sessions: {
+        [sessionID1]: {
+          id: sessionID1,
+          transactions: [
+            {
+              state: "available" as const,
+              tx: {
+                privacy: "trusting" as const,
+                changes:
+                  `[{"op": "set", "key": "${coValueID2}", "value": "someValue"}]` as Stringified<
+                    string[]
+                  >,
+                madeAt: 1,
+              },
+              signature: null,
+            },
+            {
+              state: "available" as const,
+              tx: tx2,
+              signature: "signature_after2" as Signature,
+            },
+            { state: "available" as const, tx: tx3, signature: null },
+            { state: "available" as const, tx: tx4, signature: null },
+            {
+              state: "available" as const,
+              tx: tx5,
+              signature: "signature_after5" as Signature,
+            },
+          ],
+          lastVerified: -1,
+          lastAvailable: 4,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: { header: true, sessions: { [sessionID1]: 5 } },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+    [coValueID2]: {
+      id: coValueID2,
+      header: null,
+      sessions: {},
+      storageState: "unknown",
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+  } satisfies LocalNodeState["coValues"],
+  coValue2IsExtendedGroupOfCoValue1: {
+    [coValueID1]: {
+      id: coValueID1,
+      header: {
+        type: "comap",
+        ruleset: { type: "group", initialAdmin: "sealer_z1/signer_z1" },
+        meta: null,
+        uniqueness: 0,
+      },
+      sessions: {
+        [sessionID1]: {
+          id: sessionID1,
+          transactions: [
+            {
+              state: "available" as const,
+              tx: {
+                privacy: "trusting" as const,
+                changes:
+                  `[{"op": "set", "key": "parent_${coValueID2}", "value": "someValue"}]` as Stringified<
+                    string[]
+                  >,
+                madeAt: 1,
+              },
+              signature: null,
+            },
+            {
+              state: "available" as const,
+              tx: tx2,
+              signature: "signature_after2" as Signature,
+            },
+            { state: "available" as const, tx: tx3, signature: null },
+            { state: "available" as const, tx: tx4, signature: null },
+            {
+              state: "available" as const,
+              tx: tx5,
+              signature: "signature_after5" as Signature,
+            },
+          ],
+          lastVerified: -1,
+          lastAvailable: 4,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: { header: true, sessions: { [sessionID1]: 5 } },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+    [coValueID2]: {
+      id: coValueID2,
+      header: null,
+      sessions: {},
+      storageState: "unknown",
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+  } satisfies LocalNodeState["coValues"],
+  coValue2IsAccountOwnerOfCoValue1: {
+    [coValueID1]: {
+      id: coValueID1,
+      header: {
+        type: "comap",
+        ruleset: { type: "ownedByGroup", group: coValueID2 },
+        meta: null,
+        uniqueness: 0,
+      },
+      sessions: {
+        [`${coValueID2}_session1` as SessionID]: {
+          id: `${coValueID2}_session1` as SessionID,
+          transactions: [
+            { state: "available" as const, tx: tx1, signature: null },
+            {
+              state: "available" as const,
+              tx: tx2,
+              signature: signatureAfter2,
+            },
+            { state: "available" as const, tx: tx3, signature: null },
+            { state: "available" as const, tx: tx4, signature: null },
+            {
+              state: "available" as const,
+              tx: tx5,
+              signature: signatureAfter5,
+            },
+          ],
+          lastVerified: -1,
+          lastAvailable: 4,
+          lastDepsAvailable: -1,
+          lastDecrypted: -1,
+          streamingHash: null,
+        },
+      },
+      storageState: { header: true, sessions: { [sessionID1]: 5 } },
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+    [coValueID2]: {
+      id: coValueID2,
+      header: {
+        type: "comap",
+        ruleset: { type: "group", initialAdmin: "sealer_z1/signer_z1" },
+        meta: null,
+        uniqueness: 0,
+      },
+      sessions: {
+        ["sealer_z1/signer_z1_session_zInit"]: {
+          id: "sealer_z1/signer_z1_session_zInit" as const,
+          transactions: [
+            {
+              state: "verified" as const,
+              tx: {
+                privacy: "trusting" as const,
+                changes:
+                  '[{ "op": "set", "key": "sealer_z1/signer_z1", "value": "admin"}]' as Stringified<
+                    JsonValue[]
+                  >,
+                madeAt: 1,
+              },
+              validity: { type: "valid" },
+              decryptionState: {
+                type: "decrypted",
+                changes: [
+                  { op: "set", key: "sealer_z1/signer_z1", value: "admin" },
+                ],
+              },
+              stored: true,
+              signature: "signature_zSomeValidSignature",
+            },
+          ],
+          lastVerified: 0,
+          lastAvailable: 0,
+          lastDepsAvailable: 0,
+          lastDecrypted: 0,
+          streamingHash: null,
+        },
+      },
+      storageState: "unknown",
+      peerState: {},
+      listeners: {},
+      dependents: [],
+    },
+  } satisfies LocalNodeState["coValues"],
+};

packages/cojson/src/tests/localNode/storage.test.ts

@@ -0,0 +1,241 @@
+import { describe, expect, test } from "vitest";
+import { Signature } from "../../crypto/crypto.js";
+import { subscribe } from "../../localNode/actions/subscribing.js";
+import { addTransaction } from "../../localNode/handlers/addTransaction.js";
+import { onMetadataLoaded } from "../../localNode/handlers/onMetadataLoaded.js";
+import { stageLoad } from "../../localNode/stages/0_load.js";
+import {
+  CoValueHeader,
+  SessionState,
+  emptyNode,
+} from "../../localNode/structure.js";
+import {
+  coValueID1,
+  coValueID2,
+  scenarios,
+  sessionID1,
+  tx1,
+  tx2,
+  tx3,
+  tx4,
+  tx5,
+} from "./setup.js";
+
+describe("Loading from storage", () => {
+  function setupNodeWithTwoCoValues() {
+    const node = emptyNode();
+
+    const _1 = subscribe(node, coValueID1);
+    const _2 = subscribe(node, coValueID2);
+    return { node, coValueID1, coValueID2 };
+  }
+
+  test("stageLoad puts covalues of unknown storage state into pending and issues load effects", () => {
+    const { node, coValueID1, coValueID2 } = setupNodeWithTwoCoValues();
+    const { effects } = stageLoad(node);
+    expect(effects).toEqual([
+      { type: "loadMetadataFromStorage", id: coValueID1 },
+      { type: "loadMetadataFromStorage", id: coValueID2 },
+    ]);
+
+    expect(node.coValues[coValueID1].storageState).toBe("pending");
+    expect(node.coValues[coValueID2].storageState).toBe("pending");
+  });
+
+  test("when we receive metadata of a present CoValue from storage, we keep it in memory, update the storage state and add pending transactions", () => {
+    const { node, coValueID1 } = setupNodeWithTwoCoValues();
+    const header = {
+      type: "comap",
+      ruleset: { type: "unsafeAllowAll" },
+      meta: null,
+      uniqueness: 0,
+    } satisfies CoValueHeader;
+
+    const knownState = {
+      header: true,
+      sessions: {
+        [sessionID1]: 5,
+      },
+    };
+
+    onMetadataLoaded(node, coValueID1, header, knownState);
+
+    const entry = node.coValues[coValueID1];
+
+    expect(entry?.header).toEqual(header);
+    expect(entry?.storageState).toBe(knownState);
+    expect(entry?.sessions[sessionID1]?.transactions).toEqual([
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+    ]);
+  });
+
+  test("when we receive information that a CoValue is unavailable in storage, we update the storage state accordingly", () => {
+    const { node, coValueID1 } = setupNodeWithTwoCoValues();
+    const knownState = "unavailable" as const;
+
+    onMetadataLoaded(node, coValueID1, null, knownState);
+
+    expect(node.coValues[coValueID1].storageState).toBe("unavailable");
+  });
+
+  test("stageLoad requests transactions from storage if a CoValue has listeners", () => {
+    const node = emptyNode();
+
+    node.coValues = structuredClone(
+      scenarios.coValuesWithAvailableInStorageTxs,
+    );
+
+    const _ = subscribe(node, coValueID1);
+
+    const { effects } = stageLoad(node);
+    expect(effects).toEqual([
+      {
+        type: "loadTransactionsFromStorage",
+        id: coValueID1,
+        sessionID: sessionID1,
+        from: 0,
+        to: 1,
+      },
+    ]);
+
+    expect(
+      node.coValues[coValueID1].sessions[sessionID1]?.transactions,
+    ).toEqual([
+      { state: "loadingFromStorage" },
+      { state: "loadingFromStorage" },
+    ]);
+
+    expect(
+      node.coValues[coValueID2].sessions[sessionID1]?.transactions,
+    ).toEqual([
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+    ]);
+  });
+
+  test("stageLoad requests transactions from storage if a CoValue has listeners", () => {
+    const node = emptyNode();
+
+    node.coValues = structuredClone(
+      scenarios.coValuesWithAvailableInStorageTxs,
+    );
+
+    node.coValues[coValueID1].dependents.push(coValueID2);
+
+    const { effects } = stageLoad(node);
+    expect(effects).toEqual([
+      {
+        type: "loadTransactionsFromStorage",
+        id: coValueID1,
+        sessionID: sessionID1,
+        from: 0,
+        to: 1,
+      },
+    ]);
+
+    expect(
+      node.coValues[coValueID1].sessions[sessionID1]?.transactions,
+    ).toEqual([
+      { state: "loadingFromStorage" },
+      { state: "loadingFromStorage" },
+    ]);
+
+    expect(
+      node.coValues[coValueID2].sessions[sessionID1]?.transactions,
+    ).toEqual([
+      { state: "availableInStorage" },
+      { state: "availableInStorage" },
+    ]);
+  });
+
+  test.todo(
+    "stageLoad requests transactions from storage that need to be synced",
+  );
+
+  test("Transactions added from storage are added to memory", () => {
+    const { node, coValueID1 } = setupNodeWithTwoCoValues();
+    const header = {
+      type: "comap",
+      ruleset: { type: "unsafeAllowAll" },
+      meta: null,
+      uniqueness: 0,
+    } satisfies CoValueHeader;
+
+    const knownState = {
+      header: true,
+      sessions: {
+        [sessionID1]: 5,
+      },
+    };
+
+    onMetadataLoaded(node, coValueID1, header, knownState);
+
+    const { result: result1 } = addTransaction(
+      node,
+      coValueID1,
+      sessionID1,
+      0,
+      [tx1, tx2],
+      "signature_after2" as Signature,
+    );
+
+    expect(result1).toEqual({ type: "success" });
+    expect(node.coValues[coValueID1].sessions[sessionID1]).toEqual({
+      id: sessionID1,
+      lastAvailable: 1,
+      lastDepsAvailable: -1,
+      lastVerified: -1,
+      lastDecrypted: -1,
+      transactions: [
+        { state: "available", tx: tx1, signature: null },
+        {
+          state: "available",
+          tx: tx2,
+          signature: "signature_after2" as Signature,
+        },
+        { state: "availableInStorage" },
+        { state: "availableInStorage" },
+        { state: "availableInStorage" },
+      ],
+      streamingHash: null,
+    } satisfies SessionState);
+
+    const { result: result2 } = addTransaction(
+      node,
+      coValueID1,
+      sessionID1,
+      2,
+      [tx3, tx4, tx5],
+      "signature_after5" as Signature,
+    );
+
+    expect(result2).toEqual({ type: "success" });
+    expect(node.coValues[coValueID1].sessions[sessionID1]).toEqual({
+      id: sessionID1,
+      lastAvailable: 4,
+      lastDepsAvailable: -1,
+      lastVerified: -1,
+      lastDecrypted: -1,
+      transactions: [
+        { state: "available", tx: tx1, signature: null },
+        {
+          state: "available",
+          tx: tx2,
+          signature: "signature_after2" as Signature,
+        },
+        { state: "available", tx: tx3, signature: null },
+        { state: "available", tx: tx4, signature: null },
+        {
+          state: "available",
+          tx: tx5,
+          signature: "signature_after5" as Signature,
+        },
+      ],
+      streamingHash: null,
+    } satisfies SessionState);
+  });
+});

packages/cojson/src/tests/localNode/subscribing.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, test } from "vitest";
+import { RawCoID } from "../../exports.js";
+import { subscribe, unsubscribe } from "../../localNode/actions/subscribing.js";
+
+import { emptyNode } from "../../localNode/structure.js";
+
+describe("Subscribing to a CoValue", () => {
+  test("creates an empty entry if none exists yet", () => {
+    const node = emptyNode();
+    const id = "co_fakeCoValueID" as RawCoID;
+    const { listenerID } = subscribe(node, id);
+
+    expect(listenerID).toBeDefined();
+
+    expect(node.coValues[id]).toEqual({
+      id,
+      header: null,
+      sessions: {},
+      storageState: "unknown",
+      peerState: {},
+      listeners: { [listenerID]: "unknown" },
+      incomingMessages: {},
+      dependents: [],
+    });
+  });
+
+  test("adds a listener if an entry already exists", () => {
+    const node = emptyNode();
+    const id = "co_fakeCoValueID" as RawCoID;
+    const { listenerID: firstListenerID } = subscribe(node, id);
+    const { listenerID: secondListenerID } = subscribe(node, id);
+
+    expect(firstListenerID).toBeDefined();
+    expect(secondListenerID).toBeDefined();
+    expect(firstListenerID).not.toEqual(secondListenerID);
+
+    expect(node.coValues[id]).toEqual({
+      id,
+      header: null,
+      sessions: {},
+      storageState: "unknown",
+      peerState: {},
+      listeners: {
+        [firstListenerID]: "unknown",
+        [secondListenerID]: "unknown",
+      },
+      dependents: [],
+      incomingMessages: {},
+    });
+  });
+
+  test("unsubscribing from a CoValue removes the listener", () => {
+    const node = emptyNode();
+    const id = "co_fakeCoValueID" as RawCoID;
+
+    const { listenerID } = subscribe(node, id);
+    expect(node.coValues[id].listeners[listenerID]).toBe("unknown");
+
+    unsubscribe(node, id, listenerID);
+    expect(node.coValues[id].listeners[listenerID]).toBeUndefined();
+  });
+});

packages/cojson/src/tests/localNode/sync.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from "vitest";
+import { RawCoID } from "../../exports.js";
+import { subscribe } from "../../localNode/actions/subscribing.js";
+import { stageLoad } from "../../localNode/stages/0_load.js";
+import { stageSyncOut } from "../../localNode/stages/6_syncOut.js";
+import { emptyNode } from "../../localNode/structure.js";
+
+describe("Syncing", () => {
+  const node = emptyNode();
+  const coValueID1 = "co_zCoValueID1" as RawCoID;
+  const coValueID2 = "co_z" as RawCoID;
+  const _1 = subscribe(node, coValueID1);
+
+  stageLoad(node);
+
+  const _2 = subscribe(node, coValueID2);
+
+  test("stageSync doesn't do anything and causes no effects on CoValues with storage state unknown or pending", () => {
+    const coValuesBefore = structuredClone(node.coValues);
+    const { effects } = stageSyncOut(node);
+    expect(effects).toEqual([]);
+    expect(node.coValues).toEqual(coValuesBefore);
+  });
+});

packages/cojson/src/tests/localNode/testUtils.ts

@@ -0,0 +1,112 @@
+import { AgentID, RawAccountID, RawCoID, SessionID } from "../../exports.js";
+import { Stringified, stableStringify } from "../../jsonStringify.js";
+import {
+  CoValueHeader,
+  CoValueState,
+  emptyCoValueState,
+} from "../../localNode/structure.js";
+
+function getIdFromTestHeader(header: CoValueHeader): RawCoID {
+  return `co_z${header.type}${header.uniqueness}${stableStringify(header.ruleset)}`;
+}
+
+export function createTestCoMap(groupId: RawCoID | null, uniqueness: string) {
+  const header = {
+    type: "comap",
+    ruleset: groupId
+      ? { type: "ownedByGroup", group: groupId }
+      : { type: "unsafeAllowAll" },
+    meta: null,
+    uniqueness: uniqueness,
+    createdAt: null,
+  } satisfies CoValueHeader;
+
+  const coValueState = emptyCoValueState(getIdFromTestHeader(header));
+  coValueState.header = header;
+
+  return coValueState;
+}
+
+export function createTestGroup(
+  owner: RawAccountID | AgentID,
+  uniqueness: string,
+) {
+  const header = {
+    type: "comap",
+    ruleset: { type: "group", initialAdmin: owner },
+    meta: null,
+    uniqueness: uniqueness,
+    createdAt: null,
+  } satisfies CoValueHeader;
+
+  const coValueState = emptyCoValueState(getIdFromTestHeader(header));
+  coValueState.header = header;
+
+  return coValueState;
+}
+
+export function addMemberTestTransaction(
+  coValue: CoValueState,
+  member: RawCoID,
+  sessionId: SessionID,
+) {
+  if (!coValue.sessions[sessionId]) {
+    coValue.sessions[sessionId] = {
+      id: sessionId,
+      transactions: [],
+      streamingHash: null,
+      lastAvailable: -1,
+      lastDepsAvailable: -1,
+      lastVerified: -1,
+      lastDecrypted: -1,
+    };
+  }
+
+  const tx = {
+    state: "available" as const,
+    tx: {
+      privacy: "trusting" as const,
+      changes: JSON.stringify([
+        { op: "set", key: member, value: "reader" },
+      ]) as Stringified<string[]>,
+      madeAt: Date.now(),
+    },
+    signature: null,
+  };
+
+  coValue.sessions[sessionId].transactions.push(tx);
+  coValue.sessions[sessionId].lastAvailable++;
+}
+
+export function addParentGroupTestTransaction(
+  coValue: CoValueState,
+  parent: RawCoID,
+  sessionId: SessionID,
+) {
+  if (!coValue.sessions[sessionId]) {
+    coValue.sessions[sessionId] = {
+      id: sessionId,
+      transactions: [],
+      streamingHash: null,
+      lastAvailable: -1,
+      lastDepsAvailable: -1,
+      lastVerified: -1,
+      lastDecrypted: -1,
+    };
+  }
+
+  const tx = {
+    state: "available" as const,
+    tx: {
+      privacy: "trusting" as const,
+      changes: JSON.stringify([
+        { op: "set", key: `parent_${parent}`, value: "extend" },
+      ]) as Stringified<string[]>,
+      madeAt: Date.now(),
+    },
+    signature: null,
+  };
+
+  coValue.sessions[sessionId].transactions.push(tx);
+  coValue.sessions[sessionId].lastAvailable++;
+}

packages/cojson/src/tests/localNode/verify.test.ts

@@ -0,0 +1,271 @@
+import { describe, expect, test } from "vitest";
+import { Signature } from "../../crypto/crypto.js";
+import { subscribe } from "../../localNode/actions/subscribing.js";
+import { stageVerify } from "../../localNode/stages/2_verify.js";
+
+import { TransactionState, emptyNode } from "../../localNode/structure.js";
+import { MockCrypto } from "../MockCrypto.js";
+import {
+  crypto,
+  coValueID1,
+  coValueID2,
+  scenarios,
+  sessionID1,
+  signatureAfter2,
+  signatureAfter5,
+  tx1,
+  tx2,
+  tx3,
+  tx4,
+  tx5,
+} from "./setup.js";
+
+describe("stageVerify", () => {
+  test("stageVerify does nothing for CoValues without listeners or dependents", () => {
+    const node = emptyNode();
+
+    node.coValues = structuredClone(scenarios.coValue2IsGroupOfCoValue1);
+
+    const coValuesBefore = structuredClone(node.coValues);
+
+    stageVerify(node, new MockCrypto(crypto));
+    expect(node.coValues).toEqual(coValuesBefore);
+  });
+
+  test("stageVerify verifies a CoValue if it has listeners (primitive signer)", () => {
+    const node = emptyNode();
+
+    node.coValues = structuredClone(scenarios.coValue2IsGroupOfCoValue1);
+
+    const _ = subscribe(node, coValueID1);
+
+    stageVerify(node, new MockCrypto(crypto));
+
+    expect(node.coValues[coValueID1].sessions[sessionID1].lastVerified).toEqual(
+      4,
+    );
+    expect(node.coValues[coValueID1].sessions[sessionID1].transactions).toEqual(
+      [
+        {
+          state: "verified" as const,
+          tx: tx1,
+          signature: null,
+          validity: { type: "unknown" },
+          decryptionState: { type: "notDecrypted" },
+          stored: false,
+        },
+        {
+          state: "verified" as const,
+          tx: tx2,
+          signature: signatureAfter2,
+          validity: { type: "unknown" },
+          decryptionState: { type: "notDecrypted" },
+          stored: false,
+        },
+        {
+          state: "verified" as const,
+          tx: tx3,
+          signature: null,
+          validity: { type: "unknown" },
+          decryptionState: { type: "notDecrypted" },
+          stored: false,
+        },
+        {
+          state: "verified" as const,
+          tx: tx4,
+          signature: null,
+          validity: { type: "unknown" },
+          decryptionState: { type: "notDecrypted" },
+          stored: false,
+        },
+        {
+          state: "verified" as const,
+          tx: tx5,
+          signature: signatureAfter5,
+          validity: { type: "unknown" },
+          decryptionState: { type: "notDecrypted" },
+          stored: false,
+        },
+      ] satisfies TransactionState[],
+    );
+  });
+
+  test("stageVerify verifies a CoValue if it has listeners (invalid signature, primitive signer)", () => {
+    const node = emptyNode();
+
+    const coValues = structuredClone(scenarios.coValue2IsGroupOfCoValue1);
+    coValues[coValueID1].sessions[sessionID1].transactions[1].signature =
+      "signature_zInvalid1";
+    coValues[coValueID1].sessions[sessionID1].transactions[4].signature =
+      "signature_zInvalid2";
+
+    node.coValues = coValues;
+    const _ = subscribe(node, coValueID1);
+
+    stageVerify(node, new MockCrypto(crypto));
+
+    expect(node.coValues[coValueID1].sessions[sessionID1].lastVerified).toEqual(
+      4,
+    );
+    expect(node.coValues[coValueID1].sessions[sessionID1].transactions).toEqual(
+      [
+        {
+          state: "verificationFailed" as const,
+          tx: tx1,
+          signature: null,
+          reason: "Invalid signature at idx 1",
+          hash: null,
+        },
+        {
+          state: "verificationFailed" as const,
+          tx: tx2,
+          signature: "signature_zInvalid1" as Signature,
+          reason: "Invalid signature (here)",
+          hash: "hash_zEyyx6wfnEsvcc4Br2hUSApxgdmpMitin3QHtLyPDxepA",
+        },
+        {
+          state: "verificationFailed" as const,
+          tx: tx3,
+          signature: null,
+          reason: "Invalid signature at idx 1",
+          hash: null,
+        },
+        {
+          state: "verificationFailed" as const,
+          tx: tx4,
+          signature: null,
+          reason: "Invalid signature at idx 1",
+          hash: null,
+        },
+        {
+          state: "verificationFailed" as const,
+          tx: tx5,
+          signature: "signature_zInvalid2" as Signature,
+          reason: "Invalid signature at idx 1",
+          hash: null,
+        },
+      ] satisfies TransactionState[],
+    );
+  });
+
+  test("stageVerify verifies a CoValue if it has listeners (account signer)", () => {
+    const node = emptyNode();
+
+    node.coValues = structuredClone(scenarios.coValue2IsAccountOwnerOfCoValue1);
+
+    const _ = subscribe(node, coValueID1);
+
+    stageVerify(node, new MockCrypto(crypto));
+
+    expect(
+      node.coValues[coValueID1].sessions[`${coValueID2}_session1`].lastVerified,
+    ).toEqual(4);
+    expect(
+      node.coValues[coValueID1].sessions[`${coValueID2}_session1`].transactions,
+    ).toEqual([
+      {
+        state: "verified" as const,
+        tx: tx1,
+        signature: null,
+        validity: { type: "unknown" },
+        decryptionState: { type: "notDecrypted" },
+        stored: false,
+      },
+      {
+        state: "verified" as const,
+        tx: tx2,
+        signature: signatureAfter2,
+        validity: { type: "unknown" },
+        decryptionState: { type: "notDecrypted" },
+        stored: false,
+      },
+      {
+        state: "verified" as const,
+        tx: tx3,
+        signature: null,
+        validity: { type: "unknown" },
+        decryptionState: { type: "notDecrypted" },
+        stored: false,
+      },
+      {
+        state: "verified" as const,
+        tx: tx4,
+        signature: null,
+        validity: { type: "unknown" },
+        decryptionState: { type: "notDecrypted" },
+        stored: false,
+      },
+      {
+        state: "verified" as const,
+        tx: tx5,
+        signature: signatureAfter5,
+        validity: { type: "unknown" },
+        decryptionState: { type: "notDecrypted" },
+        stored: false,
+      },
+    ] satisfies TransactionState[]);
+  });
+
+  test("stageVerify verifies a CoValue if it has listeners (invalid signature, account signer)", () => {
+    const node = emptyNode();
+
+    const coValues = structuredClone(
+      scenarios.coValue2IsAccountOwnerOfCoValue1,
+    );
+    coValues[coValueID1].sessions[
+      `${coValueID2}_session1`
+    ].transactions[1].signature = "signature_zInvalid1";
+    coValues[coValueID1].sessions[
+      `${coValueID2}_session1`
+    ].transactions[4].signature = "signature_zInvalid2";
+
+    node.coValues = coValues;
+
+    const _ = subscribe(node, coValueID1);
+
+    stageVerify(node, new MockCrypto(crypto));
+
+    expect(
+      node.coValues[coValueID1].sessions[`${coValueID2}_session1`].lastVerified,
+    ).toEqual(4);
+    expect(
+      node.coValues[coValueID1].sessions[`${coValueID2}_session1`].transactions,
+    ).toEqual([
+      {
+        state: "verificationFailed" as const,
+        tx: tx1,
+        signature: null,
+        reason: "Invalid signature at idx 1",
+        hash: null,
+      },
+      {
+        state: "verificationFailed" as const,
+        tx: tx2,
+        signature: "signature_zInvalid1" as Signature,
+        reason: "Invalid signature (here)",
+        hash: "hash_zEyyx6wfnEsvcc4Br2hUSApxgdmpMitin3QHtLyPDxepA",
+      },
+      {
+        state: "verificationFailed" as const,
+        tx: tx3,
+        signature: null,
+        reason: "Invalid signature at idx 1",
+        hash: null,
+      },
+      {
+        state: "verificationFailed" as const,
+        tx: tx4,
+        signature: null,
+        reason: "Invalid signature at idx 1",
+        hash: null,
+      },
+      {
+        state: "verificationFailed" as const,
+        tx: tx5,
+        signature: "signature_zInvalid2" as Signature,
+        reason: "Invalid signature at idx 1",
+        hash: null,
+      },
+    ] satisfies TransactionState[]);
+  });
+});

packages/jazz-browser/src/OPFSFilesystem.ts

@@ -1,348 +0,0 @@
-import { BlockFilename, CryptoProvider, FileSystem, WalFilename } from "cojson";
-
-export class OPFSFilesystem
-  implements
-    FileSystem<
-      { id: number; filename: string },
-      { id: number; filename: string }
-    >
-{
-  opfsWorker: Worker;
-  callbacks: Map<number, (event: MessageEvent) => void> = new Map();
-  nextRequestId = 0;
-
-  constructor(public crypto: CryptoProvider) {
-    this.opfsWorker = new Worker(
-      URL.createObjectURL(
-        new Blob([opfsWorkerJSSrc], { type: "text/javascript" }),
-      ),
-    );
-    this.opfsWorker.onmessage = (event) => {
-      // console.log("Received from OPFS worker", event.data);
-      const handler = this.callbacks.get(event.data.requestId);
-      if (handler) {
-        handler(event);
-        this.callbacks.delete(event.data.requestId);
-      }
-    };
-  }
-
-  listFiles(): Promise<string[]> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("listFiles" + requestId + "_listFiles");
-      this.callbacks.set(requestId, (event) => {
-        performance.mark("listFilesEnd" + requestId + "_listFiles");
-        performance.measure(
-          "listFiles" + requestId + "_listFiles",
-          "listFiles" + requestId + "_listFiles",
-          "listFilesEnd" + requestId + "_listFiles",
-        );
-        resolve(event.data.fileNames);
-      });
-      this.opfsWorker.postMessage({ type: "listFiles", requestId });
-    });
-  }
-
-  openToRead(
-    filename: string,
-  ): Promise<{ handle: { id: number; filename: string }; size: number }> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("openToRead" + "_" + filename);
-      this.callbacks.set(requestId, (event) => {
-        resolve({
-          handle: { id: event.data.handle, filename },
-          size: event.data.size,
-        });
-        performance.mark("openToReadEnd" + "_" + filename);
-        performance.measure(
-          "openToRead" + "_" + filename,
-          "openToRead" + "_" + filename,
-          "openToReadEnd" + "_" + filename,
-        );
-      });
-      this.opfsWorker.postMessage({
-        type: "openToRead",
-        filename,
-        requestId,
-      });
-    });
-  }
-
-  createFile(filename: string): Promise<{ id: number; filename: string }> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("createFile" + "_" + filename);
-      this.callbacks.set(requestId, (event) => {
-        performance.mark("createFileEnd" + "_" + filename);
-        performance.measure(
-          "createFile" + "_" + filename,
-          "createFile" + "_" + filename,
-          "createFileEnd" + "_" + filename,
-        );
-        resolve({ id: event.data.handle, filename });
-      });
-      this.opfsWorker.postMessage({
-        type: "createFile",
-        filename,
-        requestId,
-      });
-    });
-  }
-
-  openToWrite(filename: string): Promise<{ id: number; filename: string }> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("openToWrite" + "_" + filename);
-      this.callbacks.set(requestId, (event) => {
-        performance.mark("openToWriteEnd" + "_" + filename);
-        performance.measure(
-          "openToWrite" + "_" + filename,
-          "openToWrite" + "_" + filename,
-          "openToWriteEnd" + "_" + filename,
-        );
-        resolve({ id: event.data.handle, filename });
-      });
-      this.opfsWorker.postMessage({
-        type: "openToWrite",
-        filename,
-        requestId,
-      });
-    });
-  }
-
-  append(
-    handle: { id: number; filename: string },
-    data: Uint8Array,
-  ): Promise<void> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("append" + "_" + handle.filename);
-      this.callbacks.set(requestId, (_) => {
-        performance.mark("appendEnd" + "_" + handle.filename);
-        performance.measure(
-          "append" + "_" + handle.filename,
-          "append" + "_" + handle.filename,
-          "appendEnd" + "_" + handle.filename,
-        );
-        resolve(undefined);
-      });
-      this.opfsWorker.postMessage({
-        type: "append",
-        handle: handle.id,
-        data,
-        requestId,
-      });
-    });
-  }
-
-  read(
-    handle: { id: number; filename: string },
-    offset: number,
-    length: number,
-  ): Promise<Uint8Array> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("read" + "_" + handle.filename);
-      this.callbacks.set(requestId, (event) => {
-        performance.mark("readEnd" + "_" + handle.filename);
-        performance.measure(
-          "read" + "_" + handle.filename,
-          "read" + "_" + handle.filename,
-          "readEnd" + "_" + handle.filename,
-        );
-        resolve(event.data.data);
-      });
-      this.opfsWorker.postMessage({
-        type: "read",
-        handle: handle.id,
-        offset,
-        length,
-        requestId,
-      });
-    });
-  }
-
-  close(handle: { id: number; filename: string }): Promise<void> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("close" + "_" + handle.filename);
-      this.callbacks.set(requestId, (_) => {
-        performance.mark("closeEnd" + "_" + handle.filename);
-        performance.measure(
-          "close" + "_" + handle.filename,
-          "close" + "_" + handle.filename,
-          "closeEnd" + "_" + handle.filename,
-        );
-        resolve(undefined);
-      });
-      this.opfsWorker.postMessage({
-        type: "close",
-        handle: handle.id,
-        requestId,
-      });
-    });
-  }
-
-  closeAndRename(
-    handle: { id: number; filename: string },
-    filename: BlockFilename,
-  ): Promise<void> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("closeAndRename" + "_" + handle.filename);
-      this.callbacks.set(requestId, () => {
-        performance.mark("closeAndRenameEnd" + "_" + handle.filename);
-        performance.measure(
-          "closeAndRename" + "_" + handle.filename,
-          "closeAndRename" + "_" + handle.filename,
-          "closeAndRenameEnd" + "_" + handle.filename,
-        );
-        resolve(undefined);
-      });
-      this.opfsWorker.postMessage({
-        type: "closeAndRename",
-        handle: handle.id,
-        filename,
-        requestId,
-      });
-    });
-  }
-
-  removeFile(filename: BlockFilename | WalFilename): Promise<void> {
-    return new Promise((resolve) => {
-      const requestId = this.nextRequestId++;
-      performance.mark("removeFile" + "_" + filename);
-      this.callbacks.set(requestId, () => {
-        performance.mark("removeFileEnd" + "_" + filename);
-        performance.measure(
-          "removeFile" + "_" + filename,
-          "removeFile" + "_" + filename,
-          "removeFileEnd" + "_" + filename,
-        );
-        resolve(undefined);
-      });
-      this.opfsWorker.postMessage({
-        type: "removeFile",
-        filename,
-        requestId,
-      });
-    });
-  }
-}
-
-const opfsWorkerJSSrc = `
-
-    let rootDirHandle;
-    const handlesByRequest = new Map();
-    const handlesByFilename = new Map();
-    const filenamesForHandles = new Map();
-
-    onmessage = async function handleEvent(event) {
-        rootDirHandle = rootDirHandle || await navigator.storage.getDirectory();
-        // console.log("Received in OPFS worker", {...event.data, data: event.data.data ? "some data of length " + event.data.data.length : undefined});
-        if (event.data.type === "listFiles") {
-            const fileNames = [];
-            for await (const entry of rootDirHandle.values()) {
-                if (entry.kind === "file") {
-                    fileNames.push(entry.name);
-                }
-            }
-            postMessage({requestId: event.data.requestId, fileNames});
-        } else if (event.data.type === "openToRead" || event.data.type === "openToWrite") {
-            let syncHandle;
-            const existingHandle = handlesByFilename.get(event.data.filename);
-            if (existingHandle) {
-                throw new Error("Handle already exists for file: " + event.data.filename);
-            } else {
-                const handle = await rootDirHandle.getFileHandle(event.data.filename);
-                try {
-                    syncHandle = await handle.createSyncAccessHandle();
-                } catch (e) {
-                    throw new Error("Couldn't open file for reading: " + event.data.filename, {cause: e});
-                }
-            }
-            handlesByRequest.set(event.data.requestId, syncHandle);
-            handlesByFilename.set(event.data.filename, syncHandle);
-            filenamesForHandles.set(syncHandle, event.data.filename);
-            let size;
-            try {
-                size = syncHandle.getSize();
-            } catch (e) {
-                throw new Error("Couldn't get size of file: " + event.data.filename, {cause: e});
-            }
-            postMessage({requestId: event.data.requestId, handle: event.data.requestId, size});
-        } else if (event.data.type === "createFile") {
-            const handle = await rootDirHandle.getFileHandle(event.data.filename, {
-                create: true,
-            });
-            let syncHandle;
-            try {
-                syncHandle = await handle.createSyncAccessHandle();
-            } catch (e) {
-                throw new Error("Couldn't create file: " + event.data.filename, {cause: e});
-            }
-            handlesByRequest.set(event.data.requestId, syncHandle);
-            handlesByFilename.set(event.data.filename, syncHandle);
-            filenamesForHandles.set(syncHandle, event.data.filename);
-            postMessage({requestId: event.data.requestId, handle: event.data.requestId, result: "done"});
-        } else if (event.data.type === "append") {
-            const writable = handlesByRequest.get(event.data.handle);
-            writable.write(event.data.data, {at: writable.getSize()});
-            writable.flush();
-            postMessage({requestId: event.data.requestId, result: "done"});
-        } else if (event.data.type === "read") {
-            const readable = handlesByRequest.get(event.data.handle);
-            const buffer = new Uint8Array(event.data.length);
-            const read = readable.read(buffer, {at: event.data.offset});
-            if (read < event.data.length) {
-                throw new Error("Couldn't read enough");
-            }
-            postMessage({requestId: event.data.requestId, data: buffer, result: "done"});
-        } else if (event.data.type === "close") {
-            const handle = handlesByRequest.get(event.data.handle);
-            // console.log("Closing handle", filenamesForHandles.get(handle), event.data.handle, handle);
-            handle.flush();
-            handle.close();
-            handlesByRequest.delete(handle);
-            const filename = filenamesForHandles.get(handle);
-            handlesByFilename.delete(filename);
-            filenamesForHandles.delete(handle);
-            postMessage({requestId: event.data.requestId, result: "done"});
-        } else if (event.data.type === "closeAndRename") {
-            const handle = handlesByRequest.get(event.data.handle);
-            handle.flush();
-            const buffer = new Uint8Array(handle.getSize());
-            const read = handle.read(buffer, {at: 0});
-            if (read < buffer.length) {
-                throw new Error("Couldn't read enough " + read + ", " + handle.getSize());
-            }
-            handle.close();
-            const oldFilename = filenamesForHandles.get(handle);
-            await rootDirHandle.removeEntry(oldFilename);
-
-            const newHandle = await rootDirHandle.getFileHandle(event.data.filename, { create: true });
-            let writable;
-            try {
-                writable = await newHandle.createSyncAccessHandle();
-            } catch (e) {
-                throw new Error("Couldn't create file (to rename to): " + event.data.filename, { cause: e })
-            }
-            writable.write(buffer);
-            writable.close();
-            postMessage({requestId: event.data.requestId, result: "done"});
-        } else if (event.data.type === "removeFile") {
-            try {
-                await rootDirHandle.removeEntry(event.data.filename);
-            } catch(e) {
-                throw new Error("Couldn't remove file: " + event.data.filename, { cause: e });
-            }
-            postMessage({requestId: event.data.requestId, result: "done"});
-        } else {
-            console.error("Unknown event type", event.data.type);
-        }
-    };
-
-    //# sourceURL=opfsWorker.js
-    `;

packages/jazz-browser/src/createBrowserContext.ts

@@ -1,4 +1,4 @@
-import { LSMStorage, LocalNode, Peer, RawAccountID } from "cojson";
+import { LocalNode, Peer, RawAccountID } from "cojson";
 import { IDBStorage } from "cojson-storage-indexeddb";
 import { WebSocketPeerWithReconnection } from "cojson-transport-ws";
 import { WasmCrypto } from "cojson/crypto/WasmCrypto";
@@ -19,8 +19,6 @@ import {
   createAnonymousJazzContext,
 } from "jazz-tools";
 import { createJazzContext } from "jazz-tools";
-import { OPFSFilesystem } from "./OPFSFilesystem.js";
-import { StorageConfig, getStorageOptions } from "./storageOptions.js";
 import { setupInspector } from "./utils/export-account-inspector.js";
 
 setupInspector();
@@ -28,7 +26,7 @@ setupInspector();
 export type BaseBrowserContextOptions = {
   sync: SyncConfig;
   reconnectionTimeout?: number;
-  storage?: StorageConfig;
+  storage?: "indexedDB";
   crypto?: CryptoProvider;
   authSecretStorage: AuthSecretStorage;
 };
@@ -50,22 +48,9 @@ async function setupPeers(options: BaseBrowserContextOptions) {
   const crypto = options.crypto || (await WasmCrypto.create());
   let node: LocalNode | undefined = undefined;
 
-  const { useSingleTabOPFS, useIndexedDB } = getStorageOptions(options.storage);
-
   const peersToLoadFrom: Peer[] = [];
 
-  if (useSingleTabOPFS) {
-    peersToLoadFrom.push(
-      await LSMStorage.asPeer({
-        fs: new OPFSFilesystem(crypto),
-        // trace: true,
-      }),
-    );
-  }
-
-  if (useIndexedDB) {
-    peersToLoadFrom.push(await IDBStorage.asPeer());
-  }
+  peersToLoadFrom.push(await IDBStorage.asPeer());
 
   if (options.sync.when === "never") {
     return {

packages/jazz-browser/src/storageOptions.ts

@@ -1,23 +0,0 @@
-type StorageOption = "indexedDB" | "singleTabOPFS";
-type CombinedStorageOption = ["singleTabOPFS", "indexedDB"];
-export type StorageConfig =
-  | StorageOption
-  | CombinedStorageOption
-  | [StorageOption];
-
-export function getStorageOptions(storage?: StorageConfig): {
-  useSingleTabOPFS: boolean;
-  useIndexedDB: boolean;
-} {
-  const useSingleTabOPFS =
-    (Array.isArray(storage) && storage.includes("singleTabOPFS")) ||
-    storage === "singleTabOPFS";
-
-  const useIndexedDB =
-    !storage ||
-    (Array.isArray(storage) && storage.includes("indexedDB")) ||
-    storage === "indexedDB" ||
-    !useSingleTabOPFS;
-
-  return { useSingleTabOPFS, useIndexedDB };
-}

packages/jazz-browser/src/tests/storageOptions.test.ts

@@ -1,74 +0,0 @@
-import { describe, expect, test } from "vitest";
-import { getStorageOptions } from "../storageOptions.js";
-import { StorageConfig } from "../storageOptions.js";
-
-describe("getStorageOptions", () => {
-  test("should default to indexedDB only when no storage option is provided", () => {
-    const result = getStorageOptions();
-    expect(result).toEqual({
-      useSingleTabOPFS: false,
-      useIndexedDB: true,
-    });
-  });
-
-  test("should enable only indexedDB when 'indexedDB' is provided", () => {
-    const result = getStorageOptions("indexedDB");
-    expect(result).toEqual({
-      useSingleTabOPFS: false,
-      useIndexedDB: true,
-    });
-  });
-
-  test("should enable only singleTabOPFS when 'singleTabOPFS' is provided", () => {
-    const result = getStorageOptions("singleTabOPFS");
-    expect(result).toEqual({
-      useSingleTabOPFS: true,
-      useIndexedDB: false,
-    });
-  });
-
-  test("should enable both when array with both options is provided", () => {
-    const result = getStorageOptions(["singleTabOPFS", "indexedDB"]);
-    expect(result).toEqual({
-      useSingleTabOPFS: true,
-      useIndexedDB: true,
-    });
-  });
-
-  test("should enable only indexedDB when array with only indexedDB is provided", () => {
-    const result = getStorageOptions(["indexedDB"]);
-    expect(result).toEqual({
-      useSingleTabOPFS: false,
-      useIndexedDB: true,
-    });
-  });
-
-  test("should enable only singleTabOPFS when array with only singleTabOPFS is provided", () => {
-    const result = getStorageOptions(["singleTabOPFS"]);
-    expect(result).toEqual({
-      useSingleTabOPFS: true,
-      useIndexedDB: false,
-    });
-  });
-
-  test("should fallback to indexedDB when singleTabOPFS is not available", () => {
-    // Testing the fallback behavior when storage is undefined
-    const result = getStorageOptions(undefined);
-    expect(result.useIndexedDB).toBe(true);
-
-    // Testing with an empty array (invalid case but should fallback safely)
-    const result2 = getStorageOptions([] as unknown as StorageConfig);
-    expect(result2.useIndexedDB).toBe(true);
-  });
-
-  // Type checking tests
-  test("should handle type checking for StorageConfig", () => {
-    // These should compile without type errors
-    getStorageOptions("indexedDB");
-    getStorageOptions("singleTabOPFS");
-    getStorageOptions(["singleTabOPFS", "indexedDB"]);
-    getStorageOptions(["indexedDB"]);
-    getStorageOptions(["singleTabOPFS"]);
-    getStorageOptions(undefined);
-  });
-});

vite.config.ts

@@ -4,7 +4,7 @@ import { defineConfig } from "vitest/config";
 export default defineConfig({
   root: "./",
   test: {
-    workspace: ["packages/*", "tests/browser-integration"],
+    workspace: ["packages/cojson"],
     coverage: {
       enabled: false,
       provider: "istanbul",