WIP: move automerge to loro

Author: rroohhh

backend/transcribee_backend/helpers/sync.py

@@ -1,6 +1,7 @@
 import asyncio
 from asyncio import Queue
-from typing import Callable
+from typing import Callable, Tuple
+import uuid
 
 from fastapi import WebSocket, WebSocketDisconnect
 from sqlmodel import Session, select
@@ -45,14 +46,17 @@ def __init__(
         self._can_write = can_write
         self._subscribed = set()
         self._msg_queue = Queue()
+        self._id = uuid.uuid4()
 
     def subscribe(self, channel: str):
         self._subscribed.add(channel)
         sync_manager.subscribe(channel, self.handle_incoming_broadcast)
 
-    async def handle_incoming_broadcast(self, channel: str, message: bytes):
+    async def handle_incoming_broadcast(self, channel: str, message: Tuple[uuid.UUID, bytes]):
         if channel in self._subscribed:
-            await self._msg_queue.put(message)
+            id, msg = message
+            if (id != self._id):
+                await self._msg_queue.put(msg)
 
     async def listener(self):
         while True:
@@ -75,15 +79,18 @@ async def broadcast_sender(self):
         # https://asgi.readthedocs.io/en/latest/specs/www.html#send-send-event
         message = [bytes([SyncMessageType.FULL_DOCUMENT])]
         for update in self._session.exec(statement):
-            message.append(update.change_bytes)
+            message.append(len(update.change_bytes).to_bytes(4) + update.change_bytes + bytes([SyncMessageType.CHANGE]))
+            # message.append(update.change_bytes)
+            print("hello", len(update.change_bytes))
+        message[-1] = message[-1][:-1]
         await self._ws.send_bytes(message)  # type: ignore
         # END
 
         await self._ws.send_bytes(bytes([SyncMessageType.CHANGE_BACKLOG_COMPLETE]))
 
         while True:
             msg = await self._msg_queue.get()
-            await self._ws.send_bytes(bytes([SyncMessageType.CHANGE]) + msg)
+            await self._ws.send_bytes(bytes([SyncMessageType.CHANGE]) + len(msg).to_bytes(4) + msg)
 
     async def run(self):
         await self._ws.accept()
@@ -126,4 +133,4 @@ async def on_message(self, message: bytes):
         self._session.add(self._doc)
 
         self._session.commit()
-        await sync_manager.broadcast(str(self._doc.id), message)
+        await sync_manager.broadcast(str(self._doc.id), (self._id, message))

frontend/package-lock.json

@@ -3,6 +3,7 @@
   "version": "0.1.0",
   "scripts": {
     "dev": "vite",
+    "profile": "vite --profile",
     "build": "npm-run-all --continue-on-error build:*",
     "build:licenses": "node scripts/generate_licenses.mjs public/LICENSES.md",
     "build:vite": "vite build",
@@ -57,13 +58,13 @@
   },
   "dependencies": {
     "@audapolis/webvtt-writer": "^1.0.6",
-    "@automerge/automerge": "~2.2.0",
-    "@automerge/automerge-wasm": "^0.15.0",
     "@fontsource/inter": "^4.5.15",
     "@podlove/html5-audio-driver": "^2.0.3",
     "@popperjs/core": "^2.11.8",
     "@zip.js/zip.js": "^2.7.31",
     "clsx": "^1.2.1",
+    "fast-equals": "^5.2.2",
+    "loro-crdt": "^1.4.4",
     "openapi-typescript-fetch": "github:bugbakery/openapi-typescript-fetch#4b5cc33983c5a658feedd628d417599db2bd672c",
     "react": "^18.2.0",
     "react-base16-styling": "^0.9.1",
@@ -77,7 +78,6 @@
     "react-popper": "^2.3.0",
     "reconnecting-websocket": "^4.4.0",
     "slate": "^0.94.1",
-    "slate-automerge-doc": "github:bugbakery/slate-automerge-doc#98d7a78f14cb4a16484847e0090f6b5439cc16b6",
     "slate-history": "^0.93.0",
     "slate-react": "^0.94.2",
     "swr": "^2.2.4",

frontend/package.json

@@ -1,54 +1,58 @@
-import { Document } from './editor/types';
-import { next as Automerge } from '@automerge/automerge';
-import * as AutomergeStable from '@automerge/automerge';
+import { Document, EditorDocument } from './editor/types';
+// import { next as Automerge } from '@automerge/automerge';
+// import * as AutomergeStable from '@automerge/automerge';
 
-function convertString(s: string): string {
-  // this typecasting is a hack to avoid having multiple types for different versions for now
-  return new AutomergeStable.Text(s.toString()) as unknown as string;
+export function documentToJSON(doc: EditorDocument): Document {
+  return doc.toJSON().root;
 }
 
-export function migrateDocument(doc: Automerge.Doc<Document>): Automerge.Doc<Document> {
-  let theDoc = doc;
-  const v1 = theDoc.version === 1;
-  const actorID = Automerge.getActorId(doc);
-  if (v1) {
-    theDoc = AutomergeStable.load(Automerge.save(doc), actorID);
-  }
-  if (theDoc.version === 2) {
-    return doc;
-  }
-  const migratedDoc = AutomergeStable.change(theDoc, (doc: Document) => {
-    switch (doc.version) {
-      case 1:
-        for (const speakerID of Object.keys(doc.speaker_names)) {
-          doc.speaker_names[speakerID] = convertString(doc.speaker_names[speakerID]);
-        }
+// function convertString(s: string): string {
+//   // this typecasting is a hack to avoid having multiple types for different versions for now
+//   return new AutomergeStable.Text(s.toString()) as unknown as string;
+// }
 
-        doc.children.forEach((paragraph) => {
-          paragraph.type = convertString(paragraph.type) as 'paragraph';
-          paragraph.speaker = paragraph.speaker ? convertString(paragraph.speaker) : null;
-          paragraph.lang = convertString(paragraph.lang.toString());
+// export function migrateDocument(doc: Automerge.Doc<Document>): Automerge.Doc<Document> {
+//   let theDoc = doc;
+//   const v1 = theDoc.version === 1;
+//   const actorID = Automerge.getActorId(doc);
+//   if (v1) {
+//     theDoc = AutomergeStable.load(Automerge.save(doc), actorID);
+//   }
+//   if (theDoc.version === 2) {
+//     return doc;
+//   }
+//   const migratedDoc = AutomergeStable.change(theDoc, (doc: Document) => {
+//     switch (doc.version) {
+//       case 1:
+//         for (const speakerID of Object.keys(doc.speaker_names)) {
+//           doc.speaker_names[speakerID] = convertString(doc.speaker_names[speakerID]);
+//         }
 
-          paragraph.children.forEach((child) => {
-            const start = child.start;
-            child.start = start;
-            const end = child.end;
-            child.end = end;
-            const conf = child.conf;
-            child.conf = conf;
-            child.text = convertString(child.text.toString());
-          });
-        });
-        doc.version = 2;
-      // falls through
-      case 2:
-        break;
-    }
-  });
+//         doc.children.forEach((paragraph) => {
+//           paragraph.type = convertString(paragraph.type) as 'paragraph';
+//           paragraph.speaker = paragraph.speaker ? convertString(paragraph.speaker) : null;
+//           paragraph.lang = convertString(paragraph.lang.toString());
 
-  if (v1) {
-    return Automerge.load(AutomergeStable.save(migratedDoc), actorID);
-  } else {
-    return migratedDoc;
-  }
-}
+//           paragraph.children.forEach((child) => {
+//             const start = child.start;
+//             child.start = start;
+//             const end = child.end;
+//             child.end = end;
+//             const conf = child.conf;
+//             child.conf = conf;
+//             child.text = convertString(child.text.toString());
+//           });
+//         });
+//         doc.version = 2;
+//       // falls through
+//       case 2:
+//         break;
+//     }
+//   });
+
+//   if (v1) {
+//     return Automerge.load(AutomergeStable.save(migratedDoc), actorID);
+//   } else {
+//     return migratedDoc;
+//   }
+// }