Merge branch 'develop' into renovate/eslint-plugin-playwright-2.x

Author: eliandoran

.github/workflows/dev.yml

@@ -39,7 +39,7 @@ jobs:
 
       - uses: nrwl/nx-set-shas@v4
       - name: Check affected
-        run: pnpm nx affected -t rebuild-deps
+        run: pnpm nx affected -t build rebuild-deps
 
   report-electron-size:
     name: Report Electron size

README.md

@@ -8,7 +8,7 @@ TriliumNext Notes is an open-source, cross-platform hierarchical note taking app
 
 See [screenshots](https://triliumnext.github.io/Docs/Wiki/screenshot-tour) for quick overview:
 
-<a href="https://triliumnext.github.io/Docs/Wiki/screenshot-tour"><img src="https://github.com/TriliumNext/Notes/blob/develop/images/screenshots/app.png?raw=true" alt="Trilium Screenshot" width="1000"></a>
+<a href="https://triliumnext.github.io/Docs/Wiki/screenshot-tour"><img src="./docs/app.png" alt="Trilium Screenshot" width="1000"></a>
 
 ## ⚠️ Why TriliumNext?
 

_regroup/package.json

@@ -35,7 +35,7 @@
     "chore:generate-openapi": "tsx bin/generate-openapi.js"
   },
   "devDependencies": {    
-    "@playwright/test": "1.51.1",
+    "@playwright/test": "1.52.0",
     "@stylistic/eslint-plugin": "4.2.0",        
     "@types/express": "5.0.1",    
     "@types/node": "22.15.3",    

apps/db-compare/README.md

@@ -0,0 +1,17 @@
+# Database compare tool
+
+> [!IMPORTANT]
+> The original implementation was signficantly out of date. While we have made the effort of updating dependencies and getting it to run, currently it only compares the old database structure (v214).
+
+To build and run manually:
+
+```sh
+nx build db-compare
+node ./apps/db-compare/dist/compare.js
+```
+
+To serve development build with arguments:
+
+```sh
+nx serve db-compare --args "apps/server/spec/db/document_v214.db" --args "apps/server/spec/db/document_v214_migrated.db"
+```

apps/db-compare/eslint.config.mjs

@@ -0,0 +1,5 @@
+import baseConfig from "../../eslint.config.mjs";
+
+export default [
+    ...baseConfig
+];

apps/db-compare/package.json

@@ -0,0 +1,75 @@
+{
+  "name": "@triliumnext/db-compare",
+  "version": "0.0.1",
+  "private": true,
+  "description": "Tool to compare content of Trilium databases. Useful for debugging sync problems.",
+  "dependencies": {
+    "colors": "1.4.0",
+    "diff": "7.0.0",
+    "sqlite": "5.1.1",
+    "sqlite3": "5.1.5"
+  },
+  "nx": {
+    "name": "db-compare",
+    "targets": {
+      "build": {
+        "executor": "@nx/esbuild:esbuild",
+        "outputs": [
+          "{options.outputPath}"
+        ],
+        "defaultConfiguration": "production",
+        "options": {
+          "platform": "node",
+          "outputPath": "apps/db-compare/dist",
+          "format": [
+            "cjs"
+          ],
+          "bundle": false,
+          "main": "apps/db-compare/src/compare.ts",
+          "tsConfig": "apps/db-compare/tsconfig.app.json",
+          "assets": [],
+          "esbuildOptions": {
+            "sourcemap": true,
+            "outExtension": {
+              ".js": ".js"
+            }
+          }
+        },
+        "configurations": {
+          "development": {},
+          "production": {
+            "esbuildOptions": {
+              "sourcemap": false,
+              "outExtension": {
+                ".js": ".js"
+              }
+            }
+          }
+        }
+      },
+      "serve": {
+        "executor": "@nx/js:node",
+        "defaultConfiguration": "development",
+        "dependsOn": [
+          "build"
+        ],
+        "options": {
+          "buildTarget": "db-compare:build",
+          "runBuildTargetDependencies": false
+        },
+        "configurations": {
+          "development": {
+            "buildTarget": "db-compare:build:development"
+          },
+          "production": {
+            "buildTarget": "db-compare:build:production"
+          }
+        }
+      }
+    }
+  },
+  "devDependencies": {
+    "@types/colors": "1.2.4",
+    "@types/diff": "^7.0.2"
+  }
+}

apps/db-compare/src/compare.ts

@@ -0,0 +1,137 @@
+"use strict";
+
+import * as jsDiff from "diff";
+import * as sqlite from "sqlite";
+import * as sqlite3 from "sqlite3";
+import sql from "./sql.js";
+
+import "colors";
+import path from "path";
+
+function printDiff(one: string, two: string) {
+    const diff = jsDiff.diffChars(one, two);
+
+    diff.forEach(function(part){
+        // green for additions, red for deletions
+        // grey for common parts
+        const color = part.added ? 'green' :
+            part.removed ? 'red' : 'grey';
+        process.stderr.write(part.value[color]);
+    });
+
+    console.log("");
+}
+
+function checkMissing(table: string, name: string, ids1: string[], ids2: string[]) {
+    const missing = ids1.filter(item => ids2.indexOf(item) < 0);
+
+    if (missing.length > 0) {
+        console.log("Missing IDs from " + name + " table " + table + ": ", missing);
+    }
+}
+
+function handleBuffer(obj: { content: Buffer | string }) {
+    if (obj && Buffer.isBuffer(obj.content)) {
+        obj.content = obj.content.toString();
+    }
+
+    return obj;
+}
+
+function compareRows(table: string, rsLeft: Record<string, any>, rsRight: Record<string, any>, column: string) {
+    const leftIds = Object.keys(rsLeft);
+    const rightIds = Object.keys(rsRight);
+
+    console.log("");
+    console.log("--------------------------------------------------------");
+    console.log(`${table} - ${leftIds.length}/${rightIds.length}`);
+
+    checkMissing(table, "right", leftIds, rightIds);
+    checkMissing(table, "left", rightIds, leftIds);
+
+    const commonIds = leftIds.filter(item => rightIds.includes(item));
+
+    for (const id of commonIds) {
+        const valueLeft = handleBuffer(rsLeft[id]);
+        const valueRight = handleBuffer(rsRight[id]);
+
+        const left = JSON.stringify(valueLeft, null, 2);
+        const right = JSON.stringify(valueRight, null, 2);
+
+        if (left !== right) {
+            console.log("Table " + table + " row with " + column + "=" + id + " differs:");
+            console.log("Left: ", left);
+            console.log("Right: ", right);
+            printDiff(left, right);
+        }
+    }
+}
+
+async function main() {
+    const dbLeftPath = process.argv.at(-2);
+    const dbRightPath = process.argv.at(-1);
+
+    if (process.argv.length < 4 || !dbLeftPath || !dbRightPath) {
+        console.log(`Usage: ${process.argv[0]} ${process.argv[1]} path/to/first.db path/to/second.db`);
+        process.exit(1);
+    }
+
+    let dbLeft: sqlite.Database;
+    let dbRight: sqlite.Database;
+
+    try {
+        dbLeft = await sqlite.open({filename: dbLeftPath, driver: sqlite3.Database});
+    } catch (e: any) {
+        console.error(`Could not load first database at ${path.resolve(dbRightPath)} due to: ${e.message}`);
+        process.exit(2);
+    }
+
+    try {
+        dbRight = await sqlite.open({filename: dbRightPath, driver: sqlite3.Database});
+    } catch (e: any) {
+        console.error(`Could not load second database at ${path.resolve(dbRightPath)} due to: ${e.message}`);
+        process.exit(3);
+    }
+
+    async function compare(table: string, column: string, query: string) {
+        const rsLeft = await sql.getIndexed(dbLeft, column, query);
+        const rsRight = await sql.getIndexed(dbRight, column, query);
+
+        compareRows(table, rsLeft, rsRight, column);
+    }
+
+    await compare("branches", "branchId",
+        "SELECT branchId, noteId, parentNoteId, notePosition, utcDateModified, isDeleted, prefix FROM branches");
+
+    await compare("notes", "noteId",
+        "SELECT noteId, title, dateCreated, utcDateCreated, isProtected, isDeleted FROM notes WHERE isDeleted = 0");
+
+    await compare("note_contents", "noteId",
+       "SELECT note_contents.noteId, note_contents.content FROM note_contents JOIN notes USING(noteId) WHERE isDeleted = 0");
+
+    await compare("note_revisions", "noteRevisionId",
+        "SELECT noteRevisionId, noteId, title, dateCreated, dateLastEdited, utcDateCreated, utcDateLastEdited, isProtected FROM note_revisions");
+
+    await compare("note_revision_contents", "noteRevisionId",
+        "SELECT noteRevisionId, content FROM note_revision_contents");
+
+    await compare("options", "name",
+            `SELECT name, value, utcDateModified FROM options WHERE isSynced = 1`);
+
+    await compare("attributes", "attributeId",
+        "SELECT attributeId, noteId, type, name, value FROM attributes");
+
+    await compare("etapi_tokens", "etapiTokenId",
+        "SELECT etapiTokenId, name, tokenHash, utcDateCreated, utcDateModified, isDeleted FROM etapi_tokens");
+
+    await compare("entity_changes", "uniqueId",
+        "SELECT entityName || '-' || entityId AS uniqueId, hash, isErased, utcDateChanged FROM entity_changes WHERE isSynced = 1");
+}
+
+(async () => {
+    try {
+        await main();
+    } catch (e) {
+        console.error(e);
+    }
+})();

apps/db-compare/src/sql.ts

@@ -0,0 +1,90 @@
+"use strict";
+
+import type { Database } from "sqlite";
+
+async function getSingleResult(db: Database, query: string, params: any[] = []) {
+    return await wrap(db, async db => db.get(query, ...params));
+}
+
+async function getSingleResultOrNull(db: Database, query: string, params: any[] = []) {
+    const all = await wrap(db, async db => db.all(query, ...params));
+
+    return all.length > 0 ? all[0] : null;
+}
+
+async function getSingleValue(db: Database, query: string, params: any[] = []) {
+    const row = await getSingleResultOrNull(db, query, params);
+
+    if (!row) {
+        return null;
+    }
+
+    return row[Object.keys(row)[0]];
+}
+
+async function getResults(db: Database, query: string, params: any[] = []) {
+    return await wrap(db, async db => db.all(query, ...params));
+}
+
+async function getIndexed(db: Database, column: string, query: string, params: any[] = []) {
+    const results = await getResults(db, query, params);
+
+    const map: Record<string, any> = {};
+
+    for (const row of results) {
+        map[row[column]] = row;
+    }
+
+    return map;
+}
+
+async function getMap(db: Database, query: string, params: any[] = []) {
+    const map: Record<string, any> = {};
+    const results = await getResults(db, query, params);
+
+    for (const row of results) {
+        const keys = Object.keys(row);
+
+        map[row[keys[0]]] = row[keys[1]];
+    }
+
+    return map;
+}
+
+async function getFlattenedResults(db: Database, key: string, query: string, params: any[] = []) {
+    const list = [];
+    const result = await getResults(db, query, params);
+
+    for (const row of result) {
+        list.push(row[key]);
+    }
+
+    return list;
+}
+
+async function execute(db: Database, query: string, params: any[] = []) {
+    return await wrap(db, async db => db.run(query, ...params));
+}
+
+async function wrap<T>(db: Database, func: (db: Database) => Promise<T>) {
+    const thisError = new Error();
+
+    try {
+        return await func(db);
+    } catch (e: any) {
+        console.error("Error executing query. Inner exception: " + e.stack + thisError.stack);
+
+        throw thisError;
+    }
+}
+
+export default {
+    getSingleValue,
+    getSingleResult,
+    getSingleResultOrNull,
+    getResults,
+    getIndexed,
+    getMap,
+    getFlattenedResults,
+    execute
+};

apps/db-compare/tsconfig.app.json

@@ -0,0 +1,20 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "types": [
+      "node"
+    ],
+    "rootDir": "src",
+    "tsBuildInfoFile": "dist/tsconfig.app.tsbuildinfo",
+    "verbatimModuleSyntax": false
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "eslint.config.js",
+    "eslint.config.cjs",
+    "eslint.config.mjs"
+  ]
+}

apps/db-compare/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.app.json"
+    }
+  ]
+}