Fix Knex transaction errors in prod

.env.example

@@ -29,3 +29,12 @@ PORT=3333
 
 # These have defaults hard-coded, but are being overridden
 CACHE_VIEWS=false
+
+# SQLite durability/concurrency notes:
+# - WAL improves concurrent reads/writes for the default SQLite deployment.
+# - FULL is the safer default for production durability.
+# - NORMAL reduces fsync pressure, but may lose the last committed transactions
+#   on sudden power loss or kernel crash.
+DB_SQLITE_JOURNAL_MODE=WAL
+DB_SQLITE_SYNCHRONOUS=FULL
+DB_BUSY_TIMEOUT=5000

README.md

@@ -117,6 +117,9 @@ Ferdium-server's configuration is saved inside an `.env` file. Besides AdonisJS'
 - `CONNECT_WITH_FRANZ` (`true` or `false`, default: `true`): Whether to enable connections to the Franz server. By enabling this option, Ferdium-server can:
   - Show the full Franz recipe library instead of only custom recipes
   - Import Franz accounts
+- `DB_SQLITE_JOURNAL_MODE` (`DELETE`, `TRUNCATE`, `PERSIST`, `MEMORY`, `WAL`, or `OFF`, default: `WAL`): SQLite journal mode. `WAL` improves concurrency for the default single-file deployment.
+- `DB_SQLITE_SYNCHRONOUS` (`OFF`, `NORMAL`, `FULL`, or `EXTRA`, default: `FULL`): SQLite synchronous mode. `FULL` is the safer default for production durability. `NORMAL` reduces write latency but can lose the last committed transactions on sudden power loss.
+- `DB_BUSY_TIMEOUT` (milliseconds, default: `5000`): How long SQLite should wait for a locked database before failing a query.
 </details>
 <details>
 <summary>Importing your Franz/Ferdi account</summary>

app/Controllers/Http/Dashboard/DataController.ts

@@ -1,4 +1,6 @@
 import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
+import Service from 'App/Models/Service';
+import Workspace from 'App/Models/Workspace';
 
 export default class DataController {
   /**
@@ -7,8 +9,16 @@ export default class DataController {
   public async show({ view, auth }: HttpContextContract) {
     const { user } = auth;
 
-    const services = await user?.related('services').query();
-    const workspaces = await user?.related('workspaces').query();
+    let services: Service[] = [];
+    let workspaces: Workspace[] = [];
+
+    if (user) {
+      services = await Service.query().where('userId', user.id).orderBy('id', 'asc');
+      workspaces = await Workspace.query()
+        .where('userId', user.id)
+        .orderBy('order', 'asc')
+        .orderBy('id', 'asc');
+    }
 
     return view.render('dashboard/data', {
       username: user?.username,

app/Controllers/Http/Dashboard/ExportController.ts

@@ -1,4 +1,6 @@
 import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
+import Service from 'App/Models/Service';
+import Workspace from 'App/Models/Workspace';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function deepParseToJSON(obj: any): Record<string, unknown> {
@@ -37,8 +39,13 @@ export default class ExportController {
    */
   public async show({ auth, response }: HttpContextContract) {
     const user = auth.user!;
-    const services = await user.related('services').query();
-    const workspaces = await user.related('workspaces').query();
+    const services = await Service.query()
+      .where('userId', user.id)
+      .orderBy('id', 'asc');
+    const workspaces = await Workspace.query()
+      .where('userId', user.id)
+      .orderBy('order', 'asc')
+      .orderBy('id', 'asc');
 
     const exportData = {
       username: user.username,

app/Controllers/Http/ServiceController.ts

@@ -13,6 +13,10 @@ const createSchema = schema.create({
   recipeId: schema.string(),
 });
 
+async function loadUserServices(userId: number) {
+  return Service.query().where('userId', userId).orderBy('id', 'asc');
+}
+
 export default class ServiceController {
   // Create a new service for user
   public async create({ request, response, auth }: HttpContextContract) {
@@ -85,7 +89,7 @@ export default class ServiceController {
     }
 
     const { id } = user;
-    const services = await user.related('services').query();
+    const services = await loadUserServices(user.id);
 
     // Convert to array with all data Franz wants
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -94,6 +98,11 @@ export default class ServiceController {
         typeof service.settings === 'string'
           ? JSON.parse(service.settings)
           : service.settings;
+      // eslint-disable-next-line unicorn/no-null
+      let iconUrl: string | null = null;
+      if (settings.iconId) {
+        iconUrl = `${url}/v1/icon/${settings.iconId}`;
+      }
 
       return {
         customRecipe: false,
@@ -107,10 +116,7 @@ export default class ServiceController {
         spellcheckerLanguage: '',
         workspaces: [],
         ...settings,
-        iconUrl: settings.iconId
-          ? `${url}/v1/icon/${settings.iconId}`
-          : // eslint-disable-next-line unicorn/no-null
-          null,
+        iconUrl,
         id: service.serviceId,
         name: service.name,
         recipeId: service.recipeId,
@@ -309,14 +315,19 @@ export default class ServiceController {
     }
 
     // Get new services
-    const services = await user.related('services').query();
+    const services = await loadUserServices(user.id);
     // Convert to array with all data Franz wants
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const servicesArray = services.map((service: any) => {
       const settings =
         typeof service.settings === 'string'
           ? JSON.parse(service.settings)
           : service.settings;
+      // eslint-disable-next-line unicorn/no-null
+      let iconUrl: string | null = null;
+      if (settings.iconId) {
+        iconUrl = `${url}/v1/icon/${settings.iconId}`;
+      }
 
       return {
         customRecipe: false,
@@ -330,10 +341,7 @@ export default class ServiceController {
         spellcheckerLanguage: '',
         workspaces: [],
         ...settings,
-        iconUrl: settings.iconId
-          ? `${url}/v1/icon/${settings.iconId}`
-          : // eslint-disable-next-line unicorn/no-null
-          null,
+        iconUrl,
         id: service.serviceId,
         name: service.name,
         recipeId: service.recipeId,

app/Controllers/Http/WorkspaceController.ts

@@ -1,5 +1,6 @@
 import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext';
 import { validator, schema } from '@ioc:Adonis/Core/Validator';
+import Database from '@ioc:Adonis/Lucid/Database';
 import Workspace from 'App/Models/Workspace';
 import { v4 as uuid } from 'uuid';
 
@@ -38,16 +39,21 @@ export default class WorkspaceController {
     }
 
     // Get new, unused uuid
-    let workspaceId;
+    let workspaceId: string;
+    let existingWorkspace: Workspace | null;
     do {
       workspaceId = uuid();
-    } while (
-      // eslint-disable-next-line unicorn/no-await-expression-member, no-await-in-loop
-      (await Workspace.query().where('workspaceId', workspaceId)).length > 0
-    );
+      // eslint-disable-next-line no-await-in-loop
+      existingWorkspace = await Workspace.query()
+        .where('workspaceId', workspaceId)
+        .first();
+    } while (existingWorkspace);
 
-    // eslint-disable-next-line unicorn/no-await-expression-member
-    const order = (await user.related('workspaces').query()).length;
+    const workspaceCount = await Database.from('workspaces')
+      .where('userId', user.id)
+      .count('* as total')
+      .first();
+    const order = Number(workspaceCount?.total ?? 0);
 
     await Workspace.create({
       userId: user.id,
@@ -163,7 +169,10 @@ export default class WorkspaceController {
       return response.unauthorized('Missing or invalid api token');
     }
 
-    const workspaces = await user.related('workspaces').query();
+    const workspaces = await Workspace.query()
+      .where('userId', user.id)
+      .orderBy('order', 'asc')
+      .orderBy('id', 'asc');
     // Convert to array with all data Franz wants
     let workspacesArray: object[] = [];
     if (workspaces) {

app/Models/User.ts

@@ -88,9 +88,8 @@ export default class User extends BaseModel {
   }
 
   private async generateToken(user: User, type: string): Promise<string> {
-    const query = user
-      .related('tokens')
-      .query()
+    const query = Token.query()
+      .where('user_id', user.id)
       .where('type', type)
       .where('is_revoked', false)
       .where(
@@ -106,7 +105,7 @@ export default class User extends BaseModel {
 
     const token = Encryption.encrypt(randtoken.generate(16));
 
-    await user.related('tokens').create({ type, token });
+    await Token.create({ user_id: user.id, type, token });
 
     return token;
   }

config/database.ts

@@ -10,6 +10,99 @@ import path from 'node:path';
 import Env from '@ioc:Adonis/Core/Env';
 import { DatabaseConfig } from '@ioc:Adonis/Lucid/Database';
 
+interface SqliteConnection {
+  run: (query: string, callback: (error: Error | null) => void) => void;
+}
+
+type SqlitePoolCallback = (
+  error: Error | null,
+  connection: SqliteConnection,
+) => void;
+
+const SQLITE_JOURNAL_MODE_VALUES = new Set([
+  'DELETE',
+  'TRUNCATE',
+  'PERSIST',
+  'MEMORY',
+  'WAL',
+  'OFF',
+]);
+const SQLITE_SYNCHRONOUS_VALUES = new Set(['OFF', 'NORMAL', 'FULL', 'EXTRA']);
+const SQLITE_BUSY_TIMEOUT_DEFAULT = 5000;
+const SQLITE_JOURNAL_MODE_DEFAULT = 'WAL';
+const SQLITE_SYNCHRONOUS_DEFAULT = 'FULL';
+
+function getSqlitePragmaValue(
+  envName: string,
+  defaultValue: string,
+  allowedValues: Set<string>,
+) {
+  const configuredValue = Env.get(envName, defaultValue).trim().toUpperCase();
+
+  if (!allowedValues.has(configuredValue)) {
+    return defaultValue;
+  }
+
+  return configuredValue;
+}
+
+const sqliteBusyTimeoutEnv = Env.get(
+  'DB_BUSY_TIMEOUT',
+  SQLITE_BUSY_TIMEOUT_DEFAULT.toString(),
+);
+const sqliteBusyTimeoutParsed = Number.parseInt(sqliteBusyTimeoutEnv, 10);
+const sqliteBusyTimeout =
+  Number.isFinite(sqliteBusyTimeoutParsed) && sqliteBusyTimeoutParsed > 0
+    ? sqliteBusyTimeoutParsed
+    : SQLITE_BUSY_TIMEOUT_DEFAULT;
+const sqliteJournalMode = getSqlitePragmaValue(
+  'DB_SQLITE_JOURNAL_MODE',
+  SQLITE_JOURNAL_MODE_DEFAULT,
+  SQLITE_JOURNAL_MODE_VALUES,
+);
+const sqliteSynchronous = getSqlitePragmaValue(
+  'DB_SQLITE_SYNCHRONOUS',
+  SQLITE_SYNCHRONOUS_DEFAULT,
+  SQLITE_SYNCHRONOUS_VALUES,
+);
+
+function configureSqliteConnection(
+  conn: SqliteConnection,
+  cb: SqlitePoolCallback,
+) {
+  return conn.run('PRAGMA foreign_keys = ON', (error: Error | null) => {
+    if (error) {
+      cb(error, conn);
+      return;
+    }
+
+    conn.run(
+      `PRAGMA journal_mode = ${sqliteJournalMode}`,
+      (journalModeError: Error | null) => {
+        if (journalModeError) {
+          cb(journalModeError, conn);
+          return;
+        }
+
+        conn.run(
+          `PRAGMA synchronous = ${sqliteSynchronous}`,
+          (synchronousError: Error | null) => {
+            if (synchronousError) {
+              cb(synchronousError, conn);
+              return;
+            }
+
+            conn.run(
+              `PRAGMA busy_timeout = ${sqliteBusyTimeout}`,
+              (busyTimeoutError: Error | null) => cb(busyTimeoutError, conn),
+            );
+          },
+        );
+      },
+    );
+  });
+}
+
 const databaseConfig: DatabaseConfig = {
   /*
   |--------------------------------------------------------------------------
@@ -44,11 +137,11 @@ const databaseConfig: DatabaseConfig = {
         ),
       },
       pool: {
-        afterCreate: (conn, cb) => {
-          conn.run('PRAGMA foreign_keys=true', cb);
-        },
-        min: Number.parseInt(Env.get('DB_POOL_MIN', '2'), 10),
-        max: Number.parseInt(Env.get('DB_POOL_MAX', '20'), 10),
+        afterCreate: configureSqliteConnection,
+        // SQLite is a single-file database. Keeping one shared connection
+        // avoids starving the pool with blocked readers/writers under load.
+        min: 1,
+        max: 1,
       },
       migrations: {
         naturalSort: true,

data/.gitignore

@@ -1 +1,4 @@
 ferdium.sqlite
+ferdium.sqlite-shm
+ferdium.sqlite-wal
+ferdium-*.log

database/migrations/1774667000000_add_database_hot_path_indexes.ts

@@ -0,0 +1,39 @@
+import BaseSchema from '@ioc:Adonis/Lucid/Schema';
+
+export default class extends BaseSchema {
+  public async up(): Promise<void> {
+    this.schema.alterTable('services', table => {
+      table.index(['userId'], 'services_user_id_index');
+      table.index(['serviceId'], 'services_service_id_index');
+    });
+
+    this.schema.alterTable('workspaces', table => {
+      table.index(['userId'], 'workspaces_user_id_index');
+    });
+
+    this.schema.alterTable('tokens', table => {
+      table.index(
+        ['user_id', 'type', 'is_revoked', 'updated_at'],
+        'tokens_user_id_type_is_revoked_updated_at_index',
+      );
+    });
+  }
+
+  public async down(): Promise<void> {
+    this.schema.alterTable('tokens', table => {
+      table.dropIndex(
+        ['user_id', 'type', 'is_revoked', 'updated_at'],
+        'tokens_user_id_type_is_revoked_updated_at_index',
+      );
+    });
+
+    this.schema.alterTable('workspaces', table => {
+      table.dropIndex(['userId'], 'workspaces_user_id_index');
+    });
+
+    this.schema.alterTable('services', table => {
+      table.dropIndex(['serviceId'], 'services_service_id_index');
+      table.dropIndex(['userId'], 'services_user_id_index');
+    });
+  }
+}