From 0cac5f0383ea6cd23f3b8c90fe2da12e9b1b050e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Oliveira?= Date: Sat, 28 Mar 2026 03:42:33 +0000 Subject: [PATCH 1/3] Fix Knex transaction errors in prod --- .../Http/Dashboard/DataController.ts | 14 ++++- .../Http/Dashboard/ExportController.ts | 11 +++- app/Controllers/Http/ServiceController.ts | 28 +++++---- app/Controllers/Http/WorkspaceController.ts | 23 +++++--- app/Models/User.ts | 7 +-- config/database.ts | 58 +++++++++++++++++-- ...667000000_add_database_hot_path_indexes.ts | 39 +++++++++++++ tests/functional/dashboard/data.spec.ts | 16 +++++ 8 files changed, 166 insertions(+), 30 deletions(-) create mode 100644 database/migrations/1774667000000_add_database_hot_path_indexes.ts diff --git a/app/Controllers/Http/Dashboard/DataController.ts b/app/Controllers/Http/Dashboard/DataController.ts index f77702f0..a4d04566 100644 --- a/app/Controllers/Http/Dashboard/DataController.ts +++ b/app/Controllers/Http/Dashboard/DataController.ts @@ -1,4 +1,6 @@ import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext'; +import Service from 'App/Models/Service'; +import Workspace from 'App/Models/Workspace'; export default class DataController { /** @@ -7,8 +9,16 @@ export default class DataController { public async show({ view, auth }: HttpContextContract) { const { user } = auth; - const services = await user?.related('services').query(); - const workspaces = await user?.related('workspaces').query(); + let services: Service[] = []; + let workspaces: Workspace[] = []; + + if (user) { + services = await Service.query().where('userId', user.id).orderBy('id', 'asc'); + workspaces = await Workspace.query() + .where('userId', user.id) + .orderBy('order', 'asc') + .orderBy('id', 'asc'); + } return view.render('dashboard/data', { username: user?.username, diff --git a/app/Controllers/Http/Dashboard/ExportController.ts b/app/Controllers/Http/Dashboard/ExportController.ts index 7155eab7..ac8b19a3 100644 --- a/app/Controllers/Http/Dashboard/ExportController.ts +++ b/app/Controllers/Http/Dashboard/ExportController.ts @@ -1,4 +1,6 @@ import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext'; +import Service from 'App/Models/Service'; +import Workspace from 'App/Models/Workspace'; // eslint-disable-next-line @typescript-eslint/no-explicit-any function deepParseToJSON(obj: any): Record { @@ -37,8 +39,13 @@ export default class ExportController { */ public async show({ auth, response }: HttpContextContract) { const user = auth.user!; - const services = await user.related('services').query(); - const workspaces = await user.related('workspaces').query(); + const services = await Service.query() + .where('userId', user.id) + .orderBy('id', 'asc'); + const workspaces = await Workspace.query() + .where('userId', user.id) + .orderBy('order', 'asc') + .orderBy('id', 'asc'); const exportData = { username: user.username, diff --git a/app/Controllers/Http/ServiceController.ts b/app/Controllers/Http/ServiceController.ts index 76e72e4b..e0b93e93 100644 --- a/app/Controllers/Http/ServiceController.ts +++ b/app/Controllers/Http/ServiceController.ts @@ -13,6 +13,10 @@ const createSchema = schema.create({ recipeId: schema.string(), }); +async function loadUserServices(userId: number) { + return Service.query().where('userId', userId).orderBy('id', 'asc'); +} + export default class ServiceController { // Create a new service for user public async create({ request, response, auth }: HttpContextContract) { @@ -85,7 +89,7 @@ export default class ServiceController { } const { id } = user; - const services = await user.related('services').query(); + const services = await loadUserServices(user.id); // Convert to array with all data Franz wants // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -94,6 +98,11 @@ export default class ServiceController { typeof service.settings === 'string' ? JSON.parse(service.settings) : service.settings; + // eslint-disable-next-line unicorn/no-null + let iconUrl: string | null = null; + if (settings.iconId) { + iconUrl = `${url}/v1/icon/${settings.iconId}`; + } return { customRecipe: false, @@ -107,10 +116,7 @@ export default class ServiceController { spellcheckerLanguage: '', workspaces: [], ...settings, - iconUrl: settings.iconId - ? `${url}/v1/icon/${settings.iconId}` - : // eslint-disable-next-line unicorn/no-null - null, + iconUrl, id: service.serviceId, name: service.name, recipeId: service.recipeId, @@ -309,7 +315,7 @@ export default class ServiceController { } // Get new services - const services = await user.related('services').query(); + const services = await loadUserServices(user.id); // Convert to array with all data Franz wants // eslint-disable-next-line @typescript-eslint/no-explicit-any const servicesArray = services.map((service: any) => { @@ -317,6 +323,11 @@ export default class ServiceController { typeof service.settings === 'string' ? JSON.parse(service.settings) : service.settings; + // eslint-disable-next-line unicorn/no-null + let iconUrl: string | null = null; + if (settings.iconId) { + iconUrl = `${url}/v1/icon/${settings.iconId}`; + } return { customRecipe: false, @@ -330,10 +341,7 @@ export default class ServiceController { spellcheckerLanguage: '', workspaces: [], ...settings, - iconUrl: settings.iconId - ? `${url}/v1/icon/${settings.iconId}` - : // eslint-disable-next-line unicorn/no-null - null, + iconUrl, id: service.serviceId, name: service.name, recipeId: service.recipeId, diff --git a/app/Controllers/Http/WorkspaceController.ts b/app/Controllers/Http/WorkspaceController.ts index 70af343e..2d3eb935 100644 --- a/app/Controllers/Http/WorkspaceController.ts +++ b/app/Controllers/Http/WorkspaceController.ts @@ -1,5 +1,6 @@ import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext'; import { validator, schema } from '@ioc:Adonis/Core/Validator'; +import Database from '@ioc:Adonis/Lucid/Database'; import Workspace from 'App/Models/Workspace'; import { v4 as uuid } from 'uuid'; @@ -39,15 +40,20 @@ export default class WorkspaceController { // Get new, unused uuid let workspaceId; + let existingWorkspace; do { workspaceId = uuid(); - } while ( - // eslint-disable-next-line unicorn/no-await-expression-member, no-await-in-loop - (await Workspace.query().where('workspaceId', workspaceId)).length > 0 - ); + // eslint-disable-next-line no-await-in-loop + existingWorkspace = await Workspace.query() + .where('workspaceId', workspaceId) + .first(); + } while (existingWorkspace); - // eslint-disable-next-line unicorn/no-await-expression-member - const order = (await user.related('workspaces').query()).length; + const workspaceCount = await Database.from('workspaces') + .where('userId', user.id) + .count('* as total') + .first(); + const order = Number(workspaceCount?.total ?? 0); await Workspace.create({ userId: user.id, @@ -163,7 +169,10 @@ export default class WorkspaceController { return response.unauthorized('Missing or invalid api token'); } - const workspaces = await user.related('workspaces').query(); + const workspaces = await Workspace.query() + .where('userId', user.id) + .orderBy('order', 'asc') + .orderBy('id', 'asc'); // Convert to array with all data Franz wants let workspacesArray: object[] = []; if (workspaces) { diff --git a/app/Models/User.ts b/app/Models/User.ts index 0b8e688a..b501dd6b 100644 --- a/app/Models/User.ts +++ b/app/Models/User.ts @@ -88,9 +88,8 @@ export default class User extends BaseModel { } private async generateToken(user: User, type: string): Promise { - const query = user - .related('tokens') - .query() + const query = Token.query() + .where('user_id', user.id) .where('type', type) .where('is_revoked', false) .where( @@ -106,7 +105,7 @@ export default class User extends BaseModel { const token = Encryption.encrypt(randtoken.generate(16)); - await user.related('tokens').create({ type, token }); + await Token.create({ user_id: user.id, type, token }); return token; } diff --git a/config/database.ts b/config/database.ts index eb5f5235..9e19e35c 100644 --- a/config/database.ts +++ b/config/database.ts @@ -10,6 +10,54 @@ import path from 'node:path'; import Env from '@ioc:Adonis/Core/Env'; import { DatabaseConfig } from '@ioc:Adonis/Lucid/Database'; +interface SqliteConnection { + run: (query: string, callback: (error: Error | null) => void) => void; +} + +type SqlitePoolCallback = ( + error: Error | null, + connection: SqliteConnection, +) => void; + +const sqliteBusyTimeout = Number.parseInt( + Env.get('DB_BUSY_TIMEOUT', '5000'), + 10, +); + +function configureSqliteConnection( + conn: SqliteConnection, + cb: SqlitePoolCallback, +) { + return conn.run('PRAGMA foreign_keys = ON', (error: Error | null) => { + if (error) { + cb(error, conn); + return; + } + + conn.run('PRAGMA journal_mode = WAL', (journalModeError: Error | null) => { + if (journalModeError) { + cb(journalModeError, conn); + return; + } + + conn.run( + 'PRAGMA synchronous = NORMAL', + (synchronousError: Error | null) => { + if (synchronousError) { + cb(synchronousError, conn); + return; + } + + conn.run( + `PRAGMA busy_timeout = ${sqliteBusyTimeout}`, + (busyTimeoutError: Error | null) => cb(busyTimeoutError, conn), + ); + }, + ); + }); + }); +} + const databaseConfig: DatabaseConfig = { /* |-------------------------------------------------------------------------- @@ -44,11 +92,11 @@ const databaseConfig: DatabaseConfig = { ), }, pool: { - afterCreate: (conn, cb) => { - conn.run('PRAGMA foreign_keys=true', cb); - }, - min: Number.parseInt(Env.get('DB_POOL_MIN', '2'), 10), - max: Number.parseInt(Env.get('DB_POOL_MAX', '20'), 10), + afterCreate: configureSqliteConnection, + // SQLite is a single-file database. Keeping one shared connection + // avoids starving the pool with blocked readers/writers under load. + min: 1, + max: 1, }, migrations: { naturalSort: true, diff --git a/database/migrations/1774667000000_add_database_hot_path_indexes.ts b/database/migrations/1774667000000_add_database_hot_path_indexes.ts new file mode 100644 index 00000000..fdd34958 --- /dev/null +++ b/database/migrations/1774667000000_add_database_hot_path_indexes.ts @@ -0,0 +1,39 @@ +import BaseSchema from '@ioc:Adonis/Lucid/Schema'; + +export default class extends BaseSchema { + public async up(): Promise { + this.schema.alterTable('services', table => { + table.index(['userId'], 'services_user_id_index'); + table.index(['serviceId'], 'services_service_id_index'); + }); + + this.schema.alterTable('workspaces', table => { + table.index(['userId'], 'workspaces_user_id_index'); + }); + + this.schema.alterTable('tokens', table => { + table.index( + ['user_id', 'type', 'is_revoked', 'updated_at'], + 'tokens_user_id_type_is_revoked_updated_at_index', + ); + }); + } + + public async down(): Promise { + this.schema.alterTable('tokens', table => { + table.dropIndex( + ['user_id', 'type', 'is_revoked', 'updated_at'], + 'tokens_user_id_type_is_revoked_updated_at_index', + ); + }); + + this.schema.alterTable('workspaces', table => { + table.dropIndex(['userId'], 'workspaces_user_id_index'); + }); + + this.schema.alterTable('services', table => { + table.dropIndex(['serviceId'], 'services_service_id_index'); + table.dropIndex(['userId'], 'services_user_id_index'); + }); + } +} diff --git a/tests/functional/dashboard/data.spec.ts b/tests/functional/dashboard/data.spec.ts index 1a0e7adb..4b9732e6 100644 --- a/tests/functional/dashboard/data.spec.ts +++ b/tests/functional/dashboard/data.spec.ts @@ -26,6 +26,22 @@ test.group('Dashboard / Data page', () => { ); }); + test('handles concurrent requests without exhausting the database connection pool', async ({ + client, + }) => { + const user = await UserFactory.with('services', 10) + .with('workspaces', 10) + .create(); + const responses = await Promise.all( + Array.from({ length: 25 }, () => client.get('/user/data').loginAs(user)), + ); + + for (const response of responses) { + response.assertStatus(200); + response.assertTextIncludes(user.email); + } + }); + // TODO: Add test to include services. // TODO: Add test to include workspaces. }); From e31aeb77c76f287e88d78093b1ad6015ac4dc7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Oliveira?= <37463445+SpecialAro@users.noreply.github.com> Date: Sat, 28 Mar 2026 04:01:02 +0000 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- app/Controllers/Http/WorkspaceController.ts | 4 ++-- config/database.ts | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/Controllers/Http/WorkspaceController.ts b/app/Controllers/Http/WorkspaceController.ts index 2d3eb935..63621a83 100644 --- a/app/Controllers/Http/WorkspaceController.ts +++ b/app/Controllers/Http/WorkspaceController.ts @@ -39,8 +39,8 @@ export default class WorkspaceController { } // Get new, unused uuid - let workspaceId; - let existingWorkspace; + let workspaceId: string; + let existingWorkspace: Workspace | null; do { workspaceId = uuid(); // eslint-disable-next-line no-await-in-loop diff --git a/config/database.ts b/config/database.ts index 9e19e35c..8e47d183 100644 --- a/config/database.ts +++ b/config/database.ts @@ -19,10 +19,16 @@ type SqlitePoolCallback = ( connection: SqliteConnection, ) => void; -const sqliteBusyTimeout = Number.parseInt( - Env.get('DB_BUSY_TIMEOUT', '5000'), - 10, +const SQLITE_BUSY_TIMEOUT_DEFAULT = 5000; +const sqliteBusyTimeoutEnv = Env.get( + 'DB_BUSY_TIMEOUT', + SQLITE_BUSY_TIMEOUT_DEFAULT.toString(), ); +const sqliteBusyTimeoutParsed = Number.parseInt(sqliteBusyTimeoutEnv, 10); +const sqliteBusyTimeout = + Number.isFinite(sqliteBusyTimeoutParsed) && sqliteBusyTimeoutParsed > 0 + ? sqliteBusyTimeoutParsed + : SQLITE_BUSY_TIMEOUT_DEFAULT; function configureSqliteConnection( conn: SqliteConnection, From 7201748e49c09dfc0b5346906564b18be59fd574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Oliveira?= Date: Sat, 28 Mar 2026 04:13:13 +0000 Subject: [PATCH 3/3] Change defaults --- .env.example | 9 ++++++ README.md | 3 ++ config/database.ts | 81 ++++++++++++++++++++++++++++++++++------------ data/.gitignore | 3 ++ 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index a288fde1..06e681b7 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,12 @@ PORT=3333 # These have defaults hard-coded, but are being overridden CACHE_VIEWS=false + +# SQLite durability/concurrency notes: +# - WAL improves concurrent reads/writes for the default SQLite deployment. +# - FULL is the safer default for production durability. +# - NORMAL reduces fsync pressure, but may lose the last committed transactions +# on sudden power loss or kernel crash. +DB_SQLITE_JOURNAL_MODE=WAL +DB_SQLITE_SYNCHRONOUS=FULL +DB_BUSY_TIMEOUT=5000 diff --git a/README.md b/README.md index 23cf137a..22abab69 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,9 @@ Ferdium-server's configuration is saved inside an `.env` file. Besides AdonisJS' - `CONNECT_WITH_FRANZ` (`true` or `false`, default: `true`): Whether to enable connections to the Franz server. By enabling this option, Ferdium-server can: - Show the full Franz recipe library instead of only custom recipes - Import Franz accounts +- `DB_SQLITE_JOURNAL_MODE` (`DELETE`, `TRUNCATE`, `PERSIST`, `MEMORY`, `WAL`, or `OFF`, default: `WAL`): SQLite journal mode. `WAL` improves concurrency for the default single-file deployment. +- `DB_SQLITE_SYNCHRONOUS` (`OFF`, `NORMAL`, `FULL`, or `EXTRA`, default: `FULL`): SQLite synchronous mode. `FULL` is the safer default for production durability. `NORMAL` reduces write latency but can lose the last committed transactions on sudden power loss. +- `DB_BUSY_TIMEOUT` (milliseconds, default: `5000`): How long SQLite should wait for a locked database before failing a query.
Importing your Franz/Ferdi account diff --git a/config/database.ts b/config/database.ts index 8e47d183..c92945c8 100644 --- a/config/database.ts +++ b/config/database.ts @@ -19,7 +19,33 @@ type SqlitePoolCallback = ( connection: SqliteConnection, ) => void; +const SQLITE_JOURNAL_MODE_VALUES = new Set([ + 'DELETE', + 'TRUNCATE', + 'PERSIST', + 'MEMORY', + 'WAL', + 'OFF', +]); +const SQLITE_SYNCHRONOUS_VALUES = new Set(['OFF', 'NORMAL', 'FULL', 'EXTRA']); const SQLITE_BUSY_TIMEOUT_DEFAULT = 5000; +const SQLITE_JOURNAL_MODE_DEFAULT = 'WAL'; +const SQLITE_SYNCHRONOUS_DEFAULT = 'FULL'; + +function getSqlitePragmaValue( + envName: string, + defaultValue: string, + allowedValues: Set, +) { + const configuredValue = Env.get(envName, defaultValue).trim().toUpperCase(); + + if (!allowedValues.has(configuredValue)) { + return defaultValue; + } + + return configuredValue; +} + const sqliteBusyTimeoutEnv = Env.get( 'DB_BUSY_TIMEOUT', SQLITE_BUSY_TIMEOUT_DEFAULT.toString(), @@ -29,6 +55,16 @@ const sqliteBusyTimeout = Number.isFinite(sqliteBusyTimeoutParsed) && sqliteBusyTimeoutParsed > 0 ? sqliteBusyTimeoutParsed : SQLITE_BUSY_TIMEOUT_DEFAULT; +const sqliteJournalMode = getSqlitePragmaValue( + 'DB_SQLITE_JOURNAL_MODE', + SQLITE_JOURNAL_MODE_DEFAULT, + SQLITE_JOURNAL_MODE_VALUES, +); +const sqliteSynchronous = getSqlitePragmaValue( + 'DB_SQLITE_SYNCHRONOUS', + SQLITE_SYNCHRONOUS_DEFAULT, + SQLITE_SYNCHRONOUS_VALUES, +); function configureSqliteConnection( conn: SqliteConnection, @@ -40,27 +76,30 @@ function configureSqliteConnection( return; } - conn.run('PRAGMA journal_mode = WAL', (journalModeError: Error | null) => { - if (journalModeError) { - cb(journalModeError, conn); - return; - } - - conn.run( - 'PRAGMA synchronous = NORMAL', - (synchronousError: Error | null) => { - if (synchronousError) { - cb(synchronousError, conn); - return; - } - - conn.run( - `PRAGMA busy_timeout = ${sqliteBusyTimeout}`, - (busyTimeoutError: Error | null) => cb(busyTimeoutError, conn), - ); - }, - ); - }); + conn.run( + `PRAGMA journal_mode = ${sqliteJournalMode}`, + (journalModeError: Error | null) => { + if (journalModeError) { + cb(journalModeError, conn); + return; + } + + conn.run( + `PRAGMA synchronous = ${sqliteSynchronous}`, + (synchronousError: Error | null) => { + if (synchronousError) { + cb(synchronousError, conn); + return; + } + + conn.run( + `PRAGMA busy_timeout = ${sqliteBusyTimeout}`, + (busyTimeoutError: Error | null) => cb(busyTimeoutError, conn), + ); + }, + ); + }, + ); }); } diff --git a/data/.gitignore b/data/.gitignore index 61500541..b798f404 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1 +1,4 @@ ferdium.sqlite +ferdium.sqlite-shm +ferdium.sqlite-wal +ferdium-*.log