upgrade to postgresql

Author: Tricked-dev

Cargo.lock

@@ -40,17 +40,15 @@ dotenv = { version = "0.15.0", features = ["clap"] }
 color-eyre = "0.6.3"
 vesper = "0.13.0"
 once_cell = { version = "1.19.0", features = ["parking_lot"] }
-serde_rusqlite = "0.39.0"
 openrouter_api = { git = "https://github.com/socrates8300/openrouter_api.git", rev = "96f8bd7f5ec219abb2335ac16110835e15487c66" }
 thiserror = "2.0.12"
-r2d2_sqlite = "0.30.0"
-r2d2 = "0.8.10"
-wb_sqlite = "0.2.1"
-rusqlite = { version = "0.36.0", features = ["bundled"] }
 num-format = "0.4.4"
 fasteval = "0.2.4"
 png = "0.17"
 axum = "0.8"
 tera = "1"
 tempfile = "3"
 urlencoding = "2.1"
+tokio-postgres = "0.7.16"
+deadpool-postgres = { version = "0.14.1", features = ["rt_tokio_1"] }
+postgres-from-row = "0.5.2"

Cargo.toml

@@ -0,0 +1,79 @@
+# SQLite → PostgreSQL Migration
+
+## Overview
+
+The bot uses PostgreSQL with `tokio-postgres` + `deadpool-postgres`. On every startup
+`run_migrations` runs both SQL files in order — idempotent, so safe to run repeatedly.
+
+```
+migrations/
+  001_init.sql          — base schema (mirrors SQLite layout, used for fresh installs)
+  002_improve_types.sql — upgrades to PG-native types (runs after pgloader or after 001)
+  pgloader.load         — pgloader script for one-time SQLite data migration
+```
+
+---
+
+## Fresh install (no existing data)
+
+1. Create a PostgreSQL database.
+2. Set `DATABASE_URL` and start the bot — migrations run automatically on startup.
+
+```bash
+DATABASE_URL=postgres://user:pass@localhost/trickedbot cargo run
+```
+
+---
+
+## Migrating from existing SQLite (`database.db`)
+
+### 1. Install pgloader
+
+```bash
+nix-shell -p pgloader   # or: apt install pgloader
+```
+
+### 2. Edit `migrations/pgloader.load`
+
+Fill in the real paths:
+
+```
+FROM sqlite:///absolute/path/to/database.db
+INTO postgresql://user:pass@localhost/trickedbot
+```
+
+### 3. Run pgloader
+
+```bash
+pgloader migrations/pgloader.load
+```
+
+pgloader creates the tables and loads all data. The CAST rules handle:
+- `user.id` — TEXT in SQLite (rusqlite stored u64 as strings) → BIGINT
+- `memory.user_id` — stored as integer in SQLite → BIGINT
+- all other integer columns → BIGINT (002 downcasts level/xp back to INT)
+
+### 4. Start the bot
+
+```bash
+DATABASE_URL=postgres://user:pass@localhost/trickedbot cargo run
+```
+
+On startup `run_migrations` runs:
+- **001** — no-op (tables already exist from pgloader)
+- **002** — upgrades types in-place:
+  - `user.id` TEXT → BIGINT (if not already done by pgloader CAST)
+  - `user.level`, `user.xp` BIGINT → INT (pgloader over-widens all integers)
+  - `memory.user_id` TEXT → BIGINT (if not already done)
+  - `math_question.answer` real → double precision
+  - adds FK `memory.user_id → user.id ON DELETE CASCADE` (drops orphaned memories first)
+
+All 002 steps check current column types before acting — safe to re-run.
+
+---
+
+## Notes
+
+- `math_question` — pgloader snake_cases the SQLite table name `MathQuestion`; the bot queries `math_question` accordingly.
+- Orphaned memories (referencing users who left) are automatically deleted before the FK is added.
+- `DATABASE_FILE` / `--database-file` no longer exists; use `DATABASE_URL` / `--database-url`.

docs/MIGRATION.md

@@ -0,0 +1,28 @@
+-- Initial schema matching the original SQLite structure.
+-- Keeps memory.user_id as TEXT to match SQLite's original layout.
+-- Run 002_improve_types.sql after loading to upgrade to native PG types.
+
+CREATE TABLE IF NOT EXISTS "user" (
+    id             BIGINT PRIMARY KEY,
+    level          INT    NOT NULL DEFAULT 0,
+    xp             INT    NOT NULL DEFAULT 0,
+    social_credit  BIGINT NOT NULL DEFAULT 0,
+    name           TEXT   NOT NULL DEFAULT '',
+    relationship   TEXT   NOT NULL DEFAULT '',
+    example_input  TEXT   NOT NULL DEFAULT '',
+    example_output TEXT   NOT NULL DEFAULT ''
+);
+
+CREATE TABLE IF NOT EXISTS memory (
+    id      BIGSERIAL PRIMARY KEY,
+    user_id TEXT      NOT NULL,
+    content TEXT      NOT NULL,
+    key     TEXT      NOT NULL,
+    UNIQUE (user_id, key)
+);
+
+CREATE TABLE IF NOT EXISTS mathquestion (
+    id       SERIAL           PRIMARY KEY,
+    question TEXT             NOT NULL,
+    answer   DOUBLE PRECISION NOT NULL
+);

migrations/001_init.sql

@@ -0,0 +1,75 @@
+-- Upgrade schema from SQLite-compatible types to PostgreSQL-native types.
+-- Idempotent: safe to run on any state (fresh, post-pgloader, or already upgraded).
+
+-- 1. Convert user.id from TEXT to BIGINT (it was stored as string by rusqlite)
+DO $$ DECLARE
+  pk_name text;
+BEGIN
+  IF (SELECT data_type FROM information_schema.columns
+      WHERE table_name = 'user' AND column_name = 'id') = 'text' THEN
+
+    -- Dynamically find and drop the PK so we can change the column type
+    SELECT c.conname INTO pk_name
+    FROM pg_constraint c
+    JOIN pg_class t ON c.conrelid = t.oid
+    WHERE t.relname = 'user' AND c.contype = 'p';
+
+    IF pk_name IS NOT NULL THEN
+      EXECUTE format('ALTER TABLE "user" DROP CONSTRAINT %I', pk_name);
+    END IF;
+
+    ALTER TABLE "user" ALTER COLUMN id TYPE BIGINT USING id::BIGINT;
+    ALTER TABLE "user" ADD PRIMARY KEY (id);
+  END IF;
+END $$;
+
+-- 2. Downcast level/xp from BIGINT (pgloader global cast) back to INT (Rust i32)
+DO $$ BEGIN
+  IF (SELECT data_type FROM information_schema.columns
+      WHERE table_name = 'user' AND column_name = 'level') = 'bigint' THEN
+    ALTER TABLE "user" ALTER COLUMN level TYPE INT USING level::INT;
+  END IF;
+END $$;
+
+DO $$ BEGIN
+  IF (SELECT data_type FROM information_schema.columns
+      WHERE table_name = 'user' AND column_name = 'xp') = 'bigint' THEN
+    ALTER TABLE "user" ALTER COLUMN xp TYPE INT USING xp::INT;
+  END IF;
+END $$;
+
+-- 3. Convert memory.user_id from TEXT to BIGINT (if not already done by pgloader CAST)
+DO $$ BEGIN
+  IF (SELECT data_type FROM information_schema.columns
+      WHERE table_name = 'memory' AND column_name = 'user_id') = 'text' THEN
+
+    ALTER TABLE memory DROP CONSTRAINT IF EXISTS memory_user_id_key;
+    ALTER TABLE memory DROP CONSTRAINT IF EXISTS memory_user_key_unique;
+
+    ALTER TABLE memory ALTER COLUMN user_id TYPE BIGINT USING user_id::BIGINT;
+
+    ALTER TABLE memory ADD CONSTRAINT memory_user_key_unique UNIQUE (user_id, key);
+  END IF;
+END $$;
+
+-- 4. Fix math_question.answer from real (float4) to double precision (float8)
+DO $$ BEGIN
+  IF (SELECT data_type FROM information_schema.columns
+      WHERE table_name = 'math_question' AND column_name = 'answer') = 'real' THEN
+    ALTER TABLE math_question ALTER COLUMN answer TYPE DOUBLE PRECISION USING answer::DOUBLE PRECISION;
+  END IF;
+END $$;
+
+-- 5. Add FK memory.user_id -> "user".id with CASCADE delete
+DO $$ BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_constraint c
+    JOIN pg_class t ON c.conrelid = t.oid
+    WHERE t.relname = 'memory' AND c.contype = 'f'
+  ) THEN
+    -- Remove orphaned memories (users who left/were deleted) before adding FK
+    DELETE FROM memory WHERE user_id NOT IN (SELECT id FROM "user");
+    ALTER TABLE memory ADD CONSTRAINT memory_user_id_fkey
+      FOREIGN KEY (user_id) REFERENCES "user"(id) ON DELETE CASCADE;
+  END IF;
+END $$;

migrations/002_improve_types.sql

@@ -0,0 +1,12 @@
+LOAD DATABASE
+    FROM sqlite:///path/to/database.db
+    INTO postgresql://user:pass@localhost/trickedbot
+
+WITH include drop, create tables, create indexes, reset sequences
+
+CAST
+    type integer to bigint,
+    column user.id to bigint using (lambda (v) (parse-integer v)),
+    column memory.user_id to bigint
+
+ALTER SCHEMA 'main' RENAME TO 'public';

migrations/pgloader.load

@@ -4,8 +4,8 @@ use openrouter_api::{
     types::chat::{ChatCompletionRequest, Message, MessageContent},
     OpenRouterClient,
 };
-use r2d2_sqlite::SqliteConnectionManager;
-use serde_rusqlite::from_row;
+use deadpool_postgres::Pool;
+use crate::db;
 use tokio::sync::mpsc;
 
 use crate::{
@@ -79,54 +79,6 @@ fn build_authors_note() -> &'static str {
 **Current Mode:** Trickster (unless Sebook detected)]"#
 }
 
-/// Retrieves relevant memories for the user from the database
-/// Reference: AGENT_GUIDE.md section on Memory Context Injection
-fn get_user_memories(database: &r2d2::Pool<SqliteConnectionManager>, user_id: u64) -> Result<Vec<Memory>> {
-    let db = database.get()?;
-    let mut statement = db.prepare("SELECT * FROM memory WHERE user_id = ? ORDER BY id DESC LIMIT 5")?;
-
-    let memories: Vec<Memory> = statement
-        .query_map([user_id.to_string()], |row| {
-            from_row::<Memory>(row).map_err(|_| rusqlite::Error::QueryReturnedNoRows)
-        })?
-        .filter_map(Result::ok)
-        .collect();
-
-    Ok(memories)
-}
-
-/// Fetches relationships for users mentioned in the conversation
-fn get_users_with_relationships(database: &r2d2::Pool<SqliteConnectionManager>, context: &str) -> Result<Vec<(String, String)>> {
-    let db = database.get()?;
-    let mut statement = db.prepare("SELECT name, relationship FROM user WHERE relationship != ''")?;
-
-    let users: Vec<(String, String)> = statement
-        .query_map([], |row| {
-            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
-        })?
-        .filter_map(Result::ok)
-        .filter(|(name, _)| context.contains(name.as_str())) // Only include users in the conversation
-        .collect();
-
-    Ok(users)
-}
-
-/// Fetches examples for users mentioned in the conversation
-fn get_users_with_examples(database: &r2d2::Pool<SqliteConnectionManager>, context: &str) -> Result<Vec<(String, String, String)>> {
-    let db = database.get()?;
-    let mut statement = db.prepare("SELECT name, example_input, example_output FROM user WHERE example_input != '' AND example_output != ''")?;
-
-    let users: Vec<(String, String, String)> = statement
-        .query_map([], |row| {
-            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?))
-        })?
-        .filter_map(Result::ok)
-        .filter(|(name, _, _)| context.contains(name.as_str())) // Only include users in the conversation
-        .collect();
-
-    Ok(users)
-}
-
 /// Formats memories into natural language for prompt injection with usage guidelines
 fn format_memories(memories: &[Memory]) -> String {
     if memories.is_empty() {
@@ -218,52 +170,6 @@ Quality over quantity - one great response beats three mediocre fragments."#,
     )
 }
 
-/// Get user from database or return default
-fn get_user_or_default(database: &r2d2::Pool<SqliteConnectionManager>, user_id: u64) -> User {
-    database
-        .get()
-        .ok()
-        .and_then(|db| {
-            db.prepare("SELECT * FROM user WHERE id = ?").ok().and_then(|mut stmt| {
-                stmt.query_one([user_id.to_string()], |row| {
-                    from_row::<User>(row).map_err(|_| rusqlite::Error::QueryReturnedNoRows)
-                })
-                .ok()
-            })
-        })
-        .unwrap_or_else(|| User {
-            id: user_id,
-            level: 0,
-            xp: 0,
-            social_credit: 0,
-            name: "Unknown".to_owned(),
-            relationship: String::new(),
-            example_input: String::new(),
-            example_output: String::new(),
-        })
-}
-
-/// Replace user mentions in context with actual usernames
-fn replace_mentions(
-    context: String,
-    user_mentions: &HashMap<String, u64>,
-    database: &r2d2::Pool<SqliteConnectionManager>,
-) -> String {
-    let mut processed = context;
-    for (mention, user_id) in user_mentions {
-        if let Ok(db) = database.get() {
-            if let Ok(mut stmt) = db.prepare("SELECT * FROM user WHERE id = ?") {
-                if let Ok(user) = stmt.query_one([user_id.to_string()], |row| {
-                    from_row::<User>(row).map_err(|_| rusqlite::Error::QueryReturnedNoRows)
-                }) {
-                    processed = processed.replace(mention, &user.name);
-                }
-            }
-        }
-    }
-    processed
-}
-
 /// Stream AI response chunks through a channel
 async fn stream_ai_response(
     client: OpenRouterClient<openrouter_api::Ready>,
@@ -313,7 +219,7 @@ async fn stream_ai_response(
 }
 
 pub async fn main(
-    database: r2d2::Pool<SqliteConnectionManager>,
+    database: Pool,
     user_id: u64,
     message: &str,
     context: &str,
@@ -339,14 +245,32 @@ pub async fn main(
         )?;
 
     // Process context and get user info
-    let processed_context = replace_mentions(context.to_string(), &user_mentions, &database);
-    let user = get_user_or_default(&database, user_id);
-    let memories = get_user_memories(&database, user_id)?;
+    // Replace user mentions with names
+    let mut processed_context = context.to_string();
+    for (mention, &user_id_ref) in &user_mentions {
+        if let Ok(Some(u)) = db::get_user(&database, user_id_ref).await {
+            processed_context = processed_context.replace(mention, &u.name);
+        }
+    }
+
+    let user = db::get_user(&database, user_id).await?.unwrap_or_else(|| crate::database::User {
+        id: user_id as i64,
+        level: 0,
+        xp: 0,
+        social_credit: 0,
+        name: "Unknown".to_owned(),
+        relationship: String::new(),
+        example_input: String::new(),
+        example_output: String::new(),
+    });
+    let memories = db::get_memories(&database, user_id).await.unwrap_or_default();
     let formatted_memories = format_memories(&memories);
 
     // Fetch dynamic relationship and example data for users in the conversation
-    let users_with_relationships = get_users_with_relationships(&database, &processed_context).unwrap_or_default();
-    let users_with_examples = get_users_with_examples(&database, &processed_context).unwrap_or_default();
+    let users_with_relationships = db::get_users_with_relationships(&database).await.unwrap_or_default()
+        .into_iter().filter(|(name, _)| processed_context.contains(name.as_str())).collect::<Vec<_>>();
+    let users_with_examples = db::get_users_with_examples(&database).await.unwrap_or_default()
+        .into_iter().filter(|(name, _, _)| processed_context.contains(name.as_str())).collect::<Vec<_>>();
 
     // Build prompt
     let system_prompt = build_character_prompt(