feat(sidecar): JSON-family storage backend — plain / JSON-LD / NDJSON (V-L2-F3, #146) (#148)

Implement a json sidecar store covering plain JSON, JSON-LD, and NDJSON,
with full parity to the SQLite runtime path (provenance incl. forks,
temporal versioning, drift, gc). `[sidecar].storage = "json"` + a new
`[sidecar].format` key; `sidecar::StorageKind::resolve` is the single
backend resolver. Re-opens the capability dropped in #112/#144.

Closes #146

Author: hyperpolymath

src/abi/mod.rs

@@ -460,8 +460,12 @@ impl AccessPolicy {
 /// with validation and path resolution.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SidecarConfig {
-    /// Storage backend: "sqlite" (default) or "postgres"/"postgresql".
+    /// Storage backend: "sqlite" (default), "postgres"/"postgresql", or
+    /// "json" (see `format`).
     pub storage: String,
+    /// On-disk encoding for the `json` store: "plain" (default), "ld"
+    /// (JSON-LD), or "ndjson". Ignored for sql backends. V-L2-F3 (#146).
+    pub format: String,
     /// File path for the sidecar database.
     pub path: String,
 }
@@ -471,6 +475,7 @@ impl SidecarConfig {
     pub fn default_sqlite() -> Self {
         Self {
             storage: "sqlite".to_string(),
+            format: "plain".to_string(),
             path: ".verisim/sidecar.db".to_string(),
         }
     }

src/codegen/overlay.rs

@@ -67,47 +67,24 @@ fn must_validate_identifier(name: &str) -> &str {
 // SQL dialect (V-L2-F1, #45)
 // ---------------------------------------------------------------------------
 
-/// The SQL dialect the sidecar DDL is emitted for. Selected from the
-/// manifest's `[sidecar].storage`. The table bodies are written in the
-/// portable subset both engines accept (`CREATE TABLE IF NOT EXISTS`,
-/// `CHECK`, partial unique indexes, `CURRENT_TIMESTAMP`); the only
-/// genuinely dialect-divergent fragment is the metadata upsert
-/// (`INSERT OR IGNORE` vs `INSERT … ON CONFLICT DO NOTHING`), which lives
-/// in the [`sqlite`] / [`postgres`] modules.
+/// The SQL dialect the sidecar DDL is emitted for. Selected (via
+/// [`crate::sidecar::StorageKind`]) from the manifest's `[sidecar].storage`.
+/// The table bodies are written in the portable subset both engines accept
+/// (`CREATE TABLE IF NOT EXISTS`, `CHECK`, partial unique indexes,
+/// `CURRENT_TIMESTAMP`); the only genuinely dialect-divergent fragment is
+/// the metadata upsert (`INSERT OR IGNORE` vs `INSERT … ON CONFLICT DO
+/// NOTHING`), which lives in the [`sqlite`] / [`postgres`] modules.
 ///
-/// [`from_storage`](SqlDialect::from_storage) is the single source of
-/// truth for which `[sidecar].storage` values are accepted; `generate`,
-/// `validate`, and `doctor` all defer to it.
+/// Storage-string resolution lives in [`crate::sidecar::StorageKind::resolve`]
+/// (the single source of truth) — it maps `sqlite`/`postgres` to a dialect
+/// via [`StorageKind::sql_dialect`](crate::sidecar::StorageKind::sql_dialect)
+/// and `json` to the non-SQL [`crate::sidecar::json`] store.
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum SqlDialect {
     Sqlite,
     Postgres,
 }
 
-impl SqlDialect {
-    /// Map a `[sidecar].storage` value to a dialect (case-insensitive):
-    /// `sqlite` → [`SqlDialect::Sqlite`]; `postgres`/`postgresql` →
-    /// [`SqlDialect::Postgres`]. Every other value is rejected rather than
-    /// silently emitting SQLite DDL regardless of the backend (V-L2-F1).
-    ///
-    /// The octad data layer is intrinsically relational (hash-chains under
-    /// `BEGIN IMMEDIATE`, partial-unique temporal indexes, `CHECK`
-    /// constraints, recursive-CTE lineage acyclicity), so the
-    /// never-implemented `json` document store was dropped rather than
-    /// built (V-L2-F2, #112). It is now an unsupported value like any
-    /// other — no special-casing, no "coming soon" pointer.
-    pub fn from_storage(storage: &str) -> anyhow::Result<Self> {
-        match storage.to_lowercase().as_str() {
-            "sqlite" => Ok(SqlDialect::Sqlite),
-            "postgres" | "postgresql" => Ok(SqlDialect::Postgres),
-            other => anyhow::bail!(
-                "unsupported [sidecar].storage {other:?}; supported values are \
-                 \"sqlite\" (default) and \"postgres\"/\"postgresql\"."
-            ),
-        }
-    }
-}
-
 // ---------------------------------------------------------------------------
 // Overlay generation
 // ---------------------------------------------------------------------------
@@ -870,33 +847,7 @@ mod tests {
         assert!(!p.contains("Seed metadata from parsed schema"));
     }
 
-    #[test]
-    fn test_storage_to_dialect_mapping() {
-        assert_eq!(
-            SqlDialect::from_storage("sqlite").unwrap(),
-            SqlDialect::Sqlite
-        );
-        assert_eq!(
-            SqlDialect::from_storage("postgres").unwrap(),
-            SqlDialect::Postgres
-        );
-        assert_eq!(
-            SqlDialect::from_storage("PostgreSQL").unwrap(),
-            SqlDialect::Postgres
-        );
-        // V-L2-F2 (#112): the json store was dropped, never implemented. It
-        // is now rejected like any other unsupported value, and the error
-        // advertises only the supported stores — it must NOT imply json is
-        // planned (no "#112" / "not implemented" pointer).
-        let json_err = SqlDialect::from_storage("json").unwrap_err().to_string();
-        assert!(
-            json_err.contains("unsupported") && json_err.contains("sqlite"),
-            "json must be rejected as an unsupported store, got: {json_err}"
-        );
-        assert!(
-            !json_err.contains("#112") && !json_err.to_lowercase().contains("not implemented"),
-            "the dropped json store must not be advertised as planned, got: {json_err}"
-        );
-        assert!(SqlDialect::from_storage("mariadb").is_err());
-    }
+    // Storage-string resolution (incl. the json family) is owned by
+    // `crate::sidecar::StorageKind` and tested there; `SqlDialect` is now a
+    // plain dialect tag with no string parsing of its own.
 }

src/gc.rs

@@ -39,19 +39,52 @@ impl GcReport {
 }
 
 /// Purge sidecar rows older than the retention bound. `dry_run = true`
-/// reports what would be deleted without changing the DB.
+/// reports what would be deleted without changing the store.
 ///
-/// Returns `Err` if the sidecar storage is not SQLite (unsupported in
-/// this cut) or if the file is unreachable.
+/// Dispatches on the resolved [`StorageKind`](crate::sidecar::StorageKind):
+/// the `sqlite` and `json` (plain/ld/ndjson) backends are implemented;
+/// `postgres` gc is not yet. Returns `Err` for an unsupported storage or an
+/// unreachable store.
 pub fn run_gc(manifest: &Manifest, dry_run: bool) -> Result<GcReport> {
-    if manifest.sidecar.storage != "sqlite" {
-        bail!(
-            "verisimiser gc currently only supports the SQLite sidecar backend; \
-             [sidecar].storage is {:?}",
-            manifest.sidecar.storage
-        );
+    use crate::sidecar::StorageKind;
+    match StorageKind::resolve(&manifest.sidecar.storage, &manifest.sidecar.format)? {
+        StorageKind::Sqlite => run_gc_sqlite(manifest, dry_run),
+        StorageKind::Json(format) => run_gc_json(manifest, dry_run, format),
+        StorageKind::Postgres => bail!(
+            "verisimiser gc supports the sqlite and json sidecar backends; \
+             gc for [sidecar].storage = \"postgres\" is not yet implemented"
+        ),
     }
+}
 
+/// JSON-family gc: load the store, purge in memory, persist iff applying.
+/// The per-dimension semantics (incl. keeping the current temporal version)
+/// live in [`crate::sidecar::json::JsonStore::gc_purge`].
+fn run_gc_json(
+    manifest: &Manifest,
+    dry_run: bool,
+    format: crate::sidecar::JsonFormat,
+) -> Result<GcReport> {
+    let sidecar_path = &manifest.sidecar.path;
+    let mut store = crate::sidecar::json::JsonStore::open(sidecar_path, format)
+        .with_context(|| format!("opening json sidecar at {}", sidecar_path))?;
+    let counts = store.gc_purge(&manifest.retention, dry_run);
+    if !dry_run {
+        store
+            .save()
+            .with_context(|| format!("saving json sidecar at {}", sidecar_path))?;
+    }
+    Ok(GcReport {
+        sidecar: sidecar_path.clone(),
+        dry_run,
+        provenance_deleted: counts.provenance,
+        temporal_deleted: counts.temporal,
+        lineage_deleted: counts.lineage,
+    })
+}
+
+/// SQLite gc (the reference path).
+fn run_gc_sqlite(manifest: &Manifest, dry_run: bool) -> Result<GcReport> {
     let sidecar_path = &manifest.sidecar.path;
     let conn = Connection::open(sidecar_path)
         .with_context(|| format!("opening sidecar at {}", sidecar_path))?;
@@ -148,6 +181,7 @@ mod tests {
         .unwrap();
         m.sidecar = SidecarConfig {
             storage: storage.to_string(),
+            format: "plain".to_string(),
             path: sidecar_path.to_string(),
         };
         m.retention = retention;
@@ -308,15 +342,67 @@ mod tests {
     }
 
     #[test]
-    fn gc_rejects_non_sqlite_backend() {
-        // `postgres` is a valid generate-time dialect, but gc is SQLite-only
-        // and must refuse rather than silently no-op. (The `json` value was
-        // dropped as a storage option entirely in V-L2-F2 / #112.)
+    fn gc_rejects_postgres_backend() {
+        // `postgres` is a valid generate-time dialect, but gc is not yet
+        // implemented for it and must refuse rather than silently no-op.
+        // (The json family *is* now supported — see the json gc test below.)
         let m = fixture("/dev/null", RetentionConfig::default(), "postgres");
         let err = run_gc(&m, true).unwrap_err();
         assert!(
-            err.to_string().contains("only supports the SQLite sidecar"),
-            "expected explicit unsupported-backend error; got: {err}"
+            err.to_string().contains("not yet implemented"),
+            "expected explicit postgres-unsupported error; got: {err}"
+        );
+    }
+
+    #[test]
+    fn gc_json_backend_purges_old_rows_and_persists() {
+        use crate::sidecar::JsonFormat;
+        use crate::sidecar::json::{JsonStore, ProvenanceRow, SidecarData, encode};
+
+        let dir = tempfile::tempdir().unwrap();
+        let sidecar = dir.path().join("sidecar.json");
+        let sidecar_str = sidecar.to_str().unwrap();
+
+        // Seed one aged + one fresh provenance row directly (deterministic
+        // timestamps; the append API always stamps "now").
+        let aged = ProvenanceRow {
+            hash: "old".into(),
+            previous_hash: String::new(),
+            entity_id: "e".into(),
+            table_name: "t".into(),
+            operation: "insert".into(),
+            actor: "a".into(),
+            timestamp: "2020-01-01T00:00:00+00:00".into(),
+            before_snapshot: None,
+            transformation: None,
+        };
+        let fresh = ProvenanceRow {
+            hash: "new".into(),
+            timestamp: "9999-01-01T00:00:00+00:00".into(),
+            ..aged.clone()
+        };
+        let data = SidecarData {
+            provenance_log: vec![aged, fresh],
+            ..Default::default()
+        };
+        std::fs::write(&sidecar, encode(&data, JsonFormat::Plain).unwrap()).unwrap();
+
+        let m = fixture(
+            sidecar_str,
+            RetentionConfig {
+                provenance_days: 30,
+                temporal_days: 30,
+                lineage_days: 30,
+            },
+            "json",
         );
+        let report = run_gc(&m, false).unwrap();
+        assert_eq!(report.provenance_deleted, 1, "old provenance row purged");
+        assert_eq!(report.total(), 1);
+
+        // The purge was persisted: reopening shows only the fresh row.
+        let reopened = JsonStore::open(&sidecar, JsonFormat::Plain).unwrap();
+        assert_eq!(reopened.data().provenance_log.len(), 1);
+        assert_eq!(reopened.data().provenance_log[0].hash, "new");
     }
 }

src/lib.rs

@@ -13,6 +13,7 @@ pub mod doctor;
 pub mod gc;
 pub mod intercept;
 pub mod manifest;
+pub mod sidecar;
 pub mod tier1;
 pub mod tier2;