smoketest and fixture

Author: joshua-spacetime

crates/smoketests/fixtures/README.md

@@ -44,3 +44,55 @@ CARGO_NET_OFFLINE=true CARGO_TARGET_DIR="$TMP/target-old" \
 cp "$TMP/target-old/wasm32-unknown-unknown/release/upgrade_old_module.wasm" \
   crates/smoketests/fixtures/upgrade_old_module_v1.wasm
 ```
+
+## `stale-view-backing-table-v2.6.0`
+
+This is a standalone data-dir fixture created by `spacetimedb-standalone` `v2.6.0`.
+It contains database identity
+`c200f6ec405075e508c2ed6474019332d6a2a46c69614306cc4bd980e0b8b767`, published
+from `crates/smoketests/modules/views-sql`.
+
+The fixture exists to cover startup repair for view backing tables created before
+the `arg_hash` internal column. The persisted sender-scoped view backing tables
+begin with `sender`, and anonymous view backing tables do not have an internal
+argument column.
+
+The checked-in fixture intentionally keeps only the control database entry and
+commitlog segment required to replay that database state. The current server can
+recreate config, metadata, commitlog offset indexes (`*.stdb.ofs`), snapshots,
+lock files, and program byte storage from defaults and the commitlog during
+startup.
+
+To regenerate it:
+
+```bash
+TMP="$(mktemp -d)"
+git archive --format=tar v2.6.0 | tar -xf - -C "$TMP"
+
+cargo build --manifest-path "$TMP/Cargo.toml" \
+  -p spacetimedb-standalone --release \
+  --features spacetimedb-standalone/allow_loopback_http_for_tests
+
+"$TMP/target/release/spacetimedb-standalone" start \
+  --data-dir "$TMP/data" \
+  --jwt-key-dir "$TMP/keys" \
+  --listen-addr 127.0.0.1:0 \
+  --non-interactive
+
+# In another shell, publish to the printed server URL.
+CARGO_HOME="$TMP/cargo-home" CARGO_TARGET_DIR="$TMP/module-target" \
+  target/release/spacetimedb-cli --config-path "$TMP/client-config.toml" publish \
+  --server "http://127.0.0.1:<PORT>" \
+  --module-path crates/smoketests/modules/views-sql \
+  --yes stale-view-backing-table-v26
+
+# Stop the old server, update the identity constant if it changed, then copy the
+# minimal fixture.
+FIXTURE=crates/smoketests/fixtures/stale-view-backing-table-v2.6.0
+rsync -am \
+  --include='*/' \
+  --include='control-db/db' \
+  --include='replicas/1/clog/*.stdb.log' \
+  --exclude='*' \
+  "$TMP/data/" "$FIXTURE/"
+```

crates/smoketests/fixtures/stale-view-backing-table-v2.6.0/.gitignore

@@ -0,0 +1,4 @@
+!replicas/
+!replicas/1/
+!replicas/1/clog/
+!replicas/1/clog/*.stdb.log

crates/smoketests/fixtures/stale-view-backing-table-v2.6.0/control-db/db

@@ -198,6 +198,25 @@ pub fn patch_module_cargo_to_local_bindings(module_dir: &Path) -> Result<()> {
     Ok(())
 }
 
+fn copy_dir_contents(src: &Path, dst: &Path) -> Result<()> {
+    fs::create_dir_all(dst).with_context(|| format!("Failed to create {}", dst.display()))?;
+    for entry in fs::read_dir(src).with_context(|| format!("Failed to read {}", src.display()))? {
+        let entry = entry?;
+        let src_path = entry.path();
+        let dst_path = dst.join(entry.file_name());
+        let file_type = entry.file_type()?;
+        if file_type.is_dir() {
+            copy_dir_contents(&src_path, &dst_path)?;
+        } else if file_type.is_file() {
+            fs::copy(&src_path, &dst_path)
+                .with_context(|| format!("Failed to copy {} to {}", src_path.display(), dst_path.display()))?;
+        } else {
+            bail!("Unsupported fixture entry type at {}", src_path.display());
+        }
+    }
+    Ok(())
+}
+
 /// Returns the shared target directory for smoketest module builds.
 ///
 /// All tests share this directory to cache compiled dependencies. The warmup step
@@ -448,6 +467,8 @@ pub struct Smoketest {
     /// The SpacetimeDB server guard (stops server on drop).
     /// None when running against a remote server.
     pub guard: Option<SpacetimeDbGuard>,
+    /// Owns a copied fixture data directory, if this smoketest was started from one.
+    _data_dir_fixture: Option<tempfile::TempDir>,
     /// Temporary directory containing the module project.
     pub project_dir: tempfile::TempDir,
     /// Additional features for the spacetimedb bindings dependency.
@@ -718,13 +739,19 @@ impl<'a> SubscribeBuilder<'a> {
 pub struct SmoketestBuilder {
     module_code: Option<String>,
     precompiled_module: Option<String>,
+    data_dir_fixture: Option<DataDirFixture>,
     bindings_features: Vec<String>,
     extra_deps: String,
     autopublish: bool,
     pg_port: Option<u16>,
     server_url_override: Option<String>,
 }
 
+struct DataDirFixture {
+    path: PathBuf,
+    database_identity: String,
+}
+
 impl Default for SmoketestBuilder {
     fn default() -> Self {
         Self::new()
@@ -737,6 +764,7 @@ impl SmoketestBuilder {
         Self {
             module_code: None,
             precompiled_module: None,
+            data_dir_fixture: None,
             bindings_features: vec!["unstable".to_string()],
             extra_deps: String::new(),
             autopublish: true,
@@ -750,6 +778,18 @@ impl SmoketestBuilder {
         self
     }
 
+    /// Starts the local server from a copy of a persisted standalone data directory fixture.
+    ///
+    /// The fixture directory is copied to a temporary directory before startup so tests can
+    /// freely mutate it. Tests using this should normally also call `autopublish(false)`.
+    pub fn data_dir_fixture(mut self, path: impl AsRef<Path>, database_identity: impl Into<String>) -> Self {
+        self.data_dir_fixture = Some(DataDirFixture {
+            path: path.as_ref().to_path_buf(),
+            database_identity: database_identity.into(),
+        });
+        self
+    }
+
     /// Enables the PostgreSQL wire protocol on the specified port.
     pub fn pg_port(mut self, port: u16) -> Self {
         self.pg_port = Some(port);
@@ -822,20 +862,45 @@ impl SmoketestBuilder {
         let _ = ensure_binaries_built();
         let build_start = Instant::now();
 
+        let fixture_identity = self
+            .data_dir_fixture
+            .as_ref()
+            .map(|fixture| fixture.database_identity.clone());
+
         // Check if we're running against a remote server
-        let (guard, server_url) = if let Some(url) = self.server_url_override {
+        let (guard, server_url, data_dir_fixture) = if let Some(fixture) = self.data_dir_fixture.as_ref() {
+            if self.server_url_override.is_some() || remote_server_url().is_some() {
+                panic!("data_dir_fixture requires a local server managed by the smoketest harness");
+            }
+
+            let temp_dir = tempfile::tempdir().expect("Failed to create temp data fixture directory");
+            copy_dir_contents(&fixture.path, temp_dir.path()).unwrap_or_else(|err| {
+                panic!(
+                    "failed to copy data dir fixture from {} to {}: {err:#}",
+                    fixture.path.display(),
+                    temp_dir.path().display()
+                )
+            });
+
+            let guard = timed!(
+                "server spawn from data dir fixture",
+                SpacetimeDbGuard::spawn_with_data_dir(temp_dir.path().to_path_buf(), self.pg_port)
+            );
+            let url = guard.host_url.clone();
+            (Some(guard), url, Some(temp_dir))
+        } else if let Some(url) = self.server_url_override {
             eprintln!("[REMOTE] Using explicit server URL: {}", url);
-            (None, url)
+            (None, url, None)
         } else if let Some(remote_url) = remote_server_url() {
             eprintln!("[REMOTE] Using remote server: {}", remote_url);
-            (None, remote_url)
+            (None, remote_url, None)
         } else {
             let guard = timed!(
                 "server spawn",
                 SpacetimeDbGuard::spawn_in_temp_data_dir_with_pg_port(self.pg_port)
             );
             let url = guard.host_url.clone();
-            (Some(guard), url)
+            (Some(guard), url, None)
         };
 
         let project_dir = tempfile::tempdir().expect("Failed to create temp project directory");
@@ -863,8 +928,9 @@ impl SmoketestBuilder {
         let config_path = project_dir.path().join("config.toml");
         let mut smoketest = Smoketest {
             guard,
+            _data_dir_fixture: data_dir_fixture,
             project_dir,
-            database_identity: None,
+            database_identity: fixture_identity,
             server_url,
             config_path,
             module_name,

crates/smoketests/fixtures/stale-view-backing-table-v2.6.0/replicas/1/clog/00000000000000000000.stdb.log

@@ -1,5 +1,12 @@
+use std::path::PathBuf;
+
 use serde_json::{json, Value};
-use spacetimedb_smoketests::{require_dotnet, require_pnpm, ModuleLanguage, Smoketest};
+use spacetimedb_smoketests::{
+    require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest,
+};
+
+const STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY: &str =
+    "c200f6ec405075e508c2ed6474019332d6a2a46c69614306cc4bd980e0b8b767";
 
 const TS_VIEWS_SUBSCRIBE_MODULE: &str = r#"import { schema, t, table } from "spacetimedb/server";
 
@@ -218,6 +225,27 @@ fn project_fields(events: Vec<Value>, view_name: &str, projected_fields: &[&str]
         .collect()
 }
 
+fn stale_view_backing_table_fixture() -> PathBuf {
+    workspace_root()
+        .join("crates")
+        .join("smoketests")
+        .join("fixtures")
+        .join("stale-view-backing-table-v2.6.0")
+}
+
+fn stale_view_backing_table_test() -> Smoketest {
+    let test = Smoketest::builder()
+        .data_dir_fixture(
+            stale_view_backing_table_fixture(),
+            STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY,
+        )
+        .autopublish(false)
+        .build();
+
+    test.new_identity().unwrap();
+    test
+}
+
 fn assert_count_view_refresh_behavior(test: &Smoketest, view_name: &str, id: &str, value: &str, updated_value: &str) {
     let query = format!("select * from {view_name}");
     let sub = test.subscribe(&[&query]).expect_rows(2).background().unwrap();
@@ -621,6 +649,105 @@ fn test_view_primary_key_auto_migration_disconnects_clients() {
     );
 }
 
+#[test]
+fn test_repair_stale_sender_scoped_view_backing_table_on_startup() {
+    require_local_server!();
+
+    let test = stale_view_backing_table_test();
+
+    test.assert_sql(
+        "SELECT * FROM player",
+        r#" id | level
+----+-------"#,
+    );
+
+    let sender_view_sub = test
+        .subscribe(&["select * from player"])
+        .expect_rows(2)
+        .background()
+        .unwrap();
+
+    test.call("set_player_state", &["42", "1"]).unwrap();
+    test.call("set_player_state", &["42", "2"]).unwrap();
+
+    let sender_view_events = sender_view_sub.collect().unwrap();
+    let sender_view_projection = project_fields(sender_view_events, "player", &["id", "level"]);
+    assert_eq!(
+        serde_json::json!(sender_view_projection),
+        json!([
+            {
+                "player": {
+                    "deletes": [],
+                    "inserts": [{ "id": 42, "level": 1 }]
+                }
+            },
+            {
+                "player": {
+                    "deletes": [{ "id": 42, "level": 1 }],
+                    "inserts": [{ "id": 42, "level": 2 }]
+                }
+            }
+        ])
+    );
+
+    test.assert_sql(
+        "SELECT * FROM player",
+        r#" id | level
+----+-------
+ 42 | 2"#,
+    );
+}
+
+#[test]
+fn test_repair_stale_anonymous_view_backing_table_on_startup() {
+    require_local_server!();
+
+    let test = stale_view_backing_table_test();
+
+    test.assert_sql(
+        "SELECT * FROM player_and_level",
+        r#" id | level
+----+-------"#,
+    );
+
+    let anonymous_view_sub = test
+        .subscribe(&["select * from player_and_level"])
+        .expect_rows(2)
+        .background()
+        .unwrap();
+
+    test.call("add_player_level", &["1", "2"]).unwrap();
+    test.call("add_player_level", &["2", "2"]).unwrap();
+
+    let anonymous_view_events = anonymous_view_sub.collect().unwrap();
+    let anonymous_view_projection = project_fields(anonymous_view_events, "player_and_level", &["id", "level"]);
+    assert_eq!(
+        serde_json::json!(anonymous_view_projection),
+        json!([
+            {
+                "player_and_level": {
+                    "deletes": [],
+                    "inserts": [{ "id": 1, "level": 2 }]
+                }
+            },
+            {
+                "player_and_level": {
+                    "deletes": [],
+                    "inserts": [{ "id": 2, "level": 2 }]
+                }
+            }
+        ])
+    );
+
+    test.assert_sql(
+        "SELECT * FROM player_and_level",
+        r#" id | level
+----+-------
+ 1  | 2
+ 2  | 2"#,
+    );
+}
+
 #[test]
 fn test_view_accessibility() {
     let test = Smoketest::builder().precompiled_module("views-callable").build();