Auto-migration rules for views (#3484)

# Description of Changes

<!-- Please describe your change, mention any related tickets, and so on
here. -->

Defines auto-migration rules for views.

> What are they?

Answer: You can always auto-migrate a view. It's just a matter of
whether you have to disconnect clients. Removal of a view or changing
the return type in a breaking way requires that we disconnect clients.

TODO: Even if a view is updated in a completely compatible way, we still
have to wipe or dirty the read set in order to force a re-computation
because the behavior could have changed.

# API and ABI breaking changes

<!-- If this is an API or ABI breaking change, please apply the
corresponding GitHub label. -->

None

# Expected complexity level and risk

<!--
How complicated do you think these changes are? Grade on a scale from 1
to 5,
where 1 is a trivial change, and 5 is a deep-reaching and complex
change.

This complexity rating applies not only to the complexity apparent in
the diff,
but also to its interactions with existing and future code.

If you answered more than a 2, explain what is complex about the PR,
and what other components it interacts with in potentially concerning
ways. -->

2.5

Not just a mechanical change.

Most of the time you can think of views as though they were tables.
However for auto-migration there is a key distinction. Views don't have
observably persistent state. While we can persist state internally, we
can always derive a view's result set from the database state. Hence
auto-migration rules are not as strict.

# Testing

<!-- Describe any testing you've done, and any testing you'd like your
reviewers to do,
so that you're confident that all the changes work as expected! -->

- [x] Auto-migration planner tests

Author: joshua-spacetime

crates/core/src/db/relational_db.rs

@@ -1055,13 +1055,30 @@ impl RelationalDB {
         Ok(self.inner.create_table_mut_tx(tx, schema)?)
     }
 
-    pub fn create_view_table(
+    pub fn drop_table(&self, tx: &mut MutTx, table_id: TableId) -> Result<(), DBError> {
+        let table_name = self
+            .table_name_from_id_mut(tx, table_id)?
+            .map(|name| name.to_string())
+            .unwrap_or_default();
+        Ok(self.inner.drop_table_mut_tx(tx, table_id).map(|_| {
+            DB_METRICS
+                .rdb_num_table_rows
+                .with_label_values(&self.database_identity, &table_id.into(), &table_name)
+                .set(0)
+        })?)
+    }
+
+    pub fn create_view(
         &self,
         tx: &mut MutTx,
         module_def: &ModuleDef,
         view_def: &ViewDef,
     ) -> Result<(ViewId, TableId), DBError> {
-        Ok(tx.create_view_with_backing_table(module_def, view_def)?)
+        Ok(tx.create_view(module_def, view_def)?)
+    }
+
+    pub fn drop_view(&self, tx: &mut MutTx, view_id: ViewId) -> Result<(), DBError> {
+        Ok(tx.drop_view(view_id)?)
     }
 
     pub fn create_table_for_test_with_the_works(
@@ -1141,19 +1158,6 @@ impl RelationalDB {
         self.create_table_for_test_with_the_works(name, schema, &indexes[..], &[], StAccess::Public)
     }
 
-    pub fn drop_table(&self, tx: &mut MutTx, table_id: TableId) -> Result<(), DBError> {
-        let table_name = self
-            .table_name_from_id_mut(tx, table_id)?
-            .map(|name| name.to_string())
-            .unwrap_or_default();
-        Ok(self.inner.drop_table_mut_tx(tx, table_id).map(|_| {
-            DB_METRICS
-                .rdb_num_table_rows
-                .with_label_values(&self.database_identity, &table_id.into(), &table_name)
-                .set(0)
-        })?)
-    }
-
     /// Rename a table.
     ///
     /// Sets the name of the table to `new_name` regardless of the previous value. This is a
@@ -1164,6 +1168,10 @@ impl RelationalDB {
         Ok(self.inner.rename_table_mut_tx(tx, table_id, new_name)?)
     }
 
+    pub fn view_id_from_name_mut(&self, tx: &MutTx, view_name: &str) -> Result<Option<ViewId>, DBError> {
+        Ok(self.inner.view_id_from_name_mut_tx(tx, view_name)?)
+    }
+
     pub fn table_id_from_name_mut(&self, tx: &MutTx, table_name: &str) -> Result<Option<TableId>, DBError> {
         Ok(self.inner.table_id_from_name_mut_tx(tx, table_name)?)
     }

crates/core/src/db/update.rs

@@ -7,7 +7,7 @@ use spacetimedb_lib::identity::AuthCtx;
 use spacetimedb_lib::AlgebraicValue;
 use spacetimedb_primitives::{ColSet, TableId};
 use spacetimedb_schema::auto_migrate::{AutoMigratePlan, ManualMigratePlan, MigratePlan};
-use spacetimedb_schema::def::TableDef;
+use spacetimedb_schema::def::{TableDef, ViewDef};
 use spacetimedb_schema::schema::{column_schemas_from_defs, IndexSchema, Schema, SequenceSchema, TableSchema};
 
 /// The logger used for by [`update_database`] and friends.
@@ -137,6 +137,17 @@ fn auto_migrate_database(
 
                 stdb.create_table(tx, table_schema)?;
             }
+            spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name) => {
+                let view_def: &ViewDef = plan.new.expect_lookup(view_name);
+                stdb.create_view(tx, plan.new, view_def)?;
+            }
+            spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name) => {
+                let view_id = stdb.view_id_from_name_mut(tx, view_name)?.unwrap();
+                stdb.drop_view(tx, view_id)?;
+            }
+            spacetimedb_schema::auto_migrate::AutoMigrateStep::UpdateView(_) => {
+                unimplemented!("Recompute view and update its backing table")
+            }
             spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(index_name) => {
                 let table_def = plan.new.stored_in_table_def(index_name).unwrap();
                 let index_def = table_def.indexes.get(index_name).unwrap();

crates/core/src/host/module_host.rs

@@ -422,7 +422,7 @@ pub fn create_table_from_view_def(
     module_def: &ModuleDef,
     view_def: &ViewDef,
 ) -> anyhow::Result<()> {
-    stdb.create_view_table(tx, module_def, view_def)
+    stdb.create_view(tx, module_def, view_def)
         .with_context(|| format!("failed to create table for view {}", &view_def.name))?;
     Ok(())
 }

crates/datastore/src/locking_tx_datastore/datastore.rs

@@ -35,7 +35,7 @@ use spacetimedb_durability::TxOffset;
 use spacetimedb_lib::{db::auth::StAccess, metrics::ExecutionMetrics};
 use spacetimedb_lib::{ConnectionId, Identity};
 use spacetimedb_paths::server::SnapshotDirPath;
-use spacetimedb_primitives::{ColList, ConstraintId, IndexId, SequenceId, TableId};
+use spacetimedb_primitives::{ColList, ConstraintId, IndexId, SequenceId, TableId, ViewId};
 use spacetimedb_sats::{
     algebraic_value::de::ValueDeserializer, bsatn, buffer::BufReader, AlgebraicValue, ProductValue,
 };
@@ -512,6 +512,10 @@ impl MutTxDatastore for Locking {
         tx.rename_table(table_id, new_name)
     }
 
+    fn view_id_from_name_mut_tx(&self, tx: &Self::MutTx, view_name: &str) -> Result<Option<ViewId>> {
+        tx.view_id_from_name(view_name)
+    }
+
     fn table_id_from_name_mut_tx(&self, tx: &Self::MutTx, table_name: &str) -> Result<Option<TableId>> {
         tx.table_id_from_name(table_name)
     }

crates/datastore/src/locking_tx_datastore/mut_tx.rs

@@ -9,8 +9,9 @@ use super::{
     SharedMutexGuard, SharedWriteGuard,
 };
 use crate::system_tables::{
-    system_tables, ConnectionIdViaU128, StConnectionCredentialsFields, StConnectionCredentialsRow, StViewFields,
-    StViewParamRow, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, ST_VIEW_PARAM_ID,
+    system_tables, ConnectionIdViaU128, StConnectionCredentialsFields, StConnectionCredentialsRow, StViewColumnFields,
+    StViewFields, StViewParamFields, StViewParamRow, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID,
+    ST_VIEW_PARAM_ID,
 };
 use crate::traits::{InsertFlags, RowTypeForTable, TxData, UpdateFlags};
 use crate::{
@@ -46,7 +47,7 @@ use spacetimedb_sats::{
     AlgebraicType, AlgebraicValue, ProductType, ProductValue, WithTypespace,
 };
 use spacetimedb_schema::{
-    def::{ColumnDef, ModuleDef, ViewDef},
+    def::{ModuleDef, ViewColumnDef, ViewDef},
     schema::{ColumnSchema, ConstraintSchema, IndexSchema, RowLevelSecuritySchema, SequenceSchema, TableSchema},
 };
 use spacetimedb_table::{
@@ -184,7 +185,7 @@ impl MutTxId {
         Ok(())
     }
 
-    /// Create a backing table for a view.
+    /// Create a backing table for a view and update the system tables.
     ///
     /// Requires:
     /// - Everything [`Self::create_table`] requires.
@@ -193,11 +194,7 @@ impl MutTxId {
     /// - Everything [`Self::create_table`] ensures.
     /// - The returned [`ViewId`] is unique and not [`ViewId::SENTINEL`].
     /// - All view metadata maintained by the datastore is created atomically
-    pub fn create_view_with_backing_table(
-        &mut self,
-        module_def: &ModuleDef,
-        view_def: &ViewDef,
-    ) -> Result<(ViewId, TableId)> {
+    pub fn create_view(&mut self, module_def: &ModuleDef, view_def: &ViewDef) -> Result<(ViewId, TableId)> {
         let table_schema = TableSchema::from_view_def(module_def, view_def);
         let table_id = self.create_table(table_schema)?;
 
@@ -206,16 +203,35 @@ impl MutTxId {
             is_anonymous,
             is_public,
             params,
-            columns,
+            return_columns,
             ..
         } = view_def;
 
         let view_id = self.insert_into_st_view(name.clone().into(), table_id, *is_public, *is_anonymous)?;
         self.insert_into_st_view_param(view_id, params)?;
-        self.insert_into_st_view_column(view_id, columns)?;
+        self.insert_into_st_view_column(view_id, return_columns)?;
         Ok((view_id, table_id))
     }
 
+    /// Drop the backing table of a view and update the system tables.
+    pub fn drop_view(&mut self, view_id: ViewId) -> Result<()> {
+        // Drop the view's metadata
+        self.drop_st_view(view_id)?;
+        self.drop_st_view_param(view_id)?;
+        self.drop_st_view_column(view_id)?;
+
+        // Drop the view's backing table if materialized
+        if let StViewRow {
+            table_id: Some(table_id),
+            ..
+        } = self.lookup_st_view(view_id)?
+        {
+            return self.drop_table(table_id);
+        };
+
+        Ok(())
+    }
+
     /// Create a table.
     ///
     /// Requires:
@@ -320,6 +336,15 @@ impl MutTxId {
         })
     }
 
+    fn lookup_st_view(&self, view_id: ViewId) -> Result<StViewRow> {
+        let row = self
+            .iter_by_col_eq(ST_VIEW_ID, StViewFields::ViewId, &view_id.into())?
+            .next()
+            .ok_or_else(|| TableError::IdNotFound(SystemTable::st_view, view_id.into()))?;
+
+        StViewRow::try_from(row)
+    }
+
     /// Insert a row into `st_view`, auto-increments and returns the [`ViewId`].
     fn insert_into_st_view(
         &mut self,
@@ -365,7 +390,7 @@ impl MutTxId {
     }
 
     /// For each column or field returned in a view, insert a row into `st_view_column`.
-    fn insert_into_st_view_column(&mut self, view_id: ViewId, columns: &[ColumnDef]) -> Result<()> {
+    fn insert_into_st_view_column(&mut self, view_id: ViewId, columns: &[ViewColumnDef]) -> Result<()> {
         for def in columns {
             self.insert_via_serialize_bsatn(
                 ST_VIEW_COLUMN_ID,
@@ -425,6 +450,26 @@ impl MutTxId {
         self.delete_col_eq(ST_COLUMN_ID, StColumnFields::TableId.col_id(), &table_id.into())
     }
 
+    /// Drops the row in `st_table` for this `table_id`
+    fn drop_st_table(&mut self, table_id: TableId) -> Result<()> {
+        self.delete_col_eq(ST_TABLE_ID, StTableFields::TableId.col_id(), &table_id.into())
+    }
+
+    /// Drops the row in `st_view` for this `view_id`
+    fn drop_st_view(&mut self, view_id: ViewId) -> Result<()> {
+        self.delete_col_eq(ST_VIEW_ID, StViewFields::ViewId.col_id(), &view_id.into())
+    }
+
+    /// Drops the rows in `st_view_param` for this `view_id`
+    fn drop_st_view_param(&mut self, view_id: ViewId) -> Result<()> {
+        self.delete_col_eq(ST_VIEW_PARAM_ID, StViewParamFields::ViewId.col_id(), &view_id.into())
+    }
+
+    /// Drops the rows in `st_view_column` for this `view_id`
+    fn drop_st_view_column(&mut self, view_id: ViewId) -> Result<()> {
+        self.delete_col_eq(ST_VIEW_COLUMN_ID, StViewColumnFields::ViewId.col_id(), &view_id.into())
+    }
+
     pub fn drop_table(&mut self, table_id: TableId) -> Result<()> {
         self.clear_table(table_id)?;
 
@@ -443,7 +488,7 @@ impl MutTxId {
         }
 
         // Drop the table and their columns
-        self.delete_col_eq(ST_TABLE_ID, StTableFields::TableId.col_id(), &table_id.into())?;
+        self.drop_st_table(table_id)?;
         self.drop_st_column(table_id)?;
 
         if let Some(schedule) = &schema.schedule {
@@ -492,6 +537,14 @@ impl MutTxId {
         Ok(ret)
     }
 
+    pub fn view_id_from_name(&self, view_name: &str) -> Result<Option<ViewId>> {
+        let view_name = &view_name.into();
+        let row = self
+            .iter_by_col_eq(ST_VIEW_ID, StViewFields::ViewName, view_name)?
+            .next();
+        Ok(row.map(|row| row.read_col(StViewFields::ViewId).unwrap()))
+    }
+
     pub fn table_id_from_name(&self, table_name: &str) -> Result<Option<TableId>> {
         let table_name = &table_name.into();
         let row = self

crates/datastore/src/system_tables.rs

@@ -107,6 +107,7 @@ pub const ST_RESERVED_SEQUENCE_RANGE: u32 = 4096;
 #[derive(Debug, Display)]
 pub enum SystemTable {
     st_table,
+    st_view,
     st_column,
     st_sequence,
     st_index,
@@ -749,6 +750,13 @@ pub struct StViewRow {
     pub is_anonymous: bool,
 }
 
+impl TryFrom<RowRef<'_>> for StViewRow {
+    type Error = DatastoreError;
+    fn try_from(row: RowRef<'_>) -> Result<Self, DatastoreError> {
+        read_via_bsatn(row)
+    }
+}
+
 /// A wrapper around `AlgebraicType` that acts like `AlgegbraicType::bytes()` for serialization purposes.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct AlgebraicTypeViaBytes(pub AlgebraicType);

crates/datastore/src/traits.rs

@@ -509,6 +509,7 @@ pub trait MutTxDatastore: TxDatastore + MutTx {
     fn schema_for_table_mut_tx(&self, tx: &Self::MutTx, table_id: TableId) -> Result<Arc<TableSchema>>;
     fn drop_table_mut_tx(&self, tx: &mut Self::MutTx, table_id: TableId) -> Result<()>;
     fn rename_table_mut_tx(&self, tx: &mut Self::MutTx, table_id: TableId, new_name: &str) -> Result<()>;
+    fn view_id_from_name_mut_tx(&self, tx: &Self::MutTx, view_name: &str) -> Result<Option<ViewId>>;
     fn table_id_from_name_mut_tx(&self, tx: &Self::MutTx, table_name: &str) -> Result<Option<TableId>>;
     fn table_id_exists_mut_tx(&self, tx: &Self::MutTx, table_id: &TableId) -> bool;
     fn table_name_from_id_mut_tx<'a>(&'a self, tx: &'a Self::MutTx, table_id: TableId) -> Result<Option<Cow<'a, str>>>;

crates/sats/src/convert.rs

@@ -1,7 +1,7 @@
 use crate::sum_value::SumTag;
 use crate::{i256, u256};
 use crate::{AlgebraicType, AlgebraicValue, ProductType, ProductValue};
-use spacetimedb_primitives::{ColId, ConstraintId, IndexId, ScheduleId, SequenceId, TableId};
+use spacetimedb_primitives::{ColId, ConstraintId, IndexId, ScheduleId, SequenceId, TableId, ViewId};
 
 impl crate::Value for AlgebraicValue {
     type Type = AlgebraicType;
@@ -64,6 +64,7 @@ macro_rules! system_id {
     };
 }
 system_id!(TableId);
+system_id!(ViewId);
 system_id!(ColId);
 system_id!(SequenceId);
 system_id!(IndexId);