Auto-migrate: Allow adding new variants at the tail

crates/bench/benches/delete_table.rs

@@ -9,7 +9,8 @@ use rand::{prelude::*, seq::SliceRandom};
 use smallvec::SmallVec;
 use spacetimedb::db::datastore::locking_tx_datastore::delete_table;
 use spacetimedb_data_structures::map::HashSet;
-use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, Size, SquashedOffset};
+use spacetimedb_sats::layout::Size;
+use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, SquashedOffset};
 use std::collections::BTreeSet;
 
 fn time<R>(body: impl FnOnce() -> R) -> Duration {

crates/bench/benches/index.rs

@@ -7,7 +7,8 @@ use criterion::{
 use foldhash::{HashSet, HashSetExt};
 use itertools::Itertools as _;
 use spacetimedb_data_structures::map::{Entry, HashMap};
-use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, Size, SquashedOffset};
+use spacetimedb_sats::layout::Size;
+use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, SquashedOffset};
 use spacetimedb_table::table_index::unique_direct_index::UniqueDirectIndex;
 use spacetimedb_table::table_index::uniquemap::UniqueMap;
 

crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap

@@ -117,6 +117,7 @@ spacetimedb
 │   │   │   └── keccak
 │   │   ├── smallvec
 │   │   ├── spacetimedb_bindings_macro (*)
+│   │   ├── spacetimedb_data_structures (*)
 │   │   ├── spacetimedb_primitives (*)
 │   │   └── thiserror (*)
 │   └── thiserror (*)

crates/codegen/src/csharp.rs

@@ -12,13 +12,13 @@ use crate::util::{
     type_ref_name,
 };
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_primitives::ColId;
 use spacetimedb_schema::def::{BTreeAlgorithm, IndexAlgorithm, ModuleDef, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
 use spacetimedb_schema::type_for_generate::{
-    AlgebraicTypeDef, AlgebraicTypeUse, PlainEnumTypeDef, PrimitiveType, ProductTypeDef, SumTypeDef,
-    TypespaceForGenerate,
+    AlgebraicTypeDef, AlgebraicTypeUse, PlainEnumTypeDef, ProductTypeDef, SumTypeDef, TypespaceForGenerate,
 };
 
 const INDENT: &str = "    ";
@@ -187,7 +187,7 @@ const REDUCER_EVENTS: &str = r#"
                 return Event;
             }
         }
-        
+
         /// <summary>
         /// Access to tables in the client cache, which stores a read-only replica of the remote database state.
         ///
@@ -368,11 +368,11 @@ const REDUCER_EVENTS: &str = r#"
 
         /// <summary>
         /// Subscribe to the following SQL queries.
-        /// 
+        ///
         /// This method returns immediately, with the data not yet added to the DbConnection.
         /// The provided callbacks will be invoked once the data is returned from the remote server.
         /// Data from all the provided queries will be returned at the same time.
-        /// 
+        ///
         /// See the SpacetimeDB SQL docs for more information on SQL syntax:
         /// <a href="https://spacetimedb.com/docs/sql">https://spacetimedb.com/docs/sql</a>
         /// </summary>

crates/codegen/src/rust.rs

@@ -3,11 +3,12 @@ use super::util::{collect_case, iter_reducers, print_lines, type_ref_name};
 use super::Lang;
 use crate::util::{iter_tables, iter_types, iter_unique_cols, print_auto_generated_file_comment};
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::sats::AlgebraicTypeRef;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
-use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse, PrimitiveType};
+use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse};
 use std::collections::BTreeSet;
 use std::fmt::{self, Write};
 use std::ops::Deref;

crates/codegen/src/typescript.rs

@@ -8,11 +8,12 @@ use std::fmt::{self, Write};
 use std::ops::Deref;
 
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::sats::AlgebraicTypeRef;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
-use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse, PrimitiveType};
+use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse};
 
 use super::code_indenter::{CodeIndenter, Indenter};
 use super::Lang;

crates/codegen/src/util.rs

@@ -8,6 +8,7 @@ use std::{
 use super::code_indenter::Indenter;
 use convert_case::{Case, Casing};
 use itertools::Itertools;
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::version;
 use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef};
 use spacetimedb_primitives::ColList;
@@ -20,7 +21,7 @@ use spacetimedb_schema::{
 use spacetimedb_schema::{
     def::{ModuleDef, ReducerDef},
     identifier::Identifier,
-    type_for_generate::{AlgebraicTypeUse, PrimitiveType},
+    type_for_generate::AlgebraicTypeUse,
 };
 
 /// Turns a closure `f: Fn(&mut Formatter) -> Result` into `fmt::Display`.

crates/codegen/tests/snapshots/codegen__codegen_csharp.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/cli/tests/codegen.rs
+source: crates/codegen/tests/codegen.rs
 expression: outfiles
 ---
 "Reducers/Add.g.cs" = '''
@@ -1215,7 +1215,7 @@ namespace SpacetimeDB
                     return Event;
                 }
             }
-            
+
             /// <summary>
             /// Access to tables in the client cache, which stores a read-only replica of the remote database state.
             ///
@@ -1396,11 +1396,11 @@ namespace SpacetimeDB
 
             /// <summary>
             /// Subscribe to the following SQL queries.
-            /// 
+            ///
             /// This method returns immediately, with the data not yet added to the DbConnection.
             /// The provided callbacks will be invoked once the data is returned from the remote server.
             /// Data from all the provided queries will be returned at the same time.
-            /// 
+            ///
             /// See the SpacetimeDB SQL docs for more information on SQL syntax:
             /// <a href="https://spacetimedb.com/docs/sql">https://spacetimedb.com/docs/sql</a>
             /// </summary>

crates/core/src/db/datastore/error.rs

@@ -82,6 +82,8 @@ pub enum TableError {
     Duplicate(#[from] table::DuplicateError),
     #[error(transparent)]
     ReadColTypeError(#[from] read_column::TypeError),
+    #[error(transparent)]
+    ChangeColumnsError(#[from] table::ChangeColumnsError),
 }
 
 #[derive(Error, Debug, PartialEq, Eq)]

crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs

@@ -25,22 +25,22 @@ use crate::{
     execution_context::ExecutionContext,
 };
 use anyhow::anyhow;
-use core::ops::RangeBounds;
+use core::{convert::Infallible, ops::RangeBounds};
 use itertools::Itertools;
 use spacetimedb_data_structures::map::{HashSet, IntMap};
 use spacetimedb_lib::{
     db::auth::{StAccess, StTableType},
     Identity,
 };
 use spacetimedb_primitives::{ColList, ColSet, IndexId, TableId};
+use spacetimedb_sats::memory_usage::MemoryUsage;
 use spacetimedb_sats::{AlgebraicValue, ProductValue};
 use spacetimedb_schema::{def::IndexAlgorithm, schema::TableSchema};
 use spacetimedb_table::{
     blob_store::{BlobStore, HashMapBlobStore},
     indexes::{RowPointer, SquashedOffset},
     page_pool::PagePool,
     table::{IndexScanRangeIter, InsertError, RowRef, Table, TableAndIndex},
-    MemoryUsage,
 };
 use std::collections::BTreeMap;
 use std::sync::Arc;
@@ -686,6 +686,24 @@ impl CommittedState {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.table_access = access);
             }
+            // A table's row type was changed. Change back to the old one.
+            // The row representation of old rows hasn't changed,
+            // so it's safe to not rewrite the rows and merely change the type back.
+            TableAlterRowType(table_id, column_schemas) => {
+                let table = self.tables.get_mut(&table_id)?;
+                // SAFETY:
+                // Let the "old" type/schema be the one in `column_schemas`.
+                // Let the "new" type/schema be the one used by the table which we are rolling back.
+                // There's no need to validate "old",
+                // as it was the row type prior to the change which we're rolling back.
+                // We can use "old", as this is the commit table,
+                // which is immutable to row addition during a transaction,
+                // and thus will only have rows compatible with it.
+                // The rows in the tx state might not be, as they may use e.g., a new variant.
+                // However, we don't care about that, as the tx state is being discarded.
+                unsafe { table.change_columns_to_unchecked(column_schemas, |_, _, _| Ok::<_, Infallible>(())) }
+                    .unwrap_or_else(|e| match e {});
+            }
             // A constraint was removed. Add it back.
             ConstraintRemoved(table_id, constraint_schema) => {
                 let table = self.tables.get_mut(&table_id)?;