Auto-migrate: Allow adding new variants at the tail

crates/bench/benches/delete_table.rs

@@ -9,7 +9,8 @@ use rand::{prelude::*, seq::SliceRandom};
 use smallvec::SmallVec;
 use spacetimedb::db::datastore::locking_tx_datastore::delete_table;
 use spacetimedb_data_structures::map::HashSet;
-use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, Size, SquashedOffset};
+use spacetimedb_sats::layout::Size;
+use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, SquashedOffset};
 use std::collections::BTreeSet;
 
 fn time<R>(body: impl FnOnce() -> R) -> Duration {

crates/bench/benches/index.rs

@@ -7,7 +7,8 @@ use criterion::{
 use foldhash::{HashSet, HashSetExt};
 use itertools::Itertools as _;
 use spacetimedb_data_structures::map::{Entry, HashMap};
-use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, Size, SquashedOffset};
+use spacetimedb_sats::layout::Size;
+use spacetimedb_table::indexes::{PageIndex, PageOffset, RowPointer, SquashedOffset};
 use spacetimedb_table::table_index::unique_direct_index::UniqueDirectIndex;
 use spacetimedb_table::table_index::uniquemap::UniqueMap;
 

crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap

@@ -117,6 +117,7 @@ spacetimedb
 │   │   │   └── keccak
 │   │   ├── smallvec
 │   │   ├── spacetimedb_bindings_macro (*)
+│   │   ├── spacetimedb_data_structures (*)
 │   │   ├── spacetimedb_primitives (*)
 │   │   └── thiserror (*)
 │   └── thiserror (*)

crates/codegen/src/csharp.rs

@@ -12,13 +12,13 @@ use crate::util::{
     type_ref_name,
 };
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_primitives::ColId;
 use spacetimedb_schema::def::{BTreeAlgorithm, IndexAlgorithm, ModuleDef, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
 use spacetimedb_schema::type_for_generate::{
-    AlgebraicTypeDef, AlgebraicTypeUse, PlainEnumTypeDef, PrimitiveType, ProductTypeDef, SumTypeDef,
-    TypespaceForGenerate,
+    AlgebraicTypeDef, AlgebraicTypeUse, PlainEnumTypeDef, ProductTypeDef, SumTypeDef, TypespaceForGenerate,
 };
 
 const INDENT: &str = "    ";
@@ -187,7 +187,7 @@ const REDUCER_EVENTS: &str = r#"
                 return Event;
             }
         }
-        
+
         /// <summary>
         /// Access to tables in the client cache, which stores a read-only replica of the remote database state.
         ///
@@ -368,11 +368,11 @@ const REDUCER_EVENTS: &str = r#"
 
         /// <summary>
         /// Subscribe to the following SQL queries.
-        /// 
+        ///
         /// This method returns immediately, with the data not yet added to the DbConnection.
         /// The provided callbacks will be invoked once the data is returned from the remote server.
         /// Data from all the provided queries will be returned at the same time.
-        /// 
+        ///
         /// See the SpacetimeDB SQL docs for more information on SQL syntax:
         /// <a href="https://spacetimedb.com/docs/sql">https://spacetimedb.com/docs/sql</a>
         /// </summary>

crates/codegen/src/rust.rs

@@ -3,11 +3,12 @@ use super::util::{collect_case, iter_reducers, print_lines, type_ref_name};
 use super::Lang;
 use crate::util::{iter_tables, iter_types, iter_unique_cols, print_auto_generated_file_comment};
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::sats::AlgebraicTypeRef;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
-use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse, PrimitiveType};
+use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse};
 use std::collections::BTreeSet;
 use std::fmt::{self, Write};
 use std::ops::Deref;

crates/codegen/src/typescript.rs

@@ -8,11 +8,12 @@ use std::fmt::{self, Write};
 use std::ops::Deref;
 
 use convert_case::{Case, Casing};
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::sats::AlgebraicTypeRef;
 use spacetimedb_schema::def::{ModuleDef, ReducerDef, ScopedTypeName, TableDef, TypeDef};
 use spacetimedb_schema::identifier::Identifier;
 use spacetimedb_schema::schema::{Schema, TableSchema};
-use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse, PrimitiveType};
+use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse};
 
 use super::code_indenter::{CodeIndenter, Indenter};
 use super::Lang;

crates/codegen/src/util.rs

@@ -8,6 +8,7 @@ use std::{
 use super::code_indenter::Indenter;
 use convert_case::{Case, Casing};
 use itertools::Itertools;
+use spacetimedb_lib::sats::layout::PrimitiveType;
 use spacetimedb_lib::version;
 use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef};
 use spacetimedb_primitives::ColList;
@@ -20,7 +21,7 @@ use spacetimedb_schema::{
 use spacetimedb_schema::{
     def::{ModuleDef, ReducerDef},
     identifier::Identifier,
-    type_for_generate::{AlgebraicTypeUse, PrimitiveType},
+    type_for_generate::AlgebraicTypeUse,
 };
 
 /// Turns a closure `f: Fn(&mut Formatter) -> Result` into `fmt::Display`.

crates/codegen/tests/snapshots/codegen__codegen_csharp.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/cli/tests/codegen.rs
+source: crates/codegen/tests/codegen.rs
 expression: outfiles
 ---
 "Reducers/Add.g.cs" = '''
@@ -1215,7 +1215,7 @@ namespace SpacetimeDB
                     return Event;
                 }
             }
-            
+
             /// <summary>
             /// Access to tables in the client cache, which stores a read-only replica of the remote database state.
             ///
@@ -1396,11 +1396,11 @@ namespace SpacetimeDB
 
             /// <summary>
             /// Subscribe to the following SQL queries.
-            /// 
+            ///
             /// This method returns immediately, with the data not yet added to the DbConnection.
             /// The provided callbacks will be invoked once the data is returned from the remote server.
             /// Data from all the provided queries will be returned at the same time.
-            /// 
+            ///
             /// See the SpacetimeDB SQL docs for more information on SQL syntax:
             /// <a href="https://spacetimedb.com/docs/sql">https://spacetimedb.com/docs/sql</a>
             /// </summary>

crates/core/src/db/datastore/error.rs

@@ -82,6 +82,8 @@ pub enum TableError {
     Duplicate(#[from] table::DuplicateError),
     #[error(transparent)]
     ReadColTypeError(#[from] read_column::TypeError),
+    #[error(transparent)]
+    ChangeColumnsError(#[from] table::ChangeColumnsError),
 }
 
 #[derive(Error, Debug, PartialEq, Eq)]

crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs

@@ -25,22 +25,22 @@ use crate::{
     execution_context::ExecutionContext,
 };
 use anyhow::anyhow;
-use core::ops::RangeBounds;
+use core::{convert::Infallible, ops::RangeBounds};
 use itertools::Itertools;
 use spacetimedb_data_structures::map::{HashSet, IntMap};
 use spacetimedb_lib::{
     db::auth::{StAccess, StTableType},
     Identity,
 };
 use spacetimedb_primitives::{ColList, ColSet, IndexId, TableId};
+use spacetimedb_sats::memory_usage::MemoryUsage;
 use spacetimedb_sats::{AlgebraicValue, ProductValue};
 use spacetimedb_schema::{def::IndexAlgorithm, schema::TableSchema};
 use spacetimedb_table::{
     blob_store::{BlobStore, HashMapBlobStore},
     indexes::{RowPointer, SquashedOffset},
     page_pool::PagePool,
     table::{IndexScanRangeIter, InsertError, RowRef, Table, TableAndIndex},
-    MemoryUsage,
 };
 use std::collections::BTreeMap;
 use std::sync::Arc;
@@ -652,57 +652,59 @@ impl CommittedState {
     ) -> Option<()> {
         use PendingSchemaChange::*;
         match change {
-            // An index was removed. Add it back.
             IndexRemoved(table_id, index_id, table_index, index_schema) => {
                 let table = self.tables.get_mut(&table_id)?;
                 // SAFETY: `table_index` was derived from `table`.
                 unsafe { table.add_index(index_id, table_index) };
                 table.with_mut_schema(|s| s.update_index(index_schema));
                 self.index_id_map.insert(index_id, table_id);
             }
-            // An index was added. Remove it.
             IndexAdded(table_id, index_id, pointer_map) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.delete_index(&self.blob_store, index_id, pointer_map);
                 table.with_mut_schema(|s| s.remove_index(index_id));
                 self.index_id_map.remove(&index_id);
             }
-            // A table was removed. Add it back.
             TableRemoved(table_id, table) => {
                 // We don't need to deal with sub-components.
                 // That is, we don't need to add back indices and such.
                 // Instead, there will be separate pending schema changes like `IndexRemoved`.
                 self.tables.insert(table_id, table);
             }
-            // A table was added. Remove it.
             TableAdded(table_id) => {
                 // We don't need to deal with sub-components.
                 // That is, we don't need to remove indices and such.
                 // Instead, there will be separate pending schema changes like `IndexAdded`.
                 self.tables.remove(&table_id);
             }
-            // A table's access was changed. Change back to the old one.
             TableAlterAccess(table_id, access) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.table_access = access);
             }
-            // A constraint was removed. Add it back.
+            TableAlterRowType(table_id, column_schemas) => {
+                let table = self.tables.get_mut(&table_id)?;
+                // SAFETY: `validate = false`.
+                // We must ensure that `column_schemas` is compatible with the existing rows of `table`.
+                // This is the commit table,
+                // so new rows exploiting the changed schema, e.g., by using a new variant,
+                // have not been added here as the commit table is immutable to row addition
+                // during a transaction.
+                unsafe { table.change_columns_to_unchecked(column_schemas, |_, _, _| Ok::<_, Infallible>(())) }
+                    .unwrap_or_else(|e| match e {});
+            }
             ConstraintRemoved(table_id, constraint_schema) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.update_constraint(constraint_schema));
             }
-            // A constraint was added. Remove it.
             ConstraintAdded(table_id, constraint_id) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.remove_constraint(constraint_id));
             }
-            // A sequence was removed. Add it back.
             SequenceRemoved(table_id, seq, schema) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.update_sequence(schema));
                 seq_state.insert(seq);
             }
-            // A sequence was added. Remove it.
             SequenceAdded(table_id, sequence_id) => {
                 let table = self.tables.get_mut(&table_id)?;
                 table.with_mut_schema(|s| s.remove_sequence(sequence_id));