Noa/ts module host changes (#3388)

# Description of Changes

Host-side changes extracted from #3327 

I added AUTO_INC_OVERFLOW even though we don't currently ever return it,
in order to future-proof so it's already there when we start emitting
it.

Prepublish was failing because it was expecting a wasm module
unconditionally, so now it takes ?host_type.

I tweaked JS deser to accept null/undefined when the unit type or an
option type is expected.

I switched to bsatn, because the native sats->js translator wasn't
matching what js was expecting.

I renamed the sys module: my thinking is that `spacetime:` as a scheme
will help disambiguate it, and maybe it could also be used for IMC in
the future or something? And I believe we had discussed wanting this to
be versioned, similar to wasm imports.

Trying to get a borrowed str from deserialize_js doesn't work, because
v8 strings don't store utf8.

# Testing

<!-- Describe any testing you've done, and any testing you'd like your
reviewers to do,
so that you're confident that all the changes work as expected! -->

- [x] All this was done in the course of getting an actual typescript
module to successfully publish.

Author: coolreader18

crates/bindings-csharp/Runtime/Exceptions.cs

@@ -72,6 +72,11 @@ public class NoSpaceException : StdbException
     public override string Message => "The provided bytes sink has no more room left";
 }
 
+public class AutoIncOverflowException : StdbException
+{
+    public override string Message => "The auto-increment sequence overflowed";
+}
+
 public class UnknownException : StdbException
 {
     private readonly Errno code;

crates/bindings-csharp/Runtime/Internal/FFI.cs

@@ -36,6 +36,7 @@ public enum Errno : short
     SCHEDULE_AT_DELAY_TOO_LONG = 13,
     INDEX_NOT_UNIQUE = 14,
     NO_SUCH_ROW = 15,
+    AUTO_INC_OVERFLOW = 16,
 }
 
 #pragma warning disable IDE1006 // Naming Styles - Not applicable to FFI stuff.
@@ -96,6 +97,7 @@ public static CheckedStatus ConvertToManaged(Errno status)
                     Errno.SCHEDULE_AT_DELAY_TOO_LONG => new ScheduleAtDelayTooLongException(),
                     Errno.INDEX_NOT_UNIQUE => new IndexNotUniqueException(),
                     Errno.NO_SUCH_ROW => new NoSuchRowException(),
+                    Errno.AUTO_INC_OVERFLOW => new AutoIncOverflowException(),
                     _ => new UnknownException(status),
                 };
             }

crates/bindings/src/table.rs

@@ -1009,7 +1009,7 @@ fn insert<T: Table>(mut row: T::Row, mut buf: IterBuf) -> Result<T::Row, TryInse
             sys::Errno::UNIQUE_ALREADY_EXISTS => {
                 T::UniqueConstraintViolation::get().map(TryInsertError::UniqueConstraintViolation)
             }
-            // sys::Errno::AUTO_INC_OVERFLOW => Tbl::AutoIncOverflow::get().map(TryInsertError::AutoIncOverflow),
+            sys::Errno::AUTO_INC_OVERFLOW => T::AutoIncOverflow::get().map(TryInsertError::AutoIncOverflow),
             _ => None,
         };
         err.unwrap_or_else(|| panic!("unexpected insertion error: {e}"))

crates/cli/src/subcommands/publish.rs

@@ -169,6 +169,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E
                 &client,
                 &database_host,
                 &domain.to_string(),
+                host_type,
                 &program_bytes,
                 &auth_header,
                 break_clients_flag,
@@ -275,16 +276,27 @@ pub fn pretty_print_style_from_env() -> PrettyPrintStyle {
 
 /// Applies pre-publish logic: checking for migration plan, prompting user, and
 /// modifying the request builder accordingly.
+#[allow(clippy::too_many_arguments)]
 async fn apply_pre_publish_if_needed(
     mut builder: reqwest::RequestBuilder,
     client: &reqwest::Client,
     base_url: &str,
     domain: &String,
+    host_type: &str,
     program_bytes: &[u8],
     auth_header: &AuthHeader,
     break_clients_flag: bool,
 ) -> Result<reqwest::RequestBuilder, anyhow::Error> {
-    if let Some(pre) = call_pre_publish(client, base_url, &domain.to_string(), program_bytes, auth_header).await? {
+    if let Some(pre) = call_pre_publish(
+        client,
+        base_url,
+        &domain.to_string(),
+        host_type,
+        program_bytes,
+        auth_header,
+    )
+    .await?
+    {
         println!("{}", pre.migrate_plan);
 
         if pre.break_clients
@@ -310,12 +322,15 @@ async fn call_pre_publish(
     client: &reqwest::Client,
     database_host: &str,
     domain: &String,
+    host_type: &str,
     program_bytes: &[u8],
     auth_header: &AuthHeader,
 ) -> Result<Option<PrePublishResult>, anyhow::Error> {
     let mut builder = client.post(format!("{database_host}/v1/database/{domain}/pre_publish"));
     let style = pretty_print_style_from_env();
-    builder = builder.query(&[("pretty_print_style", style)]);
+    builder = builder
+        .query(&[("pretty_print_style", style)])
+        .query(&[("host_type", host_type)]);
 
     builder = add_auth_header_opt(builder, auth_header);
 

crates/client-api/src/routes/database.rs

@@ -701,15 +701,29 @@ pub struct PrePublishParams {
 pub struct PrePublishQueryParams {
     #[serde(default)]
     style: PrettyPrintStyle,
+    #[serde(default)]
+    host_type: HostType,
 }
 
 pub async fn pre_publish<S: NodeDelegate + ControlStateDelegate>(
     State(ctx): State<S>,
     Path(PrePublishParams { name_or_identity }): Path<PrePublishParams>,
-    Query(PrePublishQueryParams { style }): Query<PrePublishQueryParams>,
+    Query(PrePublishQueryParams { style, host_type }): Query<PrePublishQueryParams>,
     Extension(auth): Extension<SpacetimeAuth>,
     body: Bytes,
 ) -> axum::response::Result<axum::Json<PrePublishResult>> {
+    // Feature gate V8 modules.
+    // The host must've been compiled with the `unstable` feature.
+    // TODO(v8): ungate this when V8 is ready to ship.
+    #[cfg(not(feature = "unstable"))]
+    if host_type == HostType::Js {
+        return Err((
+            StatusCode::BAD_REQUEST,
+            "JS host type requires a host with unstable features",
+        )
+            .into());
+    }
+
     // User should not be able to print migration plans for a database that they do not own
     let database_identity = resolve_and_authenticate(&ctx, &name_or_identity, &auth).await?;
     let style = match style {
@@ -723,7 +737,7 @@ pub async fn pre_publish<S: NodeDelegate + ControlStateDelegate>(
                 database_identity,
                 program_bytes: body.into(),
                 num_replicas: None,
-                host_type: HostType::Wasm,
+                host_type,
             },
             style,
         )

crates/core/src/host/v8/de.rs

@@ -131,6 +131,11 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco
     deserialize_primitive!(deserialize_f32, f32);
 
     fn deserialize_product<V: ProductVisitor<'de>>(self, visitor: V) -> Result<V::Output, Self::Error> {
+        // In `ProductType.serializeValue()` in the TS SDK, null/undefined is accepted for the unit type.
+        if visitor.product_len() == 0 && self.input.is_null_or_undefined() {
+            return visitor.visit_seq_product(de::UnitAccess::new());
+        }
+
         let object = cast!(
             self.common.scope,
             self.input,
@@ -149,6 +154,17 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco
 
     fn deserialize_sum<V: SumVisitor<'de>>(self, visitor: V) -> Result<V::Output, Self::Error> {
         let scope = &*self.common.scope;
+
+        // In `SumType.serializeValue()` in the TS SDK, option is treated specially -
+        // null/undefined marks none, any other value `x` is `some(x)`.
+        if visitor.is_option() {
+            return if self.input.is_null_or_undefined() {
+                visitor.visit_sum(de::NoneAccess::new())
+            } else {
+                visitor.visit_sum(de::SomeAccess::new(self))
+            };
+        }
+
         let sum_name = visitor.sum_name().unwrap_or("<unknown>");
 
         // We expect a canonical representation of a sum value in JS to be

crates/core/src/host/v8/error.rs

@@ -15,7 +15,7 @@ pub(super) type ValueResult<'scope, T> = Result<T, ExceptionValue<'scope>>;
 ///
 /// Newtyped for additional type safety and to track JS exceptions in the type system.
 #[derive(Debug)]
-pub(super) struct ExceptionValue<'scope>(Local<'scope, Value>);
+pub(super) struct ExceptionValue<'scope>(pub(super) Local<'scope, Value>);
 
 /// Error types that can convert into JS exception values.
 pub(super) trait IntoException<'scope> {

crates/core/src/host/v8/mod.rs

@@ -26,7 +26,7 @@ use core::{ptr, str};
 use spacetimedb_client_api_messages::energy::ReducerBudget;
 use spacetimedb_datastore::locking_tx_datastore::MutTxId;
 use spacetimedb_datastore::traits::Program;
-use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp};
+use spacetimedb_lib::{bsatn, ConnectionId, Identity, RawModuleDef, Timestamp};
 use spacetimedb_schema::auto_migrate::MigrationPolicy;
 use std::sync::{Arc, LazyLock};
 use std::time::Instant;
@@ -415,7 +415,7 @@ fn eval_module<'scope>(
     script_id: i32,
     code: &str,
     resolve_deps: impl MapFnTo<ResolveModuleCallback<'scope>>,
-) -> ExcResult<(Local<'scope, v8::Module>, Local<'scope, Value>)> {
+) -> ExcResult<(Local<'scope, v8::Module>, Local<'scope, v8::Promise>)> {
     // Get the source map, if any.
     let source_map_url = find_source_map(code)
         .map(|sm| sm.into_string(scope))
@@ -454,14 +454,28 @@ fn eval_module<'scope>(
     // Evaluate the module.
     let value = module.evaluate(scope).ok_or_else(exception_already_thrown)?;
 
+    if module.get_status() == v8::ModuleStatus::Errored {
+        // If there's an exception while evaluating the code of the module, `evaluate()` won't
+        // throw, but instead the status will be `Errored` and the exception can be obtained from
+        // `get_exception()`.
+        return Err(error::ExceptionValue(module.get_exception()).throw(scope));
+    }
+
+    let value = value.cast::<v8::Promise>();
+    if value.state() == v8::PromiseState::Pending {
+        // If the user were to put top-level `await new Promise((resolve) => { /* do nothing */ })`
+        // the module value would never actually resolve. For now, reject this entirely.
+        return Err(error::TypeError("module has top-level await and is pending").throw(scope));
+    }
+
     Ok((module, value))
 }
 
 /// Compile, instantiate, and evaluate the user module with `code`.
 fn eval_user_module<'scope>(
     scope: &PinScope<'scope, '_>,
     code: &str,
-) -> ExcResult<(Local<'scope, v8::Module>, Local<'scope, Value>)> {
+) -> ExcResult<(Local<'scope, v8::Module>, Local<'scope, v8::Promise>)> {
     let name = str_from_ident!(spacetimedb_module).string(scope).into();
     eval_module(scope, name, 0, code, resolve_sys_module)
 }
@@ -614,7 +628,7 @@ fn call_call_reducer<'scope>(
     let sender = serialize_to_js(scope, &sender.to_u256())?;
     let conn_id: v8::Local<'_, v8::Value> = serialize_to_js(scope, &conn_id.to_u128())?;
     let timestamp = serialize_to_js(scope, &timestamp)?;
-    let reducer_args = serialize_to_js(scope, &reducer_args.tuple.elements)?;
+    let reducer_args = serialize_to_js(scope, reducer_args.get_bsatn())?;
     let args = &[reducer_id, sender, conn_id, timestamp, reducer_args];
 
     // Get the function on the global proxy object and convert to a function.
@@ -677,8 +691,16 @@ fn call_describe_module<'scope>(
     let raw_mod_js = call_free_fun(scope, fun, &[])?;
 
     // Deserialize the raw module.
-    let raw_mod: RawModuleDef = deserialize_js(scope, raw_mod_js)?;
-    Ok(raw_mod)
+    let raw_mod = cast!(
+        scope,
+        raw_mod_js,
+        v8::Uint8Array,
+        "bytes return from __describe_module__"
+    )
+    .map_err(|e| e.throw(scope))?;
+
+    let bytes = raw_mod.get_contents(&mut []);
+    bsatn::from_slice::<RawModuleDef>(bytes).map_err(|_e| error::TypeError("invalid bsatn module def").throw(scope))
 }
 
 #[cfg(test)]
@@ -779,19 +801,7 @@ js error Uncaught Error: foobar
     fn call_describe_module_works() {
         let code = r#"
             function __describe_module__() {
-                return {
-                    "tag": "V9",
-                    "value": {
-                        "typespace": {
-                            "types": [],
-                        },
-                        "tables": [],
-                        "reducers": [],
-                        "types": [],
-                        "misc_exports": [],
-                        "row_level_security": [],
-                    },
-                };
+                return new Uint8Array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
             }
         "#;
         let raw_mod = with_script_catch(code, call_describe_module).map_err(|e| e.to_string());

crates/core/src/host/v8/syscall.rs

@@ -104,7 +104,7 @@ fn register_sys_module<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope
     )
 }
 
-const SYS_MODULE_NAME: &StringConst = str_from_ident!(spacetimedb_sys);
+const SYS_MODULE_NAME: &StringConst = &StringConst::new("spacetime:sys@1.0");
 
 /// The return type of a module -> host syscall.
 pub(super) type FnRet<'scope> = ExcResult<Local<'scope, Value>>;
@@ -258,8 +258,8 @@ fn with_span<'scope, R>(
 /// Throws a `TypeError` if:
 /// - `name` is not `string`.
 fn table_id_from_name(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult<TableId> {
-    let name: &str = deserialize_js(scope, args.get(0))?;
-    Ok(env_on_isolate(scope).instance_env.table_id_from_name(name)?)
+    let name: String = deserialize_js(scope, args.get(0))?;
+    Ok(env_on_isolate(scope).instance_env.table_id_from_name(&name)?)
 }
 
 /// Module ABI that finds the `IndexId` for an index name.
@@ -294,8 +294,8 @@ fn table_id_from_name(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArgume
 /// Throws a `TypeError`:
 /// - if `name` is not `string`.
 fn index_id_from_name(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult<IndexId> {
-    let name: &str = deserialize_js(scope, args.get(0))?;
-    Ok(env_on_isolate(scope).instance_env.index_id_from_name(name)?)
+    let name: String = deserialize_js(scope, args.get(0))?;
+    Ok(env_on_isolate(scope).instance_env.index_id_from_name(&name)?)
 }
 
 /// Module ABI that returns the number of rows currently in table identified by `table_id`.

crates/primitives/src/errno.rs

@@ -22,6 +22,7 @@ macro_rules! errnos {
             SCHEDULE_AT_DELAY_TOO_LONG(13, "Specified delay in scheduling row was too long"),
             INDEX_NOT_UNIQUE(14, "The index was not unique"),
             NO_SUCH_ROW(15, "The row was not found, e.g., in an update call"),
+            AUTO_INC_OVERFLOW(16, "The auto-increment sequence overflowed"),
         );
     };
 }