Re-land spacetime lock/unlock to prevent accidental database deletion (#4888)

Re-lands #4502 on current `master` after the revert in #4881.

## Summary
- restore `spacetime lock` / `spacetime unlock`
- block deleting locked databases
- restore the database-lock smoketests

## Validation
- `cargo fmt --all --check`
- `cargo check -p spacetimedb-cli -p spacetimedb-client-api -p
spacetimedb-standalone`

---------

Co-authored-by: clockwork-labs-bot <clockwork-labs-bot@users.noreply.github.com>
Co-authored-by: Tyler Cloutier <cloutiertyler@aol.com>
Co-authored-by: Tyler Cloutier <cloutiertyler@users.noreply.github.com>
Co-authored-by: Zeke Foppa <bfops@users.noreply.github.com>

Author: 3 people

crates/cli/src/lib.rs

@@ -38,6 +38,8 @@ pub fn get_subcommands() -> Vec<Command> {
         server::cli(),
         subscribe::cli(),
         start::cli(),
+        lock::cli(),
+        unlock::cli(),
         subcommands::version::cli(),
     ]
 }
@@ -67,6 +69,8 @@ pub async fn exec_subcommand(
         "start" => return start::exec(config, paths, args).await,
         "login" => login::exec(config, args).await,
         "logout" => logout::exec(config, args).await,
+        "lock" => lock::exec(config, args).await,
+        "unlock" => unlock::exec(config, args).await,
         "version" => return subcommands::version::exec(paths, root_dir, args).await,
         unknown => Err(anyhow::anyhow!("Invalid subcommand: {unknown}")),
     }

crates/cli/src/subcommands/lock.rs

@@ -0,0 +1,58 @@
+use crate::common_args;
+use crate::config::Config;
+use crate::subcommands::db_arg_resolution::{load_config_db_targets, resolve_database_arg};
+use crate::util::{add_auth_header_opt, database_identity, get_auth_header};
+use clap::{Arg, ArgMatches};
+
+pub fn cli() -> clap::Command {
+    clap::Command::new("lock")
+        .about("Lock a database to prevent accidental deletion")
+        .long_about(
+            "Lock a database to prevent it from being deleted.\n\n\
+             A locked database cannot be deleted until it is unlocked with `spacetime unlock`.\n\
+             This is a safety mechanism to protect production databases from accidental deletion.",
+        )
+        .arg(
+            Arg::new("database")
+                .required(false)
+                .help("The name or identity of the database to lock"),
+        )
+        .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database"))
+        .arg(
+            Arg::new("no_config")
+                .long("no-config")
+                .action(clap::ArgAction::SetTrue)
+                .help("Ignore spacetime.json configuration"),
+        )
+        .after_help("Run `spacetime help lock` for more detailed information.\n")
+}
+
+pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> {
+    let server_from_cli = args.get_one::<String>("server").map(|s| s.as_ref());
+    let no_config = args.get_flag("no_config");
+    let database_arg = args.get_one::<String>("database").map(|s| s.as_str());
+    let config_targets = load_config_db_targets(no_config)?;
+    let resolved = resolve_database_arg(
+        database_arg,
+        config_targets.as_deref(),
+        "spacetime lock [database] [--no-config]",
+    )?;
+    let server = server_from_cli.or(resolved.server.as_deref());
+
+    let identity = database_identity(&config, &resolved.database, server).await?;
+    let host_url = config.get_host_url(server)?;
+    let auth_header = get_auth_header(&mut config, false, server, true).await?;
+    let client = reqwest::Client::new();
+
+    let mut builder = client.post(format!("{host_url}/v1/database/{identity}/lock"));
+    builder = add_auth_header_opt(builder, &auth_header);
+
+    let response = builder.send().await?;
+    response.error_for_status()?;
+
+    println!(
+        "Database {} is now locked. It cannot be deleted until unlocked.",
+        identity
+    );
+    Ok(())
+}

crates/cli/src/subcommands/mod.rs

@@ -8,6 +8,7 @@ pub mod dns;
 pub mod generate;
 pub mod init;
 pub mod list;
+pub mod lock;
 pub mod login;
 pub mod logout;
 pub mod logs;
@@ -17,4 +18,5 @@ pub mod server;
 pub mod sql;
 pub mod start;
 pub mod subscribe;
+pub mod unlock;
 pub mod version;

crates/cli/src/subcommands/unlock.rs

@@ -0,0 +1,54 @@
+use crate::common_args;
+use crate::config::Config;
+use crate::subcommands::db_arg_resolution::{load_config_db_targets, resolve_database_arg};
+use crate::util::{add_auth_header_opt, database_identity, get_auth_header};
+use clap::{Arg, ArgMatches};
+
+pub fn cli() -> clap::Command {
+    clap::Command::new("unlock")
+        .about("Unlock a database to allow deletion")
+        .long_about(
+            "Unlock a database that was previously locked with `spacetime lock`.\n\n\
+             After unlocking, the database can be deleted normally with `spacetime delete`.",
+        )
+        .arg(
+            Arg::new("database")
+                .required(false)
+                .help("The name or identity of the database to unlock"),
+        )
+        .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database"))
+        .arg(
+            Arg::new("no_config")
+                .long("no-config")
+                .action(clap::ArgAction::SetTrue)
+                .help("Ignore spacetime.json configuration"),
+        )
+        .after_help("Run `spacetime help unlock` for more detailed information.\n")
+}
+
+pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> {
+    let server_from_cli = args.get_one::<String>("server").map(|s| s.as_ref());
+    let no_config = args.get_flag("no_config");
+    let database_arg = args.get_one::<String>("database").map(|s| s.as_str());
+    let config_targets = load_config_db_targets(no_config)?;
+    let resolved = resolve_database_arg(
+        database_arg,
+        config_targets.as_deref(),
+        "spacetime unlock [database] [--no-config]",
+    )?;
+    let server = server_from_cli.or(resolved.server.as_deref());
+
+    let identity = database_identity(&config, &resolved.database, server).await?;
+    let host_url = config.get_host_url(server)?;
+    let auth_header = get_auth_header(&mut config, false, server, true).await?;
+    let client = reqwest::Client::new();
+
+    let mut builder = client.post(format!("{host_url}/v1/database/{identity}/unlock"));
+    builder = add_auth_header_opt(builder, &auth_header);
+
+    let response = builder.send().await?;
+    response.error_for_status()?;
+
+    println!("Database {} is now unlocked.", identity);
+    Ok(())
+}

crates/client-api/src/lib.rs

@@ -281,6 +281,9 @@ pub trait ControlStateReadAccess {
     async fn lookup_database_identity(&self, domain: &str) -> anyhow::Result<Option<Identity>>;
     async fn reverse_lookup(&self, database_identity: &Identity) -> anyhow::Result<Vec<DomainName>>;
     async fn lookup_namespace_owner(&self, name: &str) -> anyhow::Result<Option<Identity>>;
+
+    // Locks
+    async fn is_database_locked(&self, database_identity: &Identity) -> anyhow::Result<bool>;
 }
 
 /// Write operations on the SpacetimeDB control plane.
@@ -337,6 +340,14 @@ pub trait ControlStateWriteAccess: Send + Sync {
         owner_identity: &Identity,
         domain_names: &[DomainName],
     ) -> anyhow::Result<SetDomainsResult>;
+
+    // Locks
+    async fn set_database_lock(
+        &self,
+        caller_identity: &Identity,
+        database_identity: &Identity,
+        locked: bool,
+    ) -> anyhow::Result<()>;
 }
 
 #[async_trait]
@@ -392,6 +403,10 @@ impl<T: ControlStateReadAccess + Send + Sync + Sync + ?Sized> ControlStateReadAc
     async fn lookup_namespace_owner(&self, name: &str) -> anyhow::Result<Option<Identity>> {
         (**self).lookup_namespace_owner(name).await
     }
+
+    async fn is_database_locked(&self, database_identity: &Identity) -> anyhow::Result<bool> {
+        (**self).is_database_locked(database_identity).await
+    }
 }
 
 #[async_trait]
@@ -447,6 +462,17 @@ impl<T: ControlStateWriteAccess + ?Sized> ControlStateWriteAccess for Arc<T> {
             .replace_dns_records(database_identity, owner_identity, domain_names)
             .await
     }
+
+    async fn set_database_lock(
+        &self,
+        caller_identity: &Identity,
+        database_identity: &Identity,
+        locked: bool,
+    ) -> anyhow::Result<()> {
+        (**self)
+            .set_database_lock(caller_identity, database_identity, locked)
+            .await
+    }
 }
 
 #[async_trait]

crates/client-api/src/routes/database.rs

@@ -848,6 +848,14 @@ pub async fn reset<S: NodeDelegate + ControlStateDelegate + Authorization>(
     ctx.authorize_action(auth.claims.identity, database.database_identity, Action::ResetDatabase)
         .await?;
 
+    if ctx.is_database_locked(&database_identity).await.map_err(log_and_500)? {
+        return Err((
+            StatusCode::FORBIDDEN,
+            "Database is locked and cannot be reset with --delete-data. Run `spacetime unlock` first.",
+        )
+            .into());
+    }
+
     let num_replicas = num_replicas.map(validate_replication_factor).transpose()?.flatten();
     ctx.reset_database(
         &auth.claims.identity,
@@ -1323,13 +1331,62 @@ pub async fn delete_database<S: ControlStateDelegate + Authorization>(
 
     ctx.authorize_action(auth.claims.identity, database_identity, Action::DeleteDatabase)
         .await?;
+
+    if ctx.is_database_locked(&database_identity).await.map_err(log_and_500)? {
+        return Err((
+            StatusCode::FORBIDDEN,
+            "Database is locked and cannot be deleted. Run `spacetime unlock` first.",
+        )
+            .into());
+    }
+
     ctx.delete_database(&auth.claims.identity, &database_identity)
         .await
         .map_err(log_and_500)?;
 
     Ok(())
 }
 
+pub async fn lock_database<S: ControlStateDelegate + Authorization>(
+    State(ctx): State<S>,
+    Path(DeleteDatabaseParams { name_or_identity }): Path<DeleteDatabaseParams>,
+    Extension(auth): Extension<SpacetimeAuth>,
+) -> axum::response::Result<impl IntoResponse> {
+    let database_identity = name_or_identity.resolve(&ctx).await?;
+    let Some(_database) = worker_ctx_find_database(&ctx, &database_identity).await? else {
+        return Err(StatusCode::NOT_FOUND.into());
+    };
+
+    ctx.authorize_action(auth.claims.identity, database_identity, Action::DeleteDatabase)
+        .await?;
+
+    ctx.set_database_lock(&auth.claims.identity, &database_identity, true)
+        .await
+        .map_err(log_and_500)?;
+
+    Ok(())
+}
+
+pub async fn unlock_database<S: ControlStateDelegate + Authorization>(
+    State(ctx): State<S>,
+    Path(DeleteDatabaseParams { name_or_identity }): Path<DeleteDatabaseParams>,
+    Extension(auth): Extension<SpacetimeAuth>,
+) -> axum::response::Result<impl IntoResponse> {
+    let database_identity = name_or_identity.resolve(&ctx).await?;
+    let Some(_database) = worker_ctx_find_database(&ctx, &database_identity).await? else {
+        return Err(StatusCode::NOT_FOUND.into());
+    };
+
+    ctx.authorize_action(auth.claims.identity, database_identity, Action::DeleteDatabase)
+        .await?;
+
+    ctx.set_database_lock(&auth.claims.identity, &database_identity, false)
+        .await
+        .map_err(log_and_500)?;
+
+    Ok(())
+}
+
 #[derive(Deserialize)]
 pub struct AddNameParams {
     name_or_identity: NameOrIdentity,
@@ -1498,6 +1555,10 @@ pub struct DatabaseRoutes<S> {
     pub db_reset: MethodRouter<S>,
     /// GET: /database/: name_or_identity/unstable/timestamp
     pub timestamp_get: MethodRouter<S>,
+    /// POST: /database/:name_or_identity/lock
+    pub lock_post: MethodRouter<S>,
+    /// POST: /database/:name_or_identity/unlock
+    pub unlock_post: MethodRouter<S>,
     /// ANY: /database/:name_or_identity/route
     pub http_route_root: MethodRouter<S>,
     /// ANY: /database/:name_or_identity/route/
@@ -1529,6 +1590,8 @@ where
             pre_publish: post(pre_publish::<S>),
             db_reset: put(reset::<S>),
             timestamp_get: get(get_timestamp::<S>),
+            lock_post: post(lock_database::<S>),
+            unlock_post: post(unlock_database::<S>),
             http_route_root: any(handle_http_route_root::<S>),
             http_route_root_slash: any(handle_http_route_root_slash::<S>),
             http_route: any(handle_http_route::<S>),
@@ -1556,7 +1619,9 @@ where
             .route("/sql", self.sql_post)
             .route("/unstable/timestamp", self.timestamp_get)
             .route("/pre_publish", self.pre_publish)
-            .route("/reset", self.db_reset);
+            .route("/reset", self.db_reset)
+            .route("/lock", self.lock_post)
+            .route("/unlock", self.unlock_post);
 
         let authed_root_router = axum::Router::new().route(
             "/",
@@ -1761,6 +1826,10 @@ mod tests {
         async fn lookup_namespace_owner(&self, _name: &str) -> anyhow::Result<Option<Identity>> {
             Ok(None)
         }
+
+        async fn is_database_locked(&self, _database_identity: &Identity) -> anyhow::Result<bool> {
+            Ok(false)
+        }
     }
 
     #[async_trait]
@@ -1790,6 +1859,15 @@ mod tests {
             Err(anyhow::anyhow!("unused"))
         }
 
+        async fn set_database_lock(
+            &self,
+            _caller_identity: &Identity,
+            _database_identity: &Identity,
+            _locked: bool,
+        ) -> anyhow::Result<()> {
+            Err(anyhow::anyhow!("unused"))
+        }
+
         async fn reset_database(&self, _caller_identity: &Identity, _spec: DatabaseResetDef) -> anyhow::Result<()> {
             Err(anyhow::anyhow!("unused"))
         }