Merge pull request #82 from hotdata-dev/feat/databases-managed-cli

Add managed databases CLI

Author: eddietejeda

README.md

@@ -65,6 +65,7 @@ API key priority (lowest to highest): config file → `HOTDATA_API_KEY` env var
 | `auth` | `login`, `status`, `logout` | `login` or bare `auth` opens browser login; `status` / `logout` manage the saved profile |
 | `workspaces` | `list`, `set` | Manage workspaces |
 | `connections` | `list`, `create`, `refresh`, `new` | Manage connections |
+| `databases` | `list`, `create`, `delete`, `tables` | Managed databases (create and load tables via parquet) |
 | `tables` | `list` | List tables and columns |
 | `datasets` | `list`, `create`, `update` | Manage uploaded datasets |
 | `context` | `list`, `show`, `pull`, `push` | Workspace Markdown context (e.g. data model `DATAMODEL`) via the context API |
@@ -127,6 +128,34 @@ hotdata connections create list <type_name> --format json
 hotdata connections create --name "my-conn" --type postgres --config '{"host":"...","port":5432,...}'
 ```
 
+## Databases
+
+Managed databases are Hotdata-owned catalogs you create and populate yourself (no remote source to sync). Query them with SQL as `database_name.schema.table` — the database name is the connection name.
+
+```sh
+hotdata databases list [-w <id>] [-o table|json|yaml]
+hotdata databases create --name <name> [--table <table> ...] [--schema public] [-o table|json|yaml]
+hotdata databases <name_or_id> [-o table|json|yaml]
+hotdata databases delete <name_or_id>
+
+hotdata databases tables list <database> [--schema <name>] [-o table|json|yaml]
+hotdata databases tables load <database> <table> --file ./data.parquet [--schema public]
+hotdata databases tables load <database> <table> --upload-id <id> [--schema public]
+hotdata databases tables delete <database> <table> [--schema public]
+```
+
+- `create` registers a managed connection (`source_type: managed`) with no external credentials. Use `--table` to declare tables up front (required before `tables load` on the current API).
+- `tables load` uploads a **parquet** file (or uses a staged `upload_id` from `POST /v1/files`) and publishes it as the table generation (`replace` mode).
+- For CSV/JSON uploads without a managed database, use `hotdata datasets create` instead (`datasets.main.*`).
+
+Example:
+
+```sh
+hotdata databases create --name sales --table orders
+hotdata databases tables load sales orders --file ./orders.parquet
+hotdata query "SELECT count(*) FROM sales.public.orders"
+```
+
 ## Tables
 
 ```sh

skills/hotdata/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: hotdata
-description: Use this skill when the user wants to run hotdata CLI commands, query the Hotdata API, list workspaces, list connections, create connections, list tables, manage datasets, execute SQL queries, inspect query run history, search tables, manage indexes, manage sandboxes, manage workspace context and stored docs such as context:DATAMODEL via the context API (`hotdata context`), install or update the bundled agent skills (`hotdata skills`), generate shell completions (`hotdata completions`), or interact with the hotdata service. Activate when the user says "run hotdata", "query hotdata", "list workspaces", "list connections", "create a connection", "list tables", "list datasets", "create a dataset", "upload a dataset", "execute a query", "search a table", "list indexes", "create an index", "list query runs", "list past queries", "query history", "list sandboxes", "create a sandbox", "run a sandbox", "workspace context", "pull context", "push context", "data model", "context:DATAMODEL", or asks you to use the hotdata CLI.
+description: Use this skill when the user wants to run hotdata CLI commands, query the Hotdata API, list workspaces, list connections, create connections, list or create managed databases, load parquet into database tables, list tables, manage datasets, execute SQL queries, inspect query run history, search tables, manage indexes, manage sandboxes, manage workspace context and stored docs such as context:DATAMODEL via the context API (`hotdata context`), install or update the bundled agent skills (`hotdata skills`), generate shell completions (`hotdata completions`), or interact with the hotdata service. Activate when the user says "run hotdata", "query hotdata", "list workspaces", "list connections", "create a connection", "list databases", "create a database", "managed database", "load parquet", "list tables", "list datasets", "create a dataset", "upload a dataset", "execute a query", "search a table", "list indexes", "create an index", "list query runs", "list past queries", "query history", "list sandboxes", "create a sandbox", "run a sandbox", "workspace context", "pull context", "push context", "data model", "context:DATAMODEL", or asks you to use the hotdata CLI.
 version: 0.2.2
 ---
 
@@ -73,14 +73,14 @@ These are **patterns** built from the commands below—not separate CLI subcomma
 
 - **Model (`context:DATAMODEL`)** — The **shared** Markdown semantic map of the workspace (entities, keys, joins across connections). **Store and read it only via workspace context** (`hotdata context list`, then `hotdata context show DATAMODEL` **only when listed**, `context push DATAMODEL`); refresh using `connections`, `connections refresh`, `tables list`, and `datasets list`. For a **deep** pass (connector enrichment, indexes, per-table detail), see [references/MODEL_BUILD.md](references/MODEL_BUILD.md). Contrast **analysis modeling** in sandboxes or chat (see [Analysis modeling vs context:DATAMODEL](#analysis-modeling-vs-contextdatamodel)).
 - **History** — Inspect prior activity via `hotdata queries list` (query runs) and `hotdata results list` / `results <id>` (row data).
-- **Chain** — Follow-ups via **`datasets create`** then `query` against `datasets.<schema>.<table>`.
+- **Chain** — Follow-ups via **`datasets create`** then `query` against `datasets.<schema>.<table>`, or via **`databases create`** + **`databases tables load`** (parquet) then `query` against `<database>.<schema>.<table>`.
 - **Indexes** — Review SQL and schema, compare to existing indexes, create **sorted**, **bm25**, or **vector** indexes when it clearly helps; see [references/WORKFLOWS.md](references/WORKFLOWS.md#indexes).
 
 Full step-by-step procedures: [references/WORKFLOWS.md](references/WORKFLOWS.md).
 
 ## Available Commands
 
-Top-level subcommands (each detailed below): **`auth`**, **`datasets`**, **`query`**, **`workspaces`**, **`connections`**, **`tables`**, **`skills`**, **`results`**, **`jobs`**, **`indexes`**, **`embedding-providers`**, **`search`**, **`queries`**, **`sandbox`**, **`context`**, **`completions`**.
+Top-level subcommands (each detailed below): **`auth`**, **`datasets`**, **`query`**, **`workspaces`**, **`connections`**, **`databases`**, **`tables`**, **`skills`**, **`results`**, **`jobs`**, **`indexes`**, **`embedding-providers`**, **`search`**, **`queries`**, **`sandbox`**, **`context`**, **`completions`**.
 
 Global CLI options: **`--api-key`**, **`-v` / `--version`**, **`-h` / `--help`**. Hidden developer flag: **`--debug`** (verbose HTTP logs).
 
@@ -167,6 +167,43 @@ hotdata connections create \
 - Fields with `"type": "array"` must be JSON arrays (e.g. `"spreadsheet_ids": ["abc", "def"]`).
 - Nested `oneOf` fields must be a JSON object including a `"type"` discriminator field matching the chosen variant's `const` value.
 
+### Managed databases (`databases`)
+
+**Managed databases** are Hotdata-owned catalogs (`source_type: managed`) you create and populate yourself—no remote source to sync. Query them in SQL as **`<database_name>.<schema>.<table>`** (the database name is the connection name). Prefer **`hotdata databases`** over **`hotdata connections create --type managed`** for this workflow.
+
+**Parquet vs datasets:** `databases tables load` accepts **parquet only**. For CSV/JSON uploads without a managed database, use **`hotdata datasets create`**.
+
+**Declare tables at create time:** On the current API, each table must be declared with **`--table`** when creating the database before **`tables load`** will succeed. If load fails with *not declared*, recreate with `--table` or add declaration support when the API allows it.
+
+```
+hotdata databases list [--workspace-id <workspace_id>] [--output table|json|yaml]
+hotdata databases create --name <name> [--table <table> ...] [--schema public] [--workspace-id <workspace_id>] [--output table|json|yaml]
+hotdata databases <name_or_id> [--workspace-id <workspace_id>] [--output table|json|yaml]
+hotdata databases delete <name_or_id> [--workspace-id <workspace_id>]
+
+hotdata databases tables list <database> [--schema <name>] [--workspace-id <workspace_id>] [--output table|json|yaml]
+hotdata databases tables load <database> <table> --file ./data.parquet [--schema public] [--workspace-id <workspace_id>]
+hotdata databases tables load <database> <table> --upload-id <id> [--schema public] [--workspace-id <workspace_id>]
+hotdata databases tables delete <database> <table> [--schema public] [--workspace-id <workspace_id>]
+```
+
+- `list` — managed databases only (filters `source_type: managed` from connections).
+- `create` — registers a managed connection with optional `config.schemas[].tables[]` from repeated **`--table`**. Default schema is **`public`**.
+- `<name_or_id>` — inspect one database (name, id, table counts, SQL prefix hint).
+- `delete` — removes the managed database and its tables.
+- `tables list` — tables with `TABLE` (`<database>.<schema>.<table>`), `SYNCED`, `LAST_SYNC` (via `information_schema`).
+- `tables load` — uploads a local **parquet** file (or uses **`--upload-id`** from a prior `POST /v1/files` staging) and publishes with **`replace`** mode. **`--file`** and **`--upload-id`** are mutually exclusive.
+- `tables delete` — drops a table from the managed database.
+- Resolving by **name** or **connection id** works for all subcommands that take `<database>` or `<name_or_id>`. Non-managed connections error with a hint to use **`hotdata connections`**.
+
+Example:
+
+```
+hotdata databases create --name sales --table orders
+hotdata databases tables load sales orders --file ./orders.parquet
+hotdata query "SELECT count(*) FROM sales.public.orders"
+```
+
 ### List Tables and Columns
 ```
 hotdata tables list [--workspace-id <workspace_id>] [--connection-id <connection_id>] [--schema <pattern>] [--table <pattern>] [--limit <int>] [--cursor <cursor>] [--output table|json|yaml]
@@ -499,6 +536,24 @@ Use a sandbox to explore tables and capture **analysis-oriented** notes in sandb
    hotdata query "SELECT \"CustomerName\" FROM datasets.main.my_csv LIMIT 10"
    ```
 
+## Workflow: Creating a managed database (parquet)
+
+1. Create the database and declare tables up front:
+   ```
+   hotdata databases create --name mydb --table events --table users
+   ```
+2. Load parquet into each table:
+   ```
+   hotdata databases tables load mydb events --file ./events.parquet
+   ```
+3. Confirm tables and query:
+   ```
+   hotdata databases tables list mydb
+   hotdata query "SELECT * FROM mydb.public.events LIMIT 10"
+   ```
+
+For CSV/JSON file uploads, use **`hotdata datasets create`** instead.
+
 ## Workflow: Creating a Connection
 
 1. List available connection types:

src/command.rs

@@ -69,6 +69,23 @@ pub enum Commands {
         command: Option<ConnectionsCommands>,
     },
 
+    /// Managed databases you create and populate with tables (parquet uploads)
+    Databases {
+        /// Database name or connection ID (omit to use a subcommand)
+        name_or_id: Option<String>,
+
+        /// Workspace ID (defaults to first workspace from login)
+        #[arg(long, short = 'w', global = true)]
+        workspace_id: Option<String>,
+
+        /// Output format
+        #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])]
+        output: String,
+
+        #[command(subcommand)]
+        command: Option<DatabasesCommands>,
+    },
+
     /// Manage tables in a workspace
     Tables {
         #[command(subcommand)]
@@ -515,6 +532,98 @@ pub enum ConnectionsCreateCommands {
     },
 }
 
+#[derive(Subcommand)]
+pub enum DatabasesCommands {
+    /// List managed databases in the workspace
+    List {
+        /// Output format
+        #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])]
+        output: String,
+    },
+
+    /// Create a new managed database
+    Create {
+        /// Database name (used as the connection name in SQL: `name.schema.table`)
+        #[arg(long)]
+        name: String,
+
+        /// Schema for tables declared at create time (default: public)
+        #[arg(long, default_value = "public")]
+        schema: String,
+
+        /// Table to declare up front (repeatable). Required before load on current API.
+        #[arg(long = "table")]
+        tables: Vec<String>,
+
+        /// Output format
+        #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])]
+        output: String,
+    },
+
+    /// Delete a managed database and its tables
+    Delete {
+        /// Database name or connection ID
+        name_or_id: String,
+    },
+
+    /// Manage tables inside a managed database
+    Tables {
+        #[command(subcommand)]
+        command: DatabaseTablesCommands,
+    },
+}
+
+#[derive(Subcommand)]
+pub enum DatabaseTablesCommands {
+    /// List tables in a managed database
+    List {
+        /// Database name or connection ID
+        database: String,
+
+        /// Filter by schema name
+        #[arg(long)]
+        schema: Option<String>,
+
+        /// Output format
+        #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])]
+        output: String,
+    },
+
+    /// Load a parquet file into a table (creates or replaces the table)
+    Load {
+        /// Database name or connection ID
+        database: String,
+
+        /// Table name
+        table: String,
+
+        /// Schema name (default: public)
+        #[arg(long, default_value = "public")]
+        schema: String,
+
+        /// Path to a local parquet file to upload and load
+        #[arg(long, conflicts_with = "upload_id")]
+        file: Option<String>,
+
+        /// Use a previously staged upload ID from `POST /v1/files` instead of uploading
+        #[arg(long)]
+        upload_id: Option<String>,
+    },
+
+    /// Delete a table from a managed database
+    Delete {
+        /// Database name or connection ID
+        database: String,
+
+        /// Table name
+        table: String,
+
+        /// Schema name (default: public)
+        #[arg(long, default_value = "public")]
+        schema: String,
+    },
+}
+
 #[derive(Subcommand)]
 pub enum ConnectionsCommands {
     /// Interactively create a new connection