diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 44eb9e8266..807ad8e211 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,3 +1,5 @@ +# Developer Guide (How to build Perspective from this repo) + This guide will teach you everything you need to know to get started hacking on the Perspective codebase. Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for contribution guidelines. @@ -13,9 +15,9 @@ Perspective is organized as a and uses [lerna](https://lernajs.io/) to manage dependencies. This guide provides instructions for both the JavaScript and Python libraries. -To switch your development toolchain between the two, use `yarn setup`. Once the -setup script has been run, common commands like `yarn build` and `yarn test` -automatically call the correct build and test tools. +To switch your development toolchain between the two, use `pnpm run setup`. Once +the setup script has been run, common commands like `pnpm run build` and +`pnpm run test` automatically call the correct build and test tools. ### System Dependencies @@ -64,7 +66,7 @@ required. `Perspective.js` specifies its Emscripten version dependency in `package.json`, and the correct version of Emscripten will be installed with other JS -dependencies by running `yarn`. +dependencies by running `pnpm install`. #### Building via local EMSDK @@ -93,14 +95,14 @@ To install a specific version of Emscripten (e.g. `2.0.6`): ## `perspective-python` To build the Python library, first configure your project to build Python via -`yarn setup`. Then, install the requirements corresponding to your version of -python, e.g. +`pnpm run setup`. Then, install the requirements corresponding to your version +of python, e.g. ```bash pip install -r rust/perspective-python/requirements.txt ``` -`perspective-python` supports Python 3.8 and upwards. +`perspective-python` supports Python 3.11 and upwards. ### `perspective-jupyterlab` @@ -150,10 +152,10 @@ PATH=$(brew --prefix llvm@17)/bin:$PATH building if you have `brew`-installed versions of libraries, such as `flatbuffers`. -### Windows 10 +### Windows 10+ You need to use bash in order to build Perspective packages. To successfully -build on Windows 10, enable +build on Windows 10+, enable [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) (WSL) and install the Linux distribution of your choice. diff --git a/README.md b/README.md index 52cccc9771..1653dcce8a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ - +

@@ -20,7 +20,7 @@ large and streaming datasets. Build user-configurable reports, dashboards, notebooks, and applications with a high-performance query engine compiled to WebAssembly, Python, and Rust. -### Features +## Features - A framework-agnostic user interface packaged as a [Custom Element](https://developer.mozilla.org/en-US/docs/Web/Web_Components/Using_custom_elements), @@ -43,21 +43,24 @@ WebAssembly, Python, and Rust. - A [JupyterLab](https://jupyter.org/) widget and Python client library for interactive data analysis in notebooks. -### Documentation +## Documentation - [Project Site](https://perspective-dev.github.io/) - [User Guide](https://perspective-dev.github.io/guide/) -- Python API - - [`perspective`](https://perspective-dev.github.io/python/index.html) - - [`perspective.widget`](https://perspective-dev.github.io/python/perspective/widget.html) - - [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.htm) - - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlett.htm) - - [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.htm) - JavaScript API - [`@perspective-dev/client` Browser](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html) - [`@perspective-dev/client` Node.js](https://perspective-dev.github.io/node/modules/src_ts_perspective.node.ts.html) - - [`@perspective-dev/viewer`](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) - - [`@perspective-dev/react`](https://perspective-dev.github.io/react/index.html) + - [`@perspective-dev/client` Clickhouse Virtual Server](https://perspective-dev.github.io/browser/modules/dist_esm_virtual_servers_clickhouse.js.html) + - [`@perspective-dev/client` DuckDB Virtual Server](https://perspective-dev.github.io/browser/modules/dist_esm_virtual_servers_duckdb.js.html) + - [`@perspective-dev/viewer` Web Component](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) +- Python API + - [`perspective`](https://perspective-dev.github.io/python/index.html) + - [`perspective.widget`](https://perspective-dev.github.io/python/perspective/widget.html) + - [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.html) + - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlett.html) + - [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.html) + - [`perspective.virtual_servers.clickhouse`](https://perspective-dev.github.io/python/perspective/virtual_servers/clickhouse.html) + - [`perspective.virtual_servers.duckdb`](https://perspective-dev.github.io/python/perspective/virtual_servers/duckdb.html) - Rust API - [`perspective`](https://docs.rs/perspective/latest/perspective/) - [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/) @@ -66,13 +69,13 @@ WebAssembly, Python, and Rust. - [`perspective-js`](https://docs.rs/perspective-js/latest/perspective_js/) - [`perspective-viewer`](https://docs.rs/perspective-viewer/latest/perspective_viewer/) -### Examples +## Examples -
editablefileduckdb
fractalmarketraycasting
evictionsnypdstreaming
covidwebcammovies
superstorecitibikeolympics
dataset
+
editablefileduckdb
fractalmarketraycasting
evictionsnypdstreaming
covidwebcammovies
superstorecitibikeolympics
dataset
-### Media +## Media diff --git a/docs/compose.yml b/docs/compose.yml new file mode 100644 index 0000000000..801c87a176 --- /dev/null +++ b/docs/compose.yml @@ -0,0 +1,16 @@ +services: + mdbook: + container_name: mdbook + image: peaceiris/mdbook:v0.5.0 + stdin_open: true + tty: true + ports: + - 3000:3000 + - 3001:3001 + volumes: + - ${PWD}/..:/repo + working_dir: /repo/docs + command: + - serve + - --hostname + - "0.0.0.0" diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 3182a073ef..83321c4475 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -81,23 +81,6 @@ const config = { /** @type {import('@docusaurus/preset-classic').Options} */ ({ docs: false, - // docs: { - // // sidebarPath: require.resolve("./sidebars.js"), - // docItemComponent: require.resolve( - // "./src/components/DocItem" - // ), - // // Please change this to your repo. - // // Remove this to remove the "edit this page" links. - // // editUrl: - // // "https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/", - // }, - // blog: { - // showReadingTime: true, - // // Please change this to your repo. - // // Remove this to remove the "edit this page" links. - // editUrl: - // "https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/", - // }, theme: { customCss: require.resolve("./src/css/custom.css"), }, @@ -112,106 +95,15 @@ const config = { defaultMode: "dark", }, navbar: { - // title: "Perspective", logo: { alt: "Perspective", src: "svg/perspective-logo-light.svg", }, items: [ { - type: "dropdown", + type: "html", + value: `Docs`, position: "right", - label: "Docs", - items: [ - { - type: "html", - value: `User Guide`, - }, - { - type: "html", - value: 'Python API', - }, - { - type: "html", - value: `perspective`, - }, - { - type: "html", - value: `perspective.widget`, - }, - { - type: "html", - value: `perspective.handlers.aiohttp`, - }, - { - type: "html", - value: `perspective.handlers.starlette`, - }, - { - type: "html", - value: `perspective.handlers.tornado`, - }, - { - type: "html", - value: 'JavaScript API', - }, - { - type: "html", - value: link( - "@perspective-dev/client Browser", - "/browser/modules/src_ts_perspective.browser.ts.html", - ), - }, - { - type: "html", - value: link( - "@perspective-dev/client Node.js", - "/node/modules/src_ts_perspective.node.ts.html", - ), - }, - { - type: "html", - value: link( - "@perspective-dev/viewer", - "/viewer/modules/perspective-viewer.html", - ), - }, - { - type: "html", - value: link( - "@perspective-dev/react", - "/react/index.html", - ), - }, - { - type: "html", - value: 'Rust API', - }, - { - type: "html", - value: `perspective`, - }, - { - type: "html", - value: `perspective-client`, - }, - { - type: "html", - value: `perspective-server`, - }, - { - type: "html", - value: `perspective-js`, - }, - { - type: "html", - value: `perspective-python`, - }, - { - type: "html", - value: `perspective-viewer`, - }, - ], }, { to: "/examples", @@ -226,134 +118,7 @@ const config = { ], }, footer: { - links: [ - // { - // title: "Docs", - // items: [ - // { - // type: "html", - // html: `User Guide`, - // }, - // { - // type: "html", - // html: 'Python API', - // }, - // { - // type: "html", - // html: `perspective`, - // }, - // { - // type: "html", - // html: `perspective.widget`, - // }, - // { - // type: "html", - // html: `perspective.handlers.aiohttp`, - // }, - // { - // type: "html", - // html: `perspective.handlers.starlette`, - // }, - // { - // type: "html", - // html: `perspective.handlers.tornado`, - // }, - // { - // type: "html", - // html: 'JavaScript API', - // }, - // { - // type: "html", - // html: link( - // "@perspective-dev/client Browser", - // "/browser/modules/src_ts_perspective.browser.ts.html", - // ), - // }, - // { - // type: "html", - // html: link( - // "@perspective-dev/client Node.js", - // "/node/modules/src_ts_perspective.node.ts.html", - // ), - // }, - // { - // type: "html", - // html: link( - // "@perspective-dev/viewer", - // "/viewer/modules/perspective-viewer.html", - // ), - // }, - // { - // type: "html", - // html: link( - // "@perspective-dev/react", - // "/react/index.html", - // ), - // }, - // { - // type: "html", - // html: 'Rust API', - // }, - // { - // type: "html", - // html: `perspective`, - // }, - // { - // type: "html", - // html: `perspective-client`, - // }, - // { - // type: "html", - // html: `perspective-server`, - // }, - // { - // type: "html", - // html: `perspective-js`, - // }, - // { - // type: "html", - // html: `perspective-python`, - // }, - // { - // type: "html", - // html: `perspective-viewer`, - // }, - // ], - // }, - // { - // title: "OpenJS Foundation", - // items: [ - // { - // label: "Foundation", - // href: "https://openjsf.org/", - // }, - // { - // label: "Terms of Use", - // href: "https://terms-of-use.openjsf.org/", - // }, - // { - // label: "Privacy Policy", - // href: "https://privacy-policy.openjsf.org/", - // }, - // { - // label: "Bylaws", - // href: "https://bylaws.openjsf.org/", - // }, - // { - // label: "Trademark Policy", - // href: "https://trademark-policy.openjsf.org/", - // }, - // { - // label: "Trademark List", - // href: "https://trademark-list.openjsf.org/", - // }, - // { - // label: "Cookie Policy", - // href: "https://www.linuxfoundation.org/cookies/", - // }, - // ], - // }, - ], + links: [], logo: { alt: "OpenJS Foundation Logo", src: "img/openjs_foundation-logo-horizontal-black.png", diff --git a/docs/md/FAQ.md b/docs/md/FAQ.md new file mode 100644 index 0000000000..9d5705da51 --- /dev/null +++ b/docs/md/FAQ.md @@ -0,0 +1,539 @@ +# FAQ + +## Installation + +### Python installation fails on Windows + +Python wheels are published for supported Python versions and platforms. On +Windows, ensure you have a compatible Python version and architecture. Install +with: + +```bash +pip install perspective-python +``` + +If you encounter C++ binding errors or link errors, make sure you are using a +supported Python version and that your `pip` is up to date. Pre-built wheels +eliminate the need for a C++ compiler in most cases. + + + +### Python `import perspective` fails with `ImportError` or undefined symbol + +This typically happens when the C++ shared library (`libpsp.so`) cannot be found +or was built against a different Python version. Ensure your Python version +matches the installed wheel. On Linux, verify that required system libraries are +present. If you see errors about `libpsp.so` or undefined symbols, try +reinstalling in a clean virtual environment. + + + +### Python installation fails on macOS + +On Apple Silicon (M1/M2/M3), make sure you are using a native ARM Python build, +not one running under Rosetta. The published wheels include `aarch64` variants +for supported platforms. + + + +### How do I install Perspective in a Docker container? + +Perspective's Python wheels are built against `manylinux_2_28` containers (see +[`.github/workflows/build.yaml`](../../.github/workflows/build.yaml)), so they +are compatible with most Linux distributions based on glibc 2.28+ (e.g., Debian +10+, Ubuntu 20.04+, RHEL 8+). Use a compatible base image: + +```dockerfile +FROM python:3.12-slim +RUN pip install perspective-python +``` + +Alpine Linux uses musl instead of glibc and is **not** compatible with the +published wheels. + + + +## JavaScript Bundling + +### How do I use Perspective with Vite, Webpack, or esbuild? + +Perspective no longer exports bundler plugins. Instead, you must manually +bootstrap the WASM binaries using your bundler's asset handling. See +[Importing with or without a bundler](./how_to/javascript/importing.md) for +complete examples for Vite, Webpack, esbuild, CDN, and inline builds. + + + +## Framework Integration + +### How do I use Perspective with React? + +Perspective provides a dedicated +[React component](./how_to/javascript/react.md). You must also still initialize +Perspective's WebAssembly as per your bundler — see +[Importing with or without a bundler](./how_to/javascript/importing.md). + + + +### How do I use Perspective with Next.js? + +Perspective relies on Web Workers and WASM, which require client-side rendering. +Use dynamic imports with `ssr: false` in Next.js to load Perspective components +only on the client. + + + +### How do I use Perspective with Vue.js/Angular/etc? + +As a standard Web Component, `` works in most JavaScript web +frameworks directly via standard HTML/DOM APIs, but does not have dedicated +integration libraries for these frameworks. + + + +## Expressions + +### How do I create computed/expression columns? + +Use the [`expressions`](./explanation/view/config/expressions.md) config option +in your `View` to define new columns with ExprTK syntax, which must then be +_used_ somewhere else in your config (like `columns`) to actually be visible & +calculated. In ``, expression columns can be created from +the UI column sidebar by clicking the "New Column" button. + + + +### Can I reference one expression column from another? + +No, you must duplicate calculations that are shared between expression columns. + + + +### Can I do date arithmetic in expressions? + +Yes, but they must be converted to `float` values first (`integer` is an `i32` +which is too small). See +[Expressions](./explanation/view/config/expressions.md). + + + +### Can I do rolling sums or cumulative calculations? + +Not in Perspective's built-in engine, but as an alternative, DuckDB supports +[rolling and cumulative sums via `WINDOW` functions](https://duckdb.org/docs/stable/sql/functions/window_functions), +and DuckDB now has +[native Perspective Virtual Server support](./explanation/virtual_servers.md) +which allows arbitrary DuckDB queries (as a `TABLE` or `VIEW`) to be +`` `Table`s. + + + +## Filters + +### Can I compose filters with OR logic? + +Perspective +[filters](./explanation/view/config/selection_and_ordering.md#filter) are +composed with AND logic by default. As an alternative, you can use +[expression columns](./explanation/view/config/expressions.md) to create a +boolean column that encodes your OR logic (or any arbitrary multi-column +predicate), then filter on that column: + +```javascript +const view = await table.view({ + expressions: { + or_filter: + "if (\"State\" == 'Texas') true; else if (\"State\" == 'California') true; else false", + }, + filter: [["or_filter", "==", true]], +}); +``` + + + +### How do I update filters programmatically? + +Set the [`filter`](./explanation/view/config/selection_and_ordering.md#filter) +property on a `View` config, or use the `` +[`.restore()`](./how_to/javascript/save_restore.md) method to update filters at +runtime. + + + +### Does date filtering support ranges? + +Date columns can be +[filtered](./explanation/view/config/selection_and_ordering.md#filter) with +comparison operators (`>`, `<`, `>=`, `<=`) to achieve range-based filtering. +Apply two filters on the same date column for a range. + + + +## JupyterLab + +### `PerspectiveWidget` is not loading in JupyterLab + +See the [`PerspectiveWidget` guide](./how_to/python/jupyterlab.md) for full +setup details. Ensure the JupyterLab extension version matches your +`perspective-python` version. Make sure you are using a compatible JupyterLab +for your Perspective version (JupyterLab 4+ currently). + +Check that the extension is enabled with `jupyter labextension list`. + + + +## Memory and Performance + +### Perspective has a memory leak + +Maybe, but please review the +[Cleaning up resources](./how_to/javascript/deleting.md) docs carefully before +opening an Issue reporting it (and of course review +[`CONTRIBUTING.md`](https://github.com/perspective-dev/perspective/blob/master/CONTRIBUTING.md) +before opening _any_ Issue). Ensure you call `.delete()` on Views, Tables, and +`` instances when they are no longer needed, in reverse +dependency order. + + + +### How many rows can Perspective's built-in engine handle? + +Perspective is designed for large datasets and can handle millions of rows +depending on the number of columns and available memory. Performance also +significantly depends on column types (`"string"` being slower and larger than +other types due to dictionary interning). + +For larger datasets or out-of-memory virtualized datasets, see +[Virtual Servers](./explanation/virtual_servers.md). + + + +### How do I control threading in `perspective-python`? + +The Python library uses a thread pool internally. For advanced threading +control, consult the +[multithreading documentation](./how_to/python/multithreading.md). + + + +## Theming and Styling + +### How do I enable dark theme? + +Import `themes.css` (see [Theming](./how_to/javascript/theming.md)) and set the +theme via `restore()`: + +```javascript +await viewer.restore({ theme: "Pro Dark" }); +``` + +Or import just the dark theme directly: +`import "@perspective-dev/viewer/dist/css/pro-dark.css";` + + + +### Can I create a custom cell renderer for the datagrid? + +The datagrid plugin supports custom styling via +[`column_config`](https://perspective-dev.github.io/viewer/types/src_ts_ts-rs_ColumnConfigValues.ts.ColumnConfigValues.html) +and CSS custom properties, but custom cell renderers require building a custom +plugin. + + + +### How do I customize chart colors? + +Chart colors can be customized via +[CSS custom properties](./how_to/javascript/theming.md#custom-themes) on the +`` element. + + + +## Streaming and Real-Time Updates + +### How do I stream data into a Perspective table? + +Use [`table.update()`](./explanation/table/update_and_remove.md) to push new +data incrementally. For [indexed](./explanation/table/options.md) tables, +updates with matching index values will replace existing rows. + + + +### `table.update()` raises "No Running Event Loop" + +Perspective 3+ is now threadsafe by default and no longer requires special loop +integration. + + + +### How do I listen for data updates? + +Use `view.on_update()` to register a callback that fires when the underlying +table data changes. See [Listening for events](./how_to/javascript/events.md) +and [Advanced View Operations](./explanation/view/advanced.md#update-callbacks). + + + +## Server Architecture + +### What is the difference between Client-only, Client/Server, and Server-only modes? + +- **Client-only**: The Perspective engine runs entirely in the browser via WASM. + Best for small to medium datasets. +- **Client/Server (replicated)**: Data is hosted on a server and replicated to + the client. The client has a full copy and performs queries locally. +- **Server-only**: All queries are executed on the server. The client only + renders results. Best for very large datasets. + +See [Data Architecture](./explanation/architecture.md) for detailed explanations +of each mode. + + + +### How do I set up WebSocket authentication? + +The [`WebSocketServer`](./how_to/javascript/nodejs_server.md) does not include +built-in authentication. Implement authentication at the transport layer (e.g., +via middleware in your HTTP server) before the WebSocket upgrade. For more +complex needs, `WebSocketServer` is a simple example server based on the +`node:http` module which can serve as a starting point for a custom server. + + + +### Can I bind Perspective to a database? + +Perspective supports [Virtual Servers](./explanation/virtual_servers.md) that +proxy queries to external data sources, with built-in implementations for e.g. +[DuckDB](./how_to/javascript/virtual_server/duckdb.md). + + + +## Aggregation + +### Can I apply multiple aggregates to the same column? + +Yes, by creating a duplicate/alias for your column via +[`expressions`](./explanation/view/config/expressions.md): + +```javascript +await viewer.restore({ + columns: ["Sales", "Sales 2"], + expresions: { "Sales 2": '"Sales"' }, + aggregate: { + Sales: "sum", + "Sales 2": "avg", + }, +}); +``` + + + +### Can I compute a ratio between aggregated columns? + +Use [expression columns](./explanation/view/config/expressions.md) on an +aggregated View to compute ratios. Define an expression that divides one column +by another. + + + +## Data Loading and Arrow + +### How do I load Apache Arrow data into Perspective? + +Perspective natively accepts +[Apache Arrow format](./explanation/table/loading_data.md). Pass an +`ArrayBuffer` containing Arrow IPC data directly to `table()` or +`table.update()`. + + + +### What data formats does Perspective accept? + +Perspective accepts (see [Loading data](./explanation/table/loading_data.md)): + +- **JavaScript**: JSON (row-oriented or column-oriented objects), CSV strings, + Apache Arrow `ArrayBuffer` +- **Python**: `dict`, `list`, `pandas.DataFrame`, `pyarrow.Table`, CSV strings, + Apache Arrow bytes + + + +### CSV update fails but CSV creation works + +When updating a table created with a schema, ensure the CSV column names and +types match the schema exactly. Mismatched column names or types will cause +update failures. + + + +## Export + +### Can I export the viewer to HTML, PNG or PDF? + +HTML and PNG exports are available via `viewer.export("html")` and +`viewer.export("png")`, respectively. For PDF, render the viewer and use browser +or headless browser screenshot capabilities. + + + +### Can I export data to Excel? + +Perspective does not have built-in Excel export. Export data via +`view.to_csv()`, `view.to_json()`, or `view.to_arrow()` (see +[Serializing data](./how_to/javascript/serializing.md)) and convert to Excel +using a library like `xlsx` (JavaScript) or `openpyxl` (Python). + + + +### How do I copy data from a cell or row? + +Use the `"text"` export mode when data is selected: +`await viewer.export("text")`. + + + +## Table Operations + +### `table.remove()` does not update the viewer + +The [`remove()`](./explanation/table/update_and_remove.md) method requires an +[indexed](./explanation/table/options.md) table. Ensure your table was created +with an `index` option, and pass the index values to remove. + + + +## Viewer Configuration + +### How do I save and restore the viewer state? + +Use +[`viewer.save()` and `viewer.restore()`](./how_to/javascript/save_restore.md) to +serialize and deserialize the full viewer configuration. + + + +### Can I hide the configuration panel? + +The settings panel can be toggled programmatically via +`await viewer.restore({ settings: false })`. + + + +### Can I collapse row groups by default? + +Row group can be closed imperatively via +[`view.set_depth()`](./explanation/view/advanced.md). Expansion state is not +persisted or configurable via the `save`/`restore` API currently. + + + +## Internationalization + +### Can I change the UI language? + +Perspective's UI text is defined via CSS variables, which can be customized per +theme. See the +[Icons and Translation](./how_to/javascript/theming.md#icons-and-translation) +section of the theming guide for details. + + + +## Rust + +### How do I build Perspective from Rust? + +See the [Getting Started](./how_to/rust.md) guide for Rust. The Rust crate wraps +the C++ engine and requires a C++ toolchain. You need `cmake` installed and on +your path to build the engine. + + + +## Miscellaneous + +### Can I use Perspective without ``? + +Yes. The `perspective` library (data engine) can be used independently for +server-side data processing without any UI. Use +[`table()` and `view()`](./how_to/javascript/worker.md) directly to query data. + + + +### Can I use Perspective in Pyodide? + +There is an emscripten wheel +[published via Releases](https://github.com/perspective-dev/perspective/releases), +but it must be downloaded and hosted manually and is only built for specific +pyodide versions. + + + +### How do I handle row selection events? + +Listen for +[`perspective-click` and `perspective-select`](./how_to/javascript/events.md) +events on the `` element. + + diff --git a/docs/md/SUMMARY.md b/docs/md/SUMMARY.md index 8c8b2478a1..bc44896839 100644 --- a/docs/md/SUMMARY.md +++ b/docs/md/SUMMARY.md @@ -2,63 +2,74 @@ [What is Perspective](./perspective.md) -# Overview +# Concepts -- [Data Architecture](./explanation/architecture.md) - - [Client-only](./explanation/architecture/client_only.md) - - [Client/Server replicated](./explanation/architecture/client_server.md) - - [Server only](./explanation/architecture/server_only.md) -- [`Table`](./explanation/table.md) - - [Construct an empty `Table` from a schema](./explanation/table/constructing_schema.md) - - [Schema and column types](./explanation/table/schema.md) - - [Loading data](./explanation/table/loading_data.md) - - [`index` and `limit` options](./explanation/table/options.md) - - [`update()` and `remove()` streaming methods](./explanation/table/update_and_remove.md) - - [`clear()` and `replace()` start-over methods](./explanation/table/clear_and_replace.md) -- [`View`](./explanation/view.md) - - [Querying data](./explanation/view/querying.md) - - [`group_by`](./explanation/view/config/group_by.md) - - [`split_by`](./explanation/view/config/split_by.md) - - [`aggregates`](./explanation/view/config/aggregates.md) - - [`columns`](./explanation/view/config/columns.md) - - [`sort`](./explanation/view/config/sort.md) - - [`filter`](./explanation/view/config/filter.md) - - [`expressions`](./explanation/view/config/expressions.md) - - [Flattening a View into a Table](./explanation/view/config/flattening.md) -- [JavaScript](./explanation/javascript.md) - - [Module Structure](./explanation/javascript_module_structure.md) - - [Build options](./explanation/javascript_builds.md) -- [Python](./explanation/python.md) +- [Data Architecture](./explanation/architecture.md) + - [Client-only](./explanation/architecture/client_only.md) + - [Client/Server replicated](./explanation/architecture/client_server.md) + - [Server only](./explanation/architecture/server_only.md) +- [Virtual Servers](./explanation/virtual_servers.md) +- [`Table`](./explanation/table.md) + - [Schema and column types](./explanation/table/schema.md) + - [Loading data](./explanation/table/loading_data.md) + - [Construct an empty `Table` from a schema](./explanation/table/constructing_schema.md) + - [`index` and `limit` options](./explanation/table/options.md) + - [`update()` and `remove()` streaming methods](./explanation/table/update_and_remove.md) + - [`clear()` and `replace()` start-over methods](./explanation/table/clear_and_replace.md) +- [`View`](./explanation/view.md) + - [Querying data](./explanation/view/querying.md) + - [Grouping and Pivots](./explanation/view/config/grouping_and_pivots.md) + - [Selection and Ordering](./explanation/view/config/selection_and_ordering.md) + - [`expressions`](./explanation/view/config/expressions.md) + - [Advanced View Operations](./explanation/view/advanced.md) -# Getting Started +# JavaScript -- [Rust](./how_to/rust.md) -- [JavaScript](./how_to/javascript.md) - - [Installation via NPM](./how_to/javascript/installation.md) - - [Importing with or without a bundler](./how_to/javascript/importing.md) - - [`perspective` data engine library](./how_to/javascript/worker.md) - - [Serializing data](./how_to/javascript/serializing.md) - - [Cleaning up resources](./how_to/javascript/deleting.md) - - [Hosting a `WebSocketServer` in Node.js](./how_to/javascript/nodejs_server.md) - - [Using a dedicated `Worker`, `SharedWorker`, or `ServiceWorker`](./how_to/javascript/custom_worker.md) - - [`perspective-viewer` Custom Element library](./how_to/javascript/viewer.md) - - [Theming](./how_to/javascript/theming.md) - - [Custom Themes](./how_to/javascript/custom_themes.md) - - [Loading data from a `Table`](./how_to/javascript/loading_data.md) - - [Loading data from a virtual `Table`](./how_to/javascript/loading_virtual_data.md) - - [Saving and restoring UI state](./how_to/javascript/save_restore.md) - - [Listening for events](./how_to/javascript/events.md) - - [Plugin render limits](./how_to/javascript/plugin_settings.md) - - [React Component](./how_to/javascript/react.md) -- [Python](./how_to/python.md) - - [Installation](./how_to/python/installation.md) - - [Loading data into a `Table`](./how_to/python/table.md) - - [Callbacks and events](./how_to/python/callbacks.md) - - [Multithreading](./how_to/python/multithreading.md) - - [Hosting a WebSocket server](./how_to/python/websocket.md) - - [`PerspectiveWidget` for JupyterLab](./how_to/python/jupyterlab.md) - - [Tutorial: A `tornado` server with virtual `perspective-viewer`](./tutorials/python/tornado.md) +- [Installation and Module Structure](./how_to/javascript/installation.md) +- [Importing with or without a bundler](./how_to/javascript/importing.md) +- [`perspective` data engine library](./how_to/javascript/worker.md) + - [Serializing data](./how_to/javascript/serializing.md) + - [Cleaning up resources](./how_to/javascript/deleting.md) + - [Hosting a `WebSocketServer` in Node.js](./how_to/javascript/nodejs_server.md) + - [Customizing `perspective.worker()`](./how_to/javascript/custom_worker.md) +- [`perspective-viewer` Custom Element library](./how_to/javascript/viewer.md) + - [Loading data](./how_to/javascript/loading_data.md) + - [Theming](./how_to/javascript/theming.md) + - [Saving and restoring UI state](./how_to/javascript/save_restore.md) + - [Listening for events](./how_to/javascript/events.md) + - [Plugin render limits](./how_to/javascript/plugin_settings.md) +- [Virtual Servers](./how_to/javascript/virtual_server.md) + - [DuckDB](./how_to/javascript/virtual_server/duckdb.md) + - [ClickHouse](./how_to/javascript/virtual_server/clickhouse.md) + - [Custom](./how_to/javascript/virtual_server/custom.md) +- [React Component](./how_to/javascript/react.md) -# API +# Python -- [Crate documentation on `docs.rs` ](./api_reference.md) +- [Overview](./explanation/python.md) +- [Installation](./how_to/python/installation.md) +- [Loading data into a `Table`](./how_to/python/table.md) +- [Callbacks and events](./how_to/python/callbacks.md) +- [Multithreading](./how_to/python/multithreading.md) +- [Hosting a WebSocket server](./how_to/python/websocket.md) +- [`PerspectiveWidget` for JupyterLab](./how_to/python/jupyterlab.md) +- [Virtual Servers](./how_to/python/virtual_server.md) + - [DuckDB](./how_to/python/virtual_server/duckdb.md) + - [ClickHouse](./how_to/python/virtual_server/clickhouse.md) + - [Custom](./how_to/python/virtual_server/custom.md) + +# Rust + +- [Getting Started](./how_to/rust.md) + +# Tutorials + +- [A `tornado` server in Python](./tutorials/python/tornado.md) + +# API Reference + +- [API Reference](./api_reference.md) + +# FAQ + +- [FAQ](./FAQ.md) diff --git a/docs/md/api_reference.md b/docs/md/api_reference.md index c5836cf30b..d91655b8d8 100644 --- a/docs/md/api_reference.md +++ b/docs/md/api_reference.md @@ -2,8 +2,21 @@ Perspective's complete API is hosted on `docs.rs`: -- [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/index.html) - covers `Table` and `View` data engine API methods common for Rust, - JavaScript and Python. -- [`perspective-rs`](https://docs.rs/perspective-client/latest/perspective_client/index.html) - adds Rust-specific documentation for the Rust crate entrypoint. +- Python API + - [`perspective`](https://perspective-dev.github.io/python/index.html) + - [`perspective.widget`](https://perspective-dev.github.io/python/perspective/widget.html) + - [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.htm) + - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlett.htm) + - [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.htm) +- JavaScript API + - [`@perspective-dev/client` Browser](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html) + - [`@perspective-dev/client` Node.js](https://perspective-dev.github.io/node/modules/src_ts_perspective.node.ts.html) + - [`@perspective-dev/viewer`](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html) + - [`@perspective-dev/react`](https://perspective-dev.github.io/react/index.html) +- Rust API + - [`perspective`](https://docs.rs/perspective/latest/perspective/) + - [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/) + - [`perspective-server`](https://docs.rs/perspective-server/latest/perspective_server/) + - [`perspective-python`](https://docs.rs/perspective-python/latest/perspective_python/) + - [`perspective-js`](https://docs.rs/perspective-js/latest/perspective_js/) + - [`perspective-viewer`](https://docs.rs/perspective-viewer/latest/perspective_viewer/) diff --git a/docs/md/explanation.md b/docs/md/explanation.md index b1303b40c8..68208631b6 100644 --- a/docs/md/explanation.md +++ b/docs/md/explanation.md @@ -1 +1,4 @@ -# Explanation +# Overview + +This section covers Perspective's core concepts: data architecture patterns, +the `Table` and `View` data model, and language-specific module details. diff --git a/docs/md/explanation/architecture/client_only.md b/docs/md/explanation/architecture/client_only.md index 7db35eca0e..ca800eb5a1 100644 --- a/docs/md/explanation/architecture/client_only.md +++ b/docs/md/explanation/architecture/client_only.md @@ -27,7 +27,7 @@ As the client-only design starts with creating a client-side Perspective `Table`, data can be provided by any standard web service in any Perspective compatible format (JSON, CSV or Apache Arrow). -#### Javascript client +## Javascript client ```javascript const worker = await perspective.worker(); diff --git a/docs/md/explanation/architecture/client_server.md b/docs/md/explanation/architecture/client_server.md index f6df4408eb..cd7e60a3ad 100644 --- a/docs/md/explanation/architecture/client_server.md +++ b/docs/md/explanation/architecture/client_server.md @@ -19,7 +19,7 @@ performance on the client is very good and identical to client-only architecture. Updates and edits are seamlessly synchonized across clients via their virtual server counterparts using websockets and Apache Arrow. -#### Python and Tornado server +## Python and Tornado server ```python from perspective import Server, PerspectiveTornadoHandler @@ -39,9 +39,9 @@ loop = tornado.ioloop.IOLoop.current() loop.start() ``` -#### Javascript client +## Javascript client -Perspective's websocket client interfaces with the Python server. then +Perspective's websocket client interfaces with the Python server, then _replicates_ the server-side Table. ```javascript diff --git a/docs/md/explanation/javascript.md b/docs/md/explanation/javascript.md deleted file mode 100644 index dee52a411f..0000000000 --- a/docs/md/explanation/javascript.md +++ /dev/null @@ -1,15 +0,0 @@ -Perspective's JavaScript library offers a configurable UI powered by the same -fast streaming data engine, just re-compiled to WebAssembly. A simple example -which loads an [Apache Arrow](https://arrow.apache.org/) and computes a "Group -By" operation, returning a new Arrow: - -```javascript -import perspective from "@perspective-dev/client"; - -const table = await perspective.table(apache_arrow_data); -const view = await table.view({ group_by: ["CounterParty", "Security"] }); -const arrow = await view.to_arrow(); -``` - -[More Examples](https://github.com/perspective-dev/perspective/tree/master/examples) -are available on GitHub. diff --git a/docs/md/explanation/javascript_builds.md b/docs/md/explanation/javascript_builds.md deleted file mode 100644 index 627783884c..0000000000 --- a/docs/md/explanation/javascript_builds.md +++ /dev/null @@ -1,42 +0,0 @@ -# JavaScript Builds - -Perspective requires the browser to have access to Perspective's `.wasm` -binaries _in addition_ to the bundled `.js` files, and as a result the build -process requires a few extra steps. To ease integration, Perspective's NPM -releases come with multiple prebuilt configurations. - -## Browser - -### ESM Builds - -The recommended builds for production use are packaged as ES Modules and require -a _bootstrapping_ step in order to acquire the `.wasm` binaries and initialize -Perspective's JavaScript with them. However, because they have no hard-coded -dependencies on the `.wasm` paths, they are ideal for use with JavaScript -bundlers such as ESBuild, Rollup, Vite or Webpack. - -### CDN Builds - -Perspective's CDN builds are good for non-bundled scenarios, such as importing -directly from a ` ``` + +## Attributes + +`` can be configured via HTML attributes or JavaScript +properties. When set as attributes, the viewer will apply the configuration on +initialization: + +```html + + +``` + +## UI Features + +The viewer provides an interactive side panel with: + +- **Column list** - drag and drop columns to configure `group_by`, `split_by`, + `sort`, and `filter` fields. +- **New Column** button - opens an expression editor for creating computed + columns via the [expression language](../../explanation/view/config/expressions.md). +- **Plugin selector** - switch between visualization plugins such as Datagrid, + X/Y Line, X/Y Scatter, Treemap, Sunburst, and Heatmap. +- **Theme** selector - toggle between available themes. +- **Export** - download the current view as CSV or Arrow. +- **Copy** - copy the current view to the clipboard. +- **Reset** - restore the viewer to its default configuration. + +## Methods + +Key methods on the `` element: + +| Method | Description | +|---|---| +| `load(table)` | Bind a `Table` to the viewer | +| `restore(config)` | Apply a saved configuration object | +| `save()` | Serialize the current configuration | +| `reset(all)` | Reset configuration (pass `true` to also reset expressions) | +| `getTable()` | Get the bound `Table` | +| `flush()` | Wait for any pending UI updates to complete | diff --git a/docs/md/how_to/javascript/virtual_server.md b/docs/md/how_to/javascript/virtual_server.md new file mode 100644 index 0000000000..741a3d85be --- /dev/null +++ b/docs/md/how_to/javascript/virtual_server.md @@ -0,0 +1,19 @@ +# Virtual Servers + +Perspective's Virtual Server feature lets you connect `` to +external data sources without loading data into Perspective's built-in engine. +Instead, queries are translated and executed natively by the external database. + +For a detailed explanation of how virtual servers work, see the +[Virtual Servers](../../explanation/virtual_servers.md) concepts page. + +Perspective ships with built-in virtual server implementations for: + +- [**DuckDB**](./virtual_server/duckdb.md) — query DuckDB databases in-browser + via `@duckdb/duckdb-wasm`, or on the server via Node.js. +- [**ClickHouse**](./virtual_server/clickhouse.md) — query a ClickHouse server + directly from the browser or from Node.js. + +You can also [**implement your own**](./virtual_server/custom.md) virtual server +to connect Perspective to any data source by implementing the +`VirtualServerHandler` interface. diff --git a/docs/md/how_to/javascript/virtual_server/clickhouse.md b/docs/md/how_to/javascript/virtual_server/clickhouse.md new file mode 100644 index 0000000000..af70a7f56c --- /dev/null +++ b/docs/md/how_to/javascript/virtual_server/clickhouse.md @@ -0,0 +1,43 @@ +# ClickHouse Virtual Server + +Perspective provides a built-in virtual server for +[ClickHouse](https://clickhouse.com/), allowing `` to query +ClickHouse tables directly from the browser. + +For server-side Python usage, see the +[Python ClickHouse guide](../../python/virtual_server/clickhouse.md). + +## Installation + +```bash +npm install @perspective-dev/client @perspective-dev/viewer @clickhouse/client-web +``` + +## Usage + +Connect to a ClickHouse instance and bind it to a Perspective viewer: + +```javascript +import perspective from "@perspective-dev/client"; +import "@perspective-dev/viewer"; +import { createClient } from "@clickhouse/client-web"; + +// Connect to ClickHouse +const clickhouseClient = createClient({ + url: "http://localhost:8123", + database: "default", +}); + +// Create a Perspective virtual server backed by ClickHouse +const handler = perspective.ClickhouseHandler(clickhouseClient); +const messageHandler = perspective.createMessageHandler(handler); + +// Connect a viewer +const client = await perspective.worker(messageHandler); +const table = await client.open_table("my_table"); +document.getElementById("viewer").load(table); +``` + +## Examples + +- [Browser ClickHouse example](https://github.com/perspective-dev/perspective/tree/master/examples/esbuild-clickhouse-virtual) diff --git a/docs/md/how_to/javascript/virtual_server/custom.md b/docs/md/how_to/javascript/virtual_server/custom.md new file mode 100644 index 0000000000..38e7a3a1c2 --- /dev/null +++ b/docs/md/how_to/javascript/virtual_server/custom.md @@ -0,0 +1,80 @@ +# Implementing a custom Virtual Server + +You can connect Perspective to any data source by implementing the +`VirtualServerHandler` interface and passing it to `createMessageHandler()`. + +For background on virtual servers, see the +[Virtual Servers overview](../../../explanation/virtual_servers.md). + +## Example + +```typescript +import perspective from "@perspective-dev/client"; +import type { + VirtualServerHandler, + ColumnType, + ViewConfig, + ViewWindow, + VirtualDataSlice, +} from "@perspective-dev/client"; + +const handler = { + async getHostedTables(): Promise { + return ["my_table"]; + }, + + async tableSchema(tableId: string): Promise> { + return { name: "string", price: "float", date: "date" }; + }, + + async tableSize(tableId: string): Promise { + return 1000; + }, + + async tableMakeView( + tableId: string, + viewId: string, + config: ViewConfig, + ): Promise { + // Translate `config` (group_by, sort, filter, etc.) into a query + // against your data source. Store the query keyed by `viewId` + // for later data retrieval. + }, + + async viewDelete(viewId: string): Promise { + // Clean up resources for this view + }, + + async viewGetData( + viewId: string, + config: ViewConfig, + schema: Record, + viewport: ViewWindow, + dataSlice: VirtualDataSlice, + ): Promise { + // Query your data source using `config` and `viewport` for the + // row/column window. Push columnar results via `dataSlice.setCol()`. + }, + + getFeatures() { + return { + group_by: true, + sort: true, + filter_ops: { + string: ["==", "!=", "contains", "is null", "is not null"], + float: ["==", "!=", ">", "<", ">=", "<="], + }, + aggregates: { + float: ["sum", "avg", "count", "min", "max"], + string: ["count", "any"], + }, + }; + }, +} satisfies VirtualServerHandler; + +// Create a message handler and use it like a worker +const messageHandler = perspective.createMessageHandler(handler); +const client = await perspective.worker(messageHandler); +const table = await client.open_table("my_table"); +document.getElementById("viewer").load(table); +``` diff --git a/docs/md/how_to/javascript/virtual_server/duckdb.md b/docs/md/how_to/javascript/virtual_server/duckdb.md new file mode 100644 index 0000000000..23bfbe219e --- /dev/null +++ b/docs/md/how_to/javascript/virtual_server/duckdb.md @@ -0,0 +1,49 @@ +# DuckDB Virtual Server + +Perspective provides a built-in virtual server for +[DuckDB](https://duckdb.org/), allowing `` to query +DuckDB-WASM databases directly in the browser. + +For server-side Python usage, see the +[Python DuckDB guide](../../python/virtual_server/duckdb.md). + +## Installation + +```bash +npm install @perspective-dev/client @perspective-dev/viewer @duckdb/duckdb-wasm +``` + +## Usage + +Initialize DuckDB-WASM, load data, and connect it to a Perspective viewer: + +```javascript +import perspective from "@perspective-dev/client"; +import "@perspective-dev/viewer"; +import * as duckdb from "@duckdb/duckdb-wasm"; + +// Initialize DuckDB-WASM +const DUCKDB_BUNDLES = duckdb.getJsDelivrBundles(); +const bundle = await duckdb.selectBundle(DUCKDB_BUNDLES); +const worker = await duckdb.createWorker(bundle.mainWorker); +const logger = new duckdb.ConsoleLogger(); +const db = new duckdb.AsyncDuckDB(logger, worker); +await db.instantiate(bundle.mainModule); + +// Load data into DuckDB +const conn = await db.connect(); +await conn.query(`CREATE TABLE my_table AS SELECT * FROM 'data.parquet'`); + +// Create a Perspective virtual server backed by DuckDB +const handler = perspective.DuckDBHandler(db); +const messageHandler = perspective.createMessageHandler(handler); + +// Connect a viewer +const client = await perspective.worker(messageHandler); +const table = await client.open_table("my_table"); +document.getElementById("viewer").load(table); +``` + +## Examples + +- [Browser DuckDB example](https://github.com/perspective-dev/perspective/tree/master/examples/esbuild-duckdb-virtual) diff --git a/docs/md/how_to/python.md b/docs/md/how_to/python.md index 4193ef4bec..f124b1ccb2 100644 --- a/docs/md/how_to/python.md +++ b/docs/md/how_to/python.md @@ -1 +1,4 @@ # Python + +Guides for using `perspective-python`, including data loading, callbacks, +multithreading, WebSocket servers, and JupyterLab integration. diff --git a/docs/md/how_to/python/table.md b/docs/md/how_to/python/table.md index cf1dd54363..2436664486 100644 --- a/docs/md/how_to/python/table.md +++ b/docs/md/how_to/python/table.md @@ -69,16 +69,13 @@ table = perspective.table(data, index="index") ## Time Zone Handling -When parsing `"datetime"` strings, times are assumed _local time_ unless an -explicit timezone offset is parsed. All `"datetime"` columns (regardless of -input time zone) are _output_ to the user as `datetime.datetime` objects in -_local time_ according to the Python runtime. - -This behavior is consistent with Perspective's behavior in JavaScript. For more -details, see this in-depth -[explanation](https://github.com/perspective-dev/perspective/pull/867) of -`perspective-python` semantics around time zone handling. - -``` - -``` +When parsing `"datetime"` strings, times without an explicit timezone offset are +interpreted as _UTC_. Strings with a timezone offset (e.g., `+05:00`) are +converted to UTC. All `"datetime"` values are stored internally as milliseconds +since the Unix epoch, and are _output_ as integer timestamps (milliseconds since +epoch) from methods like `to_columns()` and `to_json()`. + +Python `datetime` objects are serialized to strings before parsing. Naive +`datetime` objects (without `tzinfo`) produce strings without timezone +information and are therefore treated as UTC. Timezone-aware `datetime` objects +include their offset in the serialized string, which is used to convert to UTC. diff --git a/docs/md/how_to/python/virtual_server.md b/docs/md/how_to/python/virtual_server.md new file mode 100644 index 0000000000..4b4f6b5f98 --- /dev/null +++ b/docs/md/how_to/python/virtual_server.md @@ -0,0 +1,18 @@ +# Virtual Servers + +Perspective's Virtual Server feature lets you connect `` to +external data sources without loading data into Perspective's built-in engine. +Instead, queries are translated and executed natively by the external database. + +For a detailed explanation of how virtual servers work, see the +[Virtual Servers](../../explanation/virtual_servers.md) concepts page. + +Perspective ships with built-in virtual server implementations for: + +- [**DuckDB**](./virtual_server/duckdb.md) — query DuckDB databases using the + `duckdb` Python package. +- [**ClickHouse**](./virtual_server/clickhouse.md) — query a ClickHouse server + using the `clickhouse-connect` Python package. + +You can also [**implement your own**](./virtual_server/custom.md) virtual server +to connect Perspective to any data source by subclassing `VirtualServerHandler`. diff --git a/docs/md/how_to/python/virtual_server/clickhouse.md b/docs/md/how_to/python/virtual_server/clickhouse.md new file mode 100644 index 0000000000..3078d2be6e --- /dev/null +++ b/docs/md/how_to/python/virtual_server/clickhouse.md @@ -0,0 +1,52 @@ +# ClickHouse Virtual Server + +Perspective provides a built-in virtual server for +[ClickHouse](https://clickhouse.com/), allowing `` clients +to query a ClickHouse server over WebSocket. + +For browser-only usage, see the +[JavaScript ClickHouse guide](../../javascript/virtual_server/clickhouse.md). + +## Installation + +```bash +pip install perspective-python clickhouse-connect +``` + +## Usage + +Create a server that exposes ClickHouse tables to browser clients: + +```python +import clickhouse_connect +import tornado.web +import tornado.ioloop +from perspective import ClickhouseVirtualServer +from perspective.handlers.tornado import PerspectiveTornadoHandler + +# Connect to ClickHouse +client = clickhouse_connect.get_client(host="localhost") + +# Create virtual server backed by ClickHouse +server = ClickhouseVirtualServer(client) + +# Serve over WebSocket +app = tornado.web.Application([ + (r"/websocket", PerspectiveTornadoHandler, {"perspective_server": server}), +]) + +app.listen(8080) +tornado.ioloop.IOLoop.current().start() +``` + +Connect from the browser: + +```javascript +const websocket = await perspective.websocket("ws://localhost:8080/websocket"); +const table = await websocket.open_table("my_table"); +document.getElementById("viewer").load(table); +``` + +## Examples + +- [Python ClickHouse example](https://github.com/perspective-dev/perspective/tree/master/examples/python-clickhouse-virtual) diff --git a/docs/md/how_to/python/virtual_server/custom.md b/docs/md/how_to/python/virtual_server/custom.md new file mode 100644 index 0000000000..c7c66e2c4d --- /dev/null +++ b/docs/md/how_to/python/virtual_server/custom.md @@ -0,0 +1,64 @@ +# Implementing a custom Virtual Server + +You can connect Perspective to any data source by subclassing +`VirtualServerHandler` and wrapping it with `VirtualServer`. + +For background on virtual servers, see the +[Virtual Servers overview](../../../explanation/virtual_servers.md). + +## Example + +```python +from perspective import VirtualServerHandler, VirtualServer + +class MyModel(VirtualServerHandler): + def get_features(self): + return { + "group_by": True, + "split_by": False, + "sort": True, + "filter_ops": { + "string": ["==", "!=", "contains"], + "float": ["==", "!=", ">", "<"], + }, + "aggregates": { + "float": ["sum", "avg", "count"], + "string": ["count"], + }, + } + + def get_hosted_tables(self): + return ["my_table"] + + def table_schema(self, table_name): + return {"name": "string", "price": "float"} + + def table_size(self, table_name): + return 1000 + + def table_make_view(self, table_name, view_id, config): + # Translate `config` (group_by, sort, filter, etc.) into a + # query against your data source. Store the query keyed by + # `view_id` for later data retrieval. + pass + + def view_delete(self, view_id): + # Clean up resources for this view + pass + + def view_get_data(self, view_id, start_row, end_row, start_col, end_col, ctx): + # Execute the stored query with the given row/column window. + # Push results via `ctx`. + pass +``` + +The `VirtualServer` instance can then be passed to a Tornado, Starlette, or +AIOHTTP handler just like a regular `Server`: + +```python +from perspective.handlers.tornado import PerspectiveTornadoHandler + +app = tornado.web.Application([ + (r"/websocket", PerspectiveTornadoHandler, {"perspective_server": VirtualServer(MyModel)}), +]) +``` diff --git a/docs/md/how_to/python/virtual_server/duckdb.md b/docs/md/how_to/python/virtual_server/duckdb.md new file mode 100644 index 0000000000..8a040ede30 --- /dev/null +++ b/docs/md/how_to/python/virtual_server/duckdb.md @@ -0,0 +1,53 @@ +# DuckDB Virtual Server + +Perspective provides a built-in virtual server for +[DuckDB](https://duckdb.org/), allowing `` clients to query +a server-side DuckDB database over WebSocket. + +For browser-only usage via DuckDB-WASM, see the +[JavaScript DuckDB guide](../../javascript/virtual_server/duckdb.md). + +## Installation + +```bash +pip install perspective-python duckdb +``` + +## Usage + +Create a server that exposes a DuckDB database to browser clients: + +```python +import duckdb +import tornado.web +import tornado.ioloop +from perspective import DuckDBVirtualServer +from perspective.handlers.tornado import PerspectiveTornadoHandler + +# Create DuckDB connection and load data +conn = duckdb.connect() +conn.execute("CREATE TABLE my_table AS SELECT * FROM 'data.parquet'") + +# Create virtual server backed by DuckDB +server = DuckDBVirtualServer(conn) + +# Serve over WebSocket +app = tornado.web.Application([ + (r"/websocket", PerspectiveTornadoHandler, {"perspective_server": server}), +]) + +app.listen(8080) +tornado.ioloop.IOLoop.current().start() +``` + +Connect from the browser: + +```javascript +const websocket = await perspective.websocket("ws://localhost:8080/websocket"); +const table = await websocket.open_table("my_table"); +document.getElementById("viewer").load(table); +``` + +## Examples + +- [Python DuckDB example](https://github.com/perspective-dev/perspective/tree/master/examples/python-duckdb-virtual) diff --git a/docs/md/how_to/python/websocket.md b/docs/md/how_to/python/websocket.md index 71762030a1..e7e932cf0e 100644 --- a/docs/md/how_to/python/websocket.md +++ b/docs/md/how_to/python/websocket.md @@ -22,8 +22,8 @@ for i in range(10): The `name` provided is important, as it enables Perspective in JavaScript to look up a `Table` and get a handle to it over the network. Otherwise, `name` -will be assigned randomlu and the `Client` must look this up with -`CLient.get_hosted_table_names()` +will be assigned randomly and the `Client` must look this up with +`Client.get_hosted_table_names()` ## Client/Server Replicated Mode @@ -46,7 +46,7 @@ _*server.py*_ ```python from perspective import Server -from perspective.hadnlers.tornado import PerspectiveTornadoHandler +from perspective.handlers.tornado import PerspectiveTornadoHandler # Create an instance of Server, and host a Table SERVER = Server() @@ -113,7 +113,7 @@ The server setup is identical to instead of creating a `View`, the client calls `load(server_table)`: In Python, use `Server` and `PerspectiveTornadoHandler` to create a websocket server that exposes a `Table`. In this example, `table` is a proxy for the `Table` we -created on the server. All API methods are available on _proxies_, the.g.us +created on the server. All API methods are available on _proxies_, e.g. calling `view()`, `schema()`, `update()` on `table` will pass those operations to the Python `Table`, execute the commands, and return the result back to Javascript. diff --git a/docs/md/javascript.md b/docs/md/javascript.md index 52258b5098..0539dd6cc1 100644 --- a/docs/md/javascript.md +++ b/docs/md/javascript.md @@ -1 +1,3 @@ # JavaScript + +Overview and module structure for Perspective's JavaScript packages. diff --git a/docs/package.json b/docs/package.json index 91084355ae..4896ad46d4 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,16 +3,17 @@ "version": "4.2.0", "private": true, "scripts": { - "build": "node build.js && docusaurus build", + "build": "npm run mdbook && node build.js && docusaurus build", "docusaurus": "docusaurus", "start": "docusaurus start", - "docs": "node build.js && docusaurus build", + "docs": "npm run build && docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clean": "docusaurus clear", "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", - "write-heading-ids": "docusaurus write-heading-ids" + "write-heading-ids": "docusaurus write-heading-ids", + "mdbook": "docker compose run --rm mdbook build" }, "dependencies": { "@docusaurus/core": "^3.7.0", diff --git a/examples/blocks/index.mjs b/examples/blocks/index.mjs index e042be2f1e..ec6259de59 100644 --- a/examples/blocks/index.mjs +++ b/examples/blocks/index.mjs @@ -107,7 +107,7 @@ function generate_readme() { .join("")}${row .map( (y) => - ``, + ``, ) .join("")}`, ) diff --git a/rust/perspective-client/src/rust/config/expressions.rs b/rust/perspective-client/src/rust/config/expressions.rs index ad0ccac47c..c851c33362 100644 --- a/rust/perspective-client/src/rust/config/expressions.rs +++ b/rust/perspective-client/src/rust/config/expressions.rs @@ -61,8 +61,8 @@ //! //! #### Expression Column Name //! -//! Expressions can be _named_ by providing a comment as the first line of the -//! expression. This name will be used in the `` UI when +//! Expressions are named by their key in the `ViewConfig::expressions` +//! `HashMap`. This name will be used in the `` UI when //! referring to the column, but will also be used in the API when specifying //! e.g. `group_by` or `sort` fields. When creating a new column via //! ``'s expression editor, new columns will get a default diff --git a/rust/perspective-js/docs.mjs b/rust/perspective-js/docs.mjs index 2bbee117d3..284a5c03ef 100644 --- a/rust/perspective-js/docs.mjs +++ b/rust/perspective-js/docs.mjs @@ -12,5 +12,6 @@ import "zx/globals"; +await $`typedoc --tsconfig tsconfig.virtual_servers.json --out ../../docs/static/browser/virtual_servers`; await $`typedoc --tsconfig tsconfig.browser.json --out ../../docs/static/browser`; await $`typedoc --tsconfig tsconfig.node.json --out ../../docs/static/node`; diff --git a/rust/perspective-js/tsconfig.json b/rust/perspective-js/tsconfig.json index 186a6d6c4b..fb19376bc4 100644 --- a/rust/perspective-js/tsconfig.json +++ b/rust/perspective-js/tsconfig.json @@ -16,6 +16,7 @@ "./src/ts/perspective.node.ts", "./src/ts/perspective.cdn.ts", "./src/ts/virtual_servers/duckdb.ts", + "./src/ts/virtual_servers/clickhouse.ts", "./test/js/*.ts" ] } diff --git a/rust/perspective-js/tsconfig.virtual_servers.json b/rust/perspective-js/tsconfig.virtual_servers.json new file mode 100644 index 0000000000..ecec138c6c --- /dev/null +++ b/rust/perspective-js/tsconfig.virtual_servers.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "strict": true, + "module": "NodeNext", + "target": "ESNext", + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "./dist/esm", + "rootDir": ".", + "moduleResolution": "NodeNext", + "skipLibCheck": true, + "resolveJsonModule": true + }, + "include": [ + "./src/ts/virtual_servers/duckdb.ts", + "./src/ts/virtual_servers/clickhouse.ts", + ] +} diff --git a/rust/perspective-python/docs.mjs b/rust/perspective-python/docs.mjs index 6e795309dc..ec25a5d245 100644 --- a/rust/perspective-python/docs.mjs +++ b/rust/perspective-python/docs.mjs @@ -11,6 +11,19 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import * as fs from "fs"; +import "zx/globals"; + +const FILES = [ + "perspective/__init__.py", + "perspective/handlers/aiohttp.py", + "perspective/handlers/starlette.py", + "perspective/handlers/tornado.py", + "perspective/widget/__init__.py", + "perspective/virtual_servers/clickhouse.py", + "perspective/virtual_servers/duckdb.py", +]; + +await $`pdoc ${FILES.join(" ")} -o ../../docs/static/python`; let content = fs.readFileSync("../../docs/static/python/perspective.html"); const reg = /\[([a-zA-Z0-9:_]+)<\/code>\]/g; @@ -36,4 +49,4 @@ content = content.toString().replaceAll(reg2, function (_, arg) { return arg; }); -fs.writeFileSync("../../docs/static/python/perspective.html", content); +fs.writeFileSync("../../docs/static/python/index.html", content); diff --git a/rust/perspective-python/package.json b/rust/perspective-python/package.json index 7c99d794b1..82a3439625 100644 --- a/rust/perspective-python/package.json +++ b/rust/perspective-python/package.json @@ -13,7 +13,7 @@ "build": "node ./build.mjs", "clean": "node ./clean.mjs", "test": "node test.mjs", - "docs": "pdoc perspective/__init__.py perspective/handlers/aiohttp.py, perspective/handlers/starlette.py perspective/handlers/tornado.py perspective/widget/__init__.py -o ../../docs/static/python && node docs.mjs" + "docs": "node docs.mjs" }, "devDependencies": { "@perspective-dev/scripts": "workspace:*", diff --git a/rust/perspective-python/perspective/virtual_servers/__init__.py b/rust/perspective-python/perspective/virtual_servers/__init__.py index 841ff7ab32..b50e4b62bb 100644 --- a/rust/perspective-python/perspective/virtual_servers/__init__.py +++ b/rust/perspective-python/perspective/virtual_servers/__init__.py @@ -11,7 +11,7 @@ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -class VirtualSessionModel: +class VirtualServerHandler: """ An interface for implementing a Perspective `VirtualServer`. It operates thusly: diff --git a/rust/perspective-python/perspective/virtual_servers/clickhouse.py b/rust/perspective-python/perspective/virtual_servers/clickhouse.py index 53ef74cd63..2c185b78c4 100644 --- a/rust/perspective-python/perspective/virtual_servers/clickhouse.py +++ b/rust/perspective-python/perspective/virtual_servers/clickhouse.py @@ -15,7 +15,7 @@ from datetime import datetime from loguru import logger -from perspective.virtual_servers import VirtualSessionModel +from perspective.virtual_servers import VirtualServerHandler NUMBER_AGGS = [ @@ -69,7 +69,7 @@ class ClickhouseVirtualSession: def __init__(self, callback, db): - self.session = perspective.VirtualServer(ClickhouseVirtualSessionModel(db)) + self.session = perspective.VirtualServer(ClickhouseVirtualServerHandler(db)) self.callback = callback def handle_request(self, msg): @@ -84,9 +84,9 @@ def new_session(self, callback): return ClickhouseVirtualSession(callback, self.db) -class ClickhouseVirtualSessionModel(VirtualSessionModel): +class ClickhouseVirtualServerHandler(VirtualServerHandler): """ - An implementation of a `perspective.VirtualSessionModel` for ClickHouse. + An implementation of a `perspective.VirtualServerHandler` for ClickHouse. """ def __init__(self, db): diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index f1b2567e6b..7282c86c2d 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -16,7 +16,7 @@ from datetime import datetime from loguru import logger -from perspective.virtual_servers import VirtualSessionModel +from perspective.virtual_servers import VirtualServerHandler NUMBER_AGGS = [ @@ -80,7 +80,7 @@ class DuckDBVirtualSession: def __init__(self, callback, db): - self.session = perspective.VirtualServer(DuckDBVirtualSessionModel(db)) + self.session = perspective.VirtualServer(DuckDBVirtualServerHandler(db)) self.callback = callback def handle_request(self, msg): @@ -95,9 +95,9 @@ def new_session(self, callback): return DuckDBVirtualSession(callback, self.db) -class DuckDBVirtualSessionModel(VirtualSessionModel): +class DuckDBVirtualServerHandler(VirtualServerHandler): """ - An implementation of a `perspective.VirtualSessionModel` for DuckDB. + An implementation of a `perspective.VirtualServerHandler` for DuckDB. """ def __init__(self, db):