Add IDE to CLI and package (#636)

* Add IDE to CLI and package

* Update CI

* Update Pyodide workers

* Change prod to dist

* PR feedback

Author: vchan

.circleci/config.yml

@@ -16,6 +16,7 @@ jobs:
   publish:
     docker:
       - image: cimg/python:3.10
+    resource_class: small
     steps:
       - checkout
       - run:
@@ -25,6 +26,7 @@ jobs:
   gh-release:
     docker:
       - image: cimg/node:16.14
+    resource_class: small
     steps:
       - run:
           name: Create release on GitHub
@@ -36,6 +38,42 @@ jobs:
             CONTINUE_ON_ERROR="false" \
             npx https://github.com/TobikoData/circleci-gh-conventional-release
 
+  generate-openapi-spec:
+    docker:
+      - image: cimg/python:3.11
+    resource_class: small
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: make install-dev
+      - run:
+          name: Generate OpenAPI spec
+          command: python web/server/openapi.py
+      - persist_to_workspace:
+          root: web/client
+          paths:
+            - openapi.json
+
+  ide-build:
+    docker:
+      - image: cimg/node:19.8
+    resource_class: small
+    steps:
+      - checkout
+      - attach_workspace:
+          at: web/client
+      - run:
+          name: Install packages
+          command: npm --prefix web/client ci
+      - run:
+          name: Build IDE
+          command: npm --prefix web/client run build
+      - persist_to_workspace:
+          root: web/client
+          paths:
+            - dist
+
 workflows:
   setup-workflow:
     jobs:
@@ -50,7 +88,12 @@ workflows:
     jobs:
       - gh-release:
           <<: *on_tag_filter
-      - publish:
-          <<: *on_tag_filter
+      - generate-openapi-spec:
           requires:
             - gh-release
+      - ide-build:
+          requires:
+            - generate-openapi-spec
+      - publish:
+          requires:
+            - ide-build

.gitignore

@@ -149,6 +149,10 @@ examples/airflow/warehouse
 **/.DS_Store
 **/node_modules
 /web/client/src/api/client.ts
+web/client/dist
 
 # Version file
 sqlmesh/_version.py
+
+# Autogenerated OpenAPI spec
+openapi.json

docker-compose.yml

@@ -1,5 +1,15 @@
 version: "3.11"
 services:
+  generate-openapi-spec:
+    image: tobiko-api
+    container_name: generate-openapi-spec
+    working_dir: /sqlmesh
+    build:
+      dockerfile: Dockerfile.api
+    command: python web/server/openapi.py
+    volumes:
+      - .:/sqlmesh
+
   app:
     image: tobiko-app
     container_name: tobiko-app
@@ -16,7 +26,7 @@ services:
     networks:
       - tobiko-development
     depends_on:
-      - api
+      - generate-openapi-spec
 
   api:
     image: tobiko-api

setup.py

@@ -14,7 +14,8 @@
     author="TobikoData Inc.",
     author_email="engineering@tobikodata.com",
     license="Apache License 2.0",
-    packages=find_packages(include=["sqlmesh", "sqlmesh.*"]),
+    packages=find_packages(include=["sqlmesh", "sqlmesh.*", "web*"]),
+    package_data={"web": ["client/dist/**"]},
     entry_points={
         "console_scripts": [
             "sqlmesh = sqlmesh.cli.main:cli",

sqlmesh/cli/main.py

@@ -11,6 +11,7 @@
 from sqlmesh.cli.example_project import ProjectTemplate, init_example_project
 from sqlmesh.core.context import Context
 from sqlmesh.utils.date import TimeLike
+from sqlmesh.utils.errors import MissingDependencyError
 
 
 @click.group(no_args_is_help=True)
@@ -308,5 +309,37 @@ def version() -> None:
         print("Version is not available")
 
 
+@cli.command("ide")
+@click.option(
+    "--host",
+    type=str,
+    help="Bind socket to this host. Default: 127.0.0.1",
+)
+@click.option(
+    "--port",
+    type=int,
+    help="Bind socket to this port. Default: 8000",
+)
+@click.pass_obj
+@error_handler
+def ide(
+    obj: Context,
+    host: t.Optional[str],
+    port: t.Optional[int],
+) -> None:
+    """Start a browser-based SQLMesh IDE."""
+    try:
+        import uvicorn
+    except ModuleNotFoundError as e:
+        raise MissingDependencyError(
+            "Missing IDE dependencies. Run `pip install sqlmesh[web]` to install them."
+        ) from e
+
+    host = host or "127.0.0.1"
+    port = 8000 if port is None else port
+    os.environ["PROJECT_PATH"] = str(obj.path)
+    uvicorn.run("web.server.main:app", host=host, port=port, log_level="info")
+
+
 if __name__ == "__main__":
     cli()

web/client/orval.config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from 'orval'
 
 export default defineConfig({
   'sqlmesh-api': {
-    input: 'http://api:8000/openapi.json',
+    input: './openapi.json',
     output: {
       prettier: true,
       target: './src/api/client.ts',

web/client/src/workers/index.ts

@@ -1,8 +1,5 @@
 const sqlglotWorker = new Worker(
   new URL('./sqlglot/worker.ts', import.meta.url),
-  {
-    type: 'module',
-  },
 )
 
 export { sqlglotWorker }

web/client/src/workers/sqlglot/worker.ts

@@ -1,16 +1,15 @@
-import 'https://cdn.jsdelivr.net/pyodide/v0.22.1/full/pyodide.js'
-
-export {}
-
 const global = self as any
+global.importScripts('https://cdn.jsdelivr.net/pyodide/v0.22.1/full/pyodide.js')
 
 async function loadPyodideAndPackages(): Promise<any[]> {
   global.pyodide = await global.loadPyodide()
   await global.pyodide.loadPackage('micropip')
 
   const micropip = global.pyodide.pyimport('micropip')
   await micropip.install('sqlglot')
-  const file = await (await fetch('./sqlglot.py')).text()
+  const file = await (
+    await fetch(new URL('./sqlglot.py', import.meta.url))
+  ).text()
 
   global.postMessage({ topic: 'init' })
 

web/server/main.py

@@ -1,6 +1,9 @@
 import asyncio
+import pathlib
 
 from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
 
 from web.server.api.endpoints import api_router
 from web.server.console import ApiConsole
@@ -9,6 +12,7 @@
 api_console = ApiConsole()
 
 app.include_router(api_router, prefix="/api")
+WEB_DIRECTORY = pathlib.Path(__file__).parent.parent
 
 
 @app.on_event("startup")
@@ -32,3 +36,21 @@ def shutdown_event() -> None:
 @app.get("/health")
 def health() -> str:
     return "ok"
+
+
+@app.get("/")
+def index() -> HTMLResponse:
+    with open(WEB_DIRECTORY / "client/dist/index.html", "r", encoding="utf-8") as f:
+        return HTMLResponse(f.read())
+
+
+app.mount(
+    "/assets",
+    StaticFiles(directory=WEB_DIRECTORY / "client/dist/assets", check_dir=False),
+    name="assets",
+)
+app.mount(
+    "/favicons",
+    StaticFiles(directory=WEB_DIRECTORY / "client/dist/favicons", check_dir=False),
+    name="favicons",
+)

web/server/openapi.py

@@ -0,0 +1,12 @@
+import json
+
+from web.server.main import app
+
+
+def generate_openapi_spec(path: str) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(app.openapi(), f)
+
+
+if __name__ == "__main__":
+    generate_openapi_spec("web/client/openapi.json")