diff --git a/backend/routers/auto_optimize.py b/backend/routers/auto_optimize.py index 1b5bad49b..ff5255226 100644 --- a/backend/routers/auto_optimize.py +++ b/backend/routers/auto_optimize.py @@ -61,6 +61,8 @@ class TriggerRequest(BaseModel): apply_mode: str = "genie_config" levers: list[int] | None = None deploy_target: str | None = None + deploy_space_id: str | None = None + catalog_map: dict[str, str] | None = None class SchemaAccessStatus(BaseModel): @@ -857,6 +859,8 @@ async def trigger(body: TriggerRequest, request: Request): apply_mode=body.apply_mode, levers=body.levers, deploy_target=body.deploy_target, + deploy_space_id=body.deploy_space_id, + catalog_map=body.catalog_map, ) return { "runId": result.run_id, @@ -879,6 +883,101 @@ async def trigger(body: TriggerRequest, request: Request): raise HTTPException(status_code=500, detail="Failed to start optimization job.") +class DeployRequest(BaseModel): + target_workspace_url: str + target_space_id: str | None = None + catalog_map: dict[str, str] | None = None + + +@router.post("/spaces/{space_id}/deploy") +async def deploy_space(space_id: str, body: DeployRequest): + """Deploy a Genie Space config to a target workspace. + + Fetches the current space config, applies catalog remapping, and + PATCHes it to the target workspace. No UC model or optimization run + required — works with any Genie Space. + """ + import json as _json + + # 1. Fetch source space config + try: + from backend.services.genie_client import get_serialized_space + space_config = get_serialized_space(genie_space_id=space_id) + except Exception as e: + raise HTTPException(status_code=400, detail=f"Failed to fetch space config: {e}") + + # 2. Apply catalog remapping + if body.catalog_map: + remapped = 0 + for key in ("tables", "metric_views"): + for src in space_config.get("data_sources", {}).get(key, []): + ident = src.get("identifier", "") + parts = ident.replace("`", "").split(".") + if len(parts) >= 3 and parts[0] in body.catalog_map: + parts[0] = body.catalog_map[parts[0]] + src["identifier"] = ".".join(parts) + remapped += 1 + logger.info("Remapped %d catalog references for deploy", remapped) + + # 3. Connect to target workspace using the app's SP credentials. + # The SP must be registered in the target workspace. + try: + from databricks.sdk import WorkspaceClient + sp_ws = get_service_principal_client() + target_ws = WorkspaceClient( + host=body.target_workspace_url.rstrip("/"), + client_id=sp_ws.config.client_id, + client_secret=sp_ws.config.client_secret, + ) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to connect to target workspace: {e}") + + # 4. Deploy: PATCH existing space or CREATE new one + try: + if body.target_space_id: + # Update existing space + from genie_space_optimizer.common.genie_client import patch_space_config + patch_space_config(target_ws, body.target_space_id, space_config) + target_space = body.target_space_id + logger.info("PATCHed existing space %s in target workspace", target_space) + else: + # Create new space in target workspace + from backend.services.genie_client import get_genie_space + source_space = get_genie_space(genie_space_id=space_id) + display_name = source_space.get("title", "Deployed Genie Space") + + # Find a warehouse in the target workspace + warehouses = list(target_ws.warehouses.list()) + if not warehouses: + raise ValueError("No SQL warehouse found in target workspace") + target_warehouse_id = warehouses[0].id + + response = target_ws.api_client.do( + method="POST", + path="/api/2.0/genie/spaces", + body={ + "title": display_name, + "description": f"Deployed from source workspace (space {space_id})", + "parent_path": "/Shared/", + "warehouse_id": target_warehouse_id, + "serialized_space": _json.dumps(space_config), + }, + ) + target_space = response.get("space_id", "") + logger.info("Created new space %s in target workspace", target_space) + + target_host = body.target_workspace_url.rstrip("/") + return { + "status": "DEPLOYED", + "targetSpaceId": target_space, + "targetUrl": target_host, + "spaceUrl": f"{target_host}/genie/rooms/{target_space}", + } + except Exception as e: + logger.exception("Failed to deploy to target workspace: %s", e) + raise HTTPException(status_code=500, detail=f"Deployment failed: {e}") + + # --------------------------------------------------------------------------- # Pipeline step definitions — group raw sub-stages into 6 logical steps # (Ported from Genie Space Optimizer's map_stages_to_steps) @@ -1162,6 +1261,7 @@ async def get_run(run_id: RunId): "levers": levers, "links": links, "convergenceReason": run.get("convergence_reason"), + "deployTarget": run.get("deploy_target") or None, "deploymentStatus": run.get("deploy_status"), "labelingSessionUrl": run.get("labeling_session_url") or None, "labelingSessionName": run.get("labeling_session_name") or None, diff --git a/frontend/src/components/auto-optimize/RunDetailView.tsx b/frontend/src/components/auto-optimize/RunDetailView.tsx index a4fdb67a7..b8585933e 100644 --- a/frontend/src/components/auto-optimize/RunDetailView.tsx +++ b/frontend/src/components/auto-optimize/RunDetailView.tsx @@ -1,5 +1,5 @@ import { useEffect, useState } from "react" -import { ArrowLeft, Cog, UserCheck, ExternalLink } from "lucide-react" +import { ArrowLeft, Cog, UserCheck, ExternalLink, CheckCircle2, Clock, AlertTriangle } from "lucide-react" import { Card, CardContent } from "@/components/ui/card" import { Badge } from "@/components/ui/badge" import { ScoreSummary } from "@/components/auto-optimize/ScoreSummary" @@ -157,6 +157,45 @@ export function RunDetailView({ runId, onBack }: RunDetailViewProps) { )} + {/* Deployment Status Banner (when already deployed) */} + {run.deployTarget && run.deploymentStatus && ( +
+ {run.deploymentStatus === "DEPLOYED" ? ( + + ) : run.deploymentStatus === "FAILED" ? ( + + ) : ( + + )} +
+

+ {run.deploymentStatus === "DEPLOYED" ? "Deployed to target workspace" + : run.deploymentStatus === "PENDING_APPROVAL" ? "Deployment awaiting approval" + : run.deploymentStatus === "FAILED" ? "Deployment failed" + : `Deployment: ${run.deploymentStatus}`} +

+

Target: {run.deployTarget}

+
+ {run.deploymentStatus === "DEPLOYED" && ( + + Open Workspace + + )} +
+ )} + + {/* Tabs */}
+
+ {catalogMappings.map((m, i) => ( +
+ { + const next = [...catalogMappings] + next[i] = { ...next[i], source: e.target.value } + setCatalogMappings(next) + }} + placeholder="dev_catalog" + className="flex-1 px-3 py-1.5 text-sm border border-default rounded-lg bg-elevated text-primary placeholder:text-muted focus:outline-none focus:ring-1 focus:ring-accent" + /> + + { + const next = [...catalogMappings] + next[i] = { ...next[i], target: e.target.value } + setCatalogMappings(next) + }} + placeholder="prod_catalog" + className="flex-1 px-3 py-1.5 text-sm border border-default rounded-lg bg-elevated text-primary placeholder:text-muted focus:outline-none focus:ring-1 focus:ring-accent" + /> + +
+ ))} + {catalogMappings.length === 0 && ( +

No catalog mappings — table references will be used as-is

+ )} + + + + + {/* Error */} + {deployError && ( +
+ {deployError} +
+ )} + + {/* Success */} + {deploySuccess && ( +
+ +
+

+ Deployed successfully +

+

+ Space config deployed to {deploySuccess.targetUrl} (space: {deploySuccess.targetSpaceId}) +

+
+ + {deploySuccess.spaceUrl ? "Open Genie Space" : "Open Workspace"} + + +
+ )} + + {/* Deploy button */} + + + ) +} diff --git a/frontend/src/pages/SpaceDetail.tsx b/frontend/src/pages/SpaceDetail.tsx index efab5f35d..db9dec6f0 100644 --- a/frontend/src/pages/SpaceDetail.tsx +++ b/frontend/src/pages/SpaceDetail.tsx @@ -4,7 +4,7 @@ * Score tab includes an inline FixAgentPanel that slides in from the right. */ import { useState, useEffect, useRef } from "react" -import { ArrowLeft, Star, BarChart2, Clock, ExternalLink, Rocket, Play, Zap, ChevronDown, ChevronRight, Settings, RefreshCw } from "lucide-react" +import { ArrowLeft, Star, BarChart2, Clock, ExternalLink, Rocket, Play, Upload, Zap, ChevronDown, ChevronRight, Settings, RefreshCw } from "lucide-react" import { scanSpace, toggleStar, getSpaceHistory, getSpaceDetail, getActiveRunForSpace } from "@/lib/api" import { MATURITY_COLORS, getOptimizationLabel } from "@/lib/utils" import type { ScanResult, ScoreHistoryPoint, OptimizationEvent } from "@/types" @@ -13,10 +13,11 @@ import { HistoryTab } from "./HistoryTab" import { useAnalysis } from "@/hooks/useAnalysis" import { SpaceOverview } from "@/components/SpaceOverview" import { AutoOptimizeTab } from "@/components/auto-optimize/AutoOptimizeTab" +import { DeployTab } from "./DeployTab" import { FixAgentPanel } from "@/components/FixAgentPanel" -type Tab = "score" | "optimize" | "history" -const VALID_TABS: readonly string[] = ["score", "optimize", "history"] +type Tab = "score" | "optimize" | "history" | "deploy" +const VALID_TABS: readonly string[] = ["score", "optimize", "history", "deploy"] interface SpaceDetailProps { spaceId: string @@ -162,6 +163,7 @@ export function SpaceDetail({ spaceId, displayName, spaceUrl, initialTab, autoSc { id: "score", label: "Score", icon: }, { id: "optimize", label: "Optimize", icon: }, { id: "history", label: "History", icon: }, + { id: "deploy", label: "Deploy", icon: }, ] // Determine contextual action(s) based on scan results @@ -322,6 +324,9 @@ export function SpaceDetail({ spaceId, displayName, spaceUrl, initialTab, autoSc {activeTab === "history" && ( )} + {activeTab === "deploy" && ( + + )} {/* Fix agent modal overlay */} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 0b5f6681f..8814c633a 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -264,6 +264,8 @@ export interface GSOTriggerRequest { apply_mode?: "genie_config" | "uc_artifact" | "both" levers?: number[] deploy_target?: string + deploy_space_id?: string + catalog_map?: Record } export interface GSOTriggerResponse { @@ -397,6 +399,7 @@ export interface GSOPipelineRun { levers: GSOLeverStatus[] links: GSOResourceLink[] convergenceReason: string | null + deployTarget: string | null deploymentStatus: string | null labelingSessionUrl: string | null labelingSessionName: string | null diff --git a/packages/genie-space-optimizer/src/genie_space_optimizer/backend/job_launcher.py b/packages/genie-space-optimizer/src/genie_space_optimizer/backend/job_launcher.py index de08d8954..795378dfb 100644 --- a/packages/genie-space-optimizer/src/genie_space_optimizer/backend/job_launcher.py +++ b/packages/genie-space-optimizer/src/genie_space_optimizer/backend/job_launcher.py @@ -363,6 +363,8 @@ def submit_optimization( triggered_by: str = "", experiment_name: str = "", deploy_target: str = "", + deploy_space_id: str = "", + catalog_map: str = "", warehouse_id: str = "", target_benchmark_count: str = "", ) -> tuple[str, int]: @@ -393,6 +395,8 @@ def submit_optimization( "triggered_by": triggered_by, "experiment_name": experiment_name, "deploy_target": deploy_target, + "deploy_space_id": deploy_space_id, + "catalog_map": catalog_map, "warehouse_id": warehouse_id, "target_benchmark_count": target_benchmark_count, }, diff --git a/packages/genie-space-optimizer/src/genie_space_optimizer/integration/trigger.py b/packages/genie-space-optimizer/src/genie_space_optimizer/integration/trigger.py index bf881e067..0ec718cb3 100644 --- a/packages/genie-space-optimizer/src/genie_space_optimizer/integration/trigger.py +++ b/packages/genie-space-optimizer/src/genie_space_optimizer/integration/trigger.py @@ -45,6 +45,8 @@ def trigger_optimization( apply_mode: str = "genie_config", levers: list[int] | None = None, deploy_target: str | None = None, + deploy_space_id: str | None = None, + catalog_map: dict[str, str] | None = None, ) -> TriggerResult: """Trigger a GSO optimization run using SQL Warehouse for state management. @@ -227,6 +229,8 @@ def trigger_optimization( triggered_by=caller_email, experiment_name=experiment_name or "", deploy_target=deploy_target or "", + deploy_space_id=deploy_space_id or "", + catalog_map=json.dumps(catalog_map) if catalog_map else "", warehouse_id=config.warehouse_id or "", ) diff --git a/packages/genie-space-optimizer/src/genie_space_optimizer/jobs/run_cross_env_deploy.py b/packages/genie-space-optimizer/src/genie_space_optimizer/jobs/run_cross_env_deploy.py index 09bbc6a66..b1b6c45fe 100644 --- a/packages/genie-space-optimizer/src/genie_space_optimizer/jobs/run_cross_env_deploy.py +++ b/packages/genie-space-optimizer/src/genie_space_optimizer/jobs/run_cross_env_deploy.py @@ -56,11 +56,13 @@ dbutils.widgets.text("model_version", "", "Model Version") dbutils.widgets.text("target_workspace_url", "", "Target Workspace URL") dbutils.widgets.text("target_space_id", "", "Target Space ID") +dbutils.widgets.text("catalog_map", "", "Catalog Mapping (JSON)") model_name = dbutils.widgets.get("model_name").strip() model_version = dbutils.widgets.get("model_version").strip() target_workspace_url = dbutils.widgets.get("target_workspace_url").strip() target_space_id = dbutils.widgets.get("target_space_id").strip() +catalog_map_raw = dbutils.widgets.get("catalog_map").strip() _banner("Resolved Parameters") _log( @@ -139,6 +141,29 @@ _log("Config loaded", keys=list(space_config.keys())) +# Remap catalog references if catalog_map is provided +catalog_map: dict[str, str] = {} +if catalog_map_raw: + try: + catalog_map = json.loads(catalog_map_raw) + except json.JSONDecodeError: + _log("WARNING: Could not parse catalog_map JSON, skipping remapping", raw=catalog_map_raw) + +if catalog_map: + _banner("Remapping Catalog References") + remapped = 0 + for source_list_key in ("tables", "metric_views"): + for src in space_config.get("data_sources", {}).get(source_list_key, []): + ident = src.get("identifier", "") + parts = ident.replace("`", "").split(".") + if len(parts) >= 3 and parts[0] in catalog_map: + old_ident = ident + parts[0] = catalog_map[parts[0]] + src["identifier"] = ".".join(parts) + _log("Remapped", old=old_ident, new=src["identifier"]) + remapped += 1 + _log("Catalog remapping complete", remapped=remapped, mappings=catalog_map) + # COMMAND ---------- # MAGIC %md diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 7d2cacb58..02facf273 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -592,12 +592,24 @@ if lakebase_db: } } -print(json.dumps({'user_api_scopes': scopes, 'resources': list(by_name.values())})) +resources_list = list(by_name.values()) +print(json.dumps({'user_api_scopes': scopes, 'resources': resources_list}) + '|||' + json.dumps({'resources': resources_list})) ") -databricks api patch "/api/2.0/apps/$APP_NAME" \ - --profile "$PROFILE" --json "$PATCH_PAYLOAD" 2>/dev/null && \ - echo " ✓ App scopes and resources configured" || \ - echo " ⚠ Could not configure app scopes/resources" + +# Split into scopes+resources and resources-only payloads +PATCH_WITH_SCOPES="${PATCH_PAYLOAD%%|||*}" +PATCH_RESOURCES_ONLY="${PATCH_PAYLOAD##*|||}" + +# Try with scopes first; if workspace doesn't support token passthrough, retry resources-only +if databricks api patch "/api/2.0/apps/$APP_NAME" \ + --profile "$PROFILE" --json "$PATCH_WITH_SCOPES" 2>/dev/null; then + echo " ✓ App scopes and resources configured" +elif databricks api patch "/api/2.0/apps/$APP_NAME" \ + --profile "$PROFILE" --json "$PATCH_RESOURCES_ONLY" 2>/dev/null; then + echo " ✓ App resources configured (user_api_scopes not supported on this workspace)" +else + echo " ⚠ Could not configure app resources" +fi databricks apps deploy "$APP_NAME" --profile "$PROFILE" \ --source-code-path "$WS_PATH" --no-wait