Inlcude all workflows in HTML dashboard

Author: MariusWirtz

src/rushti/commands.py

@@ -1635,7 +1635,11 @@ def _stats_visualize(args) -> None:
             dynamodb_task_results_table=settings.stats.dynamodb_task_results_table,
             dynamodb_endpoint_url=settings.stats.dynamodb_endpoint_url or None,
         )
-        data = get_visualization_data(args.workflow, stats_db)
+        data = get_visualization_data(
+            args.workflow,
+            stats_db,
+            include_all_workflows=True,
+        )
 
         print(f"Generating visualizations for workflow: {args.workflow}")
         if not data.get("exists"):

src/rushti/dashboard.py

@@ -195,6 +195,7 @@ def _prepare_dashboard_data(
     runs: List[Dict[str, Any]],
     task_results: List[Dict[str, Any]],
     default_runs: int,
+    selected_workflow: Optional[str] = None,
 ) -> Dict[str, Any]:
     """Prepare all data for the dashboard template.
 
@@ -330,16 +331,38 @@ def _prepare_dashboard_data(
             }
         )
 
-    # Taskfile metadata from the most recent run
-    latest = runs[0] if runs else {}
+    # Workflow metadata from the most recent run per workflow.
+    workflow_meta: Dict[str, Dict[str, Any]] = {}
+    for run in enriched_runs:
+        wf = run.get("workflow") or ""
+        if wf not in workflow_meta:
+            workflow_meta[wf] = {
+                "taskfile_name": run.get("taskfile_name", ""),
+                "taskfile_description": run.get("taskfile_description", ""),
+                "taskfile_author": run.get("taskfile_author", ""),
+                "run_count": 0,
+            }
+        workflow_meta[wf]["run_count"] += 1
+
+    workflows = list(workflow_meta.keys())
+    effective_workflow = selected_workflow if selected_workflow in workflow_meta else ""
+    if not effective_workflow and workflows:
+        effective_workflow = workflows[0]
+
+    workflow_run_count = workflow_meta.get(effective_workflow, {}).get("run_count", 0)
+    effective_default_runs = min(default_runs, workflow_run_count)
 
     return {
-        "workflow": latest.get("workflow", ""),
-        "taskfile_name": latest.get("taskfile_name", ""),
-        "taskfile_description": latest.get("taskfile_description", ""),
-        "taskfile_author": latest.get("taskfile_author", ""),
+        "workflow": effective_workflow,
+        "workflows": workflows,
+        "workflow_meta": workflow_meta,
+        "taskfile_name": workflow_meta.get(effective_workflow, {}).get("taskfile_name", ""),
+        "taskfile_description": workflow_meta.get(effective_workflow, {}).get(
+            "taskfile_description", ""
+        ),
+        "taskfile_author": workflow_meta.get(effective_workflow, {}).get("taskfile_author", ""),
         "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-        "default_runs": min(default_runs, len(runs)),
+        "default_runs": effective_default_runs,
         "total_runs": len(runs),
         "runs": enriched_runs,
         "task_results": slim_task_results,
@@ -367,7 +390,12 @@ def generate_dashboard(
     :param dag_url: Optional relative URL to the DAG visualization HTML
     :return: Path to the generated HTML file
     """
-    data = _prepare_dashboard_data(runs, task_results, default_runs)
+    data = _prepare_dashboard_data(
+        runs,
+        task_results,
+        default_runs,
+        selected_workflow=workflow,
+    )
     data_json = json.dumps(data, default=str)
 
     # Build conditional DAG link HTML
@@ -623,6 +651,10 @@ def generate_dashboard(
             </div>
             <div class="header-right">
                 {dag_link_html}
+                <div class="run-selector">
+                    <label for="workflowSelect">Workflow</label>
+                    <select id="workflowSelect" onchange="onWorkflowChange()"></select>
+                </div>
                 <div class="run-selector">
                     <label for="runCount">Show last</label>
                     <select id="runCount" onchange="updateDashboard()"></select>
@@ -758,33 +790,78 @@ def generate_dashboard(
     ];
 
     function init() {{
-        // Build run count selector options
+        // Build workflow selector first, then run selector for the selected workflow.
+        const workflowSel = document.getElementById('workflowSelect');
+        DATA.workflows.forEach(wf => {{
+            const opt = document.createElement('option');
+            const wfRunCount = (DATA.workflow_meta[wf] && DATA.workflow_meta[wf].run_count) || 0;
+            opt.value = wf;
+            opt.textContent = `${{wf}} (${{wfRunCount}})`;
+            workflowSel.appendChild(opt);
+        }});
+        if (DATA.workflow && DATA.workflows.includes(DATA.workflow)) {{
+            workflowSel.value = DATA.workflow;
+        }}
+
+        rebuildRunCountSelector(DATA.default_runs);
+
+        document.getElementById('generatedAt').textContent = 'Generated: ' + DATA.generated_at;
+        updateDashboard();
+    }}
+
+    function getSelectedWorkflow() {{
+        return document.getElementById('workflowSelect').value;
+    }}
+
+    function getRunsForWorkflow(workflow) {{
+        return DATA.runs.filter(r => (r.workflow || '') === workflow);
+    }}
+
+    function rebuildRunCountSelector(preferredCount) {{
         const sel = document.getElementById('runCount');
-        const total = DATA.total_runs;
+        const total = getRunsForWorkflow(getSelectedWorkflow()).length;
+        sel.innerHTML = '';
+
+        if (total === 0) {{
+            const opt = document.createElement('option');
+            opt.value = 0;
+            opt.textContent = '0';
+            sel.appendChild(opt);
+            sel.value = 0;
+            return;
+        }}
+
         const options = [];
         for (let i = 1; i <= Math.min(total, 5); i++) options.push(i);
         if (total > 5) options.push(5);
         if (total > 10) options.push(10);
         if (total > 20) options.push(20);
         if (total > 5) options.push(total);
-        // Deduplicate and sort
+
         const unique = [...new Set(options)].sort((a, b) => a - b);
         unique.forEach(n => {{
             const opt = document.createElement('option');
             opt.value = n;
             opt.textContent = n === total ? `All (${{total}})` : n;
             sel.appendChild(opt);
         }});
-        sel.value = DATA.default_runs;
 
-        document.getElementById('generatedAt').textContent = 'Generated: ' + DATA.generated_at;
+        const defaultCount = preferredCount || DATA.default_runs || unique[0];
+        const boundedCount = Math.min(defaultCount, total);
+        sel.value = String(boundedCount);
+    }}
+
+    function onWorkflowChange() {{
+        rebuildRunCountSelector();
         updateDashboard();
     }}
 
     function getSelectedRuns() {{
         const n = parseInt(document.getElementById('runCount').value);
+        const workflowRuns = getRunsForWorkflow(getSelectedWorkflow());
+        if (!n) return [];
         // Runs are ordered DESC (newest first); take last N and reverse for chronological display
-        return DATA.runs.slice(0, n).reverse();
+        return workflowRuns.slice(0, n).reverse();
     }}
 
     function statusBadge(status) {{
@@ -815,25 +892,29 @@ def generate_dashboard(
 
     function updateDashboard() {{
         destroyCharts();
+        const selectedWorkflow = getSelectedWorkflow();
+        const selectedWorkflowMeta = DATA.workflow_meta[selectedWorkflow] || {{}};
+        const workflowTotalRuns = getRunsForWorkflow(selectedWorkflow).length;
         const runs = getSelectedRuns();
 
         // Header subtitle
         document.getElementById('headerSubtitle').textContent =
-            `${{DATA.workflow}}${{DATA.taskfile_name ? ' — ' + DATA.taskfile_name : ''}}`;
+            `${{selectedWorkflow}}${{selectedWorkflowMeta.taskfile_name ? ' - ' + selectedWorkflowMeta.taskfile_name : ''}}`;
 
         // Metadata
         const meta = document.getElementById('metadata');
         let metaHtml = '';
-        if (DATA.taskfile_description) metaHtml += `<span>${{DATA.taskfile_description}}</span>`;
-        if (DATA.taskfile_author) metaHtml += `<span>Author: <strong>${{DATA.taskfile_author}}</strong></span>`;
+        if (selectedWorkflowMeta.taskfile_description) metaHtml += `<span>${{selectedWorkflowMeta.taskfile_description}}</span>`;
+        if (selectedWorkflowMeta.taskfile_author) metaHtml += `<span>Author: <strong>${{selectedWorkflowMeta.taskfile_author}}</strong></span>`;
+        metaHtml += `<span>Workflow runs: <strong>${{workflowTotalRuns}}</strong></span>`;
         metaHtml += `<span>Total runs in DB: <strong>${{DATA.total_runs}}</strong></span>`;
         if (runs.length > 0) {{
             metaHtml += `<span>Date range: <strong>${{formatDate(runs[0].start_time)}} — ${{formatDate(runs[runs.length-1].start_time)}}</strong></span>`;
         }}
         meta.innerHTML = metaHtml;
 
         // Summary cards
-        updateSummaryCards(runs);
+        updateSummaryCards(runs, workflowTotalRuns);
 
         // Charts
         renderRunDurationChart(runs);
@@ -850,7 +931,7 @@ def generate_dashboard(
         updateConfigDetails(runs);
     }}
 
-    function updateSummaryCards(runs) {{
+    function updateSummaryCards(runs, workflowTotalRuns) {{
         const cards = document.getElementById('summaryCards');
         if (runs.length === 0) {{ cards.innerHTML = '<div class="no-data">No runs available</div>'; return; }}
 
@@ -866,7 +947,7 @@ def generate_dashboard(
             <div class="card">
                 <div class="card-label">Runs Shown</div>
                 <div class="card-value">${{runs.length}}</div>
-                <div class="card-sub">of ${{DATA.total_runs}} total</div>
+                <div class="card-sub">of ${{workflowTotalRuns}} in workflow</div>
             </div>
             <div class="card">
                 <div class="card-label">Tasks per Run</div>

src/rushti/db_admin.py

@@ -859,13 +859,16 @@ def show_task_history(
 def get_visualization_data(
     workflow: str,
     backend: Any = DEFAULT_DB_PATH,
+    include_all_workflows: bool = False,
 ) -> Dict[str, Any]:
     """Get all data needed for the HTML dashboard visualization.
 
     Returns all runs and their task results for a workflow.
 
     :param workflow: Workflow name to query
     :param backend: Stats backend (StatsDatabase or DynamoDBStatsDatabase), or SQLite db path
+    :param include_all_workflows: When True, return runs/tasks for all workflows
+        but keep workflow as the initial/default selection.
     :return: Dictionary with 'runs' and 'task_results' lists, or error info
     """
     if isinstance(backend, str):
@@ -877,18 +880,28 @@ def get_visualization_data(
         conn.row_factory = sqlite3.Row
         cursor = conn.cursor()
 
-        cursor.execute(
-            """
-            SELECT run_id, workflow, taskfile_path, start_time, end_time,
-                   duration_seconds, status, task_count, success_count, failure_count,
-                   taskfile_name, taskfile_description, taskfile_author,
-                   max_workers, retries, result_file, exclusive, optimize
-            FROM runs
-            WHERE workflow = ?
-            ORDER BY start_time DESC
-            """,
-            (workflow,),
-        )
+        if include_all_workflows:
+            cursor.execute("""
+                SELECT run_id, workflow, taskfile_path, start_time, end_time,
+                       duration_seconds, status, task_count, success_count, failure_count,
+                       taskfile_name, taskfile_description, taskfile_author,
+                       max_workers, retries, result_file, exclusive, optimize
+                FROM runs
+                ORDER BY start_time DESC
+                """)
+        else:
+            cursor.execute(
+                """
+                SELECT run_id, workflow, taskfile_path, start_time, end_time,
+                       duration_seconds, status, task_count, success_count, failure_count,
+                       taskfile_name, taskfile_description, taskfile_author,
+                       max_workers, retries, result_file, exclusive, optimize
+                FROM runs
+                WHERE workflow = ?
+                ORDER BY start_time DESC
+                """,
+                (workflow,),
+            )
         runs_rows = cursor.fetchall()
 
         if not runs_rows:
@@ -900,13 +913,23 @@ def get_visualization_data(
 
         runs = []
         run_ids = []
+        has_selected_workflow = False
         for row in runs_rows:
             run = dict(row)
             for field in ["exclusive", "optimize"]:
                 if field in run and run[field] is not None:
                     run[field] = bool(run[field])
             runs.append(run)
             run_ids.append(run["run_id"])
+            if run.get("workflow") == workflow:
+                has_selected_workflow = True
+
+        if include_all_workflows and workflow and not has_selected_workflow:
+            conn.close()
+            return {
+                "exists": False,
+                "message": f"No runs found for workflow: {workflow}",
+            }
 
         placeholders = ",".join("?" * len(run_ids))
         cursor.execute(
@@ -932,7 +955,10 @@ def get_visualization_data(
             "task_results": task_results,
         }
 
-    runs_summary = backend.get_runs_for_workflow(workflow) or []
+    if include_all_workflows:
+        runs_summary = backend.get_all_runs() or []
+    else:
+        runs_summary = backend.get_runs_for_workflow(workflow) or []
     if not runs_summary:
         return {
             "exists": False,
@@ -941,16 +967,20 @@ def get_visualization_data(
 
     runs: List[Dict[str, Any]] = []
     task_results: List[Dict[str, Any]] = []
+    has_selected_workflow = False
 
     for summary in runs_summary:
         run_id = summary.get("run_id")
         if not run_id:
             continue
 
         run_info = backend.get_run_info(run_id) or {}
+        run_workflow = run_info.get("workflow") or summary.get("workflow") or workflow
+        if run_workflow == workflow:
+            has_selected_workflow = True
         run = {
             "run_id": run_id,
-            "workflow": run_info.get("workflow") or summary.get("workflow") or workflow,
+            "workflow": run_workflow,
             "taskfile_path": run_info.get("taskfile_path"),
             "start_time": run_info.get("start_time") or summary.get("start_time"),
             "end_time": run_info.get("end_time") or summary.get("end_time"),
@@ -1020,6 +1050,12 @@ def get_visualization_data(
             "message": f"No runs found for workflow: {workflow}",
         }
 
+    if include_all_workflows and workflow and not has_selected_workflow:
+        return {
+            "exists": False,
+            "message": f"No runs found for workflow: {workflow}",
+        }
+
     return {
         "exists": True,
         "workflow": workflow,