Add single-user report without comparison.

Author: Benjamin Pachev

scripts/generate_user_reports.py

@@ -28,12 +28,12 @@
 # from src.analysis.frequency_analysis import FrequencyAnalysis  # noqa: E402
 from src.config.enum_constants import EfficiencyCategoryEnum  # noqa: E402
 from src.analysis.user_comparison import UserComparison  # noqa: E402
+from src.analysis.user import get_metrics as get_user_metrics  # noqa: E402
 from src.utilities.report_generation import (  # noqa: E402
     calculate_analysis_period,
     calculate_summary_statistics,
     calculate_comparison_statistics,
     generate_recommendations,
-    get_total_job_count,
 )
 
 
@@ -101,20 +101,11 @@ def generate_user_report(
     # Calculate analysis period using shared utility
     analysis_period = calculate_analysis_period(user_jobs)
 
-    # === CALCULATE SUMMARY STATISTICS USING SHARED UTILITIES ===
-    # Use the get_total_job_count function if db connection is provided
-    if db:
-        total_all_jobs = get_total_job_count(db, analysis_period)
-    else:
-        total_all_jobs = len(job_data)  # Fallback to current dataset size
-
     # Use shared utility for summary statistics
     summary_stats = calculate_summary_statistics(
         user_jobs=user_jobs,
-        all_jobs_count=total_all_jobs,
         analysis_period=analysis_period
     )
-
     # === CALCULATE COMPARISON STATISTICS ===
     # Use shared utility for comparison statistics
     if user_comparison and user_comparison._cached_all_users_metrics is not None:
@@ -271,6 +262,131 @@ def check_directory_writable(directory_path: str) -> tuple[bool, str | None]:
         return False, str(e)
 
 
+def generate_single_report(
+    user_id: str,
+    db_path: str = "./slurm_data.db",
+    output_dir: str = "./reports/user_reports",
+    template_path: str = "./reports/user_no_compare_report_template.qmd",
+    output_format: str = "html",
+) -> None:
+    """Generate a report for a single user without comparison."""
+
+    user_jobs, user_metrics = get_user_metrics(db_path, user_id)
+
+    if not len(user_jobs):
+        print("No jobs available.")
+        return
+
+    user_output_dir = os.path.join(output_dir, f"user_{user_id}")
+    os.makedirs(user_output_dir, exist_ok=True)
+
+    output_file = f"{user_id}_report.{output_format}"
+    final_output_path = os.path.join(user_output_dir, output_file)
+
+    # Check that the template exists
+    if not os.path.exists(template_path):
+        abs_template_path = os.path.abspath(template_path)
+        if os.path.exists(abs_template_path):
+            template_path = abs_template_path
+        else:
+            print(f"Error: Template file not found at: {template_path}")
+            print(f"Absolute path also not found: {abs_template_path}")
+            return None
+
+    # Calculate dates for the report
+    start_date = user_jobs["StartTime"].min().strftime("%Y-%m-%d")
+    end_date = user_jobs["StartTime"].max().strftime("%Y-%m-%d")
+
+    # Calculate analysis period using shared utility
+    analysis_period = calculate_analysis_period(user_jobs)
+
+    # Use shared utility for summary statistics
+    summary_stats = calculate_summary_statistics(
+        user_jobs=user_jobs,
+        analysis_period=analysis_period
+    )
+    print(summary_stats)
+
+    # === PREPARE GPU TYPE DATA ===
+    if "GPUType" in user_jobs.columns and "primary_gpu_type" in user_jobs.columns:
+        gpu_type_data = user_jobs["primary_gpu_type"].value_counts().reset_index()
+        gpu_type_data.columns = ["gpu_type", "job_count"]
+    else:
+        gpu_type_data = pd.DataFrame(columns=["gpu_type", "job_count"])
+
+    # Generate recommendations using shared utility
+    recommendations = generate_recommendations(user_jobs, user_metrics)
+
+    # === SAVE DATA TO JSON FOR QUARTO ===
+
+    temp_data_pkl = {
+        "user_id": user_id,
+        "start_date": start_date,
+        "end_date": end_date,
+        "analysis_period": analysis_period,
+        "summary_stats": summary_stats,
+        "gpu_type_data": gpu_type_data,
+        "recommendations": recommendations,
+        "user_data": pd.DataFrame(user_metrics),
+        "user_jobs": pd.DataFrame(user_jobs),
+    }
+
+    # get abs path to the template in the reports folder
+    template_file = os.path.join(project_root, template_path)
+
+    with tempfile.NamedTemporaryFile(suffix=".pkl", delete=False) as f:
+        pickle.dump(temp_data_pkl, f, protocol=pickle.HIGHEST_PROTOCOL)
+        temp_pickle_path = f.name
+    
+    permanent_path = os.path.join(project_root, "reports", "test_data.pkl")
+    shutil.copy(temp_pickle_path, permanent_path)
+
+    # Use absolute path for output file to avoid path resolution issues
+    output_file_abs = os.path.abspath(final_output_path)
+    
+    cmd = ["quarto", "render", template_file, "--to", output_format, "-o", output_file, "--execute"]
+    cmd.extend(["-P", f"pickle_file:{temp_pickle_path}"])
+    try:
+        # Run Quarto in the reports directory (full path)
+        reports_dir = os.path.join(project_root, "reports")
+        print("Starting Quarto rendering...")
+        print(f"Output file: {output_file_abs}")
+        print(f"Command: {' '.join(cmd)}")
+
+        result = subprocess.run(
+            cmd,
+            check=False,  # Don't raise exception, handle it manually
+            capture_output=True,
+            text=True,
+            cwd=reports_dir,
+        )
+        output_path = os.path.join(reports_dir, output_file)
+        # Check if command was successful
+        if result.returncode != 0:
+            print(f"      ❌ Quarto rendering failed (code {result.returncode})")
+            if result.stderr:
+                print(f"         Error: {result.stderr.strip()}")
+            if result.stdout:
+                print(f"         Output: {result.stdout.strip()}")
+            return None
+
+        # Check if output file was created
+        if not os.path.exists(output_path):
+            print("      ❌ Output file not created")
+            print(f"         Expected at: {output_file_abs}")
+            return None
+
+        print(f"      ✅ Report saved: {os.path.basename(final_output_path)}")
+        return final_output_path
+
+    except subprocess.CalledProcessError as e:
+        print(f"      ❌ Process error: {e}")
+        return None
+    except Exception as e:
+        print(f"      ❌ Unexpected error: {e}")
+        return None
+
+
 def generate_reports_for_specific_users(
     user_list: list,
     db_path: str = "./slurm_data.db",
@@ -485,6 +601,7 @@ def generate_all_reports(
     users_parser.add_argument(
         "--min-jobs", type=int, default=10, help="Minimum number of jobs a user must have (default: 10)"
     )
+    users_parser.add_argument("--no-compare", action="store_true", default=False)
 
     # Common arguments for both modes
     parser.add_argument(
@@ -541,14 +658,18 @@ def generate_all_reports(
         print("👥 GENERATING REPORTS FOR SPECIFIC USERS")
         print(f"   Users: {', '.join(user_list)}")
         print(f"   Minimum jobs required: {args.min_jobs}")
-        report_paths = generate_reports_for_specific_users(
-            user_list=user_list,
-            db_path=args.db_path,
-            output_dir=args.output_dir,
-            template_path=args.template,
-            min_jobs=args.min_jobs,
-            output_format=args.format,
-        )
+        if args.no_compare:
+            for user in user_list:
+                generate_single_report(user, args.db_path)
+        else:
+            report_paths = generate_reports_for_specific_users(
+                user_list=user_list,
+                db_path=args.db_path,
+                output_dir=args.output_dir,
+                template_path=args.template,
+                min_jobs=args.min_jobs,
+                output_format=args.format,
+            )
 
     print("\n🎉 ALL TASKS COMPLETED!")
     print(f"   Output directory: {args.output_dir}")

src/analysis/efficiency_analysis.py

@@ -10,7 +10,7 @@
 import subprocess
 
 import numpy as np
-from typing import Generic, TypeVar, Annotated, cast
+from typing import Generic, TypeVar, cast
 import pandas as pd
 from pathlib import Path
 from src.config.constants import DEFAULT_MIN_ELAPSED_SECONDS
@@ -66,6 +66,7 @@ def load_preprocessed_jobs_dataframe_from_duckdb(
 # Generic type for metrics enums constrained to our abstract base Enum class
 MetricsDFNameEnumT = TypeVar("MetricsDFNameEnumT", bound=MetricsDataFrameNameBase)
 
+
 class EfficiencyAnalysis(Generic[MetricsDFNameEnumT]):
     """
     Class to encapsulate the efficiency analysis of jobs based on various metrics.

src/config/snapshots/partition_info.json

@@ -14,7 +14,7 @@
     "node_count": 4,
     "maxtime": "14 days",
     "deftime": "1 hour",
-    "max_ram": 560,
+    "max_ram": 450,
     "max_cpus": 72
   },
   {
@@ -29,7 +29,7 @@
   {
     "name": "astroth-cpu",
     "type": "cpu",
-    "node_count": 9,
+    "node_count": 8,
     "maxtime": "30 days",
     "deftime": "1 hour",
     "max_ram": 1000,
@@ -92,7 +92,7 @@
   {
     "name": "cpu",
     "type": "cpu",
-    "node_count": 153,
+    "node_count": 135,
     "maxtime": "2 days",
     "deftime": "1 hour",
     "max_ram": 1510,
@@ -101,7 +101,7 @@
   {
     "name": "cpu-preempt",
     "type": "cpu",
-    "node_count": 144,
+    "node_count": 173,
     "maxtime": "2 days",
     "deftime": "1 hour",
     "max_ram": 1510,
@@ -137,7 +137,7 @@
   {
     "name": "gpu",
     "type": "gpu",
-    "node_count": 153,
+    "node_count": 157,
     "maxtime": "2 days",
     "deftime": "1 hour",
     "max_ram": 2010,
@@ -206,6 +206,15 @@
     "max_ram": 250,
     "max_cpus": 64
   },
+  {
+    "name": "ood-shared",
+    "type": "cpu",
+    "node_count": 10,
+    "maxtime": "8 hours",
+    "deftime": "1 hour",
+    "max_ram": 370,
+    "max_cpus": 40
+  },
   {
     "name": "power9",
     "type": "cpu",
@@ -281,7 +290,7 @@
   {
     "name": "uri-cpu",
     "type": "cpu",
-    "node_count": 49,
+    "node_count": 57,
     "maxtime": "30 days",
     "deftime": "1 hour",
     "max_ram": 1000,
@@ -299,7 +308,7 @@
   {
     "name": "uri-richamp",
     "type": "cpu",
-    "node_count": 5,
+    "node_count": 8,
     "maxtime": "Unlimited",
     "deftime": "1 hour",
     "max_ram": 120,
@@ -314,6 +323,15 @@
     "max_ram": 370,
     "max_cpus": 48
   },
+  {
+    "name": "workflow",
+    "type": "cpu",
+    "node_count": 1,
+    "maxtime": "2 days",
+    "deftime": "1 hour",
+    "max_ram": 370,
+    "max_cpus": 24
+  },
   {
     "name": "zhoulin-cpu",
     "type": "cpu",

src/utilities/load_and_preprocess_jobs.py

@@ -25,6 +25,7 @@ def load_and_preprocess_jobs(
     min_elapsed_seconds: int = DEFAULT_MIN_ELAPSED_SECONDS,
     random_state: pd._typing.RandomState | None = None,
     sample_size: int | None = None,
+    user: str | None = None,
 ) -> pd.DataFrame:
     """
     Load jobs DataFrame from a DuckDB database with standard filtering and preprocess it.
@@ -46,6 +47,7 @@ def load_and_preprocess_jobs(
             Defaults to DEFAULT_MIN_ELAPSED_SECONDS.
         random_state (pd._typing.RandomState, optional): Random state for reproducibility. Defaults to None.
         sample_size (int, optional): Number of rows to sample from the DataFrame. Defaults to None (no sampling).
+        user (str, optional): user ID to get jobs for
 
     Returns:
         pd.DataFrame: Preprocessed DataFrame containing the filtered job data.
@@ -84,6 +86,8 @@ def load_and_preprocess_jobs(
         if not include_failed_cancelled_jobs:
             conditions_arr.append(f"Status != '{StatusEnum.FAILED.value}'")
             conditions_arr.append(f"Status != '{StatusEnum.CANCELLED.value}'")
+        if user is not None:
+            conditions_arr.append(f"User = '{user}'")
 
         query = f"SELECT * EXCLUDE {excluded_columns} FROM {table_name} WHERE {' AND '.join(conditions_arr)}"
         jobs_df = db.fetch_query(query=query)