Added Directory Structure

Author: rajat116

README.md

@@ -47,7 +47,6 @@ See full Makefile usage [here](#makefile-usage) — from setup to linting, testi
 ├── mlruns/                  ← MLflow experiment tracking artifacts
 ├── infra/                   ← Terraform IaC for provisioning MLflow container
 ├── github_pipeline/         ← Feature engineering, inference, monitoring scripts
-├── scripts/                 ← Utility or automation scripts (e.g., setup, cleanup)
 ├── tests/                   ← Pytest-based unit/integration tests
 ├── reports/                 ← Data drift reports (JSON/HTML) from Evidently
 ├── alerts/                  ← Alert log dumps (e.g., triggered drift/anomaly alerts)
@@ -177,7 +176,7 @@ You can explore experiment runs and models in the MLflow UI.
 The model (Isolation Forest) is trained on actor-wise event features:
 
 ```bash
-python scripts/train_model.py
+python github_pipeline/train_model.py
 ```
 The latest parquet file is used automatically. Model and scaler are saved to models/.
 

dags/daily_github_inference.py

@@ -7,6 +7,20 @@
 from github_pipeline.inference import run_inference
 from github_pipeline.cleanup import run_cleanup
 
+"""
+DAG: daily_github_inference
+
+This Airflow DAG orchestrates the daily GitHub anomaly detection pipeline.
+
+Tasks:
+- 📥 Download and parse GitHub Archive logs for the previous day at 15:00 UTC
+- 🛠️ Perform feature engineering on the ingested data
+- 🧠 Run inference using the latest trained Isolation Forest model
+- 🧹 Clean up old data files, keeping only the latest relevant timestamps
+
+Each step uses PythonOperators to invoke modular functions from github_pipeline.
+"""
+
 
 def get_yesterday_fixed_hour(ds: str, hour: int = 15) -> str:
     """

dags/daily_monitoring_dag.py

@@ -7,6 +7,19 @@
 from glob import glob
 import sys
 
+"""
+DAG: daily_monitoring_dag
+
+This Airflow DAG runs daily monitoring on GitHub features.
+
+Tasks:
+- 🧪 Compare the two latest feature files to detect data drift using Evidently
+- 🚨 Trigger alerts via Slack and Email if drift or anomaly spikes are detected
+- 📊 Save drift reports as JSON and HTML in the reports/ directory
+
+This DAG complements the daily inference DAG by validating data quality and stability.
+"""
+
 # Add github_pipeline to PYTHONPATH
 sys.path.append(str(Path(__file__).resolve().parents[1] / "github_pipeline"))
 

dags/retraining_dag.py

@@ -5,6 +5,20 @@
 import sys
 import os
 
+"""
+DAG: weekly_model_retraining
+
+This Airflow DAG retrains the Isolation Forest model every week using the latest feature data.
+
+Tasks:
+- 🧠 Loads most recent actor features from data/features/
+- 🛠️ Trains an Isolation Forest model on engineered features
+- 🗃️ Logs model metrics and artifacts to MLflow
+- 💾 Saves model and anomaly scores in models/ and data/features/
+
+This DAG ensures the anomaly detection system stays up to date with evolving GitHub activity patterns.
+"""
+
 # Ensure github_pipeline is in the PYTHONPATH
 sys.path.append(
     os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "github_pipeline"))

github_pipeline/cleanup.py

@@ -10,6 +10,7 @@ def run_cleanup(verbose: bool = True):
     - Last 2 timestamps
     - The timestamp used in latest model training (stored in models/last_trained.txt)
     """
+
     folders = [
         "data/raw",
         "data/processed",

github_pipeline/feature_engineering.py

@@ -8,6 +8,14 @@
 
 
 def run_feature_engineering(timestamp: str):
+    """
+    Generates actor-level hourly features from GitHub event logs
+    for a given timestamp and saves them as a Parquet file.
+
+    Input:  data/processed/{timestamp}.parquet
+    Output: data/features/actor_features_{timestamp}.parquet
+    """
+
     input_file = f"data/processed/{timestamp}.parquet"
     output_file = f"data/features/actor_features_{timestamp}.parquet"
 

github_pipeline/inference.py

@@ -9,6 +9,13 @@
 
 
 def run_inference(timestamp: str):
+    """
+    Runs anomaly inference on actor-level features using Isolation Forest.
+
+    Input:  data/features/actor_features_{timestamp}.parquet
+    Output: data/features/actor_predictions_{timestamp}.parquet
+    """
+
     print(f"[INFO] Running inference for timestamp: {timestamp}")
 
     input_file = f"data/features/actor_features_{timestamp}.parquet"

github_pipeline/monitor.py

@@ -13,6 +13,19 @@
 
 
 def run_monitoring(reference_ts: str, current_ts: str):
+    """
+    Checks for data drift (via Evidently) and anomaly spikes between two timestamps.
+    Triggers Slack and Email alerts if drift or anomalies are detected.
+
+    Inputs:
+      - actor_features_{reference_ts}.parquet
+      - actor_features_{current_ts}.parquet
+      - actor_predictions_{current_ts}.parquet (optional for anomaly alert)
+
+    Outputs:
+      - JSON and HTML drift reports in the 'reports/' directory
+    """
+
     # === Feature Columns ===
     feature_cols = [
         "event_count",

github_pipeline/train_model.py

@@ -12,6 +12,22 @@
 
 
 def run_training(timestamp: str):
+    """
+    Trains an Isolation Forest model for anomaly detection using actor features for a given timestamp.
+
+    Steps:
+    - Loads feature parquet file.
+    - Scales the data and trains the model.
+    - Computes anomaly scores and flags.
+    - Logs parameters, metrics, and artifacts to MLflow.
+    - Saves the model and prediction results to disk and registry.
+
+    Outputs:
+    - models/isolation_forest.pkl
+    - models/last_trained.txt
+    - data/features/actor_predictions_<timestamp>.parquet
+    """
+
     print("[INFO] Starting model training...")
 
     input_file = f"data/features/actor_features_{timestamp}.parquet"