added ml-flow and infrastructure

Author: dilukshashamal

.env.example

@@ -0,0 +1,22 @@
+# Core model settings
+MODEL_SOURCE=local
+MODEL_NAME=rf_churn_model
+DEFAULT_PREDICTION_THRESHOLD=0.36
+
+# MLflow settings
+MLFLOW_TRACKING_URI=http://localhost:5000
+MLFLOW_EXPERIMENT=Telco_Churn_Prediction
+MLFLOW_MODEL_ALIAS=champion
+ALLOW_LOCAL_MODEL_FALLBACK=1
+
+# UI <-> API wiring
+API_URL=http://localhost:8000
+USE_API_INFERENCE=1
+
+# Email alerts 
+ALERT_SMTP_HOST=smtp.example.com
+ALERT_SMTP_PORT=587
+ALERT_SMTP_USER=mlops@example.com
+ALERT_SMTP_PASSWORD=change_me
+ALERT_FROM_EMAIL=mlops@example.com
+ALERT_TO_EMAILS=data-team@example.com,owner@example.com

.github/workflows/ml_pipeline.yml

@@ -0,0 +1,36 @@
+name: ml-pipeline-ci
+
+on:
+    push:
+        branches: ["main", "develop"]
+    pull_request:
+        branches: ["**"]
+
+jobs:
+    ci:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Setup Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.11"
+
+            - name: Install dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install -r requirements.txt
+
+            - name: Python syntax check
+              run: |
+                  python -m compileall src app.py
+
+            - name: Import smoke tests
+              run: |
+                  python -c "import src.deploy_model"
+                  python -c "import src.crm_integration"
+                  python -c "import src.drift_monitor"
+                  python -c "import src.ab_testing"
+                  python -c "import src.shap_analysis"

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY app.py /app/app.py
+COPY src /app/src
+COPY data /app/data
+COPY models /app/models
+COPY reports /app/reports
+
+ENV PYTHONPATH=/app/src
+
+EXPOSE 8081
+EXPOSE 8501
+
+CMD ["streamlit", "run", "app.py", "--server.port", "8501", "--server.address", "0.0.0.0"]

README.md

@@ -2,6 +2,67 @@
 
 A full end-to-end churn prediction system built on the **IBM Telco Customer Churn** dataset — from raw EDA and preprocessing all the way through model training, SHAP explainability, CRM integration, data drift monitoring, A/B testing, and an interactive **Streamlit retention dashboard**.
 
+## MLOps Upgrade (In Progress)
+
+This project now includes a production-oriented MLOps path in addition to the local notebook flow:
+
+- MLflow tracking and model registry integration in `src/deploy_model.py`
+- FastAPI batch inference service in `src/api.py`
+- API-based Streamlit scoring mode (`USE_API_INFERENCE=1`)
+- Drift job entrypoint with MLflow logging and SMTP alerts in `src/drift_monitor_job.py`
+- Containerized multi-service stack via `docker-compose.yml`
+- CI workflow in `.github/workflows/ml_pipeline.yml`
+
+### Service Architecture
+
+1. `mlflow` service: experiment tracking and model registry.
+2. `api` service: batch prediction endpoint (`POST /predict`) and health endpoint (`GET /health`).
+3. `streamlit` service: dashboard UI that calls API for scoring.
+
+### Run Containerized Stack
+
+```bash
+docker compose up --build
+```
+
+Endpoints:
+
+- MLflow UI: `http://localhost:5001`
+- FastAPI: `http://localhost:18000` (`/health`, `/predict`)
+- Streamlit: `http://localhost:18501`
+
+### Environment Variables
+
+Use `.env.example` as a template for local configuration.
+
+Key variables:
+
+- `MODEL_SOURCE` (`local` or `mlflow`)
+- `MLFLOW_TRACKING_URI`
+- `MLFLOW_EXPERIMENT`
+- `MLFLOW_MODEL_ALIAS`
+- `API_URL`
+- `USE_API_INFERENCE`
+- `ALERT_SMTP_*`
+
+### Drift Monitoring Job (Airflow/Cron Ready)
+
+Run manually:
+
+```bash
+python -m src.drift_monitor_job --reference-path data/telco_churn_processed.csv --current-path data/telco_churn_processed.csv
+```
+
+This job computes PSI, logs drift metrics to MLflow, and sends email alerts when retraining is recommended.
+
+### Quick MLflow Logging (Existing Model)
+
+Use this helper to log your current local model artifact to MLflow without opening a notebook:
+
+```bash
+./venv/python.exe scripts/log_existing_model_to_mlflow.py --tracking-uri http://localhost:5000 --experiment Telco_Churn_Prediction
+```
+
 ---
 
 ## Dashboard Preview
@@ -49,6 +110,8 @@ This project answers:
 
 ```
 churn-analyze-crm/
+├── docker-compose.yml                  # Multi-service local orchestration
+├── Dockerfile                          # Shared image for API + Streamlit services
 ├── app.py                              # Streamlit retention dashboard (entry point)
 ├── requirements.txt                    # Python dependencies
 ├── data/
@@ -61,11 +124,15 @@ churn-analyze-crm/
 │   ├── eda_prac.ipynb                  # Full EDA + preprocessing notebook
 │   └── model_evaluation.ipynb          # Model training & evaluation notebook
 ├── src/
+│   ├── api.py                          # FastAPI batch inference service
 │   ├── deploy_model.py                 # Model packaging, loading & model card
 │   ├── shap_analysis.py                # SHAP explainability (waterfall, bar, beeswarm)
 │   ├── crm_integration.py              # Customer scoring, risk tiers, CRM CSV export
 │   ├── drift_monitor.py                # PSI-based data drift detection & alerts
+│   ├── drift_monitor_job.py            # Scheduler-friendly drift monitoring entrypoint
 │   └── ab_testing.py                   # A/B test simulation & power analysis
+├── .github/workflows/
+│   └── ml_pipeline.yml                 # CI checks (syntax/import smoke)
 ├── reports/                            # Auto-generated SHAP plots & drift reports
 └── img/                                # Dashboard screenshots
 ```

app.py

@@ -9,6 +9,7 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import streamlit as st
+import requests
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
 
@@ -20,6 +21,7 @@
 
 ROOT      = os.path.dirname(__file__)
 DATA_PATH = os.path.join(ROOT, 'data', 'telco_churn_processed.csv')
+API_URL   = os.environ.get('API_URL', 'http://localhost:8000')
 
 AVG_MONTHLY_REVENUE = 65   # £ per customer (MonthlyCharges proxy)
 
@@ -87,7 +89,15 @@ def plain(name: str) -> str:
 
 @st.cache_resource(show_spinner='Loading prediction system…')
 def get_model():
-    return load_model('rf_churn_model')
+    try:
+        return load_model('rf_churn_model')
+    except Exception:
+        # API inference mode can run without local model artifacts.
+        return {
+            'model': None,
+            'feature_names': [],
+            'threshold': float(os.environ.get('DEFAULT_PREDICTION_THRESHOLD', '0.36')),
+        }
 
 
 @st.cache_data(show_spinner='Loading customer data…')
@@ -96,9 +106,15 @@ def get_data():
 
 
 def model_ready() -> bool:
+    if use_api_inference():
+        return True
     return os.path.exists(os.path.join(ROOT, 'models', 'rf_churn_model.joblib'))
 
 
+def use_api_inference() -> bool:
+    return os.environ.get('USE_API_INFERENCE', '0') == '1'
+
+
 def show_deploy_prompt():
     st.warning(
         "⚠️  The prediction system has not been set up yet.\n\n"
@@ -112,6 +128,22 @@ def show_deploy_prompt():
 @st.cache_data(show_spinner='Scoring customers…')
 def get_scored(_model, _feature_names, _threshold, _data_hash):
     df = get_data()
+    if use_api_inference():
+        try:
+            response = requests.post(
+                f'{API_URL}/predict',
+                json={'records': df.to_dict(orient='records')},
+                timeout=60,
+            )
+            response.raise_for_status()
+            payload = response.json()
+            preds = pd.DataFrame(payload.get('predictions', []), index=df.index)
+            if preds.empty:
+                raise ValueError('No predictions returned by API')
+            return pd.concat([df, preds], axis=1)
+        except Exception as exc:
+            st.warning(f'API scoring unavailable, using local model instead. ({exc})')
+
     return score_customers(df, _model, _feature_names, _threshold)
 
 
@@ -384,6 +416,14 @@ def get_scored(_model, _feature_names, _threshold, _data_hash):
     feature_names = artifact['feature_names']
     df            = get_data()
 
+    if model is None:
+        st.info(
+            'SHAP explainability requires local model access. '
+            'Run with local model artifacts (MODEL_SOURCE=local) to use this page.',
+            icon='ℹ️',
+        )
+        st.stop()
+
     X = df.drop(columns=['Churn_Numeric'])
 
     with st.sidebar:

docker-compose.yml

@@ -0,0 +1,43 @@
+version: "3.8"
+
+services:
+  mlflow:
+    image: ghcr.io/mlflow/mlflow:v2.13.2
+    command: mlflow server --host 0.0.0.0 --port 5000 --backend-store-uri sqlite:////mlrunsdb/mlflow.db --default-artifact-root /mlartifacts
+    ports:
+      - "5000:5000"
+    volumes:
+      - mlflow_data:/mlartifacts
+      - mlflow_db:/mlrunsdb
+
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: uvicorn src.api:app --host 0.0.0.0 --port 8081
+    ports:
+      - "8081:8081"
+    environment:
+      - MODEL_SOURCE=mlflow
+      - MODEL_NAME=rf_churn_model
+      - MLFLOW_TRACKING_URI=http://mlflow:5000
+      - ALLOW_LOCAL_MODEL_FALLBACK=1
+    depends_on:
+      - mlflow
+
+  streamlit:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: streamlit run app.py --server.port 8501 --server.address 0.0.0.0
+    ports:
+      - "8501:8501"
+    environment:
+      - API_URL=http://api:8081
+      - USE_API_INFERENCE=1
+    depends_on:
+      - api
+
+volumes:
+  mlflow_data:
+  mlflow_db:

models/rf_churn_model.joblib

@@ -27,7 +27,7 @@
     "PaymentMethod_Mailed check"
   ],
   "threshold": 0.3632911392405064,
-  "trained_at": "2026-03-07T15:09:07.719001",
+  "trained_at": "2026-03-15T21:04:45.844238",
   "model_type": "RandomForestClassifier",
   "n_features": 25
 }

models/rf_churn_model_meta.json

@@ -10,3 +10,9 @@ shap
 streamlit
 scipy
 joblib
+mlflow
+fastapi
+uvicorn
+requests
+python-dotenv
+docker