Merge pull request #106 from ml-stat-Sustech/development

Development

Author: Jianguo99

README.md

@@ -65,10 +65,9 @@ build the basic framework of TorchCP based on [`AdverTorch`](https://github.com/
 codebase is still under construction and maintained by [`Hongxin Wei`](https://hongxin001.github.io/)'s research group
 at SUSTech. Comments, issues, contributions, and collaborations are all welcomed!
 
-## Updates of New Version (1.1.0)
+## Updates of New Version (1.2.0)
 
-This release features a comprehensive refactoring of predictor modules, along with the addition of RC3P, EntmaxScore, and the SCPO trainer.
-This release features a comprehensive refactoring of predictor modules, along with the addition of RC3P, EntmaxScore, and the SCPO trainer.
+This release enhances functionality by introducing p-value computation, conformal predictive distributions, and expanding the NORABS score function with additional difficulty estimation methods.
 Detailed changelog can be found in the [Documentation](https://torchcp.readthedocs.io/en/latest/CHANGELOG.html).
 
 # Overview
@@ -110,7 +109,7 @@ TorchCP has implemented the following methods:
 | 2019 | [**Adaptive, Distribution-Free Prediction Intervals for Deep Networks**](https://proceedings.mlr.press/v108/kivaranovic20a.html)                | AISTATS'19           | [Link](https://github.com/yromano/cqr)                 | regression.score.cqrfm                     |                     |
 | 2019 | [**Conformalized Quantile Regression**](https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf)  | NeurIPS'19           | [Link](https://github.com/yromano/cqr)                 | regression.score.cqr                       |                     |
 | 2017 | [**Distribution-Free Predictive Inference For Regression**](https://arxiv.org/abs/1604.04173)                                                   | JASA                 | [Link](https://github.com/ryantibs/conformal)          | regression.predictor.split                 |                     |
-| 2005 | [**Inductive Confidence Machines for Regression**](https://link.springer.com/chapter/10.1007/3-540-36755-1_29)                                                   | Springer                 |   | regression.score.abs regression.score.norabs                  |                     |
+| 2005 | [**Inductive Confidence Machines for Regression**](https://link.springer.com/chapter/10.1007/3-540-36755-1_29) <br>[**Guaranteed Coverage Prediction Intervals with Gaussian Process Regression**](https://arxiv.org/abs/2310.15641)<br>[**Reliable Prediction Intervals with Regression Neural Networks**](https://arxiv.org/abs/2312.09606)                                              |                  |   | regression.score.abs regression.score.norabs                  |                     |
 
 ## Graph
 

docs/source/CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+1.2.0 (2025-08-26)  
+------------------  
+* Added Conformal Predictive Distribution in regression task.  
+* Added p-value computation in classification predictors and regression predictors.  
+* Added more tyeps of difficulty estimation in the NORABS score function.
+
 1.1.0 (2025-07-15)  
 ------------------  
 * Refactored `__init__` of predictors to include `alpha` and `device` parameters for greater flexibility.  

docs/source/torchcp.regression.rst

@@ -15,6 +15,7 @@ score function
     CQRM
     CQRR
     R2CCP
+    Sign
 
 .. autoclass:: ABS
    :members:
@@ -34,6 +35,9 @@ score function
 .. autoclass:: R2CCP
    :members:
 
+.. autoclass:: Sign
+   :members:
+
 
 .. automodule:: torchcp.regression.predictor
 
@@ -46,6 +50,7 @@ predictor
     SplitPredictor
     EnsemblePredictor
     ACIPredictor
+    ConformalPredictiveDistribution
 
 .. autoclass:: SplitPredictor
    :members:
@@ -56,6 +61,9 @@ predictor
 .. autoclass:: ACIPredictor
    :members:
 
+.. autoclass:: ConformalPredictiveDistribution
+   :members:
+
 .. automodule:: torchcp.regression.loss
 
 loss

examples/classification_splitcp_cifar100.py

@@ -62,3 +62,13 @@
 result_dict = predictor.evaluate(test_dataloader)
 print(f"Coverage Rate: {result_dict['coverage_rate']:.4f}")
 print(f"Average Set Size: {result_dict['average_size']:.4f}")
+
+#########################################
+# Predict the p-value
+#########################################
+
+p_values = predictor.predict_p(test_instances.unsqueeze(0))
+print("p_values: ", p_values)
+
+
+

examples/regression_split_norabs.py

@@ -0,0 +1,120 @@
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+
+from examples.regression_cqr_synthetic import prepare_dataset
+from torchcp.regression.predictor import SplitPredictor
+from torchcp.regression.utils import build_regression_model
+
+# Ensure these classes are correctly imported depending on your project setup:
+# Option 1: If my_norabs.py is located in the same directory:
+# from my_norabs import DifficultyEstimator, NorABS
+# Option 2: If integrated into the torchcp library:
+from torchcp.regression.score import NorABS, DifficultyEstimator
+
+
+def extract_variance_as_function(x_batch, predicts_batch):
+    """
+    Custom difficulty function that replicates the behavior of the 'variance' mode.
+
+    Args:
+        x_batch (Tensor): Input features (unused here but kept for API consistency).
+        predicts_batch (Tensor): Predictions with shape (batch_size, 2),
+                                 where the second column is interpreted as variance.
+
+    Returns:
+        Tensor: A vector representing the difficulty score based on variance.
+    """
+    if predicts_batch is None or predicts_batch.ndim != 2 or predicts_batch.shape[-1] != 2:
+        raise ValueError("This function requires `predicts_batch` to have shape (batch_size, 2).")
+    return predicts_batch[:, 1]
+
+
+def run_experiment(mode='variance'):
+    """
+    Executes a conformal prediction experiment using a lazily calibrated DifficultyEstimator.
+
+    Args:
+        mode (str): Difficulty estimation strategy.
+                    Supported values: 'variance', 'knn_residual', 'knn_label', 'knn_distance', 'function'.
+    """
+    print(f"\n--- Running Experiment in '{mode}' mode ---")
+    
+    # 1. Prepare data loaders for training, calibration, and testing
+    train_loader, cal_loader, test_loader = prepare_dataset(train_ratio=0.4, cal_ratio=0.2, batch_size=128)
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+    # Extract feature dimensionality from a sample batch to build the model
+    sample_x, _ = next(iter(train_loader))
+    feature_dim = sample_x.shape[1]
+
+    # 2. Initialize the regression model (untrained at this stage)
+    model = build_regression_model("GaussianRegressionModel")(feature_dim, 64, 0.5)
+    
+    # 3. Define the NorABS score function for difficulty-aware conformity estimation
+    score_function = NorABS(data_loader=train_loader,
+                            estimate_type=mode,
+                            k=20,
+                            scalar=True,
+                            beta=0.01,
+                            device=device,
+                            custom_function=extract_variance_as_function)
+
+    # 4. Wrap the score function and model inside a SplitPredictor instance
+    predictor = SplitPredictor(score_function=score_function, model=model, alpha=0.1, device=device)
+
+    # 5. Train the base regression model using the predictor’s utility function
+    print("Training the base regression model...")
+    predictor.train(train_loader, epochs=100, lr=0.01, device=device, verbose=True)
+    # After training, the model is stored within `predictor.model`
+
+    # 6. Calibrate the predictor on the calibration set.
+    #    This step computes nonconformity scores and determines the quantile q_hat.
+    print("Calibrating the predictor (computing q_hat)...")
+    predictor.calibrate(cal_loader)
+
+    # 7. Online evaluation on the test set (transductive protocol).
+    print("Evaluating on the test set...")
+    cover_count, total, set_size_sum = 0, 0, 0
+    
+    # Maintain a mutable calibration set that is updated sequentially
+    online_cal_dataset = list(cal_loader.dataset)
+    test_dataset = test_loader.dataset
+
+    for i in tqdm(range(len(test_dataset)), desc="Online Evaluation"):
+        x_test, y_test = test_dataset[i]
+        x_test_device = x_test.to(device).unsqueeze(0)  # Predictor expects a batch dimension
+
+        # Generate prediction intervals for the test sample
+        prediction_intervals = predictor.predict(x_test_device)[0][0]
+        
+        # Evaluate coverage and interval size
+        covered = (prediction_intervals[0] <= y_test <= prediction_intervals[1])
+        cover_count += int(covered)
+        set_size_sum += (prediction_intervals[1] - prediction_intervals[0]).item()
+        total += 1
+        
+        # Update the calibration set with the new (x, y) pair
+        online_cal_dataset.append((x_test, y_test))
+        online_cal_loader = DataLoader(online_cal_dataset, batch_size=128, shuffle=True)
+        
+        # Recalibrate the predictor with the expanded calibration set
+        predictor.calibrate(online_cal_loader)
+
+    # Compute final evaluation metrics
+    coverage_rate = cover_count / total
+    average_set_size = set_size_sum / total
+    alpha = predictor.alpha
+
+    print(f"\nResults for '{mode}' mode:")
+    print(f"Online CP Coverage Rate: {coverage_rate:.4f} (Target: {1-alpha:.2f})")
+    print(f"Online CP Average Set Size: {average_set_size:.4f}")
+
+
+if __name__ == "__main__":
+    # Run the experiment under different difficulty estimation modes
+    run_experiment(mode='variance')
+    run_experiment(mode='knn_residual')
+    run_experiment(mode='knn_label')
+    run_experiment(mode='knn_distance')
+    run_experiment(mode='function')

examples/regression_split_synthetic_cpd.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2023-present, SUSTech-ML.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import numpy as np
+import torch
+from sklearn.preprocessing import StandardScaler
+from torch.utils.data import TensorDataset
+from tqdm import tqdm
+import torch.nn as nn
+import torch.optim as optim
+
+from examples.regression_cqr_synthetic import prepare_dataset
+from torchcp.regression.predictor import ConformalPredictiveDistribution 
+from torchcp.regression.score import Sign
+from torchcp.regression.utils import build_regression_model
+
+
+if __name__ == "__main__":
+    # get dataloader
+    train_loader, cal_loader, test_loader = prepare_dataset(train_ratio=0.4, cal_ratio=0.2, batch_size=128)
+    # build regression model
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    model = build_regression_model("NonLinearNet")(next(iter(train_loader))[0].shape[1], 1, 64, 0.5).to(device)
+
+    
+    
+    
+    # train model
+    epochs = 100
+    criterion = nn.MSELoss()
+    lr = 0.01
+    optimizer = optim.Adam(model.parameters(), lr=lr)    
+        
+    for tmp_x, tmp_y in train_loader:
+        outputs = model(tmp_x.to(device))
+        loss = criterion(outputs, tmp_y.reshape(-1, 1).to(device))
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+    # CPD
+    predictor = ConformalPredictiveDistribution(score_function=Sign(), model=model)
+    predictor.calibrate(cal_loader)
+
+    x = next(iter(test_loader))[0].to(device)
+    prediction_intervals = predictor.predict(x)
+        
+    print(prediction_intervals.shape)

tests/classification/predictor/test_cpds_predictor.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2023-present, SUSTech-ML.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import torch
+import pytest
+
+from torchcp.regression.predictor import ConformalPredictiveDistribution
+from torchcp.regression.score import ABS
+from torchcp.regression.utils import build_regression_model
+from torch.utils.data import DataLoader, TensorDataset
+
+@pytest.fixture
+def mock_model():
+    return build_regression_model("NonLinearNet")(5, 1, 64, 0.5)
+
+
+@pytest.fixture
+def cpds_predictor(mock_model):
+    return ConformalPredictiveDistribution(model=mock_model)
+
+
+def test_invalid_initialization():
+    with pytest.raises(ValueError):
+        ConformalPredictiveDistribution(score_function=ABS())
+
+@pytest.fixture
+def mock_data():
+    """
+    提供用于回归预测器测试的假数据。
+    返回: (train_loader, cal_loader, test_loader)
+    - 特征维度为 5，目标为连续值
+    """
+    num_samples = 3000
+    num_features = 5
+
+    # 生成特征
+    X = torch.rand((num_samples, num_features), dtype=torch.float32)
+
+    # 生成一个平滑的连续目标（非线性 + 噪声）
+    y_wo_noise = (
+        10 * torch.sin(X[:, 0] * X[:, 1] * torch.pi)
+        + 20 * (X[:, 2] - 0.5) ** 2
+        + 10 * X[:, 3]
+        + 5 * X[:, 4]
+    )
+    noise = 0.5 * torch.randn(num_samples, dtype=torch.float32)
+    y = (y_wo_noise + noise).to(torch.float32)
+
+    # 划分 train/cal/test（40%/20%/40%）
+    indices = torch.randperm(num_samples)
+    split_index1 = int(num_samples * 0.4)
+    split_index2 = int(num_samples * 0.6)
+    part1 = indices[:split_index1]
+    part2 = indices[split_index1:split_index2]
+    part3 = indices[split_index2:]
+
+    train_dataset = TensorDataset(X[part1], y[part1])
+    cal_dataset = TensorDataset(X[part2], y[part2])
+    test_dataset = TensorDataset(X[part3], y[part3])
+
+    train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)
+    cal_loader = DataLoader(cal_dataset, batch_size=128, shuffle=False)
+    test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False)
+
+    return train_loader, cal_loader, test_loader
+
+def test_workflow(mock_data, cpds_predictor):
+    # Extract mock data
+    _, cal_dataloader, test_dataloader = mock_data
+
+    cpds_predictor.calibrate(cal_dataloader)
+    assert hasattr(cpds_predictor, "scores"), "SplitPredictor should have scores after calibration."
+
+    for x_batch, _ in test_dataloader:
+            cpds_predictor.predict(x_batch)
+
+    with pytest.raises(ValueError):
+        tmp_cpds_predictor = ConformalPredictiveDistribution()
+        for x_batch, _ in test_dataloader:
+            tmp_cpds_predictor.predict(x_batch)

tests/classification/predictor/test_split.py

@@ -166,3 +166,23 @@ def test_evaluate(predictor, mock_score_function, mock_model, mock_dataset, q_ha
     assert len(results) == 2
     assert results['coverage_rate'] == metrics('coverage_rate')(excepted_sets, mock_dataset.labels)
     assert results['average_size'] == metrics('average_size')(excepted_sets)
+
+
+def test_predict_p(predictor, mock_score_function, mock_model, mock_dataset):
+    with pytest.raises(ValueError):
+        tmp_predictor = SplitPredictor(mock_score_function)
+        tmp_predictor.predict_p(mock_dataset.x)
+
+    logits = mock_model(mock_dataset.x)
+    logits = predictor._logits_transformation(logits)
+
+    predictor.calculate_threshold(logits, mock_dataset.labels)
+    p_values = predictor.predict_p(mock_dataset.x)
+    
+    assert p_values.shape == (mock_dataset.x.shape[0], logits.shape[1])
+    
+    p_values = predictor.predict_p(mock_dataset.x, mock_dataset.labels)
+    assert p_values.shape == (mock_dataset.x.shape[0], )
+
+    p_values = predictor.predict_p(mock_dataset.x, mock_dataset.labels, True)
+    assert p_values.shape == (mock_dataset.x.shape[0], )