Merge master to dev

.gitignore

@@ -25,4 +25,6 @@ docs/buildA
 **/__pycache__/
 .vscode
 .coverage
-torch_scae.egg-info
+torch_scae.egg-info
+*.so
+*.egg-info

docs/source/CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+1.0.4 (2025-07-10)  
+------------------  
+* Refactored `__init__` of predictors to include `alpha` and `device` parameters for greater flexibility.  
+* Added EntmaxScore, SCPO and RC3P methods for classification tasks.  
+* Added the normalized residual score in regression task.
+* Added p-value computation and efficiency metrics from the paper "Criteria of Efficiency for Conformal Prediction".  
+
+
+
 1.0.2 (2025-02-17)
 ------------------
 * Refactored examples codebase for better organization and clarity
@@ -48,5 +57,5 @@ Changelog
 
 0.1.0 (2023-12-23)
 ------------------
-* Introduced CP algorithms for classification, including ConfTr, THR, APS, RAPS, SAPS, Classwise CP, Clustered CP and Weighted CP.
+* Introduced CP algorithms for classification, including ConfTr, LAC, APS, RAPS, SAPS, ClassConditional CP, Clustered CP and Weighted CP.
 * Introduced CP algorithms for regression, including ACI, ABS and CQR.

docs/source/installation.rst

@@ -1,7 +1,8 @@
 Installation
 =====================
+
 Latest version (|version|)
----------------------
+--------------------------
 
 Installing TorchCP itself
 

docs/source/torchcp.classification.rst

@@ -12,16 +12,17 @@ score function
 ..  autosummary::
     :nosignatures:
 
-    THR
+    LAC
     APS
     RAPS
     SAPS
     Margin
     TOPK
     KNN
+    EntmaxScore
 
 
-.. autoclass:: THR
+.. autoclass:: LAC
    :members:
 
 .. autoclass:: APS
@@ -42,6 +43,10 @@ score function
 .. autoclass:: KNN
    :members:
 
+.. autoclass:: EntmaxScore
+   :members:
+
+
 .. automodule:: torchcp.classification.predictor
 
 predictor
@@ -51,23 +56,28 @@ predictor
     :nosignatures:
 
     SplitPredictor
-    ClassWisePredictor
+    ClassConditionalPredictor
     ClusteredPredictor
+    RC3PPredictor
     WeightedPredictor
 
 .. autoclass:: SplitPredictor
    :members:
 
-.. autoclass:: ClassWisePredictor
+.. autoclass:: ClassConditionalPredictor
    :members:
 
 .. autoclass:: ClusteredPredictor
    :members:
 
+.. autoclass:: RC3PPredictor
+   :members:
+
 .. autoclass:: WeightedPredictor
    :members:
 
 
+
 .. automodule:: torchcp.classification.loss
 
 loss function
@@ -106,6 +116,8 @@ trainer
     TSTrainer
     OrdinalTrainer
     UncertaintyAwareTrainer
+    SCPOTrainer
+
 
 .. autoclass:: BaseTrainer
    :members:
@@ -122,6 +134,10 @@ trainer
 .. autoclass:: UncertaintyAwareTrainer
    :members:
 
+.. autoclass:: SCPOTrainer
+   :members:
+
+
 .. automodule:: torchcp.classification.utils.metrics
 
 metrics
@@ -146,6 +162,20 @@ metrics
 .. autofunction:: SSCV
 .. autofunction:: WSC
 .. autofunction:: singleton_hit_ratio
+.. autofunction:: compute_p_values
+.. autofunction:: pvalue_criterion_S
+.. autofunction:: pvalue_criterion_N
+.. autofunction:: pvalue_criterion_U
+.. autofunction:: pvalue_criterion_F
+.. autofunction:: pvalue_criterion_M
+.. autofunction:: pvalue_criterion_E
+.. autofunction:: pvalue_criterion_OU
+.. autofunction:: pvalue_criterion_OF
+.. autofunction:: pvalue_criterion_OM
+.. autofunction:: pvalue_criterion_OE
+
+
+
 
 .. automodule:: torchcp.classification.utils
 

examples/classification_conftr_cifar100.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2023-present, SUSTech-ML.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import torch
+import torchvision
+import torchvision.transforms as trn
+from transformers import set_seed
+
+from examples.utils import get_dataset_dir
+from torchcp.classification.predictor import SplitPredictor
+from torchcp.classification.score import LAC
+from torchcp.classification.trainer import ConfTrTrainer
+
+set_seed(seed=2025)
+
+#######################################
+# Preparing a calibration data and a test data
+#######################################
+transform = trn.Compose([
+    trn.ToTensor(),
+    trn.Normalize(mean=[0.5071, 0.4867, 0.4408],
+                  std=[0.2675, 0.2565, 0.2761])
+])
+dataset = torchvision.datasets.CIFAR100(
+    root=get_dataset_dir(),
+    train=False,
+    download=True,
+    transform=transform
+)
+train_dataset, cal_dataset, test_dataset = torch.utils.data.random_split(dataset, [3000, 2000, 5000])
+train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4, pin_memory=True)
+cal_dataloader = torch.utils.data.DataLoader(cal_dataset, batch_size=64, shuffle=False, num_workers=4, pin_memory=True)
+test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=4,
+                                              pin_memory=True)
+test_instance, test_label = test_dataset[0]
+test_instance = test_instance.unsqueeze(0)
+
+#######################################
+# Preparing a pytorch model
+#######################################
+init_model = torch.hub.load("chenyaofo/pytorch-cifar-models", "cifar100_resnet20", pretrained=True)
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+init_model.to(device)
+init_model.eval()
+
+#######################################
+# Conformal Training
+#######################################
+
+trainer = ConfTrTrainer(
+    model=init_model,
+    alpha=0.1,
+    device=device
+)
+
+trained_model = trainer.train(train_dataloader, num_epochs=10)
+
+########################################
+# Conformal prediction
+########################################
+
+predictor = SplitPredictor(score_function=LAC(), model=trained_model, alpha=0.1)
+predictor.calibrate(cal_dataloader)
+predict_set = predictor.predict(test_instance)
+

examples/classification_confts_cifar100.py

@@ -52,9 +52,9 @@
 init_temperature = 1.0
 
 trainer = ConfTSTrainer(
+    model=model,
     init_temperature=init_temperature,
     alpha=0.1,
-    model=model,
     device=device,
     verbose=True
 )

examples/classification_splitcp_cifar100.py

@@ -12,7 +12,8 @@
 
 from examples.utils import get_dataset_dir
 from torchcp.classification.predictor import SplitPredictor
-from torchcp.classification.score import THR
+from torchcp.classification.predictor import RC3PPredictor, ClassConditionalPredictor
+from torchcp.classification.score import LAC
 
 set_seed(seed=0)
 
@@ -47,7 +48,8 @@
 # A standard process of conformal prediction
 #######################################
 alpha = 0.1  # Significance level
-predictor = SplitPredictor(score_function=THR(), model=model)
+predictor = SplitPredictor(score_function=LAC(), model=model)
+# predictor = ClassConditionalPredictor(score_function=LAC(), model=model)
 predictor.calibrate(cal_dataloader, alpha=0.1)
 
 test_instances, test_labels = test_dataset[0]

examples/classification_splitcp_cifar100_online.py

@@ -0,0 +1,87 @@
+import torch
+import torchvision
+import torchvision.transforms as trn
+from transformers import set_seed
+from tqdm import tqdm
+
+from examples.utils import get_dataset_dir
+from torchcp.classification.predictor import SplitPredictor
+from torchcp.classification.score import LAC
+
+set_seed(seed=0)
+
+#######################################
+# Dataset preparation
+#######################################
+transform = trn.Compose([
+    trn.ToTensor(),
+    trn.Normalize(mean=[0.5071, 0.4867, 0.4408],
+                  std=[0.2675, 0.2565, 0.2761])
+])
+
+dataset = torchvision.datasets.CIFAR100(
+    root=get_dataset_dir(),
+    train=False,
+    download=True,
+    transform=transform
+)
+cal_dataset, test_dataset = torch.utils.data.random_split(dataset, [5000, 5000])  
+
+#######################################
+# Model preparation
+#######################################
+model = torch.hub.load("chenyaofo/pytorch-cifar-models", "cifar100_resnet20", pretrained=True)
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device)
+model.eval()
+
+cal_logits, cal_labels = [], []
+for x, y in cal_dataset:
+    cal_logits.append(model(x.unsqueeze(0).to(device)))
+    cal_labels.append(y)
+
+cal_logits = torch.cat(cal_logits, dim=0)
+cal_labels = torch.tensor(cal_labels)
+
+
+test_logits, test_labels = [], []
+for x, y in test_dataset:
+    test_logits.append(model(x.unsqueeze(0).to(device)))
+    test_labels.append(y)
+
+test_logits = torch.cat(test_logits, dim=0)
+test_labels = torch.tensor(test_labels)
+
+#######################################
+# Online Conformal Prediction
+#######################################
+alpha = 0.1
+
+predictor = SplitPredictor(score_function=LAC(), model=model, device=device)
+
+cover_count = 0
+total = 0
+set_size_sum = 0
+
+for i in tqdm(range(len(test_dataset))):
+    predictor.calculate_threshold(cal_logits, cal_labels, alpha)
+
+    x, y = test_dataset[i]
+    logits = model(x.unsqueeze(0).to(device))
+
+
+    pred_set = predictor.predict_with_logits(logits)[0] 
+    covered = 1 if pred_set[y] else 0
+
+    cover_count += int(covered)
+    set_size_sum += pred_set.sum()
+    total += 1
+
+    cal_logits = torch.cat((cal_logits, logits), dim=0)
+    cal_labels = torch.cat((cal_labels, torch.tensor([y])), dim=0)
+
+coverage_rate = cover_count / total
+average_set_size = set_size_sum / total
+
+print(f"Online CP Coverage Rate: {coverage_rate:.4f}")
+print(f"Online CP Average Set Size: {average_set_size:.4f}")

examples/gnn_trainer_coraml.py

@@ -107,7 +107,7 @@ def train(model, optimizer, graph_data, train_idx):
     graph_data['train_idx'] = train_idx
     graph_data['val_idx'] = test_idx
     graph_data['calib_train_idx'] = calib_train_idx
-    cf_trainer = CFGNNTrainer(graph_data, model)
+    cf_trainer = CFGNNTrainer(model, graph_data)
 
     # Train conformalized gnn
     model = cf_trainer.train()

examples/llm_ConformalLM_TriviaQA.py

@@ -9,7 +9,7 @@
 import os
 
 from examples.utils import get_others_dir
-from llm_utils import *
+from examples.llm_utils import *
 from torchcp.llm.predictor import ConformalLM
 
 if __name__ == "__main__":