rebase

Author: alfekka

examples/configs/estimators/default_estimators.yaml

@@ -100,18 +100,3 @@
 - name: CocoaPPL
 - name: CocoaMTE
 - name: SemanticDensity
-- name: SAPLMA
-  cfg:
-    model_name: '${model.path}'
-- name: MIND
-- name: Sheeps
-  cfg:
-    model_name: '${model.path}'
-- name: SATRMD
-  cfg:
-    model_name: '${model.path}'
-- name: SATRMD
-  cfg:
-    model_name: '${model.path}'
-    base_method: "TokenMahalanobis"
-- name: LookBackLens

examples/configs/estimators/default_estimators_with_supervised.yaml

@@ -86,8 +86,20 @@
     spacy_path: "en_core_web_sm"
 - name: AttentionScore
   cfg:
-    layer: 16
     gen_only: False
+- name: RAUQ
+  cfg:
+    alpha: 0.2
+    use_entropy: False
+- name: RAUQ
+  cfg:
+    alpha: 0.8
+    use_entropy: True
+- name: CSL
+- name: CocoaMSP
+- name: CocoaPPL
+- name: CocoaMTE
+- name: SemanticDensity
 - name: SAPLMA
   cfg:
     model_name: '${model.path}'

src/lm_polygraph/estimators/eigenscore.py

@@ -1,4 +1,5 @@
 import numpy as np
+from scipy.linalg import eigh
 
 from typing import Dict
 
@@ -44,7 +45,7 @@ def __call__(self, stats: Dict[str, np.ndarray]) -> np.ndarray:
                 self.J_d = np.eye(dim) - 1 / dim * np.ones((dim, dim))
             covariance = sentence_embeddings @ self.J_d @ sentence_embeddings.T
             reg_covariance = covariance + self.alpha * np.eye(covariance.shape[0])
-            eigenvalues, _ = np.linalg.eig(reg_covariance)
+            eigenvalues = eigh(reg_covariance, eigvals_only=True)
             ue.append(
                 np.mean(np.log([val if val > 0 else 1e-10 for val in eigenvalues]))
             )

src/lm_polygraph/estimators/rauq.py

@@ -40,11 +40,11 @@ def __init__(
 
         # Focus on middle third of layers which typically contain most relevant information
         if self.n_layers is not None:
-            self.layers = list(
+            self.attn_layers = list(
                 range(self.n_layers // 3, int(np.ceil(self.n_layers / 3 * 2) + 1))
             )
         else:
-            self.layers = None
+            self.attn_layers = None
 
     def __str__(self) -> str:
         """Returns a string representation of the estimator."""
@@ -108,7 +108,7 @@ def __call__(self, stats: Dict[str, np.ndarray]) -> np.ndarray:
         """
         if self.n_layers is None:
             self.n_layers = stats["model"].model.config.num_hidden_layers
-            self.layers = list(
+            self.attn_layers = list(
                 range(self.n_layers // 3, int(np.ceil(self.n_layers / 3 * 2) + 1))
             )
         if self.n_heads is None:
@@ -147,7 +147,7 @@ def __call__(self, stats: Dict[str, np.ndarray]) -> np.ndarray:
 
             # Calculate uncertainty scores for each layer
             layer_scores = []
-            for layer in self.layers:
+            for layer in self.attn_layers:
                 # Select most attentive head for current layer
                 head = attentions[idx][layer].mean(-1).argmax()
 

src/lm_polygraph/stat_calculators/attention.py

@@ -94,7 +94,7 @@ def __call__(
                             torch.float16
                         )  # numpy does not support bfloat16
                     attn_mask[:, j, :j] = stacked_attention.cpu().numpy()
-                attention_all.append(attn_mask.max(0))
+                attention_all.append(attn_mask)
         result_dict = {
             "attention_all": attention_all,
         }