From 425c7f33676e648c70ef6e91d604d3dc08b15336 Mon Sep 17 00:00:00 2001 From: Vishal Doshi Date: Fri, 31 Oct 2025 14:23:38 -0400 Subject: [PATCH 1/3] Lazily make annotator instances instead of on init. --- src/modelgauge/ensemble_annotator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/modelgauge/ensemble_annotator.py b/src/modelgauge/ensemble_annotator.py index 43874dc08..1b8fbcabf 100644 --- a/src/modelgauge/ensemble_annotator.py +++ b/src/modelgauge/ensemble_annotator.py @@ -15,11 +15,19 @@ class EnsembleAnnotator(Annotator): def __init__(self, uid, annotators: list[str], ensemble_strategy: str, secrets: RawSecrets | None = None): super().__init__(uid) - self.annotators = self._make_annotators(annotators, secrets) + self._annotator_uids: list[str] = annotators + self._secrets: RawSecrets | None = secrets + self._annotators: dict[str, Annotator] = {} if ensemble_strategy not in ENSEMBLE_STRATEGIES: raise ValueError(f"Ensemble strategy {ensemble_strategy} not recognized.") self.ensemble_strategy = ENSEMBLE_STRATEGIES[ensemble_strategy] + @property + def annotators(self) -> dict[str, Annotator]: + if not self._annotators: + self._annotators = self._make_annotators(self._annotator_uids, self._secrets) + return self._annotators + def _make_annotators(self, annotator_uids: list[str], secrets: RawSecrets | None) -> dict[str, Annotator]: if secrets is None: secrets = load_secrets_from_config() From 46d67efe166ff9298967b49e878b8f6e79d41bd1 Mon Sep 17 00:00:00 2001 From: Vishal Doshi Date: Fri, 31 Oct 2025 14:29:37 -0400 Subject: [PATCH 2/3] Add helper. --- src/modelgauge/ensemble_annotator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/modelgauge/ensemble_annotator.py b/src/modelgauge/ensemble_annotator.py index 1b8fbcabf..91756268b 100644 --- a/src/modelgauge/ensemble_annotator.py +++ b/src/modelgauge/ensemble_annotator.py @@ -15,7 +15,7 @@ class EnsembleAnnotator(Annotator): def __init__(self, uid, annotators: list[str], ensemble_strategy: str, secrets: RawSecrets | None = None): super().__init__(uid) - self._annotator_uids: list[str] = annotators + self.annotator_uids: list[str] = annotators self._secrets: RawSecrets | None = secrets self._annotators: dict[str, Annotator] = {} if ensemble_strategy not in ENSEMBLE_STRATEGIES: @@ -24,8 +24,8 @@ def __init__(self, uid, annotators: list[str], ensemble_strategy: str, secrets: @property def annotators(self) -> dict[str, Annotator]: - if not self._annotators: - self._annotators = self._make_annotators(self._annotator_uids, self._secrets) + if len(self._annotators) != len(self.annotator_uids): + self._annotators = self._make_annotators(self.annotator_uids, self._secrets) return self._annotators def _make_annotators(self, annotator_uids: list[str], secrets: RawSecrets | None) -> dict[str, Annotator]: @@ -61,3 +61,10 @@ def translate_response(self, request: dict[str, Any], response: dict[str, Any]): joined_responses=annotations, metadata=ensemble_annotation.metadata, # TODO: Merge metadata here instead of in strategy ) + + +def get_annotator_component_ids(annotator: Annotator) -> list[str]: + if isinstance(annotator, EnsembleAnnotator): + return annotator.annotator_uids + else: + return [annotator.uid] From 210972bf05cb5337c71aaf361c0ad9e9d61dd152 Mon Sep 17 00:00:00 2001 From: Vishal Doshi Date: Fri, 31 Oct 2025 16:36:40 -0400 Subject: [PATCH 3/3] Add a helper short-cut for Annotator. --- src/modelgauge/annotator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/modelgauge/annotator.py b/src/modelgauge/annotator.py index 7abc4cf84..5173baeb7 100644 --- a/src/modelgauge/annotator.py +++ b/src/modelgauge/annotator.py @@ -44,3 +44,9 @@ def annotate(self, annotation_request): def translate_response(self, request, response) -> SafetyAnnotation: """Convert the raw response into the standardized SafetyAnnotation.""" pass + + def process(self, prompt: TextPrompt | ChatPrompt, response: SUTResponse) -> SafetyAnnotation: + """End-to-end processing of a single prompt+response pair.""" + annotator_request = self.translate_prompt(prompt, response) + annotator_response = self.annotate(annotator_request) + return self.translate_response(annotator_request, annotator_response)