Merge remote-tracking branch 'origin/main' into riedgar-ms/selfask-jsonschema-01

Author: riedgar-ms

pyrit/score/audio_transcript_scorer.py

@@ -5,7 +5,6 @@
 import os
 import tempfile
 import uuid
-from abc import ABC
 from typing import Optional
 
 import av
@@ -88,7 +87,7 @@ def _audio_to_wav(input_path: str, *, sample_rate: int, channels: int) -> str:
     return output_path
 
 
-class AudioTranscriptHelper(ABC):  # noqa: B024
+class AudioTranscriptHelper:  # noqa: B024
     """
     Abstract base class for audio scorers that process audio by transcribing and scoring the text.
 

pyrit/score/float_scale/video_float_scale_scorer.py

@@ -11,15 +11,14 @@
 )
 from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
-from pyrit.score.video_scorer import _BaseVideoScorer
+from pyrit.score.video_scorer import VideoHelper
 
 if TYPE_CHECKING:
     from pyrit.score.score_aggregator_result import ScoreAggregatorResult
 
 
 class VideoFloatScaleScorer(
     FloatScaleScorer,
-    _BaseVideoScorer,
 ):
     """
     A scorer that processes videos by extracting frames and scoring them using a float scale image scorer.
@@ -48,7 +47,7 @@ def __init__(
         num_sampled_frames: Optional[int] = None,
         validator: Optional[ScorerPromptValidator] = None,
         score_aggregator: FloatScaleAggregatorFunc = FloatScaleScorerByCategory.MAX,
-        image_objective_template: Optional[str] = _BaseVideoScorer._DEFAULT_IMAGE_OBJECTIVE_TEMPLATE,
+        image_objective_template: Optional[str] = VideoHelper._DEFAULT_IMAGE_OBJECTIVE_TEMPLATE,
         audio_objective_template: Optional[str] = None,
     ) -> None:
         """
@@ -82,8 +81,7 @@ def __init__(
         """
         FloatScaleScorer.__init__(self, validator=validator or self._DEFAULT_VALIDATOR)
 
-        _BaseVideoScorer.__init__(
-            self,
+        self._video_helper = VideoHelper(
             image_capable_scorer=image_capable_scorer,
             num_sampled_frames=num_sampled_frames,
             image_objective_template=image_objective_template,
@@ -92,7 +90,7 @@ def __init__(
         self._score_aggregator = score_aggregator
 
         if audio_scorer is not None:
-            self._validate_audio_scorer(audio_scorer)
+            VideoHelper._validate_audio_scorer(audio_scorer)
         self.audio_scorer = audio_scorer
 
     def _build_identifier(self) -> ComponentIdentifier:
@@ -102,17 +100,17 @@ def _build_identifier(self) -> ComponentIdentifier:
         Returns:
             ComponentIdentifier: The identifier for this scorer.
         """
-        sub_scorer_ids = [self.image_scorer.get_identifier()]
+        sub_scorer_ids = [self._video_helper.image_scorer.get_identifier()]
         if self.audio_scorer:
             sub_scorer_ids.append(self.audio_scorer.get_identifier())
 
         return self._create_identifier(
             params={
                 "score_aggregator": self._score_aggregator.__name__,
-                "num_sampled_frames": self.num_sampled_frames,
+                "num_sampled_frames": self._video_helper.num_sampled_frames,
                 "has_audio_scorer": self.audio_scorer is not None,
-                "image_objective_template": self.image_objective_template,
-                "audio_objective_template": self.audio_objective_template,
+                "image_objective_template": self._video_helper.image_objective_template,
+                "audio_objective_template": self._video_helper.audio_objective_template,
             },
             children={
                 "sub_scorers": sub_scorer_ids,
@@ -131,14 +129,14 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Op
             List of aggregated scores for the video. Returns one score if using FloatScaleScoreAggregator,
             or multiple scores (one per category) if using FloatScaleScorerByCategory.
         """
-        frame_scores = await self._score_frames_async(message_piece=message_piece, objective=objective)
+        frame_scores = await self._video_helper._score_frames_async(message_piece=message_piece, objective=objective)
 
         all_scores = list(frame_scores)
         audio_scored = False
 
         # Score audio if audio_scorer is provided
         if self.audio_scorer:
-            audio_scores = await self._score_video_audio_async(
+            audio_scores = await self._video_helper._score_video_audio_async(
                 message_piece=message_piece, audio_scorer=self.audio_scorer, objective=objective
             )
             if audio_scores:

pyrit/score/true_false/video_true_false_scorer.py

@@ -8,10 +8,10 @@
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
 from pyrit.score.true_false.true_false_score_aggregator import TrueFalseScoreAggregator
 from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
-from pyrit.score.video_scorer import _BaseVideoScorer
+from pyrit.score.video_scorer import VideoHelper
 
 
-class VideoTrueFalseScorer(TrueFalseScorer, _BaseVideoScorer):
+class VideoTrueFalseScorer(TrueFalseScorer):
     """
     A scorer that processes videos by extracting frames and scoring them using a true/false image scorer.
 
@@ -34,7 +34,7 @@ def __init__(
         audio_scorer: Optional[TrueFalseScorer] = None,
         num_sampled_frames: Optional[int] = None,
         validator: Optional[ScorerPromptValidator] = None,
-        image_objective_template: Optional[str] = _BaseVideoScorer._DEFAULT_IMAGE_OBJECTIVE_TEMPLATE,
+        image_objective_template: Optional[str] = VideoHelper._DEFAULT_IMAGE_OBJECTIVE_TEMPLATE,
         audio_objective_template: Optional[str] = None,
     ) -> None:
         """
@@ -59,18 +59,17 @@ def __init__(
         Raises:
             ValueError: If audio_scorer is provided and does not support audio_path data type.
         """
-        _BaseVideoScorer.__init__(
-            self,
+        super().__init__(validator=validator or self._DEFAULT_VALIDATOR)
+
+        self._video_helper = VideoHelper(
             image_capable_scorer=image_capable_scorer,
             num_sampled_frames=num_sampled_frames,
             image_objective_template=image_objective_template,
             audio_objective_template=audio_objective_template,
         )
 
-        TrueFalseScorer.__init__(self, validator=validator or self._DEFAULT_VALIDATOR)
-
         if audio_scorer is not None:
-            self._validate_audio_scorer(audio_scorer)
+            VideoHelper._validate_audio_scorer(audio_scorer)
         self.audio_scorer = audio_scorer
 
     def _build_identifier(self) -> ComponentIdentifier:
@@ -80,16 +79,16 @@ def _build_identifier(self) -> ComponentIdentifier:
         Returns:
             ComponentIdentifier: The identifier for this scorer.
         """
-        sub_scorer_ids = [self.image_scorer.get_identifier()]
+        sub_scorer_ids = [self._video_helper.image_scorer.get_identifier()]
         if self.audio_scorer:
             sub_scorer_ids.append(self.audio_scorer.get_identifier())
 
         return self._create_identifier(
             params={
-                "num_sampled_frames": self.num_sampled_frames,
+                "num_sampled_frames": self._video_helper.num_sampled_frames,
                 "has_audio_scorer": self.audio_scorer is not None,
-                "image_objective_template": self.image_objective_template,
-                "audio_objective_template": self.audio_objective_template,
+                "image_objective_template": self._video_helper.image_objective_template,
+                "audio_objective_template": self._video_helper.audio_objective_template,
             },
             children={
                 "sub_scorers": sub_scorer_ids,
@@ -114,7 +113,7 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Op
         piece_id = message_piece.id if message_piece.id is not None else message_piece.original_prompt_id
 
         # Get scores for all frames and aggregate with OR (True if ANY frame matches)
-        frame_scores = await self._score_frames_async(message_piece=message_piece, objective=objective)
+        frame_scores = await self._video_helper._score_frames_async(message_piece=message_piece, objective=objective)
         frame_result = TrueFalseScoreAggregator.OR(frame_scores)
 
         # Create a Score from the frame aggregation result
@@ -132,7 +131,7 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Op
 
         # Score audio if audio_scorer is provided
         if self.audio_scorer:
-            audio_scores = await self._score_video_audio_async(
+            audio_scores = await self._video_helper._score_video_audio_async(
                 message_piece=message_piece, audio_scorer=self.audio_scorer, objective=objective
             )
             if audio_scores:

pyrit/score/video_scorer.py

@@ -6,7 +6,6 @@
 import random
 import tempfile
 import uuid
-from abc import ABC
 from typing import Optional
 
 from pyrit.memory import CentralMemory
@@ -17,13 +16,13 @@
 logger = logging.getLogger(__name__)
 
 
-class _BaseVideoScorer(ABC):  # noqa: B024
+class VideoHelper:
     """
-    Abstract base class for video scorers that process videos by extracting frames and scoring them.
+    Helper class for video scorers that process videos by extracting frames and scoring them.
 
     This class provides common functionality for extracting frames from videos and delegating
-    scoring to an image-capable scorer. Concrete implementations handle aggregation logic
-    specific to their scoring type (true/false or float scale).
+    scoring to an image-capable scorer. Used via composition by VideoTrueFalseScorer and
+    VideoFloatScaleScorer.
     """
 
     _DEFAULT_VIDEO_FRAMES_SAMPLING_NUM = 5

tests/integration/converter/test_notebooks_converter.py

@@ -13,6 +13,7 @@
 nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "converters").resolve()
 
 skipped_files = [
+    "2_audio_converters.ipynb",  # requires Azure Speech API key
     "7_human_converter.ipynb",  # requires human input
 ]
 

tests/integration/datasets/test_notebooks_datasets.py

@@ -12,10 +12,14 @@
 
 nb_directory_path = pathlib.Path(path.DOCS_CODE_PATH, "datasets").resolve()
 
+skipped_files = [
+    "2_seed_programming.ipynb",  # requires OpenAI API credentials
+]
+
 
 @pytest.mark.parametrize(
     "file_name",
-    [file for file in os.listdir(nb_directory_path) if file.endswith(".ipynb")],
+    [file for file in os.listdir(nb_directory_path) if file.endswith(".ipynb") and file not in skipped_files],
 )
 def test_execute_notebooks(file_name):
     nb_path = pathlib.Path(nb_directory_path, file_name).resolve()