|
| 1 | +# Copyright (c) Microsoft Corporation. |
| 2 | +# Licensed under the MIT license. |
| 3 | + |
| 4 | +import logging |
| 5 | +import uuid |
| 6 | +from enum import Enum |
| 7 | +from typing import Literal, Optional |
| 8 | + |
| 9 | +from pyrit.datasets.seed_datasets.remote._image_cache import ( |
| 10 | + fetch_and_cache_image_async, |
| 11 | +) |
| 12 | +from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import ( |
| 13 | + _RemoteDatasetLoader, |
| 14 | +) |
| 15 | +from pyrit.models import Seed, SeedDataset, SeedObjective, SeedPrompt |
| 16 | + |
| 17 | +logger = logging.getLogger(__name__) |
| 18 | + |
| 19 | + |
| 20 | +class SIUOCategory(Enum): |
| 21 | + """Categories in the SIUO dataset, using the exact strings from the source JSON.""" |
| 22 | + |
| 23 | + SELF_HARM = "self-harm" |
| 24 | + ILLEGAL_ACTIVITIES_CRIME = "illegal activities & crime" |
| 25 | + PRIVACY_VIOLATION = "privacy violation" |
| 26 | + MORALITY = "morality" |
| 27 | + DANGEROUS_BEHAVIOR = "dangerous behavior" |
| 28 | + DISCRIMINATION_STEREOTYPING = "discrimination & stereotyping" |
| 29 | + INFORMATION_MISINTERPRETATION = "information misinterpretation" |
| 30 | + RELIGION_BELIEFS = "religion beliefs" |
| 31 | + CONTROVERSIAL_TOPICS_POLITICS = "controversial topics, politics" |
| 32 | + |
| 33 | + |
| 34 | +_AUTHORS = [ |
| 35 | + "Siyin Wang", |
| 36 | + "Xingsong Ye", |
| 37 | + "Qinyuan Cheng", |
| 38 | + "Junwen Duan", |
| 39 | + "Shimin Li", |
| 40 | + "Jinlan Fu", |
| 41 | + "Xipeng Qiu", |
| 42 | + "Xuanjing Huang", |
| 43 | +] |
| 44 | + |
| 45 | +_GROUPS = [ |
| 46 | + "Fudan University", |
| 47 | + "National University of Singapore", |
| 48 | + "Shanghai AI Laboratory", |
| 49 | +] |
| 50 | + |
| 51 | +_DESCRIPTION = ( |
| 52 | + "A multimodal example from the SIUO (Safe Inputs but Unsafe Output) benchmark. " |
| 53 | + "Each example pairs an individually-safe image with an individually-safe text " |
| 54 | + "question; the harm only emerges when the two are combined." |
| 55 | +) |
| 56 | + |
| 57 | + |
| 58 | +class _SIUODataset(_RemoteDatasetLoader): |
| 59 | + """ |
| 60 | + Loader for the SIUO (Safe Inputs but Unsafe Output) multimodal benchmark. |
| 61 | +
|
| 62 | + SIUO probes cross-modality safety alignment in vision-language models. Each |
| 63 | + example consists of an image and a text question that are individually safe, |
| 64 | + but whose combination implies an unsafe scenario across 9 critical safety |
| 65 | + domains (self-harm, illegal activities & crime, privacy violation, morality, |
| 66 | + dangerous behavior, discrimination & stereotyping, information |
| 67 | + misinterpretation, religion beliefs, controversial topics & politics). |
| 68 | +
|
| 69 | + Each example is returned as a 3-piece group sharing a prompt_group_id: |
| 70 | + a SeedObjective carrying the text question, plus a text SeedPrompt and an |
| 71 | + image SeedPrompt (both at sequence=0) that together form a single multimodal |
| 72 | + user message delivered to the target. The dataset's safety_warning field is |
| 73 | + preserved in each prompt's metadata so downstream scorers can use it as the |
| 74 | + gold rationale. |
| 75 | +
|
| 76 | + Images are fetched from the HuggingFace mirror (sinwang/SIUO), which bundles |
| 77 | + them under images/. The GitHub repo points users at a Google Drive ZIP, but |
| 78 | + the HF mirror avoids that and lets us fetch via plain HTTPS. |
| 79 | +
|
| 80 | + Note: The first call may be slow as images are downloaded from the |
| 81 | + HuggingFace dataset. Subsequent calls reuse the local image cache. |
| 82 | +
|
| 83 | + Reference: [@wang2025siuo] |
| 84 | + Paper: https://arxiv.org/abs/2406.15279 |
| 85 | + """ |
| 86 | + |
| 87 | + HF_COMMIT_SHA: str = "024e80a01795376b9fed12f8073a12f2275f22ee" |
| 88 | + GEN_JSON_URL: str = f"https://huggingface.co/datasets/sinwang/SIUO/resolve/{HF_COMMIT_SHA}/siuo_gen.json" |
| 89 | + IMAGE_BASE_URL: str = f"https://huggingface.co/datasets/sinwang/SIUO/resolve/{HF_COMMIT_SHA}/images/" |
| 90 | + PAPER_URL: str = "https://arxiv.org/abs/2406.15279" |
| 91 | + |
| 92 | + HF_DATASET_NAME: str = "sinwang/SIUO" |
| 93 | + harm_categories: tuple[str, ...] = ( |
| 94 | + "self-harm", |
| 95 | + "illegal", |
| 96 | + "privacy", |
| 97 | + "morality", |
| 98 | + "dangerous behavior", |
| 99 | + "discrimination", |
| 100 | + "misinformation", |
| 101 | + "religion", |
| 102 | + "controversial topics", |
| 103 | + ) |
| 104 | + modalities: tuple[str, ...] = ("image", "text") |
| 105 | + size: str = "medium" |
| 106 | + tags: frozenset[str] = frozenset({"default", "safety", "multimodal"}) |
| 107 | + |
| 108 | + def __init__( |
| 109 | + self, |
| 110 | + *, |
| 111 | + source: str = GEN_JSON_URL, |
| 112 | + source_type: Literal["public_url", "file"] = "public_url", |
| 113 | + categories: Optional[list[SIUOCategory]] = None, |
| 114 | + ) -> None: |
| 115 | + """ |
| 116 | + Initialize the SIUO dataset loader. |
| 117 | +
|
| 118 | + Args: |
| 119 | + source (str): URL or file path to siuo_gen.json. Defaults to the |
| 120 | + HuggingFace mirror pinned to a commit SHA for reproducibility. |
| 121 | + source_type (Literal["public_url", "file"]): Whether source is a |
| 122 | + public URL or a local file path. Defaults to 'public_url'. |
| 123 | + categories (Optional[list[SIUOCategory]]): Optional filter; only rows |
| 124 | + whose category matches one of these enum values are included. |
| 125 | + If None, every category is included. |
| 126 | +
|
| 127 | + Raises: |
| 128 | + ValueError: If categories contains a non-SIUOCategory value. |
| 129 | + """ |
| 130 | + self.source = source |
| 131 | + self.source_type: Literal["public_url", "file"] = source_type |
| 132 | + self.categories = categories |
| 133 | + |
| 134 | + if categories is not None: |
| 135 | + self._validate_enums(categories, SIUOCategory, "SIUO category") |
| 136 | + |
| 137 | + @property |
| 138 | + def dataset_name(self) -> str: |
| 139 | + """Return the dataset name.""" |
| 140 | + return "siuo" |
| 141 | + |
| 142 | + async def fetch_dataset_async(self, *, cache: bool = True) -> SeedDataset: |
| 143 | + """ |
| 144 | + Fetch the SIUO dataset and return it as a SeedDataset. |
| 145 | +
|
| 146 | + For each kept row, produces three seeds that share a prompt_group_id: |
| 147 | + a SeedObjective whose value is the row's question, a text SeedPrompt |
| 148 | + carrying the same question at sequence=0, and an image SeedPrompt at |
| 149 | + sequence=0 whose value is the local path of the cached image. The image |
| 150 | + and text prompts together form a single multimodal user message. |
| 151 | +
|
| 152 | + Args: |
| 153 | + cache (bool): Whether to cache the fetched JSON and image files. |
| 154 | + Defaults to True. |
| 155 | +
|
| 156 | + Returns: |
| 157 | + SeedDataset: A SeedDataset containing the multimodal seed groups. |
| 158 | +
|
| 159 | + Raises: |
| 160 | + ValueError: If a row is missing a required key, or if no row passes |
| 161 | + the configured filters. |
| 162 | + """ |
| 163 | + logger.info(f"Loading SIUO dataset from {self.source}") |
| 164 | + |
| 165 | + required_keys = {"question_id", "image", "question", "category", "safety_warning"} |
| 166 | + examples = self._fetch_from_url( |
| 167 | + source=self.source, |
| 168 | + source_type=self.source_type, |
| 169 | + cache=cache, |
| 170 | + ) |
| 171 | + |
| 172 | + seeds: list[Seed] = [] |
| 173 | + failed_image_count = 0 |
| 174 | + kept_categories = {cat.value for cat in self.categories} if self.categories is not None else None |
| 175 | + |
| 176 | + for example in examples: |
| 177 | + missing_keys = required_keys - example.keys() |
| 178 | + if missing_keys: |
| 179 | + raise ValueError(f"Missing keys in example: {', '.join(sorted(missing_keys))}") |
| 180 | + |
| 181 | + category = example["category"] |
| 182 | + if kept_categories is not None and category not in kept_categories: |
| 183 | + continue |
| 184 | + |
| 185 | + try: |
| 186 | + group = await self._build_seed_group_async(example=example) |
| 187 | + except Exception as e: |
| 188 | + failed_image_count += 1 |
| 189 | + logger.warning( |
| 190 | + f"[SIUO] Failed to fetch image for question_id " |
| 191 | + f"{example.get('question_id')}: {e}. Skipping this example." |
| 192 | + ) |
| 193 | + continue |
| 194 | + |
| 195 | + seeds.extend(group) |
| 196 | + |
| 197 | + if failed_image_count > 0: |
| 198 | + logger.warning(f"[SIUO] Skipped {failed_image_count} example(s) due to image fetch failures") |
| 199 | + |
| 200 | + if not seeds: |
| 201 | + raise ValueError("SeedDataset cannot be empty. Check your filter criteria.") |
| 202 | + |
| 203 | + logger.info(f"Successfully loaded {len(seeds)} seeds from SIUO dataset") |
| 204 | + return SeedDataset(seeds=seeds, dataset_name=self.dataset_name) |
| 205 | + |
| 206 | + async def _build_seed_group_async(self, *, example: dict[str, str]) -> list[Seed]: |
| 207 | + """ |
| 208 | + Build a 3-piece (objective + text + image) seed group for a single SIUO row. |
| 209 | +
|
| 210 | + Args: |
| 211 | + example (dict[str, str]): A single example row from siuo_gen.json. |
| 212 | +
|
| 213 | + Returns: |
| 214 | + list[Seed]: [SeedObjective, text SeedPrompt, image SeedPrompt], |
| 215 | + all sharing a freshly generated prompt_group_id. The text and |
| 216 | + image prompts both use sequence=0 so they are delivered to the |
| 217 | + target as a single multimodal user message. |
| 218 | +
|
| 219 | + Raises: |
| 220 | + Exception: Propagates any failure raised while fetching the image |
| 221 | + so the caller can skip the row. |
| 222 | + """ |
| 223 | + question_id = example["question_id"] |
| 224 | + question = example["question"] |
| 225 | + category = example["category"] |
| 226 | + image_filename = example["image"] |
| 227 | + safety_warning = example["safety_warning"] |
| 228 | + |
| 229 | + image_url = f"{self.IMAGE_BASE_URL}{image_filename}" |
| 230 | + image_stem = image_filename.rsplit(".", 1)[0] |
| 231 | + local_image_path = await self._fetch_and_save_image_async( |
| 232 | + image_url=image_url, |
| 233 | + image_stem=image_stem, |
| 234 | + ) |
| 235 | + |
| 236 | + group_id = uuid.uuid4() |
| 237 | + metadata: dict[str, str | int] = { |
| 238 | + "question_id": question_id, |
| 239 | + "category": category, |
| 240 | + "safety_warning": safety_warning, |
| 241 | + } |
| 242 | + |
| 243 | + objective = SeedObjective( |
| 244 | + value=question, |
| 245 | + name=f"SIUO Objective - {question_id}", |
| 246 | + dataset_name=self.dataset_name, |
| 247 | + harm_categories=[category], |
| 248 | + description=_DESCRIPTION, |
| 249 | + authors=_AUTHORS, |
| 250 | + groups=_GROUPS, |
| 251 | + source=self.PAPER_URL, |
| 252 | + prompt_group_id=group_id, |
| 253 | + ) |
| 254 | + |
| 255 | + text_prompt = SeedPrompt( |
| 256 | + value=question, |
| 257 | + data_type="text", |
| 258 | + name=f"SIUO Text - {question_id}", |
| 259 | + dataset_name=self.dataset_name, |
| 260 | + harm_categories=[category], |
| 261 | + description=_DESCRIPTION, |
| 262 | + authors=_AUTHORS, |
| 263 | + groups=_GROUPS, |
| 264 | + source=self.PAPER_URL, |
| 265 | + prompt_group_id=group_id, |
| 266 | + sequence=0, |
| 267 | + metadata=metadata, |
| 268 | + ) |
| 269 | + |
| 270 | + image_prompt = SeedPrompt( |
| 271 | + value=local_image_path, |
| 272 | + data_type="image_path", |
| 273 | + name=f"SIUO Image - {question_id}", |
| 274 | + dataset_name=self.dataset_name, |
| 275 | + harm_categories=[category], |
| 276 | + description=_DESCRIPTION, |
| 277 | + authors=_AUTHORS, |
| 278 | + groups=_GROUPS, |
| 279 | + source=self.PAPER_URL, |
| 280 | + prompt_group_id=group_id, |
| 281 | + sequence=0, |
| 282 | + metadata={**metadata, "original_image_url": image_url}, |
| 283 | + ) |
| 284 | + |
| 285 | + return [objective, text_prompt, image_prompt] |
| 286 | + |
| 287 | + async def _fetch_and_save_image_async(self, *, image_url: str, image_stem: str) -> str: |
| 288 | + """ |
| 289 | + Fetch and cache a SIUO image. |
| 290 | +
|
| 291 | + Args: |
| 292 | + image_url (str): Full URL of the image on the HuggingFace mirror. |
| 293 | + image_stem (str): Filename stem (e.g. 'S-01') used to name the |
| 294 | + cached file. |
| 295 | +
|
| 296 | + Returns: |
| 297 | + str: Local path to the cached image. |
| 298 | +
|
| 299 | + Raises: |
| 300 | + Exception: Any error raised by the underlying HTTP fetch is |
| 301 | + propagated so the caller can skip the row. |
| 302 | + """ |
| 303 | + return await fetch_and_cache_image_async( |
| 304 | + filename=f"siuo_{image_stem}.png", |
| 305 | + image_url=image_url, |
| 306 | + log_prefix="SIUO", |
| 307 | + follow_redirects=True, |
| 308 | + ) |
0 commit comments