Move exhaustive dataset tests to e2e, keep smoke tests in integration

Replace TestSeedDatasetProviderIntegration (parameterized over all 58
providers) with TestSeedDatasetSmoke testing 3 representative providers:
- 1 local YAML provider (no network)
- 1 remote URL-based provider (_XSTestDataset)
- 1 remote HuggingFace provider (_SimpleSafetyTestsDataset)

Move the full parametric test to tests/end_to_end/test_all_datasets.py
for daily CI runs, avoiding flaky HF rate-limiting failures on every PR.

Add tests/integration/datasets/test_load_default_datasets_integration.py
to verify the LoadDefaultDatasets initializer pipeline with real data.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: rlundeen2

tests/end_to_end/test_all_datasets.py

@@ -0,0 +1,65 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+End-to-end tests that verify every registered dataset provider can be fetched.
+
+These tests download real data from HuggingFace and GitHub, are slow, and are
+subject to transient network failures.  They are intended to run daily in e2e CI,
+not on every PR.
+"""
+
+import logging
+
+import pytest
+
+from pyrit.datasets import SeedDatasetProvider
+from pyrit.datasets.seed_datasets.remote import _VLSUMultimodalDataset
+from pyrit.models import SeedDataset
+
+logger = logging.getLogger(__name__)
+
+
+def get_dataset_providers():
+    """Helper to get all registered providers for parameterization."""
+    providers = SeedDatasetProvider.get_all_providers()
+    return [(name, cls) for name, cls in providers.items()]
+
+
+class TestAllDatasets:
+    """Exhaustive test that every registered dataset provider can be fetched."""
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("name,provider_cls", get_dataset_providers())
+    async def test_fetch_dataset(self, name, provider_cls):
+        """
+        Verify that a specific registered dataset can be fetched.
+
+        This test is parameterized to run for each registered provider.
+        It verifies that:
+        1. The dataset can be downloaded/loaded without error
+        2. The result is a SeedDataset
+        3. The dataset is not empty (has seeds)
+        """
+        logger.info(f"Testing provider: {name}")
+
+        try:
+            # Use max_examples for slow providers that fetch many remote images
+            provider = provider_cls(max_examples=6) if provider_cls == _VLSUMultimodalDataset else provider_cls()
+            dataset = await provider.fetch_dataset(cache=False)
+
+            assert isinstance(dataset, SeedDataset), f"{name} did not return a SeedDataset"
+            assert len(dataset.seeds) > 0, f"{name} returned an empty dataset"
+            assert dataset.dataset_name, f"{name} has no dataset_name"
+
+            # Verify seeds have required fields
+            for seed in dataset.seeds:
+                assert seed.value, f"Seed in {name} has no value"
+                assert seed.dataset_name == dataset.dataset_name, (
+                    f"Seed dataset_name mismatch in {name}: {seed.dataset_name} != {dataset.dataset_name}"
+                )
+
+            logger.info(f"Successfully verified {name} with {len(dataset.seeds)} seeds")
+
+        except Exception as e:
+            pytest.fail(f"Failed to fetch dataset from {name}: {str(e)}")

tests/integration/datasets/test_load_default_datasets_integration.py

@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Integration test for the LoadDefaultDatasets initializer.
+
+Runs the full pipeline: discovers scenario default datasets, fetches them
+from real remote sources, and stores them in in-memory CentralMemory.
+"""
+
+import logging
+
+import pytest
+
+from pyrit.datasets import SeedDatasetProvider
+from pyrit.memory import CentralMemory
+from pyrit.setup.initializers.scenarios.load_default_datasets import LoadDefaultDatasets
+
+logger = logging.getLogger(__name__)
+
+
+class TestLoadDefaultDatasetsIntegration:
+    """Integration test that LoadDefaultDatasets loads real datasets into memory."""
+
+    @pytest.mark.asyncio
+    async def test_initialize_loads_datasets_into_memory(self):
+        """
+        Verify that LoadDefaultDatasets.initialize_async() successfully fetches
+        real datasets and stores them in CentralMemory.
+        """
+        initializer = LoadDefaultDatasets()
+        await initializer.initialize_async()
+
+        memory = CentralMemory.get_memory_instance()
+        seed_datasets = await memory.get_seed_datasets_async()
+
+        assert len(seed_datasets) > 0, "No datasets were loaded into memory"
+        logger.info(f"LoadDefaultDatasets loaded {len(seed_datasets)} datasets into memory")
+
+        # Verify basic structure of loaded datasets
+        for dataset in seed_datasets:
+            assert dataset.dataset_name, "Loaded dataset has no name"
+            assert len(dataset.seeds) > 0, f"Dataset '{dataset.dataset_name}' has no seeds"
+
+    @pytest.mark.asyncio
+    async def test_all_scenario_datasets_are_fetchable(self):
+        """
+        Verify that every dataset name referenced by registered scenarios
+        can actually be fetched from SeedDatasetProvider.
+        """
+        from pyrit.registry import ScenarioRegistry
+
+        registry = ScenarioRegistry.get_registry_singleton()
+        scenario_names = registry.get_names()
+
+        all_dataset_names: list[str] = []
+        for scenario_name in scenario_names:
+            scenario_class = registry.get_class(scenario_name)
+            if scenario_class:
+                try:
+                    datasets = scenario_class.default_dataset_config().get_default_dataset_names()
+                    all_dataset_names.extend(datasets)
+                except Exception as e:
+                    logger.warning(f"Could not get default datasets from scenario '{scenario_name}': {e}")
+
+        unique_names = list(dict.fromkeys(all_dataset_names))
+        assert len(unique_names) > 0, "No scenarios registered any default datasets"
+
+        fetched = await SeedDatasetProvider.fetch_datasets_async(dataset_names=unique_names)
+        assert len(fetched) == len(unique_names), (
+            f"Expected {len(unique_names)} datasets but fetched {len(fetched)}. "
+            f"Missing: {set(unique_names) - {d.dataset_name for d in fetched}}"
+        )

tests/integration/datasets/test_seed_dataset_provider_integration.py

@@ -10,7 +10,7 @@
 
 from pyrit.datasets import SeedDatasetProvider
 from pyrit.datasets.seed_datasets.local.local_dataset_loader import _LocalDatasetLoader
-from pyrit.datasets.seed_datasets.remote import _VLSUMultimodalDataset
+from pyrit.datasets.seed_datasets.remote import _SimpleSafetyTestsDataset, _XSTestDataset
 from pyrit.datasets.seed_datasets.seed_metadata import (
     SeedDatasetFilter,
 )
@@ -19,49 +19,49 @@
 logger = logging.getLogger(__name__)
 
 
-def get_dataset_providers():
-    """Helper to get all registered providers for parameterization."""
-    providers = SeedDatasetProvider.get_all_providers()
-    return [(name, cls) for name, cls in providers.items()]
+# Smoke-test providers covering the three distinct fetch paths:
+#   - local YAML (no network)
+#   - remote URL-based (_fetch_from_url via GitHub)
+#   - remote HuggingFace (_fetch_from_huggingface)
+_all_providers = SeedDatasetProvider.get_all_providers()
+_SMOKE_PROVIDERS: list[tuple[str, type]] = [
+    ("LocalDataset_access_shell_commands", _all_providers["LocalDataset_access_shell_commands"]),
+    ("_XSTestDataset", _XSTestDataset),
+    ("_SimpleSafetyTestsDataset", _SimpleSafetyTestsDataset),
+]
 
 
-class TestSeedDatasetProviderIntegration:
-    """Integration tests for SeedDatasetProvider."""
+class TestSeedDatasetSmoke:
+    """Smoke tests for a small representative set of dataset providers.
+
+    The exhaustive test over all providers lives in tests/end_to_end/test_all_datasets.py.
+    """
 
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("name,provider_cls", get_dataset_providers())
-    async def test_fetch_dataset_integration(self, name, provider_cls):
+    @pytest.mark.parametrize("name,provider_cls", _SMOKE_PROVIDERS, ids=[p[0] for p in _SMOKE_PROVIDERS])
+    async def test_fetch_dataset_smoke(self, name, provider_cls):
         """
-        Integration test to verify that a specific registered dataset can be fetched.
+        Verify that a representative provider can be fetched successfully.
 
-        This test is parameterized to run for each registered provider.
-        It verifies that:
-        1. The dataset can be downloaded/loaded without error
-        2. The result is a SeedDataset
-        3. The dataset is not empty (has seeds)
+        Covers one local, one URL-remote, and one HuggingFace-remote provider
+        to catch regressions in each fetch path without downloading all 58 datasets.
         """
-        logger.info(f"Testing provider: {name}")
+        logger.info(f"Smoke testing provider: {name}")
 
-        try:
-            # Use max_examples for slow providers that fetch many remote images
-            provider = provider_cls(max_examples=6) if provider_cls == _VLSUMultimodalDataset else provider_cls()
-            dataset = await provider.fetch_dataset(cache=False)
+        provider = provider_cls()
+        dataset = await provider.fetch_dataset(cache=False)
 
-            assert isinstance(dataset, SeedDataset), f"{name} did not return a SeedDataset"
-            assert len(dataset.seeds) > 0, f"{name} returned an empty dataset"
-            assert dataset.dataset_name, f"{name} has no dataset_name"
+        assert isinstance(dataset, SeedDataset), f"{name} did not return a SeedDataset"
+        assert len(dataset.seeds) > 0, f"{name} returned an empty dataset"
+        assert dataset.dataset_name, f"{name} has no dataset_name"
 
-            # Verify seeds have required fields
-            for seed in dataset.seeds:
-                assert seed.value, f"Seed in {name} has no value"
-                assert seed.dataset_name == dataset.dataset_name, (
-                    f"Seed dataset_name mismatch in {name}: {seed.dataset_name} != {dataset.dataset_name}"
-                )
-
-            logger.info(f"Successfully verified {name} with {len(dataset.seeds)} seeds")
+        for seed in dataset.seeds:
+            assert seed.value, f"Seed in {name} has no value"
+            assert seed.dataset_name == dataset.dataset_name, (
+                f"Seed dataset_name mismatch in {name}: {seed.dataset_name} != {dataset.dataset_name}"
+            )
 
-        except Exception as e:
-            pytest.fail(f"Failed to fetch dataset from {name}: {str(e)}")
+        logger.info(f"Smoke test passed for {name} with {len(dataset.seeds)} seeds")
 
 
 class TestRemoteFilteringIntegration: