Require archive requester count for totals, generate archive-level requester/asset-type summaries, and support configurable asset-type ordering (#270)

Copilot · CodyCBakerPhD · pre-commit-ci[bot] · web-flow · commit 997e8a8575f5 · 2026-05-29T02:34:27.000-04:00
* Initial plan * Fix archive totals to error when requester count is missing * Add archive requester and optional weekly asset-type aggregations * Stabilize archive asset-type column ordering * Allow archive asset type order via API and CLI * Apply suggestion from @CodyCBakerPhD * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> * Update test_generic_summaries.py * Fix CLI option wiring for asset type ordering * Remove archive asset-type defaults from generic summaries * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @CodyCBakerPhD * Simplify asset_types_in_order initialization * Fix conditions for dataset summary file paths * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update _generate_archive_summaries.py * Fix conditional statement in requester count extraction * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix type conversion for requester counts * Refactor requester_counts and archive_requester_count * Fix variable name for total requester counts * Update pyproject.toml --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,7 +12,7 @@ packages = ["src/s3_log_extraction"]
 
 [project]
 name = "s3-log-extraction"
-version="1.10.0"
+version="1.10.1"
 authors = [
   { name="Cody Baker", email="cody.c.baker.phd@gmail.com" },
 ]
diff --git a/src/s3_log_extraction/_command_line_interface/_cli.py b/src/s3_log_extraction/_command_line_interface/_cli.py
@@ -375,6 +375,13 @@ def _update_ip_coordinates_cli(cache_directory: str | None = None, use_encryptio
     type=rich_click.IntRange(min=-os.cpu_count() + 1, max=os.cpu_count()),
     default=-2,
 )
+@rich_click.option(
+    "--asset-types-in-order",
+    help="Archive mode only: comma-separated list of known asset types used for output column ordering (no spaces).",
+    required=False,
+    type=rich_click.STRING,
+    default=None,
+)
 @rich_click.option(
     "--cache",
     "cache_directory",
@@ -398,14 +405,19 @@ def _update_summaries_cli(
     pick: str | None = None,
     skip: str | None = None,
     workers: int = -2,
+    asset_types_in_order: str | None = None,
     cache_directory: str | None = None,
     use_encryption: bool = True,
 ) -> None:
     """Generate condensed summaries of activity."""
     cache_path = pathlib.Path(cache_directory) if cache_directory is not None else None
     match mode:
         case "archive":
-            generate_archive_summaries(cache_directory=cache_path)
+            parsed_asset_types_in_order = asset_types_in_order.split(",") if asset_types_in_order is not None else None
+            generate_archive_summaries(
+                cache_directory=cache_path,
+                asset_types_in_order=parsed_asset_types_in_order,
+            )
         case _:
             generate_summaries(cache_directory=cache_path, use_encryption=use_encryption)
 
diff --git a/src/s3_log_extraction/summarize/_generate_archive_summaries.py b/src/s3_log_extraction/summarize/_generate_archive_summaries.py
@@ -8,7 +8,9 @@
 
 
 @beartype.beartype
-def generate_archive_summaries(cache_directory: str | pathlib.Path | None = None) -> None:
+def generate_archive_summaries(
+    cache_directory: str | pathlib.Path | None = None, asset_types_in_order: tuple[str, ...] | list[str] | None = None
+) -> None:
     """
     Generate summaries by day and region for the entire archive from the mapped S3 logs.
 
@@ -17,7 +19,12 @@ def generate_archive_summaries(cache_directory: str | pathlib.Path | None = None
     cache_directory : path-like, optional
         The top-level cache directory from which the summary directory is derived.
         If not provided, the default cache directory is used.
+    asset_types_in_order : sequence[str], optional
+        Preferred output column ordering for known asset types in the archive
+        ``by_asset_type_per_week.tsv`` summary.
     """
+    asset_types_in_order = list(dict.fromkeys(asset_types_in_order)) if asset_types_in_order is not None else []
+
     summary_directory = get_cache_subdirectory(cache_directory=cache_directory, name="summaries")
     archive_directory = summary_directory / "archive"
     archive_directory.mkdir(exist_ok=True)
@@ -72,3 +79,45 @@ def generate_archive_summaries(cache_directory: str | pathlib.Path | None = None
     aggregated_activity_by_region.to_csv(
         path_or_buf=archive_summary_by_region_file_path, mode="w", sep="\t", header=True, index=False
     )
+
+    # Requester count (aggregated from dataset requester_count.tsv files)
+    requester_counts: list[int] = [
+        int(value)
+        for summary_file_path in summary_directory.rglob(pattern="requester_count.tsv")
+        if summary_file_path.parent.name != "archive" and "<" not in (value := summary_file_path.read_text().strip())
+    ]
+    total_requester_count: int = sum(requester_counts)
+    archive_requester_count: str = "<50" if total_requester_count < 50 else str(total_requester_count)
+
+    archive_requester_count_file_path = archive_directory / "requester_count.tsv"
+    archive_requester_count_file_path.write_text(archive_requester_count)
+
+    # Optional by_asset_type_per_week aggregation
+    all_dataset_summaries_by_asset_type_per_week = [
+        pandas.read_table(filepath_or_buffer=summary_file_path)
+        for summary_file_path in summary_directory.rglob(pattern="by_asset_type_per_week.tsv")
+        if summary_file_path.parent.name != "archive"
+    ]
+    if all_dataset_summaries_by_asset_type_per_week:
+        all_summary_data = pandas.concat(objs=all_dataset_summaries_by_asset_type_per_week, ignore_index=True)
+        all_summary_data.fillna(value=0, inplace=True)
+
+        all_asset_type_columns = [
+            column_name for column_name in all_summary_data.columns if column_name != "week_start"
+        ]
+        known_asset_type_columns = [
+            column_name for column_name in asset_types_in_order if column_name in all_asset_type_columns
+        ]
+        additional_asset_type_columns = sorted(set(all_asset_type_columns).difference(asset_types_in_order))
+        asset_type_columns = [*known_asset_type_columns, *additional_asset_type_columns]
+        if asset_type_columns:
+            archive_summary = (
+                all_summary_data.groupby(by="week_start", as_index=False)[asset_type_columns]
+                .sum()
+                .reindex(columns=["week_start", *asset_type_columns])
+            )
+            archive_summary = archive_summary.astype(dtype={column_name: "int64" for column_name in asset_type_columns})
+            archive_summary.sort_values(by="week_start", key=natsort.natsort_keygen(), inplace=True)
+
+            archive_summary_file_path = archive_directory / "by_asset_type_per_week.tsv"
+            archive_summary.to_csv(path_or_buf=archive_summary_file_path, mode="w", sep="\t", header=True, index=False)
diff --git a/src/s3_log_extraction/summarize/_generate_archive_totals.py b/src/s3_log_extraction/summarize/_generate_archive_totals.py
@@ -45,9 +45,14 @@ def generate_archive_totals(
     number_of_unique_countries = len(unique_countries)
 
     requester_count_file_path = archive_directory / "requester_count.tsv"
-    number_of_requesters: str | int = (
-        requester_count_file_path.read_text().strip() if requester_count_file_path.exists() else 0
-    )
+    if not requester_count_file_path.exists():
+        msg = (
+            f"Archive requester count file not found: {requester_count_file_path}. "
+            "Run archive summaries before archive totals."
+        )
+        raise FileNotFoundError(msg)
+
+    number_of_requesters: str | int = requester_count_file_path.read_text().strip()
     if isinstance(number_of_requesters, str) and not number_of_requesters.startswith("<"):
         number_of_requesters = int(number_of_requesters)
 
diff --git a/tests/test_generic_summaries.py b/tests/test_generic_summaries.py
@@ -91,14 +91,14 @@ def test_generate_all_dataset_totals_skips_archive(tmpdir: py.path.local):
     dataset_dir = summary_dir / "ds001161"
     dataset_dir.mkdir(parents=True)
     (dataset_dir / "by_region.tsv").write_text(
-        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n" "missing\t1194564\t4\t3\n"
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t1194564\t4\t3\n"
     )
 
     # Set up an archive summary that should be excluded
     archive_dir = summary_dir / "archive"
     archive_dir.mkdir(parents=True)
     (archive_dir / "by_region.tsv").write_text(
-        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n" "missing\t7481053\t7\t5\n"
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t7481053\t7\t5\n"
     )
 
     s3_log_extraction.summarize.generate_all_dataset_totals(cache_directory=test_dir)
@@ -108,6 +108,135 @@ def test_generate_all_dataset_totals_skips_archive(tmpdir: py.path.local):
     assert "archive" not in totals, "'archive' should be excluded from totals.json"
 
 
+@pytest.mark.ai_generated
+def test_generate_archive_totals_raises_without_archive_requester_count(tmpdir: py.path.local) -> None:
+    """Archive totals should fail if archive requester count has not been generated."""
+    test_dir = pathlib.Path(tmpdir)
+    archive_dir = test_dir / "summaries" / "archive"
+    archive_dir.mkdir(parents=True)
+    (archive_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t7481053\t7\t5\n"
+    )
+
+    with pytest.raises(FileNotFoundError, match="Archive requester count file not found"):
+        s3_log_extraction.summarize.generate_archive_totals(cache_directory=test_dir)
+
+
+@pytest.mark.ai_generated
+def test_generate_archive_summaries_aggregates_requester_count(tmpdir: py.path.local) -> None:
+    test_dir = pathlib.Path(tmpdir)
+    summary_dir = test_dir / "summaries"
+
+    ds001_dir = summary_dir / "ds001"
+    ds001_dir.mkdir(parents=True)
+    (ds001_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t10\t1\t1\n"
+    )
+    (ds001_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t10\t1\t1\n"
+    )
+    (ds001_dir / "requester_count.tsv").write_text("60\n")
+
+    ds002_dir = summary_dir / "ds002"
+    ds002_dir.mkdir(parents=True)
+    (ds002_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t40\t2\t1\n"
+    )
+    (ds002_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t40\t2\t1\n"
+    )
+    (ds002_dir / "requester_count.tsv").write_text("40\n")
+
+    s3_log_extraction.summarize.generate_archive_summaries(cache_directory=test_dir)
+
+    archive_requester_count_file_path = summary_dir / "archive" / "requester_count.tsv"
+    assert archive_requester_count_file_path.exists()
+    assert archive_requester_count_file_path.read_text().strip() == "100"
+
+
+@pytest.mark.ai_generated
+def test_generate_archive_summaries_aggregates_optional_by_asset_type_per_week(tmpdir: py.path.local) -> None:
+    test_dir = pathlib.Path(tmpdir)
+    summary_dir = test_dir / "summaries"
+
+    ds001_dir = summary_dir / "ds001"
+    ds001_dir.mkdir(parents=True)
+    (ds001_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t10\t1\t1\n"
+    )
+    (ds001_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t10\t1\t1\n"
+    )
+    (ds001_dir / "requester_count.tsv").write_text("20\n")
+    (ds001_dir / "by_asset_type_per_week.tsv").write_text(
+        "week_start\tNeurophysiology\tMiscellaneous\n2025-12-29\t1\t2\n2026-01-05\t3\t4\n"
+    )
+
+    ds002_dir = summary_dir / "ds002"
+    ds002_dir.mkdir(parents=True)
+    (ds002_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t40\t2\t1\n"
+    )
+    (ds002_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t40\t2\t1\n"
+    )
+    (ds002_dir / "requester_count.tsv").write_text("20\n")
+    (ds002_dir / "by_asset_type_per_week.tsv").write_text("week_start\tVideo\n2025-12-29\t5\n2026-01-05\t7\n")
+
+    s3_log_extraction.summarize.generate_archive_summaries(cache_directory=test_dir)
+
+    archive_file_path = summary_dir / "archive" / "by_asset_type_per_week.tsv"
+    assert archive_file_path.exists()
+    archive_summary = pandas.read_table(filepath_or_buffer=archive_file_path)
+    expected_summary = pandas.DataFrame(
+        data={
+            "week_start": ["2025-12-29", "2026-01-05"],
+            "Miscellaneous": [2, 4],
+            "Neurophysiology": [1, 3],
+            "Video": [5, 7],
+        }
+    )
+    pandas.testing.assert_frame_equal(left=archive_summary, right=expected_summary)
+
+
+@pytest.mark.ai_generated
+def test_generate_archive_summaries_accepts_custom_asset_type_order(tmpdir: py.path.local) -> None:
+    test_dir = pathlib.Path(tmpdir)
+    summary_dir = test_dir / "summaries"
+
+    ds001_dir = summary_dir / "ds001"
+    ds001_dir.mkdir(parents=True)
+    (ds001_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t10\t1\t1\n"
+    )
+    (ds001_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t10\t1\t1\n"
+    )
+    (ds001_dir / "requester_count.tsv").write_text("20\n")
+    (ds001_dir / "by_asset_type_per_week.tsv").write_text(
+        "week_start\tNeurophysiology\tMiscellaneous\n2025-12-29\t1\t2\n"
+    )
+
+    ds002_dir = summary_dir / "ds002"
+    ds002_dir.mkdir(parents=True)
+    (ds002_dir / "by_day.tsv").write_text(
+        "date\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\n2026-01-01\t40\t2\t1\n"
+    )
+    (ds002_dir / "by_region.tsv").write_text(
+        "region\tbytes_sent\tnumber_of_requests\tnumber_of_downloads\nmissing\t40\t2\t1\n"
+    )
+    (ds002_dir / "requester_count.tsv").write_text("20\n")
+    (ds002_dir / "by_asset_type_per_week.tsv").write_text("week_start\tVideo\n2025-12-29\t5\n")
+
+    s3_log_extraction.summarize.generate_archive_summaries(
+        cache_directory=test_dir, asset_types_in_order=["Video", "Neurophysiology", "Miscellaneous"]
+    )
+
+    archive_file_path = summary_dir / "archive" / "by_asset_type_per_week.tsv"
+    archive_summary = pandas.read_table(filepath_or_buffer=archive_file_path)
+    assert archive_summary.columns.tolist() == ["week_start", "Video", "Neurophysiology", "Miscellaneous"]
+
+
 @pytest.mark.ai_generated
 @pytest.mark.parametrize(
     ("count", "modulo", "minimum", "expected"),
diff --git a/tests/test_log_bucket_stats.py b/tests/test_log_bucket_stats.py
@@ -332,22 +332,36 @@ def test_update_summaries_archive_forwards_cache_directory(
     """
     ``update summaries --mode archive`` passes ``cache_directory`` directly to ``generate_archive_summaries``.
     """
-    captured: dict[str, pathlib.Path] = {}
+    captured: dict[str, pathlib.Path | list[str] | None] = {}
 
-    def _stub_generate_archive_summaries(cache_directory: pathlib.Path | str | None = None) -> None:
+    def _stub_generate_archive_summaries(
+        cache_directory: pathlib.Path | str | None = None,
+        asset_types_in_order: tuple[str, ...] | list[str] | None = None,
+    ) -> None:
         captured["cache_directory"] = pathlib.Path(cache_directory) if cache_directory is not None else None
+        captured["asset_types_in_order"] = list(asset_types_in_order) if asset_types_in_order is not None else None
 
     monkeypatch.setattr(cli_module, "generate_archive_summaries", _stub_generate_archive_summaries)
 
     cache_dir = tmp_path / "custom-cache"
     runner = CliRunner()
     result = runner.invoke(
         s3logextraction_cli,
-        ["update", "summaries", "--mode", "archive", "--cache", str(cache_dir)],
+        [
+            "update",
+            "summaries",
+            "--mode",
+            "archive",
+            "--asset-types-in-order",
+            "Video,Neurophysiology,Miscellaneous",
+            "--cache",
+            str(cache_dir),
+        ],
     )
 
     assert result.exit_code == 0, f"CLI failed: {result.output}"
     assert captured["cache_directory"] == cache_dir
+    assert captured["asset_types_in_order"] == ["Video", "Neurophysiology", "Miscellaneous"]
 
 
 @pytest.mark.ai_generated

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ packages = ["src/s3_log_extraction"]`
`12`	`12`
`13`	`13`	`[project]`
`14`	`14`	`name = "s3-log-extraction"`
`15`		`-version="1.10.0"`
	`15`	`+version="1.10.1"`
`16`	`16`	`authors = [`
`17`	`17`	`{ name="Cody Baker", email="cody.c.baker.phd@gmail.com" },`
`18`	`18`	`]`