From e4adaf672bb917249553f6a3f5e3a0db82cda3d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Jan 2026 12:32:52 +0000 Subject: [PATCH 1/2] refactor: reorganize automation directory structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorganize directory structure to better separate concerns: - automation/ → Only workflow-specific scripts used by GitHub Actions - scripts/ → One-time migrations and manual utilities - core/generators/ → Reusable business logic components Changes: - Move one-time migration scripts to scripts/: - backfill_review_metadata.py - fix_library_versions.py - migrate_metadata_format.py - migrate_to_new_structure.py - upgrade_specs.py - upgrade_specs_ai.py - Move reusable generators to core/generators/: - plot_generator.py (from automation/generators/) - Update imports in tests and documentation - Update repository.md with new directory structure - Add sections explaining automation/ vs scripts/ principles All tests pass successfully. --- {automation => core}/generators/__init__.py | 0 .../generators/plot_generator.py | 0 docs/architecture/repository.md | 62 ++++++++++++++++--- docs/concepts/claude-skill-plot-generation.md | 2 +- .../backfill_review_metadata.py | 0 .../fix_library_versions.py | 0 .../migrate_metadata_format.py | 0 .../migrate_to_new_structure.py | 0 .../scripts => scripts}/upgrade_specs.py | 0 .../scripts => scripts}/upgrade_specs_ai.py | 0 .../generators/test_plot_generator.py | 4 +- 11 files changed, 56 insertions(+), 12 deletions(-) rename {automation => core}/generators/__init__.py (100%) rename {automation => core}/generators/plot_generator.py (100%) rename {automation/scripts => scripts}/backfill_review_metadata.py (100%) rename {automation/scripts => scripts}/fix_library_versions.py (100%) rename {automation/scripts => scripts}/migrate_metadata_format.py (100%) rename {automation/scripts => scripts}/migrate_to_new_structure.py (100%) rename {automation/scripts => scripts}/upgrade_specs.py (100%) rename {automation/scripts => scripts}/upgrade_specs_ai.py (100%) rename tests/unit/{automation => core}/generators/test_plot_generator.py (98%) diff --git a/automation/generators/__init__.py b/core/generators/__init__.py similarity index 100% rename from automation/generators/__init__.py rename to core/generators/__init__.py diff --git a/automation/generators/plot_generator.py b/core/generators/plot_generator.py similarity index 100% rename from automation/generators/plot_generator.py rename to core/generators/plot_generator.py diff --git a/docs/architecture/repository.md b/docs/architecture/repository.md index 87e737ef3f..5137fd5510 100644 --- a/docs/architecture/repository.md +++ b/docs/architecture/repository.md @@ -76,11 +76,13 @@ pyplots/ ├── core/ # Shared business logic │ ├── __init__.py │ ├── config.py # Configuration (.env-based) -│ └── database/ # Database layer -│ ├── __init__.py -│ ├── connection.py # Async connection management -│ ├── models.py # SQLAlchemy ORM models -│ └── repositories.py # Repository pattern +│ ├── database/ # Database layer +│ │ ├── __init__.py +│ │ ├── connection.py # Async connection management +│ │ ├── models.py # SQLAlchemy ORM models +│ │ └── repositories.py # Repository pattern +│ └── generators/ # Reusable code generators +│ └── plot_generator.py # Plot code generation utilities │ ├── api/ # FastAPI backend │ ├── __init__.py @@ -95,9 +97,12 @@ pyplots/ │ ├── package.json │ └── Dockerfile │ -├── automation/ # Automation scripts -│ └── scripts/ -│ └── sync_to_postgres.py # Sync plots/ to database +├── automation/ # Workflow automation +│ └── scripts/ # Workflow-specific utilities +│ ├── sync_to_postgres.py # Sync plots/ to database +│ ├── workflow_utils.py # Utilities for GitHub Actions +│ ├── label_manager.py # Label operations +│ └── workflow_cli.py # CLI for workflows │ ├── tests/ # Test suite │ └── unit/ @@ -121,6 +126,15 @@ pyplots/ ├── alembic/ # Database migrations │ └── versions/ │ +├── scripts/ # One-time and manual scripts +│ ├── evaluate-plot.py # Manual plot evaluation +│ ├── regenerate-thumbnails.py # Image processing +│ ├── backfill_review_metadata.py # One-time migration +│ ├── fix_library_versions.py # One-time fix +│ ├── migrate_metadata_format.py # One-time migration +│ ├── migrate_to_new_structure.py # One-time migration +│ └── upgrade_specs*.py # Spec upgrade utilities +│ ├── docs/ # Documentation │ ├── architecture/ │ ├── workflow.md @@ -390,12 +404,42 @@ plt.savefig('plot.png', dpi=300) ### `core/` -**Purpose**: Shared business logic used by API +**Purpose**: Shared business logic and reusable components **Key Components**: - `database/connection.py` - Async database connection - `database/models.py` - SQLAlchemy ORM models - `database/repositories.py` - Repository pattern for data access +- `generators/plot_generator.py` - Reusable plot code generation utilities + +--- + +### `automation/` + +**Purpose**: Workflow-specific automation used by GitHub Actions + +**Key Components**: +- `scripts/sync_to_postgres.py` - Database sync (used by sync-postgres.yml) +- `scripts/workflow_utils.py` - Parsing and utilities for workflows +- `scripts/label_manager.py` - Label operations and transitions +- `scripts/workflow_cli.py` - CLI interface for workflow steps + +**Principle**: Only contains components actively used by workflows. One-time or manual scripts belong in `scripts/`. + +--- + +### `scripts/` + +**Purpose**: One-time migrations and manually-run utilities + +**Key Components**: +- `evaluate-plot.py` - Manual plot quality evaluation +- `regenerate-thumbnails.py` - Image processing utilities +- `backfill_review_metadata.py` - One-time backfill migration +- `migrate_*.py` - One-time structure migrations +- `upgrade_specs*.py` - Spec upgrade utilities + +**Principle**: Scripts that are not part of automated workflows. Used for maintenance, migrations, and manual operations. --- diff --git a/docs/concepts/claude-skill-plot-generation.md b/docs/concepts/claude-skill-plot-generation.md index 55326bcad3..32763a74da 100644 --- a/docs/concepts/claude-skill-plot-generation.md +++ b/docs/concepts/claude-skill-plot-generation.md @@ -500,7 +500,7 @@ claude-skill plot-generation \ ### Example 2: From Python ```python -# automation/generators/claude_generator.py +# core/generators/claude_generator.py from claude_skills import invoke_skill result = invoke_skill( diff --git a/automation/scripts/backfill_review_metadata.py b/scripts/backfill_review_metadata.py similarity index 100% rename from automation/scripts/backfill_review_metadata.py rename to scripts/backfill_review_metadata.py diff --git a/automation/scripts/fix_library_versions.py b/scripts/fix_library_versions.py similarity index 100% rename from automation/scripts/fix_library_versions.py rename to scripts/fix_library_versions.py diff --git a/automation/scripts/migrate_metadata_format.py b/scripts/migrate_metadata_format.py similarity index 100% rename from automation/scripts/migrate_metadata_format.py rename to scripts/migrate_metadata_format.py diff --git a/automation/scripts/migrate_to_new_structure.py b/scripts/migrate_to_new_structure.py similarity index 100% rename from automation/scripts/migrate_to_new_structure.py rename to scripts/migrate_to_new_structure.py diff --git a/automation/scripts/upgrade_specs.py b/scripts/upgrade_specs.py similarity index 100% rename from automation/scripts/upgrade_specs.py rename to scripts/upgrade_specs.py diff --git a/automation/scripts/upgrade_specs_ai.py b/scripts/upgrade_specs_ai.py similarity index 100% rename from automation/scripts/upgrade_specs_ai.py rename to scripts/upgrade_specs_ai.py diff --git a/tests/unit/automation/generators/test_plot_generator.py b/tests/unit/core/generators/test_plot_generator.py similarity index 98% rename from tests/unit/automation/generators/test_plot_generator.py rename to tests/unit/core/generators/test_plot_generator.py index cc77f2a71f..f556dcb35e 100644 --- a/tests/unit/automation/generators/test_plot_generator.py +++ b/tests/unit/core/generators/test_plot_generator.py @@ -1,10 +1,10 @@ -"""Tests for automation.generators.plot_generator module.""" +"""Tests for core.generators.plot_generator module.""" from unittest.mock import MagicMock, patch import pytest -from automation.generators.plot_generator import extract_and_validate_code, retry_with_backoff +from core.generators.plot_generator import extract_and_validate_code, retry_with_backoff class TestExtractAndValidateCode: From 7d1f9a608856fd319cf071a265a87cc239db54b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Jan 2026 12:34:52 +0000 Subject: [PATCH 2/2] docs: update script usage paths in documentation Update internal documentation in migration scripts to reflect new paths in scripts/ directory instead of automation/scripts/. --- scripts/backfill_review_metadata.py | 4 ++-- scripts/migrate_metadata_format.py | 2 +- scripts/migrate_to_new_structure.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/backfill_review_metadata.py b/scripts/backfill_review_metadata.py index 5af2177875..40dedf0102 100755 --- a/scripts/backfill_review_metadata.py +++ b/scripts/backfill_review_metadata.py @@ -11,8 +11,8 @@ always takes the LAST one (the one that led to merge). Usage: - python automation/scripts/backfill_review_metadata.py --dry-run - python automation/scripts/backfill_review_metadata.py --execute + python scripts/backfill_review_metadata.py --dry-run + python scripts/backfill_review_metadata.py --execute Requires: - gh CLI authenticated diff --git a/scripts/migrate_metadata_format.py b/scripts/migrate_metadata_format.py index 9227babd5f..220027c5b1 100644 --- a/scripts/migrate_metadata_format.py +++ b/scripts/migrate_metadata_format.py @@ -7,7 +7,7 @@ 2. metadata/{library}.yaml: Flatten current:, remove history, add created/updated/review 3. implementations/*.py: Update docstring header to new format -Run: uv run python automation/scripts/migrate_metadata_format.py +Run: uv run python scripts/migrate_metadata_format.py """ import re diff --git a/scripts/migrate_to_new_structure.py b/scripts/migrate_to_new_structure.py index 17d3b158e9..d513f6d51a 100644 --- a/scripts/migrate_to_new_structure.py +++ b/scripts/migrate_to_new_structure.py @@ -12,7 +12,7 @@ - metadata/{library}.yaml (per-library: preview_url, current, history) Usage: - python automation/scripts/migrate_to_new_structure.py [--dry-run] + python scripts/migrate_to_new_structure.py [--dry-run] """ import argparse