script + simplify

anakin87 · anakin87 · commit 9d2129d02f39 · 2026-02-11T15:57:51.000+01:00
diff --git a/.github/utils/copy_file_to_api_reference.py b/.github/utils/copy_file_to_api_reference.py
@@ -0,0 +1,27 @@
+"""
+Script to copy a generated markdown file to the Haystack API reference (integrations-api).
+
+Assumes that Haystack Docs Website is available in the working directory.
+Copies the file to the main API reference and the versioned API references.
+
+Usage:
+python copy_file_to_api_reference.py <generated_markdown_file>
+"""
+
+import argparse
+import os
+import shutil
+
+def copy_file_to_api_reference(generated_markdown_file: str):
+    shutil.copy(generated_markdown_file, "docs-website/reference/integrations-api/")
+    for version_dir in os.scandir("docs-website/reference_versioned_docs"):
+        if version_dir.is_dir():
+            # example: docs-website/reference_versioned_docs/version-2.17/integrations-api
+            integrations_api_ref_dir = os.path.join(version_dir.path, "integrations-api")
+            shutil.copy(generated_markdown_file, integrations_api_ref_dir)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("generated_markdown_file", type=str, help="Path to the generated markdown file")
+    args = parser.parse_args()
+    copy_file_to_api_reference(args.generated_markdown_file)
diff --git a/.github/utils/pyproject_to_requirements.py b/.github/utils/pyproject_to_requirements.py
@@ -1,5 +1,4 @@
 import argparse
-import sys
 from pathlib import Path
 import toml
 
diff --git a/.github/workflows/CI_check_api_ref.yml b/.github/workflows/CI_check_api_ref.yml
@@ -8,15 +8,15 @@ on:
 
 jobs:
   detect-changes:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     outputs:
       integrations: ${{ steps.changed.outputs.integrations }}
     steps:
       - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.13
+      - name: Set up Python
         uses: actions/setup-python@v6
         with:
           python-version: "3.13"
@@ -26,7 +26,6 @@ jobs:
         shell: python
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          RUNNER_TEMP: ${{ runner.temp }}
         run: |
           import json
           import os
@@ -44,45 +43,50 @@ jobs:
               result = subprocess.run(["git", *args], capture_output=True, text=True)
               return result.stdout.strip(), result.returncode
 
-          # Find integrations with changed .py or config files
-          diff_output, _ = git("diff", "--name-only", f"{base_sha}...HEAD",
-                               "--", "integrations/**/*.py", "integrations/**/config_docusaurus.yml")
-          candidates = sorted({Path(p).parts[1] for p in diff_output.splitlines() if p.startswith("integrations/")})
+          # Get all changed files
+          diff_output, _ = git(
+              "diff", "--name-only", f"{base_sha}...HEAD", "--", "integrations"
+          )
+          changed_files = set(diff_output.splitlines())
+
+          # Extract integration names
+          candidates = sorted({
+              Path(p).parts[1]
+              for p in changed_files
+              if p.startswith("integrations/") and len(Path(p).parts) > 1
+          })
 
-          # Create base worktree once for docstring comparison
+          # Create base worktree for docstring comparison
           base_worktree = os.path.join(runner_temp, "base")
-          _, rc = git("worktree", "add", base_worktree, base_sha)
-          has_worktree = rc == 0
+          _, return_code = git("worktree", "add", base_worktree, base_sha)
+          has_worktree = return_code == 0
 
-          result = []
+          changed_integrations = []
+          
           for integration in candidates:
               # If pydoc config changed, always include
-              config_diff, _ = git("diff", "--name-only", f"{base_sha}...HEAD",
-                                   "--", f"integrations/{integration}/pydoc/config_docusaurus.yml")
-              if config_diff:
-                  result.append(integration)
+              if f"integrations/{integration}/pydoc/config_docusaurus.yml" in changed_files:
+                  changed_integrations.append(integration)
                   continue
 
               # Otherwise, check if docstrings changed
-              pr_checksum = docstrings_checksum(Path(".").glob(f"integrations/{integration}/**/*.py"))
+              pr_docstrings_checksum = docstrings_checksum(Path(".").glob(f"integrations/{integration}/**/*.py"))
+              base_docstrings_checksum = ""
               if has_worktree:
-                  base_checksum = docstrings_checksum(Path(base_worktree).glob(f"integrations/{integration}/**/*.py"))
-              else:
-                  # Integration doesn't exist on the base branch (new integration)
-                  base_checksum = ""
+                  base_docstrings_checksum = docstrings_checksum(Path(base_worktree).glob(f"integrations/{integration}/**/*.py"))
 
-              if base_checksum != pr_checksum:
-                  result.append(integration)
+              if base_docstrings_checksum != pr_docstrings_checksum:
+                  changed_integrations.append(integration)
 
           if has_worktree:
               git("worktree", "remove", base_worktree)
 
-          print(f"Integrations with API reference changes: {json.dumps(result)}")
+          print(f"Integrations with API reference changes: {json.dumps(changed_integrations)}")
 
           with open(os.environ["GITHUB_OUTPUT"], "a") as f:
-              f.write(f"integrations={json.dumps(result)}\n")
+              f.write(f"integrations={json.dumps(changed_integrations)}\n")
 
-  generate-docs:
+  test-api-reference-build:
     needs: detect-changes
     if: needs.detect-changes.outputs.integrations != '[]'
     runs-on: ubuntu-latest
@@ -107,34 +111,18 @@ jobs:
           hatch run docs
           # Save the generated file before the Haystack checkout overwrites the working tree
           cp ${{ matrix.integration }}.md "${{ runner.temp }}/${{ matrix.integration }}.md"
+          # Also copy the Copy script to the temp directory
+          cp .github/utils/copy_file_to_api_reference.py "${{ runner.temp }}/copy_file_to_api_reference.py"
 
-      - name: Checkout Haystack repo
+      - name: Checkout Haystack Docs Website
         uses: actions/checkout@v6
         with:
           repository: deepset-ai/haystack
           ref: main
           sparse-checkout: docs-website
 
-      - name: Sync API reference
-        shell: python
-        env:
-          INTEGRATION_API_REFERENCE: ${{ runner.temp }}/${{ matrix.integration }}.md
-        run: |
-          import os
-          import shutil
-
-          api_ref_file = os.environ['INTEGRATION_API_REFERENCE']
-
-          # Copy to main API reference
-          shutil.copy(api_ref_file, "docs-website/reference/integrations-api/")
-
-          # Copy to versioned API reference
-          for version_dir in os.scandir("docs-website/reference_versioned_docs"):
-              if version_dir.is_dir():
-                  # example: docs-website/reference_versioned_docs/version-2.17/integrations-api
-                  integrations_api_ref_dir = os.path.join(version_dir.path, "integrations-api")
-                  shutil.copy(api_ref_file, integrations_api_ref_dir)
-
+      - name: Copy file to API reference
+        run: python ${{ runner.temp }}/copy_file_to_api_reference.py ${{ runner.temp }}/${{ matrix.integration }}.md
       - name: Install Node.js
         uses: actions/setup-node@v6
         with:
diff --git a/.github/workflows/CI_docusaurus_sync.yml b/.github/workflows/CI_docusaurus_sync.yml
@@ -72,37 +72,30 @@ jobs:
           ref: main
           token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
 
+      - name: Checkout this repo for the script
+        uses: actions/checkout@v6
+        with:
+          path: core-integrations
+          sparse-checkout: |
+            .github/utils/copy_file_to_api_reference.py
+
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"          
+          python-version: "3.10"
 
       - name: Download API reference artifact
         uses: actions/download-artifact@v7
         with:
           name: ${{ needs.generate-api-reference.outputs.integration_name }}
 
       - name: Sync API reference
-        shell: python
         env:
           INTEGRATION_NAME: ${{ needs.generate-api-reference.outputs.integration_name }}
         run: |
-          import os
-          import shutil
-
-          artifact_filename = os.environ['INTEGRATION_NAME']+'.md'
-
-          # Copy to main API reference
-          shutil.copy(artifact_filename, "docs-website/reference/integrations-api/")
-
-          # Copy to versioned API reference
-          for version_dir in os.scandir("docs-website/reference_versioned_docs"):
-              if version_dir.is_dir():
-                  # example: docs-website/reference_versioned_docs/version-2.17/integrations-api
-                  integrations_api_ref_dir = os.path.join(version_dir.path, "integrations-api")
-                  shutil.copy(artifact_filename, integrations_api_ref_dir)
-
-          os.remove(artifact_filename)
+          artifact_filename="${INTEGRATION_NAME}.md"
+          python core-integrations/.github/utils/copy_file_to_api_reference.py "$artifact_filename"
+          rm "$artifact_filename"
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v8

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`import argparse`
`2`		`-import sys`
`3`	`2`	`from pathlib import Path`
`4`	`3`	`import toml`
`5`	`4`