diff --git a/.github/utils/docs_search_sync.py b/.github/utils/docs_search_sync.py new file mode 100644 index 00000000000..2f35b505b26 --- /dev/null +++ b/.github/utils/docs_search_sync.py @@ -0,0 +1,120 @@ +""" +This script syncs the Haystack docs HTML files to the deepset workspace for search indexing. + +It is used in the docs_search_sync.yml workflow. + +1. Collects all HTML files from the docs and reference directories for the stable Haystack version. +2. Uploads the HTML files to the deepset workspace. + - A timestamp-based metadata field is used to track document versions in the workspace. +3. Deletes the old HTML files from the deepset workspace. + - Since most files are overwritten during upload, only a small number of deletions is expected. + - In case MAX_DELETIONS_SAFETY_LIMIT is exceeded, we block the deletion. +""" + +import os +import sys +import time +from pathlib import Path + +import requests +from deepset_cloud_sdk.workflows.sync_client.files import DeepsetCloudFile, WriteMode, list_files, upload_texts + +DEEPSET_WORKSPACE_DOCS_SEARCH = os.environ["DEEPSET_WORKSPACE_DOCS_SEARCH"] +DEEPSET_API_KEY_DOCS_SEARCH = os.environ["DEEPSET_API_KEY_DOCS_SEARCH"] + +# If there are more files to delete than this limit, it's likely that something went wrong in the upload process. +MAX_DELETIONS_SAFETY_LIMIT = 20 + + +def collect_docs_files(version: int) -> list[DeepsetCloudFile]: + """ + Collect all HTML files from the docs and reference directories. + + Returns a list of DeepsetCloudFile objects. + """ + repo_root = Path(__file__).parent.parent.parent + build_dir = repo_root / "docs-website" / "build" + # we want to exclude previous and temporarily unstable versions (2.x) and next version (next) + exclude = ("2.", "next") + + files = [] + for section in ("docs", "reference"): + for subfolder in (build_dir / section).iterdir(): + if subfolder.is_dir() and not any(x in subfolder.name for x in exclude): + for html_file in subfolder.rglob("*.html"): + files.append( + DeepsetCloudFile( + # The build produces files like docs/agents/index.html or reference/agents-api/index.html. + # For file names, we want to use the parent directory name (agents.html or agents-api.html) + name=f"{html_file.parent.name}.html", + text=html_file.read_text(), + meta={ + "type": "api-reference" if section == "reference" else "documentation", + "version": version, + }, + ) + ) + return files + + +def delete_files(file_names: list[str]) -> None: + """ + Delete files from the deepset workspace. + """ + url = f"https://api.cloud.deepset.ai/api/v1/workspaces/{DEEPSET_WORKSPACE_DOCS_SEARCH}/files" + payload = {"names": file_names} + headers = {"Accept": "application/json", "Authorization": f"Bearer {DEEPSET_API_KEY_DOCS_SEARCH}"} + response = requests.delete(url, json=payload, headers=headers, timeout=300) + response.raise_for_status() + + +if __name__ == "__main__": + version = time.time_ns() + print(f"Docs version: {version}") + + print("Collecting docs files from build directory") + dc_files = collect_docs_files(version) + print(f"Collected {len(dc_files)} docs files") + + if len(dc_files) == 0: + print("No docs files found. Something is wrong. Exiting.") + sys.exit(1) + + print("Uploading docs files to deepset") + summary = upload_texts( + workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH, + files=dc_files, + api_key=DEEPSET_API_KEY_DOCS_SEARCH, + blocking=True, # Very important to ensure that DC is up to date when we query for deletion + timeout_s=300, + show_progress=True, + write_mode=WriteMode.OVERWRITE, + enable_parallel_processing=True, + ) + print(f"Uploaded docs files to deepset\n{summary}") + if summary.failed_upload_count > 0: + print("Failed to upload some docs files. Stopping to prevent risky deletion of old files.") + sys.exit(1) + + print("Listing old docs files from deepset") + odata_filter = f"version lt '{version}'" + old_files_names = [ + f.name + for batch in list_files( + workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH, api_key=DEEPSET_API_KEY_DOCS_SEARCH, odata_filter=odata_filter + ) + for f in batch + ] + + print(f"Found {len(old_files_names)} old files to delete") + if len(old_files_names) > MAX_DELETIONS_SAFETY_LIMIT: + print( + f"Found >{MAX_DELETIONS_SAFETY_LIMIT} old files to delete. " + "Stopping because something could have gone wrong in the upload process." + ) + sys.exit(1) + + if len(old_files_names) > 0: + print("Deleting old docs files from deepset") + delete_files(old_files_names) + print("Deleted old docs files from deepset") diff --git a/.github/workflows/docs_search_sync.yml b/.github/workflows/docs_search_sync.yml new file mode 100644 index 00000000000..888a786230f --- /dev/null +++ b/.github/workflows/docs_search_sync.yml @@ -0,0 +1,65 @@ +name: Docs Search Sync + +on: + workflow_dispatch: # Activate this workflow manually + schedule: + - cron: "0 1 * * *" + +# Running this workflow multiple times in parallel can cause issues with files uploads and deletions. +concurrency: + group: docs-search-sync + cancel-in-progress: false + +jobs: + docs-search-sync: + runs-on: ubuntu-latest + + steps: + + - name: Checkout Haystack repo + uses: actions/checkout@v6 + + - name: Install Node.js + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install Docusaurus and build docs-website + working-directory: docs-website + run: | + npm install + npm run build + + - name: Install script dependencies + run: pip install deepset-cloud-sdk requests + + - name: Update new docs to Search pipeline and remove outdated docs + env: + DEEPSET_WORKSPACE_DOCS_SEARCH: ${{ secrets.DEEPSET_WORKSPACE_DOCS_SEARCH }} + DEEPSET_API_KEY_DOCS_SEARCH: ${{ secrets.DEEPSET_API_KEY_DOCS_SEARCH }} + run: python ./.github/utils/docs_search_sync.py + + - name: Send event to Datadog for nightly failures + if: failure() && github.event_name == 'schedule' + uses: masci/datadog@v1 + with: + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} + api-url: https://api.datadoghq.eu + events: | + - title: "${{ github.workflow }} workflow" + text: "Job ${{ github.job }} in branch ${{ github.ref_name }}" + alert_type: "error" + source_type_name: "Github" + host: ${{ github.repository_owner }} + tags: + - "project:${{ github.repository }}" + - "job:${{ github.job }}" + - "run_id:${{ github.run_id }}" + - "workflow:${{ github.workflow }}" + - "branch:${{ github.ref_name }}" + - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b3a5f41ef67..1867e8da1a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -227,6 +227,7 @@ jobs: python: - '**/*.py' - '!test/**' + - '!.github/**' pyproject: - 'pyproject.toml'