Merge branch 'main' into feat/adding-delete-all-docs-to-QdrantDocumentStore

Author: davidsbatista

.github/labeler.yml

@@ -174,10 +174,10 @@ integration:stackit:
       - any-glob-to-any-file: "integrations/stackit/**/*"
       - any-glob-to-any-file: ".github/workflows/stackit.yml"
 
-integration:together-ai:
+integration:togetherai:
   - changed-files:
-      - any-glob-to-any-file: "integrations/together_ai/**/*"
-      - any-glob-to-any-file: ".github/workflows/together_ai.yml"
+      - any-glob-to-any-file: "integrations/togetherai/**/*"
+      - any-glob-to-any-file: ".github/workflows/togetherai.yml"
 
 integration:unstructured-fileconverter:
   - changed-files:

.github/utils/deepset_sync.py

@@ -0,0 +1,59 @@
+import re
+import requests
+
+# * integrations/<INTEGRATION_FOLDER_NAME>-v1.0.0
+INTEGRATION_VERSION_REGEX = r"integrations/([a-zA-Z_]+)-v([0-9]\.[0-9]+\.[0-9]+)"
+
+
+def validate_version_number(tag: str):
+    """
+    Verify that a release version number follows semantic versioning rules by checking the latest version on PyPI.
+    """
+
+    matches = re.match(INTEGRATION_VERSION_REGEX, tag)
+    if not matches or len(matches.groups()) != 2:
+        raise ValueError(f"Invalid tag: {tag}")
+
+    integration_name, version_to_release = matches.groups()
+    print(f"Integration name: {integration_name}")
+    print(f"Integration version to release: {version_to_release}")
+
+    # Replace underscores with hyphens to look for the package on PyPi
+    integration_package = f"{integration_name.replace('_', '-')}-haystack"
+    print(f"Integration PyPi package: {integration_package}")
+
+    # connect to PyPi and check the latest version
+    try:
+        response = requests.get(f"https://pypi.org/pypi/{integration_package}/json")
+        response.raise_for_status()
+        latest_version = response.json()["info"]["version"]
+        print(f"Latest version on PyPI: {latest_version}")
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404 and version_to_release in ["0.0.1", "0.1.0", "1.0.0"]:
+            print("Package not found on PyPI. Assuming this is the first release and skipping version check.")
+            return
+        raise e
+
+    x, y, z = [int(i) for i in latest_version.split(".")]
+
+    acceptable_new_versions = [
+        f"{x}.{y}.{z + 1}",
+        f"{x}.{y + 1}.0",
+        f"{x + 1}.0.0",
+    ]
+    if version_to_release not in acceptable_new_versions:
+        msg = (
+            f"Invalid version to release: {version_to_release}. Acceptable new versions are: {acceptable_new_versions}"
+        )
+        raise ValueError(msg)
+    print("The version to release is acceptable.")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tag", help="Git tag in format 'integrations/<name>-v<version>'", required=True, type=str)
+    args = parser.parse_args()
+
+    validate_version_number(args.tag)

.github/utils/validate_version.py

@@ -0,0 +1,122 @@
+name: Core / Sync API reference with Docusaurus
+
+on:
+  push:
+    tags:
+      - "**-v[0-9].[0-9]+.[0-9]+"
+
+  workflow_dispatch: # Activate this workflow manually
+    inputs:
+      tag:
+        description: "Tag with this format: integrations/<INTEGRATION_FOLDER_NAME>-v1.0.0. When running this workflow manually, version is irrelevant so you can use any value."
+        required: true
+        type: string
+        default: integrations/<INTEGRATION_FOLDER_NAME>-v1.0.0
+
+env:
+  TAG: ${{ inputs.tag || github.ref_name }}
+
+jobs:
+  generate-api-reference:
+    runs-on: ubuntu-latest
+    outputs:
+      integration_name: ${{ steps.pathfinder.outputs.integration_name }}
+
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U haystack-pydoc-tools
+
+      - name: Get project folder
+        id: pathfinder
+        shell: python
+        run: |
+          import os
+          project_path = os.environ["TAG"].rsplit("-", maxsplit=1)[0]
+          integration_name = project_path.split("/")[-1]
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+            print(f'project_path={project_path}', file=f)
+            print(f'integration_name={integration_name}', file=f)
+
+      - name: Generate API reference
+        working-directory: ${{ steps.pathfinder.outputs.project_path }}
+
+        run: pydoc-markdown pydoc/config_docusaurus.yml
+
+      - name: Upload API reference artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.pathfinder.outputs.integration_name }}
+          path: ${{ steps.pathfinder.outputs.project_path }}/${{ steps.pathfinder.outputs.integration_name }}.md
+          if-no-files-found: error
+          retention-days: 1
+          overwrite: true
+
+
+  sync-api-reference:
+    runs-on: ubuntu-latest
+    needs: generate-api-reference
+    
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@v5
+        with:
+          repository: deepset-ai/haystack
+          ref: main
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"          
+
+      - name: Download API reference artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: ${{ needs.generate-api-reference.outputs.integration_name }}
+
+      - name: Sync API reference
+        shell: python
+        env:
+          INTEGRATION_NAME: ${{ needs.generate-api-reference.outputs.integration_name }}
+        run: |
+          import os
+          import shutil
+
+          artifact_filename = os.environ['INTEGRATION_NAME']+'.md'
+
+          # Copy to main API reference
+          shutil.copy(artifact_filename, "docs-website/reference/integrations-api/")
+
+          # Copy to versioned API reference
+          for version_dir in os.scandir("docs-website/reference_versioned_docs"):
+              if version_dir.is_dir():
+                  # example: docs-website/reference_versioned_docs/version-2.17/integrations-api
+                  integrations_api_ref_dir = os.path.join(version_dir.path, "integrations-api")
+                  shutil.copy(artifact_filename, integrations_api_ref_dir)
+
+          os.remove(artifact_filename)
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        env:
+          INTEGRATION_NAME: ${{ needs.generate-api-reference.outputs.integration_name }}
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Sync Core Integrations API reference (${{ env.INTEGRATION_NAME }}) on Docusaurus"
+          branch: sync-docusaurus-api-reference-${{ env.INTEGRATION_NAME }}
+          base: main
+          title: "docs: sync Core Integrations API reference (${{ env.INTEGRATION_NAME }}) on Docusaurus"
+          add-paths: |
+            docs-website
+          body: |
+            This PR syncs the Core Integrations API reference (${{ env.INTEGRATION_NAME }}) on Docusaurus. Just approve and merge it.