haystack-core-integrations/.github/workflows/CI_check_api_ref.yml at 514dfb922390be9c0c11fbbe069be91c30adc39a · deepset-ai/haystack-core-integrations · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Core / Check API reference changes

on:
  pull_request:
    paths:
      - "integrations/**/*.py"
      - "integrations/**/config_docusaurus.yml"

jobs:
  detect-changes:
    runs-on: ubuntu-slim
    outputs:
      integrations: ${{ steps.changed.outputs.integrations }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Detect integrations with API reference changes
        id: changed
        shell: python
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          import json
          import os
          import subprocess
          import sys
          from pathlib import Path

          sys.path.insert(0, ".github/utils")
          from docstrings_checksum import docstrings_checksum

          base_sha = os.environ["BASE_SHA"]
          runner_temp = os.environ["RUNNER_TEMP"]

          def git(*args):
              result = subprocess.run(["git", *args], capture_output=True, text=True)
              return result.stdout.strip(), result.returncode

          # Get all changed files
          diff_output, _ = git(
              "diff", "--name-only", f"{base_sha}...HEAD", "--", "integrations"
          )
          changed_files = set(diff_output.splitlines())

          # Extract integration names
          candidates = sorted({
              Path(p).parts[1]
              for p in changed_files
              if p.startswith("integrations/") and len(Path(p).parts) > 1
          })

          # Create base worktree for docstring comparison
          base_worktree = os.path.join(runner_temp, "base")
          _, return_code = git("worktree", "add", base_worktree, base_sha)
          has_worktree = return_code == 0

          changed_integrations = []

          for integration in candidates:
              # If pydoc config changed, always include
              if f"integrations/{integration}/pydoc/config_docusaurus.yml" in changed_files:
                  changed_integrations.append(integration)
                  continue

              # Otherwise, check if docstrings changed
              pr_docstrings_checksum = docstrings_checksum(Path(".").glob(f"integrations/{integration}/**/*.py"))
              base_docstrings_checksum = ""
              if has_worktree:
                  base_docstrings_checksum = docstrings_checksum(Path(base_worktree).glob(f"integrations/{integration}/**/*.py"))

              if base_docstrings_checksum != pr_docstrings_checksum:
                  changed_integrations.append(integration)

          if has_worktree:
              git("worktree", "remove", base_worktree)

          print(f"Integrations with API reference changes: {json.dumps(changed_integrations)}")

          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
              f.write(f"integrations={json.dumps(changed_integrations)}\n")

  test-api-reference-build:
    needs: detect-changes
    if: needs.detect-changes.outputs.integrations != '[]'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        integration: ${{ fromJson(needs.detect-changes.outputs.integrations) }}
    steps:
      - uses: actions/checkout@v6

      - name: Set up Python 3.13
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Install Hatch
        run: pip install --upgrade hatch

      - name: Generate API reference
        working-directory: integrations/${{ matrix.integration }}
        run: |
          hatch run docs
          # Save the generated file before the Haystack checkout overwrites the working tree
          cp ${{ matrix.integration }}.md "${{ runner.temp }}/${{ matrix.integration }}.md"
          # Also copy the Copy script to the temp directory
          cp ../../.github/utils/copy_file_to_api_reference.py "${{ runner.temp }}/copy_file_to_api_reference.py"

      - name: Checkout Haystack Docs Website
        uses: actions/checkout@v6
        with:
          repository: deepset-ai/haystack
          ref: main
          sparse-checkout: docs-website

      - name: Copy file to API reference
        run: python ${{ runner.temp }}/copy_file_to_api_reference.py ${{ runner.temp }}/${{ matrix.integration }}.md
      - name: Install Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "22"

      - name: Build docs-website
        working-directory: docs-website
        run: |
          npm install
          npm run build