diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6778b0493a..b0ab927c90 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,12 @@ updates: directory: '/' schedule: interval: 'daily' + cooldown: + default-days: 1 + + - package-ecosystem: 'pip' + directory: '/' + schedule: + interval: 'daily' + cooldown: + default-days: 1 diff --git a/.github/workflows/check_api_ref.yml b/.github/workflows/check_api_ref.yml index 00c671c6b6..a1ed630108 100644 --- a/.github/workflows/check_api_ref.yml +++ b/.github/workflows/check_api_ref.yml @@ -63,13 +63,18 @@ jobs: with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"needs_check={str(needs_check).lower()}\n") - - name: Install Hatch + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 if: steps.changed.outputs.needs_check == 'true' - run: pip install hatch + with: + python-version: "3.13" + + - name: Sync dev dependencies + if: steps.changed.outputs.needs_check == 'true' + run: uv sync --group dev - name: Generate API references if: steps.changed.outputs.needs_check == 'true' - run: hatch run docs + run: uv run haystack-pydoc pydoc tmp_api_reference - name: Set up Node.js if: steps.changed.outputs.needs_check == 'true' diff --git a/.github/workflows/docs-website-test-docs-snippets.yml b/.github/workflows/docs-website-test-docs-snippets.yml index 16864c84dd..1fd7445d8e 100644 --- a/.github/workflows/docs-website-test-docs-snippets.yml +++ b/.github/workflows/docs-website-test-docs-snippets.yml @@ -12,7 +12,6 @@ on: type: string env: - HATCH_VERSION: "1.16.5" PYTHON_VERSION: "3.11" jobs: @@ -32,26 +31,18 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Setup Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - python-version: '3.11' + python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync dev and test dependencies + run: uv sync --group dev --group test - name: Generate API reference for Docusaurus - run: hatch run docs - - - name: Install base dependencies - run: | - python -m pip install --upgrade pip - pip install requests toml + run: uv run haystack-pydoc pydoc tmp_api_reference - name: Run snippet tests (verbose) - shell: bash - run: | - hatch -e test env run -- python docs-website/scripts/test_python_snippets.py --verbose tmp_api_reference/ + run: uv run --with requests python docs-website/scripts/test_python_snippets.py --verbose tmp_api_reference/ notify-slack-on-failure: if: failure() && github.ref_name == 'main' diff --git a/.github/workflows/docusaurus_sync.yml b/.github/workflows/docusaurus_sync.yml index 7608187fcd..ac5f314109 100644 --- a/.github/workflows/docusaurus_sync.yml +++ b/.github/workflows/docusaurus_sync.yml @@ -11,7 +11,6 @@ on: - ".github/workflows/docusaurus_sync.yml" env: - HATCH_VERSION: "1.16.5" PYTHON_VERSION: "3.11" jobs: @@ -24,16 +23,15 @@ jobs: - name: Checkout Haystack repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync dev dependencies + run: uv sync --group dev - name: Generate API reference for Docusaurus - run: hatch run docs + run: uv run haystack-pydoc pydoc tmp_api_reference - name: Sync generated API reference to docs folder run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9cae97d6db..01b2d8d572 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -17,7 +17,6 @@ on: env: PYTHON_VERSION: "3.10" OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HATCH_VERSION: "1.16.5" # we use HF_TOKEN instead of HF_API_TOKEN to work around a Hugging Face bug # see https://github.com/deepset-ai/haystack/issues/9552 HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} @@ -29,15 +28,15 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync e2e dependencies + run: uv sync --group e2e - name: Run tests - run: hatch run e2e:test + run: uv run pytest e2e - name: Notify Slack on nightly failure if: failure() && github.event_name == 'schedule' diff --git a/.github/workflows/nightly_testpypi_release.yml b/.github/workflows/nightly_testpypi_release.yml index 7f2a5e610a..5d7b99832b 100644 --- a/.github/workflows/nightly_testpypi_release.yml +++ b/.github/workflows/nightly_testpypi_release.yml @@ -6,9 +6,6 @@ on: - cron: "0 0 * * *" workflow_dispatch: -env: - HATCH_VERSION: "1.16.5" - jobs: nightly-release: runs-on: ubuntu-latest @@ -35,11 +32,10 @@ jobs: echo "${NIGHTLY_VERSION}" > VERSION.txt echo "Building haystack-ai version: ${NIGHTLY_VERSION}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Build Haystack - run: hatch build + run: uv build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/pypi_release.yml b/.github/workflows/pypi_release.yml index ef92315463..5c1e854184 100644 --- a/.github/workflows/pypi_release.yml +++ b/.github/workflows/pypi_release.yml @@ -7,9 +7,6 @@ on: # We must not release versions tagged with -rc0 suffix - "!v[0-9]+.[0-9]+.[0-9]-rc0" -env: - HATCH_VERSION: "1.16.5" - jobs: release-on-pypi: runs-on: ubuntu-latest @@ -21,11 +18,10 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Build Haystack - run: hatch build + run: uv build - name: Publish on PyPi uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 79081c031f..c872b3023f 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -14,7 +14,6 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} PYTHON_VERSION: "3.10" - HATCH_VERSION: "1.16.5" HAYSTACK_MPS_ENABLED: false HAYSTACK_XPU_ENABLED: false @@ -131,15 +130,12 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch - shell: bash - run: | - pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync test dependencies + run: uv sync --group test - name: Run Tika if: matrix.os == 'ubuntu-latest' @@ -151,7 +147,7 @@ jobs: run: ${{ matrix.install_cmd }} - name: Run tests - run: hatch run test:integration-only-slow + run: uv run pytest --maxfail=5 -m "integration and slow" test - name: Notify Slack on nightly failure if: failure() && github.event_name == 'schedule' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index daddef7659..9f00aeeaad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,6 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} PYTHON_VERSION: "3.10" - HATCH_VERSION: "1.16.5" jobs: check-if-changed: @@ -73,15 +72,15 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync dev dependencies + run: uv sync --group dev - name: Ruff - check format and linting - run: hatch run fmt-check + run: uv run ruff check && uv run ruff format --check - name: Check presence of license header run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check @@ -92,15 +91,15 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - run: pip install hatch==${{ env.HATCH_VERSION }} + - name: Sync project + run: uv sync - name: Check imports - run: hatch run python .github/utils/check_imports.py + run: uv run python .github/utils/check_imports.py unit-tests: name: Unit / ${{ matrix.os }} @@ -117,25 +116,21 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch - shell: bash - run: | - pip install hatch==${{ env.HATCH_VERSION }} - echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" + - name: Sync test dependencies + run: uv sync --group test - name: Run - run: hatch run test:unit + run: uv run pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 id: cache if: matrix.os == 'macos-latest' with: - path: ${{ steps.hatch.outputs.env }} + path: .venv key: ${{ runner.os }}-${{ github.sha }} - name: Coveralls @@ -168,23 +163,21 @@ jobs: test/** .github/** scripts/** - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 if: steps.files.outputs.any_changed == 'true' with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch + - name: Sync test dependencies if: steps.files.outputs.any_changed == 'true' - run: | - pip install hatch==${{ env.HATCH_VERSION }} - echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" + run: uv sync --group test - name: Mypy if: steps.files.outputs.any_changed == 'true' run: | mkdir .mypy_cache - hatch run test:types + uv run mypy --install-types --non-interactive --cache-dir=.mypy_cache/ haystack test/core/ test/marshal/ test/testing/ test/tracing/ integration-tests-linux: name: Integration / ubuntu-latest @@ -194,20 +187,15 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch - shell: bash - run: | - pip install hatch==${{ env.HATCH_VERSION }} - echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" - + - name: Sync test dependencies + run: uv sync --group test - name: Run - run: hatch run test:integration-only-fast + run: uv run pytest --maxfail=5 -m "integration and not slow" test integration-tests-macos: name: Integration / macos-latest @@ -220,26 +208,21 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch - shell: bash - run: | - pip install hatch==${{ env.HATCH_VERSION }} - echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 id: cache with: - path: ${{ steps.hatch.outputs.env }} + path: .venv key: ${{ runner.os }}-${{ github.sha }} + - name: Sync test dependencies + run: uv sync --group test - name: Run - run: hatch run test:integration-only-fast + run: uv run pytest --maxfail=5 -m "integration and not slow" test integration-tests-windows: name: Integration / windows-latest @@ -252,19 +235,15 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "${{ env.PYTHON_VERSION }}" - - name: Install Hatch - id: hatch - shell: bash - run: | - pip install hatch==${{ env.HATCH_VERSION }} - echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT" + - name: Sync test dependencies + run: uv sync --group test - name: Run - run: hatch run test:integration-only-fast + run: uv run pytest --maxfail=5 -m "integration and not slow" test notify-slack-on-failure: if: failure() && github.ref_name == 'main' diff --git a/AGENTS.md b/AGENTS.md index f7dcb2afc2..794f2f96fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,21 +2,27 @@ ## Environment -Haystack uses **Hatch** for environment and dependency management. +Haystack uses **uv** for environment and dependency management in CI and for local development. Do not run `python` or `pip` directly. -Before running code on this project, you must be able to run `hatch --version` and get a correct output. +Before running code on this project, you must be able to run `uv --version` and get a correct output. -If not, ask the user where Hatch is or if they want to install it. For installation instructions, refer to https://hatch.pypa.io/latest/install/#installation. +If not, install uv by following https://docs.astral.sh/uv/getting-started/installation/. -### Run scripts with test dependencies +### Sync dependencies and run a script -hatch -e test run python SCRIPT.py +uv run python SCRIPT.py + +### Sync test dependencies and run a script + +uv sync --group test +uv run python SCRIPT.py ### Open a shell with test dependencies -hatch -e test shell +uv sync --group test +source .venv/bin/activate ### Install temporary dependencies (for experiments only) @@ -24,36 +30,41 @@ uv pip install PACKAGE ### Delete the environment -hatch env prune +rm -rf .venv ## Tests -Tests run via Hatch and support pytest arguments. +Tests run via uv and support pytest arguments. Prefer running tests on a specific module or using `-k`, since the full suite is large. ### Run unit tests -hatch run test:unit +uv sync --group test +uv run pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test ### Run integration tests -hatch run test:integration +uv sync --group test +uv run pytest --maxfail=5 -m "integration" test ## Quality Checks ### Type checking with mypy -hatch run test:types +uv sync --group test +uv run mypy --install-types --non-interactive --cache-dir=.mypy_cache/ haystack test/core/ test/marshal/ test/testing/ test/tracing/ To fix type issues, avoid `type: ignore`, casts, or assertions when possible. If they are necessary, explain why. ### Format and lint -hatch run fmt +uv sync --group dev +uv run ruff check --fix && uv run ruff format ## Release Notes Every user-facing PR (not docs, not CI) must include a release note: -hatch run release-note SHORT_DESCRIPTION +uv sync --group dev +uv run reno new SHORT_DESCRIPTION Edit the generated file in `releasenotes/notes/`. diff --git a/pyproject.toml b/pyproject.toml index 40a549bc96..f72be9fd42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -363,5 +363,101 @@ ignore = [ "test/tools/test_parameters_schema_utils.py" = ["UP007"] "test/utils/test_type_serialization.py" = ["UP006", "UP007", "UP035", "UP045"] +[dependency-groups] +dev = [ + "pre-commit", + "ruff>=0.15.0", # RUF104 (unmatched-suppression-comment) added in 0.15.0 + "toml", + "reno", + # dulwich is a reno dependency, they pin it at >=0.15.0 so pip takes ton of time to resolve the dependency tree. + # We pin it here to avoid taking too much time. + # https://opendev.org/openstack/reno/src/branch/master/requirements.txt#L7 + "dulwich>=0.21.0,<1.0.0", + "haystack-pydoc-tools", +] +test = [ + "numpy>=2", # Haystack is compatible both with numpy 1.x and 2.x, but we test with 2.x + "numba>=0.54.0", # This pin helps uv resolve the dependency tree. See https://github.com/astral-sh/uv/issues/7881 + + "pandas", # AzureOCRDocumentConverter, CSVDocumentCleaner, CSVDocumentSplitter, + # EvaluationRunResult, XLSXToDocument, and pipeline tests + + "transformers[torch, sentencepiece]>=4.57", # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators... + "huggingface_hub>=0.27.0", # Hugging Face API Generators and Embedders + "sentence-transformers>=5.0.0", # Sentence Transformers Embedders, Rankers, and SASEvaluator + "langdetect", # TextLanguageRouter and DocumentLanguageClassifier + "openai-whisper>=20231106", # LocalWhisperTranscriber + "arrow>=1.3.0", # Jinja2TimeExtension + "pillow", # ImageContent + "pypdfium2", # PDFToImageContent + + # Converters + "pypdf", # PyPDFToDocument + "pdfminer.six", # PDFMinerToDocument + "markdown-it-py", # MarkdownToDocument + "mdit_plain", # MarkdownToDocument + "tika", # TikaDocumentConverter + "azure-ai-formrecognizer>=3.2.0b2", # AzureOCRDocumentConverter + "trafilatura", # HTMLToDocument + "python-pptx", # PPTXToDocument + "python-docx", # DocxToDocument + "jq", # JSONConverter + "openpyxl", # XLSXToDocument + "tabulate", # XLSXToDocument + "python-oxmsg", # MSGToDocument + + "nltk>=3.9.1", # NLTKDocumentSplitter, RecursiveDocumentSplitter + "tiktoken", # RecursiveDocumentSplitter + + # Human in the Loop + "rich", # Console rendering for HITL + + # OpenAPI + "jsonref", # OpenAPIServiceConnector, OpenAPIServiceToFunctions + "openapi3", + "openapi-llm>=0.4.1", # OpenAPIConnector + + # Tracing + "opentelemetry-sdk", + "ddtrace", + + # Structured logging + "structlog", + + # needed in link content fetcher tests + "httpx[http2]", + + # Azure Utils + "azure-identity", + + # Test + "pytest", + "pytest-bdd", + "pytest-cov", + "pytest-asyncio", + "pytest-rerunfailures", + "coverage", + "mypy", + "pip", # mypy needs pip to install missing stub packages + "ipython", + "colorama==0.4.6", # Pipeline checkpoints test +] +e2e = [ + {include-group = "test"}, + # NamedEntityExtractor + "spacy>=3.8.13,<3.9", + "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.8.0/en_core_web_trf-3.8.0-py3-none-any.whl", +] + +[tool.uv] +# Relative duration strings in exclude-newer require uv >= 0.9.17. +required-version = ">=0.9.17" +# Exclude package versions published within the last 24 hours to protect against supply chain +# attacks via compromised dependencies. uv resolves this relative to the current clock at +# install/lock time, so no manual date updates are needed. +# First-party packages are exempted so freshly published releases are always resolvable. +exclude-newer = "24 hours" +exclude-newer-package = { haystack-experimental = "0 days", haystack-pydoc-tools = "0 days" } + [tool.coverage.run] omit = ["haystack/testing/*"]