From 36dbd7180476cb9cfb49cd3a320c0f004c3d96d0 Mon Sep 17 00:00:00 2001 From: bilgeyucel Date: Fri, 26 Jun 2026 19:04:21 +0200 Subject: [PATCH 1/5] chore: add Haystack Enterprise Components docs page and auto-update workflow --- .../workflows/update-platform-components.yml | 102 +++ .../docs/overview/platform-components.mdx | 514 ++++++++++++++ ...rieval.mdx => sentencewindowretriever.mdx} | 4 +- docs-website/sidebars.js | 3 +- ...components-docs-page-36725b437280b21d.yaml | 7 + scripts/generate_platform_components_table.py | 396 +++++++++++ scripts/tests/__init__.py | 0 ...test_generate_platform_components_table.py | 629 ++++++++++++++++++ 8 files changed, 1652 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/update-platform-components.yml create mode 100644 docs-website/docs/overview/platform-components.mdx rename docs-website/docs/pipeline-components/retrievers/{sentencewindowretrieval.mdx => sentencewindowretriever.mdx} (98%) create mode 100644 releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml create mode 100644 scripts/generate_platform_components_table.py create mode 100644 scripts/tests/__init__.py create mode 100644 scripts/tests/test_generate_platform_components_table.py diff --git a/.github/workflows/update-platform-components.yml b/.github/workflows/update-platform-components.yml new file mode 100644 index 00000000000..ee01a281ff3 --- /dev/null +++ b/.github/workflows/update-platform-components.yml @@ -0,0 +1,102 @@ +name: Update Platform Components Table + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +concurrency: + group: update-platform-components + cancel-in-progress: false + +env: + PYTHON_VERSION: "3.11" + OUTPUT_FILE: docs-website/docs/overview/platform-components.mdx + +permissions: + contents: read + +jobs: + update: + runs-on: ubuntu-slim + + steps: + - name: Checkout deepset-ai/haystack + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + # Full history so git diff against HEAD works correctly + fetch-depth: 0 + + - name: Checkout deepset-ai/haystack-core-integrations + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: deepset-ai/haystack-core-integrations + path: haystack-core-integrations + fetch-depth: 1 + sparse-checkout: integrations + sparse-checkout-cone-mode: true + + - name: Checkout platform repo (sparse — schema file only) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ secrets.SCHEMA_REPO }} + token: ${{ secrets.PLATFORM_REPO_PAT }} + path: platform-repo + sparse-checkout: | + ${{ secrets.SCHEMA_REPO_PATH }} + sparse-checkout-cone-mode: false + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Generate platform components table + run: | + python scripts/generate_platform_components_table.py \ + --haystack-src . \ + --integrations-src ./haystack-core-integrations \ + --schema ./platform-repo/${{ secrets.SCHEMA_REPO_PATH }} \ + --output ./${{ env.OUTPUT_FILE }} + + - name: Compute PR body + id: pr_body + run: | + FILE="${{ env.OUTPUT_FILE }}" + if git ls-files --error-unmatch "$FILE" 2>/dev/null; then + ADDED=$(git diff HEAD -- "$FILE" | grep -cE '^\+\| `' || true) + REMOVED=$(git diff HEAD -- "$FILE" | grep -cE '^\-\| `' || true) + SUMMARY="Component rows changed: **+${ADDED}** added, **-${REMOVED}** removed." + else + SUMMARY="Initial generation of the platform components table." + fi + + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Open or update pull request + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + commit-message: "chore: update platform components table [skip ci]" + branch: chore/update-platform-components + base: main + title: "chore: update platform components table" + add-paths: | + ${{ env.OUTPUT_FILE }} + body: ${{ steps.pr_body.outputs.body }} + labels: | + documentation + ignore-for-release-notes diff --git a/docs-website/docs/overview/platform-components.mdx b/docs-website/docs/overview/platform-components.mdx new file mode 100644 index 00000000000..ecf3447d643 --- /dev/null +++ b/docs-website/docs/overview/platform-components.mdx @@ -0,0 +1,514 @@ +--- +title: "Haystack Enterprise Components" +id: platform-components +slug: "/platform-components" +description: "A complete list of Haystack components available on the Haystack Enterprise Platform, grouped by integration partner." +--- + +# Haystack Enterprise Components + +The Haystack Enterprise Platform currently supports **224 components** and **55 integrations**. The following table lists them grouped by integration partner. + +## Core Components + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [Agent](https://docs.haystack.deepset.ai/docs/agent) | Component | ✅ Available | +| [AnswerBuilder](https://docs.haystack.deepset.ai/docs/answerbuilder) | Builder | ✅ Available | +| [AnswerJoiner](https://docs.haystack.deepset.ai/docs/answerjoiner) | Joiner | ✅ Available | +| [AutoMergingRetriever](https://docs.haystack.deepset.ai/docs/automergingretriever) | Retriever | ✅ Available | +| [AzureOCRDocumentConverter](https://docs.haystack.deepset.ai/docs/azureocrdocumentconverter) | Converter | ✅ Available | +| [AzureOpenAIChatGenerator](https://docs.haystack.deepset.ai/docs/azureopenaichatgenerator) | Generator | ✅ Available | +| [AzureOpenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/azureopenaidocumentembedder) | Embedder | ✅ Available | +| [AzureOpenAIResponsesChatGenerator](https://docs.haystack.deepset.ai/docs/azureopenairesponseschatgenerator) | Generator | ✅ Available | +| [AzureOpenAITextEmbedder](https://docs.haystack.deepset.ai/docs/azureopenaitextembedder) | Embedder | ✅ Available | +| [BranchJoiner](https://docs.haystack.deepset.ai/docs/branchjoiner) | Joiner | ✅ Available | +| [CacheChecker](https://docs.haystack.deepset.ai/docs/cachechecker) | Component | ✅ Available | +| [ChatPromptBuilder](https://docs.haystack.deepset.ai/docs/chatpromptbuilder) | Builder | ✅ Available | +| [ConditionalRouter](https://docs.haystack.deepset.ai/docs/conditionalrouter) | Router | ✅ Available | +| [CSVDocumentCleaner](https://docs.haystack.deepset.ai/docs/csvdocumentcleaner) | Preprocessor | ✅ Available | +| [CSVDocumentSplitter](https://docs.haystack.deepset.ai/docs/csvdocumentsplitter) | Preprocessor | ✅ Available | +| [CSVToDocument](https://docs.haystack.deepset.ai/docs/csvtodocument) | Converter | ✅ Available | +| [DocumentCleaner](https://docs.haystack.deepset.ai/docs/documentcleaner) | Preprocessor | ✅ Available | +| [DocumentJoiner](https://docs.haystack.deepset.ai/docs/documentjoiner) | Joiner | ✅ Available | +| [DocumentLanguageClassifier](https://docs.haystack.deepset.ai/docs/documentlanguageclassifier) | Classifier | ✅ Available | +| [DocumentLengthRouter](https://docs.haystack.deepset.ai/docs/documentlengthrouter) | Router | ✅ Available | +| [DocumentPreprocessor](https://docs.haystack.deepset.ai/docs/documentpreprocessor) | Preprocessor | ✅ Available | +| [DocumentSplitter](https://docs.haystack.deepset.ai/docs/documentsplitter) | Preprocessor | ✅ Available | +| [DocumentToImageContent](https://docs.haystack.deepset.ai/docs/documenttoimagecontent) | Converter | ✅ Available | +| [DocumentTypeRouter](https://docs.haystack.deepset.ai/docs/documenttyperouter) | Router | ✅ Available | +| [DocumentWriter](https://docs.haystack.deepset.ai/docs/documentwriter) | Writer | ✅ Available | +| [DOCXToDocument](https://docs.haystack.deepset.ai/docs/docxtodocument) | Converter | ✅ Available | +| [EmbeddingBasedDocumentSplitter](https://docs.haystack.deepset.ai/docs/embeddingbaseddocumentsplitter) | Preprocessor | ✅ Available | +| [ExtractiveReader](https://docs.haystack.deepset.ai/docs/extractivereader) | Reader | ✅ Available | +| [FallbackChatGenerator](https://docs.haystack.deepset.ai/docs/fallbackchatgenerator) | Generator | ✅ Available | +| [FileToFileContent](https://docs.haystack.deepset.ai/docs/filetofilecontent) | Converter | ✅ Available | +| [FileTypeRouter](https://docs.haystack.deepset.ai/docs/filetyperouter) | Router | ✅ Available | +| [FilterRetriever](https://docs.haystack.deepset.ai/docs/filterretriever) | Retriever | ✅ Available | +| [HierarchicalDocumentSplitter](https://docs.haystack.deepset.ai/docs/hierarchicaldocumentsplitter) | Preprocessor | ✅ Available | +| [HTMLToDocument](https://docs.haystack.deepset.ai/docs/htmltodocument) | Converter | ✅ Available | +| [HuggingFaceAPIChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapichatgenerator) | Generator | ✅ Available | +| [HuggingFaceAPIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapidocumentembedder) | Embedder | ✅ Available | +| [HuggingFaceAPITextEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapitextembedder) | Embedder | ✅ Available | +| [HuggingFaceLocalChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfacelocalchatgenerator) | Generator | ✅ Available | +| [HuggingFaceTEIRanker](https://docs.haystack.deepset.ai/docs/huggingfaceteiranker) | Ranker | ✅ Available | +| [ImageFileToDocument](https://docs.haystack.deepset.ai/docs/imagefiletodocument) | Converter | ✅ Available | +| [ImageFileToImageContent](https://docs.haystack.deepset.ai/docs/imagefiletoimagecontent) | Converter | ✅ Available | +| [JSONConverter](https://docs.haystack.deepset.ai/docs/jsonconverter) | Converter | ✅ Available | +| [JsonSchemaValidator](https://docs.haystack.deepset.ai/docs/jsonschemavalidator) | Validator | ✅ Available | +| [LinkContentFetcher](https://docs.haystack.deepset.ai/docs/linkcontentfetcher) | Fetcher | ✅ Available | +| [ListJoiner](https://docs.haystack.deepset.ai/docs/listjoiner) | Joiner | ✅ Available | +| [LLMDocumentContentExtractor](https://docs.haystack.deepset.ai/docs/llmdocumentcontentextractor) | Extractor | ✅ Available | +| [LLMMessagesRouter](https://docs.haystack.deepset.ai/docs/llmmessagesrouter) | Router | ✅ Available | +| [LLMMetadataExtractor](https://docs.haystack.deepset.ai/docs/llmmetadataextractor) | Extractor | ✅ Available | +| [LLMRanker](https://docs.haystack.deepset.ai/docs/llmranker) | Ranker | ✅ Available | +| [LostInTheMiddleRanker](https://docs.haystack.deepset.ai/docs/lostinthemiddleranker) | Ranker | ✅ Available | +| [MarkdownHeaderSplitter](https://docs.haystack.deepset.ai/docs/markdownheadersplitter) | Preprocessor | ✅ Available | +| [MarkdownToDocument](https://docs.haystack.deepset.ai/docs/markdowntodocument) | Converter | ✅ Available | +| [MetadataRouter](https://docs.haystack.deepset.ai/docs/metadatarouter) | Router | ✅ Available | +| [MetaFieldGroupingRanker](https://docs.haystack.deepset.ai/docs/metafieldgroupingranker) | Ranker | ✅ Available | +| [MetaFieldRanker](https://docs.haystack.deepset.ai/docs/metafieldranker) | Ranker | ✅ Available | +| [MSGToDocument](https://docs.haystack.deepset.ai/docs/msgtodocument) | Converter | ✅ Available | +| [MultiFileConverter](https://docs.haystack.deepset.ai/docs/multifileconverter) | Converter | ✅ Available | +| [MultiQueryEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/multiqueryembeddingretriever) | Retriever | ✅ Available | +| [MultiQueryTextRetriever](https://docs.haystack.deepset.ai/docs/multiquerytextretriever) | Retriever | ✅ Available | +| [MultiRetriever](https://docs.haystack.deepset.ai/docs/multiretriever) | Retriever | ✅ Available | +| [NamedEntityExtractor](https://docs.haystack.deepset.ai/docs/namedentityextractor) | Extractor | ✅ Available | +| [OpenAIChatGenerator](https://docs.haystack.deepset.ai/docs/openaichatgenerator) | Generator | ✅ Available | +| [OpenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/openaidocumentembedder) | Embedder | ✅ Available | +| [OpenAIResponsesChatGenerator](https://docs.haystack.deepset.ai/docs/openairesponseschatgenerator) | Generator | ✅ Available | +| [OpenAITextEmbedder](https://docs.haystack.deepset.ai/docs/openaitextembedder) | Embedder | ✅ Available | +| [OpenAPIConnector](https://docs.haystack.deepset.ai/docs/openapiconnector) | Connector | ✅ Available | +| [OpenAPIServiceConnector](https://docs.haystack.deepset.ai/docs/openapiserviceconnector) | Connector | ✅ Available | +| [OpenAPIServiceToFunctions](https://docs.haystack.deepset.ai/docs/openapiservicetofunctions) | Converter | ✅ Available | +| [OutputAdapter](https://docs.haystack.deepset.ai/docs/outputadapter) | Converter | ✅ Available | +| [PDFMinerToDocument](https://docs.haystack.deepset.ai/docs/pdfminertodocument) | Converter | ✅ Available | +| [PDFToImageContent](https://docs.haystack.deepset.ai/docs/pdftoimagecontent) | Converter | ✅ Available | +| [PPTXToDocument](https://docs.haystack.deepset.ai/docs/pptxtodocument) | Converter | ✅ Available | +| [PromptBuilder](https://docs.haystack.deepset.ai/docs/promptbuilder) | Builder | ✅ Available | +| [PyPDFToDocument](https://docs.haystack.deepset.ai/docs/pypdftodocument) | Converter | ✅ Available | +| [PythonCodeSplitter](https://docs.haystack.deepset.ai/docs/pythoncodesplitter) | Preprocessor | ✅ Available | +| [QueryExpander](https://docs.haystack.deepset.ai/docs/queryexpander) | Component | ✅ Available | +| [RecursiveDocumentSplitter](https://docs.haystack.deepset.ai/docs/recursivesplitter) | Preprocessor | ✅ Available | +| [RegexTextExtractor](https://docs.haystack.deepset.ai/docs/regextextextractor) | Extractor | ✅ Available | +| [RemoteWhisperTranscriber](https://docs.haystack.deepset.ai/docs/remotewhispertranscriber) | Audio | ✅ Available | +| [SearchApiWebSearch](https://docs.haystack.deepset.ai/docs/searchapiwebsearch) | Component | ✅ Available | +| [SentenceTransformersDiversityRanker](https://docs.haystack.deepset.ai/docs/sentencetransformersdiversityranker) | Ranker | ✅ Available | +| [SentenceTransformersDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder) | Embedder | ✅ Available | +| [SentenceTransformersDocumentImageEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentimageembedder) | Embedder | ✅ Available | +| [SentenceTransformersSimilarityRanker](https://docs.haystack.deepset.ai/docs/sentencetransformerssimilarityranker) | Ranker | ✅ Available | +| [SentenceTransformersSparseDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerssparsedocumentembedder) | Embedder | ✅ Available | +| [SentenceTransformersSparseTextEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerssparsetextembedder) | Embedder | ✅ Available | +| [SentenceTransformersTextEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerstextembedder) | Embedder | ✅ Available | +| [SentenceWindowRetriever](https://docs.haystack.deepset.ai/docs/sentencewindowretriever) | Retriever | ✅ Available | +| [SerperDevWebSearch](https://docs.haystack.deepset.ai/docs/serperdevwebsearch) | Component | ✅ Available | +| [StringJoiner](https://docs.haystack.deepset.ai/docs/stringjoiner) | Joiner | ✅ Available | +| [TextCleaner](https://docs.haystack.deepset.ai/docs/textcleaner) | Preprocessor | ✅ Available | +| [TextEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/textembeddingretriever) | Retriever | ✅ Available | +| [TextFileToDocument](https://docs.haystack.deepset.ai/docs/textfiletodocument) | Converter | ✅ Available | +| [TextLanguageRouter](https://docs.haystack.deepset.ai/docs/textlanguagerouter) | Router | ✅ Available | +| [TikaDocumentConverter](https://docs.haystack.deepset.ai/docs/tikadocumentconverter) | Converter | ✅ Available | +| [ToolInvoker](https://docs.haystack.deepset.ai/docs/toolinvoker) | Component | ✅ Available | +| [TopPSampler](https://docs.haystack.deepset.ai/docs/toppsampler) | Sampler | ✅ Available | +| [TransformersSimilarityRanker](https://docs.haystack.deepset.ai/docs/transformerssimilarityranker) | Ranker | ✅ Available | +| [TransformersTextRouter](https://docs.haystack.deepset.ai/docs/transformerstextrouter) | Router | ✅ Available | +| [TransformersZeroShotDocumentClassifier](https://docs.haystack.deepset.ai/docs/transformerszeroshotdocumentclassifier) | Classifier | ✅ Available | +| [TransformersZeroShotTextRouter](https://docs.haystack.deepset.ai/docs/transformerszeroshottextrouter) | Router | ✅ Available | +| [XLSXToDocument](https://docs.haystack.deepset.ai/docs/xlsxtodocument) | Converter | ✅ Available | + +## AIML API + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AIMLAPIChatGenerator](https://docs.haystack.deepset.ai/docs/aimllapichatgenerator) | Generator | ✅ Available | + +## AlloyDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AlloyDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/alloydbembeddingretriever) | Retriever | ✅ Available | +| [AlloyDBKeywordRetriever](https://docs.haystack.deepset.ai/docs/alloydbkeywordretriever) | Retriever | ✅ Available | + +## Amazon Bedrock + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AmazonBedrockChatGenerator](https://docs.haystack.deepset.ai/docs/amazonbedrockchatgenerator) | Generator | ✅ Available | +| [AmazonBedrockDocumentEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrockdocumentembedder) | Embedder | ✅ Available | +| [AmazonBedrockDocumentImageEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrockdocumentimageembedder) | Embedder | ✅ Available | +| [AmazonBedrockRanker](https://docs.haystack.deepset.ai/docs/amazonbedrockranker) | Ranker | ✅ Available | +| [AmazonBedrockTextEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrocktextembedder) | Embedder | ✅ Available | + +## Amazon S3 + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [S3Downloader](https://docs.haystack.deepset.ai/docs/s3downloader) | Component | ✅ Available | + +## Amazon Textract + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AmazonTextractConverter](https://docs.haystack.deepset.ai/docs/amazontextractconverter) | Converter | ✅ Available | + +## Anthropic + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AnthropicChatGenerator](https://docs.haystack.deepset.ai/docs/anthropicchatgenerator) | Generator | ✅ Available | +| AnthropicFoundryChatGenerator | Generator | ✅ Available | +| [AnthropicVertexChatGenerator](https://docs.haystack.deepset.ai/docs/anthropicvertexchatgenerator) | Generator | ✅ Available | + +## ArcadeDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ArcadeDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/arcadedbembeddingretriever) | Retriever | ✅ Available | + +## Astra + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AstraEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/astraretriever) | Retriever | ✅ Available | + +## Azure AI Search + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AzureAISearchBM25Retriever](https://docs.haystack.deepset.ai/docs/azureaisearchbm25retriever) | Retriever | ✅ Available | +| [AzureAISearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/azureaisearchembeddingretriever) | Retriever | ✅ Available | +| [AzureAISearchHybridRetriever](https://docs.haystack.deepset.ai/docs/azureaisearchhybridretriever) | Retriever | ✅ Available | + +## Azure Document Intelligence + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AzureDocumentIntelligenceConverter](https://docs.haystack.deepset.ai/docs/azuredocumentintelligenceconverter) | Converter | ✅ Available | + +## Brave + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [BraveWebSearch](https://docs.haystack.deepset.ai/docs/bravewebsearch) | Component | ✅ Available | + +## Chroma + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ChromaEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/chromaembeddingretriever) | Retriever | ✅ Available | +| [ChromaQueryTextRetriever](https://docs.haystack.deepset.ai/docs/chromaqueryretriever) | Retriever | ✅ Available | + +## Cohere + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [CohereChatGenerator](https://docs.haystack.deepset.ai/docs/coherechatgenerator) | Generator | ✅ Available | +| [CohereDocumentEmbedder](https://docs.haystack.deepset.ai/docs/coheredocumentembedder) | Embedder | ✅ Available | +| [CohereRanker](https://docs.haystack.deepset.ai/docs/cohereranker) | Ranker | ✅ Available | +| [CohereTextEmbedder](https://docs.haystack.deepset.ai/docs/coheretextembedder) | Embedder | ✅ Available | + +## Docling + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [DoclingConverter](https://docs.haystack.deepset.ai/docs/doclingconverter) | Converter | ✅ Available | +| [DoclingServeConverter](https://docs.haystack.deepset.ai/docs/doclingserveconverter) | Converter | ✅ Available | + +## Elasticsearch + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ElasticsearchBM25Retriever](https://docs.haystack.deepset.ai/docs/elasticsearchbm25retriever) | Retriever | ✅ Available | +| [ElasticsearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/elasticsearchembeddingretriever) | Retriever | ✅ Available | +| ElasticsearchHybridRetriever | Retriever | ✅ Available | +| [ElasticsearchSQLRetriever](https://docs.haystack.deepset.ai/docs/elasticsearchsqlretriever) | Retriever | ✅ Available | + +## FalkorDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FalkorDBCypherRetriever](https://docs.haystack.deepset.ai/docs/falkordbcypherretriever) | Retriever | ✅ Available | +| [FalkorDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/falkordbembeddingretriever) | Retriever | ✅ Available | + +## FastEmbed + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FastembedDocumentEmbedder](https://docs.haystack.deepset.ai/docs/fastembeddocumentembedder) | Embedder | ✅ Available | +| [FastembedRanker](https://docs.haystack.deepset.ai/docs/fastembedranker) | Ranker | ✅ Available | +| [FastembedSparseDocumentEmbedder](https://docs.haystack.deepset.ai/docs/fastembedsparsedocumentembedder) | Embedder | ✅ Available | +| [FastembedSparseTextEmbedder](https://docs.haystack.deepset.ai/docs/fastembedsparsetextembedder) | Embedder | ✅ Available | +| [FastembedTextEmbedder](https://docs.haystack.deepset.ai/docs/fastembedtextembedder) | Embedder | ✅ Available | + +## Firecrawl + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FirecrawlCrawler](https://docs.haystack.deepset.ai/docs/firecrawlcrawler) | Fetcher | ✅ Available | +| [FirecrawlWebSearch](https://docs.haystack.deepset.ai/docs/firecrawlwebsearch) | Component | ✅ Available | + +## GitHub + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [GitHubFileEditor](https://docs.haystack.deepset.ai/docs/githubfileeditor) | Connector | ✅ Available | +| [GitHubIssueCommenter](https://docs.haystack.deepset.ai/docs/githubissuecommenter) | Connector | ✅ Available | +| [GitHubIssueViewer](https://docs.haystack.deepset.ai/docs/githubissueviewer) | Connector | ✅ Available | +| [GitHubPRCreator](https://docs.haystack.deepset.ai/docs/githubprcreator) | Connector | ✅ Available | +| [GitHubRepoForker](https://docs.haystack.deepset.ai/docs/githubrepoforker) | Connector | ✅ Available | +| [GitHubRepoViewer](https://docs.haystack.deepset.ai/docs/githubrepoviewer) | Connector | ✅ Available | + +## Google Generative AI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [GoogleGenAIChatGenerator](https://docs.haystack.deepset.ai/docs/googlegenaichatgenerator) | Generator | ✅ Available | +| [GoogleGenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/googlegenaidocumentembedder) | Embedder | ✅ Available | +| [GoogleGenAITextEmbedder](https://docs.haystack.deepset.ai/docs/googlegenaitextembedder) | Embedder | ✅ Available | + +## Hugging Face API + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [HuggingFaceAPIChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapichatgenerator) | Generator | ✅ Available | +| [HuggingFaceAPIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapidocumentembedder) | Embedder | ✅ Available | +| [HuggingFaceAPITextEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapitextembedder) | Embedder | ✅ Available | +| [HuggingFaceTEIRanker](https://docs.haystack.deepset.ai/docs/huggingfaceteiranker) | Ranker | ✅ Available | + +## Jina AI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [JinaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/jinadocumentembedder) | Embedder | ✅ Available | +| [JinaRanker](https://docs.haystack.deepset.ai/docs/jinaranker) | Ranker | ✅ Available | +| [JinaReaderConnector](https://docs.haystack.deepset.ai/docs/jinareaderconnector) | Connector | ✅ Available | +| [JinaTextEmbedder](https://docs.haystack.deepset.ai/docs/jinatextembedder) | Embedder | ✅ Available | + +## Langfuse + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LangfuseConnector](https://docs.haystack.deepset.ai/docs/langfuseconnector) | Connector | ✅ Available | + +## Lara + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LaraDocumentTranslator](https://docs.haystack.deepset.ai/docs/laradocumenttranslator) | Component | ✅ Available | + +## LiteLLM + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LiteLLMChatGenerator](https://docs.haystack.deepset.ai/docs/litellmchatgenerator) | Generator | ✅ Available | + +## Llama Stack + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LlamaStackChatGenerator](https://docs.haystack.deepset.ai/docs/llamastackchatgenerator) | Generator | ✅ Available | + +## Llama.cpp + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LlamaCppChatGenerator](https://docs.haystack.deepset.ai/docs/llamacppchatgenerator) | Generator | ✅ Available | + +## MarkItDown + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MarkItDownConverter](https://docs.haystack.deepset.ai/docs/markitdownconverter) | Converter | ✅ Available | + +## Mem0 + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [Mem0MemoryRetriever](https://docs.haystack.deepset.ai/docs/mem0memoryretriever) | Retriever | ✅ Available | +| [Mem0MemoryWriter](https://docs.haystack.deepset.ai/docs/mem0memorywriter) | Writer | ✅ Available | + +## Meta Llama + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MetaLlamaChatGenerator](https://docs.haystack.deepset.ai/docs/metallamachatgenerator) | Generator | ✅ Available | + +## Mistral + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MistralChatGenerator](https://docs.haystack.deepset.ai/docs/mistralchatgenerator) | Generator | ✅ Available | +| [MistralDocumentEmbedder](https://docs.haystack.deepset.ai/docs/mistraldocumentembedder) | Embedder | ✅ Available | +| [MistralOCRDocumentConverter](https://docs.haystack.deepset.ai/docs/mistralocrdocumentconverter) | Converter | ✅ Available | +| [MistralTextEmbedder](https://docs.haystack.deepset.ai/docs/mistraltextembedder) | Embedder | ✅ Available | + +## MongoDB Atlas + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MongoDBAtlasEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/mongodbatlasembeddingretriever) | Retriever | ✅ Available | +| [MongoDBAtlasFullTextRetriever](https://docs.haystack.deepset.ai/docs/mongodbatlasfulltextretriever) | Retriever | ✅ Available | + +## NVIDIA + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [NvidiaChatGenerator](https://docs.haystack.deepset.ai/docs/nvidiachatgenerator) | Generator | ✅ Available | +| [NvidiaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/nvidiadocumentembedder) | Embedder | ✅ Available | +| [NvidiaRanker](https://docs.haystack.deepset.ai/docs/nvidiaranker) | Ranker | ✅ Available | +| [NvidiaTextEmbedder](https://docs.haystack.deepset.ai/docs/nvidiatextembedder) | Embedder | ✅ Available | + +## Ollama + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OllamaChatGenerator](https://docs.haystack.deepset.ai/docs/ollamachatgenerator) | Generator | ✅ Available | +| [OllamaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/ollamadocumentembedder) | Embedder | ✅ Available | +| [OllamaTextEmbedder](https://docs.haystack.deepset.ai/docs/ollamatextembedder) | Embedder | ✅ Available | + +## OpenRouter + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OpenRouterChatGenerator](https://docs.haystack.deepset.ai/docs/openrouterchatgenerator) | Generator | ✅ Available | + +## OpenSearch + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OpenSearchBM25Retriever](https://docs.haystack.deepset.ai/docs/opensearchbm25retriever) | Retriever | ✅ Available | +| [OpenSearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/opensearchembeddingretriever) | Retriever | ✅ Available | +| [OpenSearchHybridRetriever](https://docs.haystack.deepset.ai/docs/opensearchhybridretriever) | Retriever | ✅ Available | +| OpenSearchMetadataRetriever | Retriever | ✅ Available | +| [OpenSearchSQLRetriever](https://docs.haystack.deepset.ai/docs/opensearchsqlretriever) | Retriever | ✅ Available | + +## Oracle + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OracleEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/oracleembeddingretriever) | Retriever | ✅ Available | +| [OracleKeywordRetriever](https://docs.haystack.deepset.ai/docs/oraclekeywordretriever) | Retriever | ✅ Available | + +## Perplexity + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PerplexityChatGenerator](https://docs.haystack.deepset.ai/docs/perplexitychatgenerator) | Generator | ✅ Available | +| [PerplexityDocumentEmbedder](https://docs.haystack.deepset.ai/docs/perplexitydocumentembedder) | Embedder | ✅ Available | +| [PerplexityTextEmbedder](https://docs.haystack.deepset.ai/docs/perplexitytextembedder) | Embedder | ✅ Available | +| [PerplexityWebSearch](https://docs.haystack.deepset.ai/docs/perplexitywebsearch) | Component | ✅ Available | + +## pgvector + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PgvectorEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/pgvectorembeddingretriever) | Retriever | ✅ Available | +| [PgvectorKeywordRetriever](https://docs.haystack.deepset.ai/docs/pgvectorkeywordretriever) | Retriever | ✅ Available | + +## Pinecone + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PineconeEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/pineconedenseretriever) | Retriever | ✅ Available | + +## Presidio + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PresidioDocumentCleaner](https://docs.haystack.deepset.ai/docs/presidiodocumentcleaner) | Preprocessor | ✅ Available | +| [PresidioEntityExtractor](https://docs.haystack.deepset.ai/docs/presidioentityextractor) | Extractor | ✅ Available | +| [PresidioTextCleaner](https://docs.haystack.deepset.ai/docs/presidiotextcleaner) | Preprocessor | ✅ Available | + +## Pyversity + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PyversityRanker](https://docs.haystack.deepset.ai/docs/pyversityranker) | Ranker | ✅ Available | + +## Qdrant + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [QdrantEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/qdrantembeddingretriever) | Retriever | ✅ Available | +| [QdrantHybridRetriever](https://docs.haystack.deepset.ai/docs/qdranthybridretriever) | Retriever | ✅ Available | +| [QdrantSparseEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/qdrantsparseembeddingretriever) | Retriever | ✅ Available | + +## Snowflake + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [SnowflakeTableRetriever](https://docs.haystack.deepset.ai/docs/snowflaketableretriever) | Retriever | ✅ Available | + +## SQLAlchemy + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| SQLAlchemyTableRetriever | Retriever | ✅ Available | + +## STACKIT + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [STACKITChatGenerator](https://docs.haystack.deepset.ai/docs/stackitchatgenerator) | Generator | ✅ Available | +| [STACKITDocumentEmbedder](https://docs.haystack.deepset.ai/docs/stackitdocumentembedder) | Embedder | ✅ Available | +| [STACKITTextEmbedder](https://docs.haystack.deepset.ai/docs/stackittextembedder) | Embedder | ✅ Available | + +## Supabase + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [SupabasePgvectorEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/supabasepgvectorembeddingretriever) | Retriever | ✅ Available | +| [SupabasePgvectorKeywordRetriever](https://docs.haystack.deepset.ai/docs/supabasepgvectorkeywordretriever) | Retriever | ✅ Available | + +## Tavily + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [TavilyWebSearch](https://docs.haystack.deepset.ai/docs/tavilywebsearch) | Component | ✅ Available | + +## TogetherAI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [TogetherAIChatGenerator](https://docs.haystack.deepset.ai/docs/togetheraichatgenerator) | Generator | ✅ Available | + +## Unstructured + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [UnstructuredFileConverter](https://docs.haystack.deepset.ai/docs/unstructuredfileconverter) | Converter | ✅ Available | + +## Valkey + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ValkeyEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/valkeyembeddingretriever) | Retriever | ✅ Available | + +## Vespa + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [VespaEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/vespaembeddingretriever) | Retriever | ✅ Available | +| [VespaKeywordRetriever](https://docs.haystack.deepset.ai/docs/vespakeywordretriever) | Retriever | ✅ Available | + +## vLLM + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [VLLMChatGenerator](https://docs.haystack.deepset.ai/docs/vllmchatgenerator) | Generator | ✅ Available | +| [VLLMDocumentEmbedder](https://docs.haystack.deepset.ai/docs/vllmdocumentembedder) | Embedder | ✅ Available | +| [VLLMRanker](https://docs.haystack.deepset.ai/docs/vllmranker) | Ranker | ✅ Available | +| [VLLMTextEmbedder](https://docs.haystack.deepset.ai/docs/vllmtextembedder) | Embedder | ✅ Available | + +## Weaviate + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [WeaviateBM25Retriever](https://docs.haystack.deepset.ai/docs/weaviatebm25retriever) | Retriever | ✅ Available | +| [WeaviateEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/weaviateembeddingretriever) | Retriever | ✅ Available | + +## Weights & Biases (Weave) + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [WeaveConnector](https://docs.haystack.deepset.ai/docs/weaveconnector) | Connector | ✅ Available | diff --git a/docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx b/docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx similarity index 98% rename from docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx rename to docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx index b673661f657..e9c5de9fc97 100644 --- a/docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx +++ b/docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx @@ -1,7 +1,7 @@ --- title: "SentenceWindowRetriever" -id: sentencewindowretrieval -slug: "/sentencewindowretrieval" +id: sentencewindowretriever +slug: "/sentencewindowretriever" description: "Use this component to retrieve neighboring sentences around relevant sentences to get the full context." --- diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 4d99fa77807..beb381b1397 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -21,6 +21,7 @@ export default { 'overview/breaking-change-policy', 'overview/migration', 'overview/migrating-from-langgraphlangchain-to-haystack', + 'overview/platform-components', ], }, { @@ -602,7 +603,7 @@ export default { 'pipeline-components/retrievers/qdrantembeddingretriever', 'pipeline-components/retrievers/qdranthybridretriever', 'pipeline-components/retrievers/qdrantsparseembeddingretriever', - 'pipeline-components/retrievers/sentencewindowretrieval', + 'pipeline-components/retrievers/sentencewindowretriever', 'pipeline-components/retrievers/snowflaketableretriever', 'pipeline-components/retrievers/supabasegroongabm25retriever', 'pipeline-components/retrievers/supabasepgvectorembeddingretriever', diff --git a/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml b/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml new file mode 100644 index 00000000000..26a3868626a --- /dev/null +++ b/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added a new "Haystack Enterprise Components" page to the documentation overview, + listing all components available on the Haystack Enterprise Platform with links + to their individual docs pages. The page is auto-generated by a new GitHub Actions + workflow (``update-platform-components``) that runs on every push to ``main``. \ No newline at end of file diff --git a/scripts/generate_platform_components_table.py b/scripts/generate_platform_components_table.py new file mode 100644 index 00000000000..0846e6f9c51 --- /dev/null +++ b/scripts/generate_platform_components_table.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +""" +Generate a Haystack Enterprise Platform Components MDX table. + +Scans deepset-ai/haystack and deepset-ai/haystack-core-integrations for classes +decorated with @component, cross-references them against the platform component +schema (schema-full-component-list.json from deepset-ai/haystack-runtime), and +writes a formatted MDX page. +""" + +import argparse +import ast +import json +import logging +import sys +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + +# Maps schema `family` value / module path segment → display type name +TYPE_MAP: dict[str, str] = { + "generators": "Generator", + "retrievers": "Retriever", + "embedders": "Embedder", + "converters": "Converter", + "rankers": "Ranker", + "document_stores": "Document Store", + "preprocessors": "Preprocessor", + "readers": "Reader", + "routers": "Router", + "joiners": "Joiner", + "builders": "Builder", + "classifiers": "Classifier", + "extractors": "Extractor", + "samplers": "Sampler", + "writers": "Writer", + "connectors": "Connector", + "fetchers": "Fetcher", + "validators": "Validator", + "audio": "Audio", +} + +# Human-readable partner labels; snake_case partner keys → display name. +# Keys that map to the same value are intentionally merged into one section. +PARTNER_LABELS: dict[str, str] = { + "aimlapi": "AIML API", + "alloydb": "AlloyDB", + "amazon_bedrock": "Amazon Bedrock", + "amazon_sagemaker": "Amazon SageMaker", + "amazon_textract": "Amazon Textract", + "anthropic": "Anthropic", + "arcadedb": "ArcadeDB", + "astra": "Astra", + "azure_ai_search": "Azure AI Search", + "azure_doc_intelligence": "Azure Document Intelligence", + "brave": "Brave", + "chroma": "Chroma", + "cohere": "Cohere", + "docling": "Docling", + "docling_serve": "Docling", # merged with docling + "elasticsearch": "Elasticsearch", + "falkordb": "FalkorDB", + "fastembed": "FastEmbed", + "firecrawl": "Firecrawl", + "github": "GitHub", + "google_genai": "Google Generative AI", + "huggingface_api": "Hugging Face API", + "jina": "Jina AI", + "langfuse": "Langfuse", + "lara": "Lara", + "litellm": "LiteLLM", + "llama_cpp": "Llama.cpp", + "llama_stack": "Llama Stack", + "markitdown": "MarkItDown", + "mem0": "Mem0", + "meta_llama": "Meta Llama", + "mistral": "Mistral", + "mongodb_atlas": "MongoDB Atlas", + "nvidia": "NVIDIA", + "ollama": "Ollama", + "opensearch": "OpenSearch", + "openrouter": "OpenRouter", + "oracle": "Oracle", + "perplexity": "Perplexity", + "pgvector": "pgvector", + "pinecone": "Pinecone", + "presidio": "Presidio", + "pyversity": "Pyversity", + "qdrant": "Qdrant", + "s3": "Amazon S3", + "snowflake": "Snowflake", + "sqlalchemy": "SQLAlchemy", + "stackit": "STACKIT", + "supabase": "Supabase", + "tavily": "Tavily", + "togetherai": "TogetherAI", + "unstructured": "Unstructured", + "valkey": "Valkey", + "vespa": "Vespa", + "vllm": "vLLM", + "voyage": "Voyage AI", + "voyage_embedders": "Voyage AI", # merged with voyage + "weave": "Weights & Biases (Weave)", + "weaviate": "Weaviate", +} + +_HAYSTACK_DOCS_BASE = "https://docs.haystack.deepset.ai" +_HAYSTACK_DOCS_PREFIX = "/docs" # Docusaurus routeBasePath + +# Namespaces that must never appear in public documentation +_EXCLUDED_NAMESPACES = ( + "deepset_cloud_custom_nodes.", + "studio_internal.", + "haystack_experimental.", + "deepl_haystack.", +) + +# Individual class names to exclude regardless of namespace +_EXCLUDED_CLASS_NAMES = { + "SuperComponent", # internal base class, not a user-facing component + "LLM", # alias not present in Haystack OS +} + +# Sets allow O(1) membership tests +_PACKAGE_ROOTS: set[str] = {"haystack_integrations", "haystack"} +_COMPONENT_DECORATORS: set[str] = {"component", "super_component"} + + +# ── Component type helpers ──────────────────────────────────────────────────── + + +def infer_type(family: str | None, fqn: str) -> str: + if family and family in TYPE_MAP: + return TYPE_MAP[family] + return next((TYPE_MAP[seg] for seg in fqn.split(".") if seg in TYPE_MAP), "Component") + + +def partner_label_for(fqn: str) -> str | None: + """Return the display-name partner label for a ``haystack_integrations.*`` FQN.""" + parts = fqn.split(".") + # haystack_integrations . components . . . … + if len(parts) < 4: + return None + key = parts[3] + return PARTNER_LABELS.get(key, key.replace("_", " ").title()) + + +# ── Docs-site link map ──────────────────────────────────────────────────────── + + +def _parse_frontmatter(text: str) -> dict[str, str]: + """Extract key-value pairs from YAML-style frontmatter delimited by ``---``.""" + if not text.startswith("---"): + return {} + try: + end = text.index("---", 3) + except ValueError: + return {} + result: dict[str, str] = {} + for line in text[3:end].splitlines(): + if ":" in line: + key, _, value = line.partition(":") + result[key.strip()] = value.strip().strip('"') + return result + + +def scan_docs_links(docs_src: Path) -> dict[str, str]: + """Scan docs-website MDX files and return ``{component_title: absolute_url}``. + + Reads only the first 1 KB of each file — enough to cover the frontmatter — + avoiding the cost of loading large MDX files in full. + """ + link_map: dict[str, str] = {} + for mdx_file in docs_src.rglob("*.mdx"): + try: + with mdx_file.open(encoding="utf-8") as fh: + head = fh.read(1024) + except OSError: + continue + fm = _parse_frontmatter(head) + title = fm.get("title", "") + slug = fm.get("slug", "") + if title and slug: + link_map[title] = f"{_HAYSTACK_DOCS_BASE}{_HAYSTACK_DOCS_PREFIX}{slug}" + return link_map + + +# ── Source-scanning helpers ─────────────────────────────────────────────────── + + +def has_component_decorator(node: ast.ClassDef) -> bool: + """Return True if *node* has a ``@component`` or ``@super_component`` decorator.""" + for dec in node.decorator_list: + if isinstance(dec, ast.Name) and dec.id in _COMPONENT_DECORATORS: + return True + if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Name) and dec.func.id in _COMPONENT_DECORATORS: + return True + return False + + +def compute_module_path(file_path: Path, repo_root: Path) -> str | None: + """Convert a ``.py`` file path to a dotted Python module path.""" + try: + parts = list(file_path.relative_to(repo_root).with_suffix("").parts) + except ValueError: + return None + if parts and parts[-1] == "__init__": + parts.pop() + start = next((i for i, p in enumerate(parts) if p in _PACKAGE_ROOTS), None) + return ".".join(parts[start:]) if start is not None else None + + +def scan_for_components(scan_dir: Path, repo_root: Path) -> set[str]: + """Scan *scan_dir* for Python files with ``@component``-decorated classes.""" + found: set[str] = set() + py_files = list(scan_dir.rglob("*.py")) + logger.info("Scanned %d Python files in %s", len(py_files), scan_dir) + + for py_file in py_files: + module_path = compute_module_path(py_file, repo_root) + if module_path is None: + continue + try: + source = py_file.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + # Fast pre-filter: skip files with no decorator mention at all + if "@component" not in source and "@super_component" not in source: + continue + try: + tree = ast.parse(source, filename=str(py_file)) + except SyntaxError as exc: + logger.warning("Could not parse %s: %s", py_file, exc) + continue + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and has_component_decorator(node): + found.add(f"{module_path}.{node.name}") + + return found + + +# ── Schema loading ──────────────────────────────────────────────────────────── + + +def load_platform_components(schema_path: Path) -> dict[str, dict]: + """Parse ``schema-full-component-list.json`` and return public components.""" + all_defs: dict = json.loads(schema_path.read_text(encoding="utf-8")).get("definitions", {}).get("Components", {}) + result: dict[str, dict] = {} + for fqn, defn in all_defs.items(): + if any(fqn.startswith(ns) for ns in _EXCLUDED_NAMESPACES): + continue + if not (fqn.startswith("haystack.") or fqn.startswith("haystack_integrations.")): + continue + if fqn.split(".")[-1] in _EXCLUDED_CLASS_NAMES: + continue + type_props = defn.get("properties", {}).get("type", {}) + result[fqn] = {"title": defn.get("title", fqn.split(".")[-1]), "family": type_props.get("family", "")} + return result + + +# ── MDX generation ──────────────────────────────────────────────────────────── + + +def build_mdx( + platform_components: dict[str, dict], + source_components: set[str], + docs_link_map: dict[str, str] | None = None, +) -> str: + """Render the MDX page content.""" + _link_map = docs_link_map or {} + core_fqns: list[str] = [] + partner_components: dict[str, list[str]] = {} + visible_count = 0 + + for fqn in platform_components: + if fqn not in source_components: + logger.warning("Schema component not found in source scan: %s", fqn) + continue + cls = fqn.split(".")[-1] + if cls.endswith("Generator") and not cls.endswith("ChatGenerator"): + continue + visible_count += 1 + if fqn.startswith("haystack_integrations."): + partner_components.setdefault(partner_label_for(fqn) or "Other", []).append(fqn) + else: + core_fqns.append(fqn) + + def render_table(fqns: list[str]) -> list[str]: + rows = [ + "| Component | Type | Haystack Enterprise Platform |", + "|-----------|------|------------------------------|", + ] + for fqn in sorted(fqns, key=lambda x: x.split(".")[-1].lower()): + meta = platform_components[fqn] + name = meta["title"] + link = _link_map.get(name, "") + name_cell = f"[{name}]({link})" if link else name + rows.append(f"| {name_cell} | {infer_type(meta.get('family'), fqn)} | ✅ Available |") + return rows + + sections: list[str] = [ + "---", + 'title: "Haystack Enterprise Components"', + "id: platform-components", + 'slug: "/platform-components"', + 'description: "A complete list of Haystack components available on the Haystack Enterprise Platform, grouped by integration partner."', + "---", + "", + "# Haystack Enterprise Components", + "", + f"The Haystack Enterprise Platform currently supports **{visible_count} components**" + f" and **{len(partner_components)} integrations**." + " The following table lists them grouped by integration partner.", + "", + ] + + if core_fqns: + sections += ["## Core Components", "", *render_table(core_fqns), ""] + + for label in sorted(partner_components, key=str.lower): + sections += [f"## {label}", "", *render_table(partner_components[label]), ""] + + return "\n".join(sections) + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser( + description="Generate the Haystack Enterprise Platform Components MDX table." + ) + parser.add_argument("--haystack-src", required=True, type=Path, metavar="PATH", + help="Root of the deepset-ai/haystack checkout.") + parser.add_argument("--integrations-src", required=True, type=Path, metavar="PATH", + help="Root of the deepset-ai/haystack-core-integrations checkout.") + parser.add_argument("--schema", required=True, type=Path, metavar="PATH", + help="Path to schema-full-component-list.json from deepset-ai/haystack-runtime.") + parser.add_argument("--output", type=Path, metavar="PATH", + default=Path("docs-website/docs/overview/platform-components.mdx"), + help="Destination .mdx file path (default: docs-website/docs/overview/platform-components.mdx).") + parser.add_argument("--dry-run", action="store_true", + help="Print generated content to stdout instead of writing the file.") + + args = parser.parse_args(argv) + + for path, flag in [ + (args.haystack_src, "--haystack-src"), + (args.integrations_src, "--integrations-src"), + (args.schema, "--schema"), + ]: + if not path.exists(): + print(f"ERROR: {flag} path does not exist: {path}", file=sys.stderr) + sys.exit(1) + + try: + platform_components = load_platform_components(args.schema) + except (json.JSONDecodeError, OSError) as exc: + print(f"ERROR: Cannot load schema file: {exc}", file=sys.stderr) + sys.exit(1) + + haystack_pkg = args.haystack_src / "haystack" + source_components = scan_for_components( + haystack_pkg if haystack_pkg.is_dir() else args.haystack_src, args.haystack_src + ) + integrations_pkg = args.integrations_src / "integrations" + source_components |= scan_for_components( + integrations_pkg if integrations_pkg.is_dir() else args.integrations_src, args.integrations_src + ) + + logger.info("Platform components in schema (public namespaces): %d", len(platform_components)) + logger.info("Components matched to source scan: %d", sum(1 for f in platform_components if f in source_components)) + + docs_src = args.haystack_src / "docs-website" / "docs" + if not docs_src.is_dir(): + logger.warning("docs-website/docs not found under --haystack-src, component links will be omitted") + docs_link_map = scan_docs_links(docs_src) if docs_src.is_dir() else {} + + mdx = build_mdx(platform_components, source_components, docs_link_map) + + if args.dry_run: + print(mdx) + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(mdx, encoding="utf-8") + logger.info("Written to %s", args.output) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/tests/test_generate_platform_components_table.py b/scripts/tests/test_generate_platform_components_table.py new file mode 100644 index 00000000000..2034f263fef --- /dev/null +++ b/scripts/tests/test_generate_platform_components_table.py @@ -0,0 +1,629 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 +"""Tests for scripts/generate_platform_components_table.py.""" + +import ast +import json +import sys +import textwrap +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from generate_platform_components_table import ( + _parse_frontmatter, + build_mdx, + compute_module_path, + has_component_decorator, + infer_type, + load_platform_components, + partner_label_for, + scan_docs_links, + scan_for_components, +) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _parse_class(source: str) -> ast.ClassDef: + tree = ast.parse(textwrap.dedent(source)) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + return node + raise AssertionError("No ClassDef found in snippet") + + +def _make_schema(components: dict) -> dict: + """Wrap a flat {fqn: {title, family}} dict into the full schema envelope.""" + defs: dict = {} + for fqn, meta in components.items(): + defs[fqn] = { + "type": "object", + "title": meta["title"], + "properties": { + "type": { + "type": "string", + "const": fqn, + "origin": fqn, + "family": meta.get("family", ""), + } + }, + } + return {"type": "object", "definitions": {"Components": defs}} + + +# ── has_component_decorator ─────────────────────────────────────────────────── + + +class TestHasComponentDecorator: + def test_bare_decorator(self): + node = _parse_class( + """ + @component + class MyComponent: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_decorator_with_arguments(self): + node = _parse_class( + """ + @component(some_flag=True) + class MyComponent: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_decorator_with_positional_argument(self): + node = _parse_class( + """ + @component("value") + class MyComponent: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_no_decorator(self): + node = _parse_class( + """ + class Plain: + pass + """ + ) + assert has_component_decorator(node) is False + + def test_different_decorator(self): + node = _parse_class( + """ + @dataclass + class NotAComponent: + pass + """ + ) + assert has_component_decorator(node) is False + + def test_similar_name_not_matched(self): + """@component_something must NOT be treated as @component.""" + node = _parse_class( + """ + @component_something + class AlmostComponent: + pass + """ + ) + assert has_component_decorator(node) is False + + def test_similar_name_called_not_matched(self): + node = _parse_class( + """ + @component_factory() + class AlmostComponent: + pass + """ + ) + assert has_component_decorator(node) is False + + def test_multiple_decorators_one_matches(self): + node = _parse_class( + """ + @some_other_decorator + @component + class MyComponent: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_inline_word_in_body_not_matched(self): + node = _parse_class( + """ + @dataclass + class NotAComponent: + component = "value" + """ + ) + assert has_component_decorator(node) is False + + def test_super_component_bare_matched(self): + node = _parse_class( + """ + @super_component + class MyHybridRetriever: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_super_component_called_matched(self): + node = _parse_class( + """ + @super_component(some_flag=True) + class MyHybridRetriever: + pass + """ + ) + assert has_component_decorator(node) is True + + def test_super_component_similar_name_not_matched(self): + node = _parse_class( + """ + @super_component_factory + class NotMatched: + pass + """ + ) + assert has_component_decorator(node) is False + + +# ── compute_module_path ─────────────────────────────────────────────────────── + + +class TestComputeModulePath: + def test_haystack_package_file(self, tmp_path): + file_path = tmp_path / "haystack" / "components" / "converters" / "csv.py" + file_path.parent.mkdir(parents=True) + file_path.touch() + assert compute_module_path(file_path, tmp_path) == "haystack.components.converters.csv" + + def test_haystack_integrations_nested_file(self, tmp_path): + file_path = ( + tmp_path + / "integrations" + / "opensearch" + / "src" + / "haystack_integrations" + / "components" + / "retrievers" + / "opensearch" + / "bm25_retriever.py" + ) + file_path.parent.mkdir(parents=True) + file_path.touch() + result = compute_module_path(file_path, tmp_path) + assert result == "haystack_integrations.components.retrievers.opensearch.bm25_retriever" + + def test_init_file_strips_dunder_init(self, tmp_path): + file_path = tmp_path / "haystack" / "components" / "__init__.py" + file_path.parent.mkdir(parents=True) + file_path.touch() + assert compute_module_path(file_path, tmp_path) == "haystack.components" + + def test_unrecognised_package_returns_none(self, tmp_path): + file_path = tmp_path / "tests" / "test_helper.py" + file_path.parent.mkdir(parents=True) + file_path.touch() + assert compute_module_path(file_path, tmp_path) is None + + def test_file_outside_repo_root_returns_none(self, tmp_path): + other = tmp_path / "other" / "haystack" / "something.py" + other.parent.mkdir(parents=True) + other.touch() + assert compute_module_path(other, tmp_path / "nonexistent") is None + + +# ── scan_for_components ─────────────────────────────────────────────────────── + + +class TestScanForComponents: + def test_detects_decorated_class(self, tmp_path): + pkg = tmp_path / "haystack" / "components" / "generators" + pkg.mkdir(parents=True) + (pkg / "openai.py").write_text( + textwrap.dedent( + """ + @component + class OpenAIGenerator: + pass + """ + ) + ) + result = scan_for_components(tmp_path / "haystack", tmp_path) + assert "haystack.components.generators.openai.OpenAIGenerator" in result + + def test_ignores_undecorated_class(self, tmp_path): + pkg = tmp_path / "haystack" / "utils" + pkg.mkdir(parents=True) + (pkg / "helper.py").write_text("class NotAComponent:\n pass\n") + assert scan_for_components(tmp_path / "haystack", tmp_path) == set() + + def test_multiple_classes_one_file(self, tmp_path): + pkg = tmp_path / "haystack" / "components" / "embedders" + pkg.mkdir(parents=True) + (pkg / "sentence_transformers.py").write_text( + textwrap.dedent( + """ + @component + class SentenceTransformersDocumentEmbedder: + pass + + @component + class SentenceTransformersTextEmbedder: + pass + + class HelperClass: + pass + """ + ) + ) + result = scan_for_components(tmp_path / "haystack", tmp_path) + assert len(result) == 2 + + +# ── load_platform_components ────────────────────────────────────────────────── + + +class TestLoadPlatformComponents: + def test_loads_haystack_components(self, tmp_path): + schema = _make_schema( + { + "haystack.components.generators.openai.OpenAIGenerator": { + "title": "OpenAIGenerator", + "family": "generators", + } + } + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + result = load_platform_components(f) + assert "haystack.components.generators.openai.OpenAIGenerator" in result + assert result["haystack.components.generators.openai.OpenAIGenerator"]["title"] == "OpenAIGenerator" + assert result["haystack.components.generators.openai.OpenAIGenerator"]["family"] == "generators" + + def test_loads_integrations_components(self, tmp_path): + schema = _make_schema( + { + "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever": { + "title": "OpenSearchBM25Retriever", + "family": "retrievers", + } + } + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + result = load_platform_components(f) + assert "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever" in result + + def test_excludes_deepset_cloud_custom_nodes(self, tmp_path): + schema = _make_schema( + { + "deepset_cloud_custom_nodes.rankers.nvidia.DeepsetNvidiaRanker": { + "title": "DeepsetNvidiaRanker", + "family": "rankers", + } + } + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + assert load_platform_components(f) == {} + + def test_excludes_studio_internal(self, tmp_path): + schema = _make_schema( + {"studio_internal.foo.Bar": {"title": "Bar", "family": "generators"}} + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + assert load_platform_components(f) == {} + + def test_excludes_haystack_experimental(self, tmp_path): + schema = _make_schema( + { + "haystack_experimental.components.generators.chat.openai.OpenAIChatGenerator": { + "title": "OpenAIChatGenerator", + "family": "generators", + } + } + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + assert load_platform_components(f) == {} + + def test_excludes_deepl_haystack(self, tmp_path): + schema = _make_schema( + {"deepl_haystack.components.DeepLTextTranslator": {"title": "DeepLTextTranslator", "family": "translators"}} + ) + f = tmp_path / "schema.json" + f.write_text(json.dumps(schema)) + assert load_platform_components(f) == {} + + +# ── partner_label_for ───────────────────────────────────────────────────────── + + +class TestPartnerLabelFor: + def test_known_partner(self): + assert ( + partner_label_for( + "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever" + ) + == "OpenSearch" + ) + + def test_voyage_embedders_merged(self): + assert ( + partner_label_for( + "haystack_integrations.components.embedders.voyage_embedders.voyage_text_embedder.VoyageTextEmbedder" + ) + == "Voyage AI" + ) + + def test_voyage_ranker_merged(self): + assert ( + partner_label_for("haystack_integrations.components.rankers.voyage.ranker.VoyageRanker") + == "Voyage AI" + ) + + def test_docling_serve_merged_with_docling(self): + assert ( + partner_label_for( + "haystack_integrations.components.converters.docling_serve.converter.DoclingServeConverter" + ) + == "Docling" + ) + + def test_amazon_bedrock_label(self): + assert ( + partner_label_for( + "haystack_integrations.components.generators.amazon_bedrock.chat.chat_generator.AmazonBedrockChatGenerator" + ) + == "Amazon Bedrock" + ) + + def test_unknown_partner_title_cases(self): + result = partner_label_for("haystack_integrations.components.retrievers.some_new_partner.foo.Bar") + assert result == "Some New Partner" + + def test_returns_none_for_short_fqn(self): + assert partner_label_for("haystack_integrations.components") is None + + +# ── infer_type ──────────────────────────────────────────────────────────────── + + +class TestInferType: + @pytest.mark.parametrize( + "family,fqn,expected", + [ + ("generators", "haystack.components.generators.openai.OpenAIGenerator", "Generator"), + ("retrievers", "haystack.components.retrievers.memory.InMemoryBM25Retriever", "Retriever"), + ("embedders", "haystack.components.embedders.foo.Bar", "Embedder"), + ("converters", "haystack.components.converters.csv.CSVToDocument", "Converter"), + ("document_stores", "haystack.document_stores.foo.Bar", "Document Store"), + # Falls back to FQN path inspection when family is missing + (None, "haystack.components.rankers.foo.Bar", "Ranker"), + ("", "haystack.components.writers.foo.Bar", "Writer"), + # Unknown returns Component + (None, "haystack.components.misc.foo.Bar", "Component"), + ], + ) + def test_infer_type(self, family, fqn, expected): + assert infer_type(family, fqn) == expected + + +# ── Cross-reference logic in build_mdx ─────────────────────────────────────── + + +_PLATFORM = { + "haystack.components.generators.openai.OpenAIGenerator": { + "title": "OpenAIGenerator", + "family": "generators", + }, + "haystack.components.retrievers.memory.InMemoryBM25Retriever": { + "title": "InMemoryBM25Retriever", + "family": "retrievers", + }, + "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever": { + "title": "OpenSearchBM25Retriever", + "family": "retrievers", + }, +} + +_SOURCE = { + "haystack.components.generators.openai.OpenAIGenerator", + "haystack.components.retrievers.memory.InMemoryBM25Retriever", + "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever", +} + + +class TestBuildMdxCrossReference: + def test_schema_component_in_source_appears(self): + mdx = build_mdx(_PLATFORM, _SOURCE) + assert "InMemoryBM25Retriever" in mdx + + def test_component_not_in_source_excluded(self): + # CSVToDocument is NOT in _PLATFORM, so it must not appear + source_with_extra = _SOURCE | {"haystack.components.converters.csv.CSVToDocument"} + platform_without = {k: v for k, v in _PLATFORM.items() if "csv" not in k} + mdx = build_mdx(platform_without, source_with_extra) + assert "CSVToDocument" not in mdx + + def test_schema_component_not_in_source_excluded_from_table(self): + # Component in schema but not in source scan must not appear in the table + platform_extra = { + **_PLATFORM, + "haystack.components.foo.MissingComponent": {"title": "MissingComponent", "family": ""}, + } + mdx = build_mdx(platform_extra, _SOURCE) + assert "MissingComponent" not in mdx + + def test_core_components_section(self): + mdx = build_mdx(_PLATFORM, _SOURCE) + assert "## Core Components" in mdx + assert "InMemoryBM25Retriever" in mdx + + def test_integration_partner_section(self): + mdx = build_mdx(_PLATFORM, _SOURCE) + assert "## OpenSearch" in mdx + assert "OpenSearchBM25Retriever" in mdx + + def test_available_marker_present(self): + mdx = build_mdx(_PLATFORM, _SOURCE) + assert "✅ Available" in mdx + + def test_generator_components_excluded(self): + platform = { + "haystack.components.generators.openai.OpenAIGenerator": { + "title": "OpenAIGenerator", + "family": "generators", + }, + "haystack.components.retrievers.memory.InMemoryBM25Retriever": { + "title": "InMemoryBM25Retriever", + "family": "retrievers", + }, + } + source = set(platform.keys()) + mdx = build_mdx(platform, source) + assert "OpenAIGenerator" not in mdx + assert "InMemoryBM25Retriever" in mdx + + def test_chat_generator_components_included(self): + platform = { + "haystack.components.generators.openai.OpenAIChatGenerator": { + "title": "OpenAIChatGenerator", + "family": "generators", + }, + "haystack.components.generators.openai.OpenAIGenerator": { + "title": "OpenAIGenerator", + "family": "generators", + }, + } + source = set(platform.keys()) + mdx = build_mdx(platform, source) + assert "OpenAIChatGenerator" in mdx + assert "OpenAIGenerator" not in mdx + + def test_voyage_embedders_and_ranker_merged_under_voyage_ai(self): + platform = { + "haystack_integrations.components.embedders.voyage_embedders.voyage_text_embedder.VoyageTextEmbedder": { + "title": "VoyageTextEmbedder", + "family": "embedders", + }, + "haystack_integrations.components.rankers.voyage.ranker.VoyageRanker": { + "title": "VoyageRanker", + "family": "rankers", + }, + } + source = set(platform.keys()) + mdx = build_mdx(platform, source) + # Must appear exactly once as a section heading + assert mdx.count("## Voyage AI") == 1 + assert "VoyageTextEmbedder" in mdx + assert "VoyageRanker" in mdx + + def test_docling_serve_merged_under_docling(self): + platform = { + "haystack_integrations.components.converters.docling.converter.DoclingConverter": { + "title": "DoclingConverter", + "family": "converters", + }, + "haystack_integrations.components.converters.docling_serve.converter.DoclingServeConverter": { + "title": "DoclingServeConverter", + "family": "converters", + }, + } + source = set(platform.keys()) + mdx = build_mdx(platform, source) + assert mdx.count("## Docling") == 1 + + def test_family_field_used_for_type(self): + platform = { + "haystack.components.retrievers.memory.InMemoryBM25Retriever": { + "title": "InMemoryBM25Retriever", + "family": "retrievers", + } + } + link_map = {"InMemoryBM25Retriever": "https://docs.haystack.deepset.ai/docs/inmemorybm25retriever"} + mdx = build_mdx(platform, set(platform.keys()), docs_link_map=link_map) + assert "| [InMemoryBM25Retriever](https://docs.haystack.deepset.ai/docs/inmemorybm25retriever) | Retriever |" in mdx + + def test_family_fallback_to_fqn_path(self): + platform = { + "haystack.components.rankers.foo.MyRanker": { + "title": "MyRanker", + "family": "", # missing family + } + } + mdx = build_mdx(platform, set(platform.keys())) + assert "| MyRanker | Ranker |" in mdx + + def test_component_with_no_docs_page_is_plain_text(self): + platform = { + "haystack.components.rankers.foo.MyRanker": { + "title": "MyRanker", + "family": "rankers", + } + } + # docs_link_map has no entry for MyRanker + mdx = build_mdx(platform, set(platform.keys()), docs_link_map={}) + assert "| MyRanker | Ranker |" in mdx + assert "[MyRanker]" not in mdx + + +# ── scan_docs_links / _parse_frontmatter ───────────────────────────────────── + + +class TestScanDocsLinks: + def test_parses_title_and_slug(self, tmp_path): + mdx = tmp_path / "pipeline-components" / "retrievers" / "inmemorybm25retriever.mdx" + mdx.parent.mkdir(parents=True) + mdx.write_text('---\ntitle: "InMemoryBM25Retriever"\nid: inmemorybm25retriever\nslug: "/inmemorybm25retriever"\n---\n') + result = scan_docs_links(tmp_path) + assert result == {"InMemoryBM25Retriever": "https://docs.haystack.deepset.ai/docs/inmemorybm25retriever"} + + def test_files_without_frontmatter_skipped(self, tmp_path): + mdx = tmp_path / "no-frontmatter.mdx" + mdx.write_text("# Just a heading\n") + assert scan_docs_links(tmp_path) == {} + + def test_files_missing_slug_skipped(self, tmp_path): + mdx = tmp_path / "partial.mdx" + mdx.write_text('---\ntitle: "OnlyTitle"\n---\n') + assert scan_docs_links(tmp_path) == {} + + def test_multiple_files_all_indexed(self, tmp_path): + for name in ("foo", "bar"): + f = tmp_path / f"{name}.mdx" + f.write_text(f'---\ntitle: "{name.title()}"\nslug: "/{name}"\n---\n') + result = scan_docs_links(tmp_path) + assert len(result) == 2 + assert result["Foo"] == "https://docs.haystack.deepset.ai/docs/foo" + + +class TestParseFrontmatter: + def test_basic_key_value(self): + text = '---\ntitle: "Hello"\nslug: "/hello"\n---\nBody' + fm = _parse_frontmatter(text) + assert fm["title"] == "Hello" + assert fm["slug"] == "/hello" + + def test_no_frontmatter_returns_empty(self): + assert _parse_frontmatter("# No frontmatter") == {} + + def test_unclosed_frontmatter_returns_empty(self): + assert _parse_frontmatter("---\ntitle: orphan\n") == {} From 507f3600cdc6808391617aa51b685b906583f9b1 Mon Sep 17 00:00:00 2001 From: bilgeyucel Date: Mon, 29 Jun 2026 11:44:27 +0200 Subject: [PATCH 2/5] chore: clean up platform components PR --- ...riever.mdx => sentencewindowretrieval.mdx} | 4 +- docs-website/sidebars.js | 2 +- ...components-docs-page-36725b437280b21d.yaml | 7 - scripts/generate_platform_components_table.py | 68 +- scripts/tests/__init__.py | 0 ...test_generate_platform_components_table.py | 629 ------------------ 6 files changed, 42 insertions(+), 668 deletions(-) rename docs-website/docs/pipeline-components/retrievers/{sentencewindowretriever.mdx => sentencewindowretrieval.mdx} (98%) delete mode 100644 releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml delete mode 100644 scripts/tests/__init__.py delete mode 100644 scripts/tests/test_generate_platform_components_table.py diff --git a/docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx b/docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx similarity index 98% rename from docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx rename to docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx index e9c5de9fc97..b673661f657 100644 --- a/docs-website/docs/pipeline-components/retrievers/sentencewindowretriever.mdx +++ b/docs-website/docs/pipeline-components/retrievers/sentencewindowretrieval.mdx @@ -1,7 +1,7 @@ --- title: "SentenceWindowRetriever" -id: sentencewindowretriever -slug: "/sentencewindowretriever" +id: sentencewindowretrieval +slug: "/sentencewindowretrieval" description: "Use this component to retrieve neighboring sentences around relevant sentences to get the full context." --- diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index beb381b1397..9f618e510d2 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -603,7 +603,7 @@ export default { 'pipeline-components/retrievers/qdrantembeddingretriever', 'pipeline-components/retrievers/qdranthybridretriever', 'pipeline-components/retrievers/qdrantsparseembeddingretriever', - 'pipeline-components/retrievers/sentencewindowretriever', + 'pipeline-components/retrievers/sentencewindowretrieval', 'pipeline-components/retrievers/snowflaketableretriever', 'pipeline-components/retrievers/supabasegroongabm25retriever', 'pipeline-components/retrievers/supabasepgvectorembeddingretriever', diff --git a/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml b/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml deleted file mode 100644 index 26a3868626a..00000000000 --- a/releasenotes/notes/add-platform-components-docs-page-36725b437280b21d.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -features: - - | - Added a new "Haystack Enterprise Components" page to the documentation overview, - listing all components available on the Haystack Enterprise Platform with links - to their individual docs pages. The page is auto-generated by a new GitHub Actions - workflow (``update-platform-components``) that runs on every push to ``main``. \ No newline at end of file diff --git a/scripts/generate_platform_components_table.py b/scripts/generate_platform_components_table.py index 0846e6f9c51..7b92966c000 100644 --- a/scripts/generate_platform_components_table.py +++ b/scripts/generate_platform_components_table.py @@ -1,9 +1,6 @@ -#!/usr/bin/env python3 # SPDX-FileCopyrightText: 2022-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - """ Generate a Haystack Enterprise Platform Components MDX table. @@ -13,6 +10,8 @@ writes a formatted MDX page. """ +from __future__ import annotations + import argparse import ast import json @@ -114,17 +113,12 @@ _HAYSTACK_DOCS_PREFIX = "/docs" # Docusaurus routeBasePath # Namespaces that must never appear in public documentation -_EXCLUDED_NAMESPACES = ( - "deepset_cloud_custom_nodes.", - "studio_internal.", - "haystack_experimental.", - "deepl_haystack.", -) +_EXCLUDED_NAMESPACES = ("deepset_cloud_custom_nodes.", "studio_internal.", "haystack_experimental.", "deepl_haystack.") # Individual class names to exclude regardless of namespace _EXCLUDED_CLASS_NAMES = { "SuperComponent", # internal base class, not a user-facing component - "LLM", # alias not present in Haystack OS + "LLM", # alias not present in Haystack OS } # Sets allow O(1) membership tests @@ -136,6 +130,7 @@ def infer_type(family: str | None, fqn: str) -> str: + """Return a human-readable type string from the schema family or FQN path segments.""" if family and family in TYPE_MAP: return TYPE_MAP[family] return next((TYPE_MAP[seg] for seg in fqn.split(".") if seg in TYPE_MAP), "Component") @@ -171,7 +166,8 @@ def _parse_frontmatter(text: str) -> dict[str, str]: def scan_docs_links(docs_src: Path) -> dict[str, str]: - """Scan docs-website MDX files and return ``{component_title: absolute_url}``. + """ + Scan docs-website MDX files and return ``{component_title: absolute_url}``. Reads only the first 1 KB of each file — enough to cover the frontmatter — avoiding the cost of loading large MDX files in full. @@ -255,7 +251,7 @@ def load_platform_components(schema_path: Path) -> dict[str, dict]: for fqn, defn in all_defs.items(): if any(fqn.startswith(ns) for ns in _EXCLUDED_NAMESPACES): continue - if not (fqn.startswith("haystack.") or fqn.startswith("haystack_integrations.")): + if not fqn.startswith(("haystack.", "haystack_integrations.")): continue if fqn.split(".")[-1] in _EXCLUDED_CLASS_NAMES: continue @@ -268,9 +264,7 @@ def load_platform_components(schema_path: Path) -> dict[str, dict]: def build_mdx( - platform_components: dict[str, dict], - source_components: set[str], - docs_link_map: dict[str, str] | None = None, + platform_components: dict[str, dict], source_components: set[str], docs_link_map: dict[str, str] | None = None ) -> str: """Render the MDX page content.""" _link_map = docs_link_map or {} @@ -309,7 +303,8 @@ def render_table(fqns: list[str]) -> list[str]: 'title: "Haystack Enterprise Components"', "id: platform-components", 'slug: "/platform-components"', - 'description: "A complete list of Haystack components available on the Haystack Enterprise Platform, grouped by integration partner."', + 'description: "A complete list of Haystack components available on the Haystack Enterprise Platform,' + ' grouped by integration partner."', "---", "", "# Haystack Enterprise Components", @@ -333,20 +328,35 @@ def render_table(fqns: list[str]) -> list[str]: def main(argv: list[str] | None = None) -> None: - parser = argparse.ArgumentParser( - description="Generate the Haystack Enterprise Platform Components MDX table." + """Entry point: parse arguments, run the scan, and write the MDX file.""" + parser = argparse.ArgumentParser(description="Generate the Haystack Enterprise Platform Components MDX table.") + parser.add_argument( + "--haystack-src", required=True, type=Path, metavar="PATH", help="Root of the deepset-ai/haystack checkout." + ) + parser.add_argument( + "--integrations-src", + required=True, + type=Path, + metavar="PATH", + help="Root of the deepset-ai/haystack-core-integrations checkout.", + ) + parser.add_argument( + "--schema", + required=True, + type=Path, + metavar="PATH", + help="Path to schema JSON file in the platform repo.", + ) + parser.add_argument( + "--output", + type=Path, + metavar="PATH", + default=Path("docs-website/docs/overview/platform-components.mdx"), + help="Destination .mdx file path (default: docs-website/docs/overview/platform-components.mdx).", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print generated content to stdout instead of writing the file." ) - parser.add_argument("--haystack-src", required=True, type=Path, metavar="PATH", - help="Root of the deepset-ai/haystack checkout.") - parser.add_argument("--integrations-src", required=True, type=Path, metavar="PATH", - help="Root of the deepset-ai/haystack-core-integrations checkout.") - parser.add_argument("--schema", required=True, type=Path, metavar="PATH", - help="Path to schema-full-component-list.json from deepset-ai/haystack-runtime.") - parser.add_argument("--output", type=Path, metavar="PATH", - default=Path("docs-website/docs/overview/platform-components.mdx"), - help="Destination .mdx file path (default: docs-website/docs/overview/platform-components.mdx).") - parser.add_argument("--dry-run", action="store_true", - help="Print generated content to stdout instead of writing the file.") args = parser.parse_args(argv) diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/scripts/tests/test_generate_platform_components_table.py b/scripts/tests/test_generate_platform_components_table.py deleted file mode 100644 index 2034f263fef..00000000000 --- a/scripts/tests/test_generate_platform_components_table.py +++ /dev/null @@ -1,629 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 -"""Tests for scripts/generate_platform_components_table.py.""" - -import ast -import json -import sys -import textwrap -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from generate_platform_components_table import ( - _parse_frontmatter, - build_mdx, - compute_module_path, - has_component_decorator, - infer_type, - load_platform_components, - partner_label_for, - scan_docs_links, - scan_for_components, -) - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - - -def _parse_class(source: str) -> ast.ClassDef: - tree = ast.parse(textwrap.dedent(source)) - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - return node - raise AssertionError("No ClassDef found in snippet") - - -def _make_schema(components: dict) -> dict: - """Wrap a flat {fqn: {title, family}} dict into the full schema envelope.""" - defs: dict = {} - for fqn, meta in components.items(): - defs[fqn] = { - "type": "object", - "title": meta["title"], - "properties": { - "type": { - "type": "string", - "const": fqn, - "origin": fqn, - "family": meta.get("family", ""), - } - }, - } - return {"type": "object", "definitions": {"Components": defs}} - - -# ── has_component_decorator ─────────────────────────────────────────────────── - - -class TestHasComponentDecorator: - def test_bare_decorator(self): - node = _parse_class( - """ - @component - class MyComponent: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_decorator_with_arguments(self): - node = _parse_class( - """ - @component(some_flag=True) - class MyComponent: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_decorator_with_positional_argument(self): - node = _parse_class( - """ - @component("value") - class MyComponent: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_no_decorator(self): - node = _parse_class( - """ - class Plain: - pass - """ - ) - assert has_component_decorator(node) is False - - def test_different_decorator(self): - node = _parse_class( - """ - @dataclass - class NotAComponent: - pass - """ - ) - assert has_component_decorator(node) is False - - def test_similar_name_not_matched(self): - """@component_something must NOT be treated as @component.""" - node = _parse_class( - """ - @component_something - class AlmostComponent: - pass - """ - ) - assert has_component_decorator(node) is False - - def test_similar_name_called_not_matched(self): - node = _parse_class( - """ - @component_factory() - class AlmostComponent: - pass - """ - ) - assert has_component_decorator(node) is False - - def test_multiple_decorators_one_matches(self): - node = _parse_class( - """ - @some_other_decorator - @component - class MyComponent: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_inline_word_in_body_not_matched(self): - node = _parse_class( - """ - @dataclass - class NotAComponent: - component = "value" - """ - ) - assert has_component_decorator(node) is False - - def test_super_component_bare_matched(self): - node = _parse_class( - """ - @super_component - class MyHybridRetriever: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_super_component_called_matched(self): - node = _parse_class( - """ - @super_component(some_flag=True) - class MyHybridRetriever: - pass - """ - ) - assert has_component_decorator(node) is True - - def test_super_component_similar_name_not_matched(self): - node = _parse_class( - """ - @super_component_factory - class NotMatched: - pass - """ - ) - assert has_component_decorator(node) is False - - -# ── compute_module_path ─────────────────────────────────────────────────────── - - -class TestComputeModulePath: - def test_haystack_package_file(self, tmp_path): - file_path = tmp_path / "haystack" / "components" / "converters" / "csv.py" - file_path.parent.mkdir(parents=True) - file_path.touch() - assert compute_module_path(file_path, tmp_path) == "haystack.components.converters.csv" - - def test_haystack_integrations_nested_file(self, tmp_path): - file_path = ( - tmp_path - / "integrations" - / "opensearch" - / "src" - / "haystack_integrations" - / "components" - / "retrievers" - / "opensearch" - / "bm25_retriever.py" - ) - file_path.parent.mkdir(parents=True) - file_path.touch() - result = compute_module_path(file_path, tmp_path) - assert result == "haystack_integrations.components.retrievers.opensearch.bm25_retriever" - - def test_init_file_strips_dunder_init(self, tmp_path): - file_path = tmp_path / "haystack" / "components" / "__init__.py" - file_path.parent.mkdir(parents=True) - file_path.touch() - assert compute_module_path(file_path, tmp_path) == "haystack.components" - - def test_unrecognised_package_returns_none(self, tmp_path): - file_path = tmp_path / "tests" / "test_helper.py" - file_path.parent.mkdir(parents=True) - file_path.touch() - assert compute_module_path(file_path, tmp_path) is None - - def test_file_outside_repo_root_returns_none(self, tmp_path): - other = tmp_path / "other" / "haystack" / "something.py" - other.parent.mkdir(parents=True) - other.touch() - assert compute_module_path(other, tmp_path / "nonexistent") is None - - -# ── scan_for_components ─────────────────────────────────────────────────────── - - -class TestScanForComponents: - def test_detects_decorated_class(self, tmp_path): - pkg = tmp_path / "haystack" / "components" / "generators" - pkg.mkdir(parents=True) - (pkg / "openai.py").write_text( - textwrap.dedent( - """ - @component - class OpenAIGenerator: - pass - """ - ) - ) - result = scan_for_components(tmp_path / "haystack", tmp_path) - assert "haystack.components.generators.openai.OpenAIGenerator" in result - - def test_ignores_undecorated_class(self, tmp_path): - pkg = tmp_path / "haystack" / "utils" - pkg.mkdir(parents=True) - (pkg / "helper.py").write_text("class NotAComponent:\n pass\n") - assert scan_for_components(tmp_path / "haystack", tmp_path) == set() - - def test_multiple_classes_one_file(self, tmp_path): - pkg = tmp_path / "haystack" / "components" / "embedders" - pkg.mkdir(parents=True) - (pkg / "sentence_transformers.py").write_text( - textwrap.dedent( - """ - @component - class SentenceTransformersDocumentEmbedder: - pass - - @component - class SentenceTransformersTextEmbedder: - pass - - class HelperClass: - pass - """ - ) - ) - result = scan_for_components(tmp_path / "haystack", tmp_path) - assert len(result) == 2 - - -# ── load_platform_components ────────────────────────────────────────────────── - - -class TestLoadPlatformComponents: - def test_loads_haystack_components(self, tmp_path): - schema = _make_schema( - { - "haystack.components.generators.openai.OpenAIGenerator": { - "title": "OpenAIGenerator", - "family": "generators", - } - } - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - result = load_platform_components(f) - assert "haystack.components.generators.openai.OpenAIGenerator" in result - assert result["haystack.components.generators.openai.OpenAIGenerator"]["title"] == "OpenAIGenerator" - assert result["haystack.components.generators.openai.OpenAIGenerator"]["family"] == "generators" - - def test_loads_integrations_components(self, tmp_path): - schema = _make_schema( - { - "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever": { - "title": "OpenSearchBM25Retriever", - "family": "retrievers", - } - } - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - result = load_platform_components(f) - assert "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever" in result - - def test_excludes_deepset_cloud_custom_nodes(self, tmp_path): - schema = _make_schema( - { - "deepset_cloud_custom_nodes.rankers.nvidia.DeepsetNvidiaRanker": { - "title": "DeepsetNvidiaRanker", - "family": "rankers", - } - } - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - assert load_platform_components(f) == {} - - def test_excludes_studio_internal(self, tmp_path): - schema = _make_schema( - {"studio_internal.foo.Bar": {"title": "Bar", "family": "generators"}} - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - assert load_platform_components(f) == {} - - def test_excludes_haystack_experimental(self, tmp_path): - schema = _make_schema( - { - "haystack_experimental.components.generators.chat.openai.OpenAIChatGenerator": { - "title": "OpenAIChatGenerator", - "family": "generators", - } - } - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - assert load_platform_components(f) == {} - - def test_excludes_deepl_haystack(self, tmp_path): - schema = _make_schema( - {"deepl_haystack.components.DeepLTextTranslator": {"title": "DeepLTextTranslator", "family": "translators"}} - ) - f = tmp_path / "schema.json" - f.write_text(json.dumps(schema)) - assert load_platform_components(f) == {} - - -# ── partner_label_for ───────────────────────────────────────────────────────── - - -class TestPartnerLabelFor: - def test_known_partner(self): - assert ( - partner_label_for( - "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever" - ) - == "OpenSearch" - ) - - def test_voyage_embedders_merged(self): - assert ( - partner_label_for( - "haystack_integrations.components.embedders.voyage_embedders.voyage_text_embedder.VoyageTextEmbedder" - ) - == "Voyage AI" - ) - - def test_voyage_ranker_merged(self): - assert ( - partner_label_for("haystack_integrations.components.rankers.voyage.ranker.VoyageRanker") - == "Voyage AI" - ) - - def test_docling_serve_merged_with_docling(self): - assert ( - partner_label_for( - "haystack_integrations.components.converters.docling_serve.converter.DoclingServeConverter" - ) - == "Docling" - ) - - def test_amazon_bedrock_label(self): - assert ( - partner_label_for( - "haystack_integrations.components.generators.amazon_bedrock.chat.chat_generator.AmazonBedrockChatGenerator" - ) - == "Amazon Bedrock" - ) - - def test_unknown_partner_title_cases(self): - result = partner_label_for("haystack_integrations.components.retrievers.some_new_partner.foo.Bar") - assert result == "Some New Partner" - - def test_returns_none_for_short_fqn(self): - assert partner_label_for("haystack_integrations.components") is None - - -# ── infer_type ──────────────────────────────────────────────────────────────── - - -class TestInferType: - @pytest.mark.parametrize( - "family,fqn,expected", - [ - ("generators", "haystack.components.generators.openai.OpenAIGenerator", "Generator"), - ("retrievers", "haystack.components.retrievers.memory.InMemoryBM25Retriever", "Retriever"), - ("embedders", "haystack.components.embedders.foo.Bar", "Embedder"), - ("converters", "haystack.components.converters.csv.CSVToDocument", "Converter"), - ("document_stores", "haystack.document_stores.foo.Bar", "Document Store"), - # Falls back to FQN path inspection when family is missing - (None, "haystack.components.rankers.foo.Bar", "Ranker"), - ("", "haystack.components.writers.foo.Bar", "Writer"), - # Unknown returns Component - (None, "haystack.components.misc.foo.Bar", "Component"), - ], - ) - def test_infer_type(self, family, fqn, expected): - assert infer_type(family, fqn) == expected - - -# ── Cross-reference logic in build_mdx ─────────────────────────────────────── - - -_PLATFORM = { - "haystack.components.generators.openai.OpenAIGenerator": { - "title": "OpenAIGenerator", - "family": "generators", - }, - "haystack.components.retrievers.memory.InMemoryBM25Retriever": { - "title": "InMemoryBM25Retriever", - "family": "retrievers", - }, - "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever": { - "title": "OpenSearchBM25Retriever", - "family": "retrievers", - }, -} - -_SOURCE = { - "haystack.components.generators.openai.OpenAIGenerator", - "haystack.components.retrievers.memory.InMemoryBM25Retriever", - "haystack_integrations.components.retrievers.opensearch.bm25_retriever.OpenSearchBM25Retriever", -} - - -class TestBuildMdxCrossReference: - def test_schema_component_in_source_appears(self): - mdx = build_mdx(_PLATFORM, _SOURCE) - assert "InMemoryBM25Retriever" in mdx - - def test_component_not_in_source_excluded(self): - # CSVToDocument is NOT in _PLATFORM, so it must not appear - source_with_extra = _SOURCE | {"haystack.components.converters.csv.CSVToDocument"} - platform_without = {k: v for k, v in _PLATFORM.items() if "csv" not in k} - mdx = build_mdx(platform_without, source_with_extra) - assert "CSVToDocument" not in mdx - - def test_schema_component_not_in_source_excluded_from_table(self): - # Component in schema but not in source scan must not appear in the table - platform_extra = { - **_PLATFORM, - "haystack.components.foo.MissingComponent": {"title": "MissingComponent", "family": ""}, - } - mdx = build_mdx(platform_extra, _SOURCE) - assert "MissingComponent" not in mdx - - def test_core_components_section(self): - mdx = build_mdx(_PLATFORM, _SOURCE) - assert "## Core Components" in mdx - assert "InMemoryBM25Retriever" in mdx - - def test_integration_partner_section(self): - mdx = build_mdx(_PLATFORM, _SOURCE) - assert "## OpenSearch" in mdx - assert "OpenSearchBM25Retriever" in mdx - - def test_available_marker_present(self): - mdx = build_mdx(_PLATFORM, _SOURCE) - assert "✅ Available" in mdx - - def test_generator_components_excluded(self): - platform = { - "haystack.components.generators.openai.OpenAIGenerator": { - "title": "OpenAIGenerator", - "family": "generators", - }, - "haystack.components.retrievers.memory.InMemoryBM25Retriever": { - "title": "InMemoryBM25Retriever", - "family": "retrievers", - }, - } - source = set(platform.keys()) - mdx = build_mdx(platform, source) - assert "OpenAIGenerator" not in mdx - assert "InMemoryBM25Retriever" in mdx - - def test_chat_generator_components_included(self): - platform = { - "haystack.components.generators.openai.OpenAIChatGenerator": { - "title": "OpenAIChatGenerator", - "family": "generators", - }, - "haystack.components.generators.openai.OpenAIGenerator": { - "title": "OpenAIGenerator", - "family": "generators", - }, - } - source = set(platform.keys()) - mdx = build_mdx(platform, source) - assert "OpenAIChatGenerator" in mdx - assert "OpenAIGenerator" not in mdx - - def test_voyage_embedders_and_ranker_merged_under_voyage_ai(self): - platform = { - "haystack_integrations.components.embedders.voyage_embedders.voyage_text_embedder.VoyageTextEmbedder": { - "title": "VoyageTextEmbedder", - "family": "embedders", - }, - "haystack_integrations.components.rankers.voyage.ranker.VoyageRanker": { - "title": "VoyageRanker", - "family": "rankers", - }, - } - source = set(platform.keys()) - mdx = build_mdx(platform, source) - # Must appear exactly once as a section heading - assert mdx.count("## Voyage AI") == 1 - assert "VoyageTextEmbedder" in mdx - assert "VoyageRanker" in mdx - - def test_docling_serve_merged_under_docling(self): - platform = { - "haystack_integrations.components.converters.docling.converter.DoclingConverter": { - "title": "DoclingConverter", - "family": "converters", - }, - "haystack_integrations.components.converters.docling_serve.converter.DoclingServeConverter": { - "title": "DoclingServeConverter", - "family": "converters", - }, - } - source = set(platform.keys()) - mdx = build_mdx(platform, source) - assert mdx.count("## Docling") == 1 - - def test_family_field_used_for_type(self): - platform = { - "haystack.components.retrievers.memory.InMemoryBM25Retriever": { - "title": "InMemoryBM25Retriever", - "family": "retrievers", - } - } - link_map = {"InMemoryBM25Retriever": "https://docs.haystack.deepset.ai/docs/inmemorybm25retriever"} - mdx = build_mdx(platform, set(platform.keys()), docs_link_map=link_map) - assert "| [InMemoryBM25Retriever](https://docs.haystack.deepset.ai/docs/inmemorybm25retriever) | Retriever |" in mdx - - def test_family_fallback_to_fqn_path(self): - platform = { - "haystack.components.rankers.foo.MyRanker": { - "title": "MyRanker", - "family": "", # missing family - } - } - mdx = build_mdx(platform, set(platform.keys())) - assert "| MyRanker | Ranker |" in mdx - - def test_component_with_no_docs_page_is_plain_text(self): - platform = { - "haystack.components.rankers.foo.MyRanker": { - "title": "MyRanker", - "family": "rankers", - } - } - # docs_link_map has no entry for MyRanker - mdx = build_mdx(platform, set(platform.keys()), docs_link_map={}) - assert "| MyRanker | Ranker |" in mdx - assert "[MyRanker]" not in mdx - - -# ── scan_docs_links / _parse_frontmatter ───────────────────────────────────── - - -class TestScanDocsLinks: - def test_parses_title_and_slug(self, tmp_path): - mdx = tmp_path / "pipeline-components" / "retrievers" / "inmemorybm25retriever.mdx" - mdx.parent.mkdir(parents=True) - mdx.write_text('---\ntitle: "InMemoryBM25Retriever"\nid: inmemorybm25retriever\nslug: "/inmemorybm25retriever"\n---\n') - result = scan_docs_links(tmp_path) - assert result == {"InMemoryBM25Retriever": "https://docs.haystack.deepset.ai/docs/inmemorybm25retriever"} - - def test_files_without_frontmatter_skipped(self, tmp_path): - mdx = tmp_path / "no-frontmatter.mdx" - mdx.write_text("# Just a heading\n") - assert scan_docs_links(tmp_path) == {} - - def test_files_missing_slug_skipped(self, tmp_path): - mdx = tmp_path / "partial.mdx" - mdx.write_text('---\ntitle: "OnlyTitle"\n---\n') - assert scan_docs_links(tmp_path) == {} - - def test_multiple_files_all_indexed(self, tmp_path): - for name in ("foo", "bar"): - f = tmp_path / f"{name}.mdx" - f.write_text(f'---\ntitle: "{name.title()}"\nslug: "/{name}"\n---\n') - result = scan_docs_links(tmp_path) - assert len(result) == 2 - assert result["Foo"] == "https://docs.haystack.deepset.ai/docs/foo" - - -class TestParseFrontmatter: - def test_basic_key_value(self): - text = '---\ntitle: "Hello"\nslug: "/hello"\n---\nBody' - fm = _parse_frontmatter(text) - assert fm["title"] == "Hello" - assert fm["slug"] == "/hello" - - def test_no_frontmatter_returns_empty(self): - assert _parse_frontmatter("# No frontmatter") == {} - - def test_unclosed_frontmatter_returns_empty(self): - assert _parse_frontmatter("---\ntitle: orphan\n") == {} From a96765ff2f1ef79dd8c6db9f79227c80a2694bb6 Mon Sep 17 00:00:00 2001 From: bilgeyucel Date: Mon, 29 Jun 2026 11:55:17 +0200 Subject: [PATCH 3/5] fix: apply ruff formatting to script --- scripts/generate_platform_components_table.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/generate_platform_components_table.py b/scripts/generate_platform_components_table.py index 7b92966c000..0c93275518e 100644 --- a/scripts/generate_platform_components_table.py +++ b/scripts/generate_platform_components_table.py @@ -341,11 +341,7 @@ def main(argv: list[str] | None = None) -> None: help="Root of the deepset-ai/haystack-core-integrations checkout.", ) parser.add_argument( - "--schema", - required=True, - type=Path, - metavar="PATH", - help="Path to schema JSON file in the platform repo.", + "--schema", required=True, type=Path, metavar="PATH", help="Path to schema JSON file in the platform repo." ) parser.add_argument( "--output", From 3d0e7b014b26b6cbdabf4d5699eb51aaad35281e Mon Sep 17 00:00:00 2001 From: bilgeyucel Date: Mon, 29 Jun 2026 12:03:55 +0200 Subject: [PATCH 4/5] fix: add blank line after SPDX header for hawkeye compliance --- scripts/generate_platform_components_table.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generate_platform_components_table.py b/scripts/generate_platform_components_table.py index 0c93275518e..e3ec95ca1ed 100644 --- a/scripts/generate_platform_components_table.py +++ b/scripts/generate_platform_components_table.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2022-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 + """ Generate a Haystack Enterprise Platform Components MDX table. From cb29aff9b29f4fa963d50be2a402daf4295dcf5b Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Mon, 29 Jun 2026 14:38:04 +0200 Subject: [PATCH 5/5] ci: only run platform-components workflow on push to main The workflow opens an auto-PR against main via create-pull-request. On pull_request events it failed with a modify/delete cherry-pick conflict because docs-website/docs/overview/platform-components.mdx does not exist on main yet (this PR introduces it), and the action bases its branch on main. The pull_request trigger also has no secret access on fork PRs and would open an unrelated bot PR on every PR to main. Restrict to push:main and workflow_dispatch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/update-platform-components.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/update-platform-components.yml b/.github/workflows/update-platform-components.yml index ee01a281ff3..20b45de5413 100644 --- a/.github/workflows/update-platform-components.yml +++ b/.github/workflows/update-platform-components.yml @@ -4,9 +4,6 @@ on: push: branches: - main - pull_request: - branches: - - main workflow_dispatch: concurrency: