diff --git a/.github/workflows/update-platform-components.yml b/.github/workflows/update-platform-components.yml new file mode 100644 index 00000000000..20b45de5413 --- /dev/null +++ b/.github/workflows/update-platform-components.yml @@ -0,0 +1,99 @@ +name: Update Platform Components Table + +on: + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: update-platform-components + cancel-in-progress: false + +env: + PYTHON_VERSION: "3.11" + OUTPUT_FILE: docs-website/docs/overview/platform-components.mdx + +permissions: + contents: read + +jobs: + update: + runs-on: ubuntu-slim + + steps: + - name: Checkout deepset-ai/haystack + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + # Full history so git diff against HEAD works correctly + fetch-depth: 0 + + - name: Checkout deepset-ai/haystack-core-integrations + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: deepset-ai/haystack-core-integrations + path: haystack-core-integrations + fetch-depth: 1 + sparse-checkout: integrations + sparse-checkout-cone-mode: true + + - name: Checkout platform repo (sparse — schema file only) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ secrets.SCHEMA_REPO }} + token: ${{ secrets.PLATFORM_REPO_PAT }} + path: platform-repo + sparse-checkout: | + ${{ secrets.SCHEMA_REPO_PATH }} + sparse-checkout-cone-mode: false + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Generate platform components table + run: | + python scripts/generate_platform_components_table.py \ + --haystack-src . \ + --integrations-src ./haystack-core-integrations \ + --schema ./platform-repo/${{ secrets.SCHEMA_REPO_PATH }} \ + --output ./${{ env.OUTPUT_FILE }} + + - name: Compute PR body + id: pr_body + run: | + FILE="${{ env.OUTPUT_FILE }}" + if git ls-files --error-unmatch "$FILE" 2>/dev/null; then + ADDED=$(git diff HEAD -- "$FILE" | grep -cE '^\+\| `' || true) + REMOVED=$(git diff HEAD -- "$FILE" | grep -cE '^\-\| `' || true) + SUMMARY="Component rows changed: **+${ADDED}** added, **-${REMOVED}** removed." + else + SUMMARY="Initial generation of the platform components table." + fi + + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Open or update pull request + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + commit-message: "chore: update platform components table [skip ci]" + branch: chore/update-platform-components + base: main + title: "chore: update platform components table" + add-paths: | + ${{ env.OUTPUT_FILE }} + body: ${{ steps.pr_body.outputs.body }} + labels: | + documentation + ignore-for-release-notes diff --git a/docs-website/docs/overview/platform-components.mdx b/docs-website/docs/overview/platform-components.mdx new file mode 100644 index 00000000000..ecf3447d643 --- /dev/null +++ b/docs-website/docs/overview/platform-components.mdx @@ -0,0 +1,514 @@ +--- +title: "Haystack Enterprise Components" +id: platform-components +slug: "/platform-components" +description: "A complete list of Haystack components available on the Haystack Enterprise Platform, grouped by integration partner." +--- + +# Haystack Enterprise Components + +The Haystack Enterprise Platform currently supports **224 components** and **55 integrations**. The following table lists them grouped by integration partner. + +## Core Components + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [Agent](https://docs.haystack.deepset.ai/docs/agent) | Component | ✅ Available | +| [AnswerBuilder](https://docs.haystack.deepset.ai/docs/answerbuilder) | Builder | ✅ Available | +| [AnswerJoiner](https://docs.haystack.deepset.ai/docs/answerjoiner) | Joiner | ✅ Available | +| [AutoMergingRetriever](https://docs.haystack.deepset.ai/docs/automergingretriever) | Retriever | ✅ Available | +| [AzureOCRDocumentConverter](https://docs.haystack.deepset.ai/docs/azureocrdocumentconverter) | Converter | ✅ Available | +| [AzureOpenAIChatGenerator](https://docs.haystack.deepset.ai/docs/azureopenaichatgenerator) | Generator | ✅ Available | +| [AzureOpenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/azureopenaidocumentembedder) | Embedder | ✅ Available | +| [AzureOpenAIResponsesChatGenerator](https://docs.haystack.deepset.ai/docs/azureopenairesponseschatgenerator) | Generator | ✅ Available | +| [AzureOpenAITextEmbedder](https://docs.haystack.deepset.ai/docs/azureopenaitextembedder) | Embedder | ✅ Available | +| [BranchJoiner](https://docs.haystack.deepset.ai/docs/branchjoiner) | Joiner | ✅ Available | +| [CacheChecker](https://docs.haystack.deepset.ai/docs/cachechecker) | Component | ✅ Available | +| [ChatPromptBuilder](https://docs.haystack.deepset.ai/docs/chatpromptbuilder) | Builder | ✅ Available | +| [ConditionalRouter](https://docs.haystack.deepset.ai/docs/conditionalrouter) | Router | ✅ Available | +| [CSVDocumentCleaner](https://docs.haystack.deepset.ai/docs/csvdocumentcleaner) | Preprocessor | ✅ Available | +| [CSVDocumentSplitter](https://docs.haystack.deepset.ai/docs/csvdocumentsplitter) | Preprocessor | ✅ Available | +| [CSVToDocument](https://docs.haystack.deepset.ai/docs/csvtodocument) | Converter | ✅ Available | +| [DocumentCleaner](https://docs.haystack.deepset.ai/docs/documentcleaner) | Preprocessor | ✅ Available | +| [DocumentJoiner](https://docs.haystack.deepset.ai/docs/documentjoiner) | Joiner | ✅ Available | +| [DocumentLanguageClassifier](https://docs.haystack.deepset.ai/docs/documentlanguageclassifier) | Classifier | ✅ Available | +| [DocumentLengthRouter](https://docs.haystack.deepset.ai/docs/documentlengthrouter) | Router | ✅ Available | +| [DocumentPreprocessor](https://docs.haystack.deepset.ai/docs/documentpreprocessor) | Preprocessor | ✅ Available | +| [DocumentSplitter](https://docs.haystack.deepset.ai/docs/documentsplitter) | Preprocessor | ✅ Available | +| [DocumentToImageContent](https://docs.haystack.deepset.ai/docs/documenttoimagecontent) | Converter | ✅ Available | +| [DocumentTypeRouter](https://docs.haystack.deepset.ai/docs/documenttyperouter) | Router | ✅ Available | +| [DocumentWriter](https://docs.haystack.deepset.ai/docs/documentwriter) | Writer | ✅ Available | +| [DOCXToDocument](https://docs.haystack.deepset.ai/docs/docxtodocument) | Converter | ✅ Available | +| [EmbeddingBasedDocumentSplitter](https://docs.haystack.deepset.ai/docs/embeddingbaseddocumentsplitter) | Preprocessor | ✅ Available | +| [ExtractiveReader](https://docs.haystack.deepset.ai/docs/extractivereader) | Reader | ✅ Available | +| [FallbackChatGenerator](https://docs.haystack.deepset.ai/docs/fallbackchatgenerator) | Generator | ✅ Available | +| [FileToFileContent](https://docs.haystack.deepset.ai/docs/filetofilecontent) | Converter | ✅ Available | +| [FileTypeRouter](https://docs.haystack.deepset.ai/docs/filetyperouter) | Router | ✅ Available | +| [FilterRetriever](https://docs.haystack.deepset.ai/docs/filterretriever) | Retriever | ✅ Available | +| [HierarchicalDocumentSplitter](https://docs.haystack.deepset.ai/docs/hierarchicaldocumentsplitter) | Preprocessor | ✅ Available | +| [HTMLToDocument](https://docs.haystack.deepset.ai/docs/htmltodocument) | Converter | ✅ Available | +| [HuggingFaceAPIChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapichatgenerator) | Generator | ✅ Available | +| [HuggingFaceAPIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapidocumentembedder) | Embedder | ✅ Available | +| [HuggingFaceAPITextEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapitextembedder) | Embedder | ✅ Available | +| [HuggingFaceLocalChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfacelocalchatgenerator) | Generator | ✅ Available | +| [HuggingFaceTEIRanker](https://docs.haystack.deepset.ai/docs/huggingfaceteiranker) | Ranker | ✅ Available | +| [ImageFileToDocument](https://docs.haystack.deepset.ai/docs/imagefiletodocument) | Converter | ✅ Available | +| [ImageFileToImageContent](https://docs.haystack.deepset.ai/docs/imagefiletoimagecontent) | Converter | ✅ Available | +| [JSONConverter](https://docs.haystack.deepset.ai/docs/jsonconverter) | Converter | ✅ Available | +| [JsonSchemaValidator](https://docs.haystack.deepset.ai/docs/jsonschemavalidator) | Validator | ✅ Available | +| [LinkContentFetcher](https://docs.haystack.deepset.ai/docs/linkcontentfetcher) | Fetcher | ✅ Available | +| [ListJoiner](https://docs.haystack.deepset.ai/docs/listjoiner) | Joiner | ✅ Available | +| [LLMDocumentContentExtractor](https://docs.haystack.deepset.ai/docs/llmdocumentcontentextractor) | Extractor | ✅ Available | +| [LLMMessagesRouter](https://docs.haystack.deepset.ai/docs/llmmessagesrouter) | Router | ✅ Available | +| [LLMMetadataExtractor](https://docs.haystack.deepset.ai/docs/llmmetadataextractor) | Extractor | ✅ Available | +| [LLMRanker](https://docs.haystack.deepset.ai/docs/llmranker) | Ranker | ✅ Available | +| [LostInTheMiddleRanker](https://docs.haystack.deepset.ai/docs/lostinthemiddleranker) | Ranker | ✅ Available | +| [MarkdownHeaderSplitter](https://docs.haystack.deepset.ai/docs/markdownheadersplitter) | Preprocessor | ✅ Available | +| [MarkdownToDocument](https://docs.haystack.deepset.ai/docs/markdowntodocument) | Converter | ✅ Available | +| [MetadataRouter](https://docs.haystack.deepset.ai/docs/metadatarouter) | Router | ✅ Available | +| [MetaFieldGroupingRanker](https://docs.haystack.deepset.ai/docs/metafieldgroupingranker) | Ranker | ✅ Available | +| [MetaFieldRanker](https://docs.haystack.deepset.ai/docs/metafieldranker) | Ranker | ✅ Available | +| [MSGToDocument](https://docs.haystack.deepset.ai/docs/msgtodocument) | Converter | ✅ Available | +| [MultiFileConverter](https://docs.haystack.deepset.ai/docs/multifileconverter) | Converter | ✅ Available | +| [MultiQueryEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/multiqueryembeddingretriever) | Retriever | ✅ Available | +| [MultiQueryTextRetriever](https://docs.haystack.deepset.ai/docs/multiquerytextretriever) | Retriever | ✅ Available | +| [MultiRetriever](https://docs.haystack.deepset.ai/docs/multiretriever) | Retriever | ✅ Available | +| [NamedEntityExtractor](https://docs.haystack.deepset.ai/docs/namedentityextractor) | Extractor | ✅ Available | +| [OpenAIChatGenerator](https://docs.haystack.deepset.ai/docs/openaichatgenerator) | Generator | ✅ Available | +| [OpenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/openaidocumentembedder) | Embedder | ✅ Available | +| [OpenAIResponsesChatGenerator](https://docs.haystack.deepset.ai/docs/openairesponseschatgenerator) | Generator | ✅ Available | +| [OpenAITextEmbedder](https://docs.haystack.deepset.ai/docs/openaitextembedder) | Embedder | ✅ Available | +| [OpenAPIConnector](https://docs.haystack.deepset.ai/docs/openapiconnector) | Connector | ✅ Available | +| [OpenAPIServiceConnector](https://docs.haystack.deepset.ai/docs/openapiserviceconnector) | Connector | ✅ Available | +| [OpenAPIServiceToFunctions](https://docs.haystack.deepset.ai/docs/openapiservicetofunctions) | Converter | ✅ Available | +| [OutputAdapter](https://docs.haystack.deepset.ai/docs/outputadapter) | Converter | ✅ Available | +| [PDFMinerToDocument](https://docs.haystack.deepset.ai/docs/pdfminertodocument) | Converter | ✅ Available | +| [PDFToImageContent](https://docs.haystack.deepset.ai/docs/pdftoimagecontent) | Converter | ✅ Available | +| [PPTXToDocument](https://docs.haystack.deepset.ai/docs/pptxtodocument) | Converter | ✅ Available | +| [PromptBuilder](https://docs.haystack.deepset.ai/docs/promptbuilder) | Builder | ✅ Available | +| [PyPDFToDocument](https://docs.haystack.deepset.ai/docs/pypdftodocument) | Converter | ✅ Available | +| [PythonCodeSplitter](https://docs.haystack.deepset.ai/docs/pythoncodesplitter) | Preprocessor | ✅ Available | +| [QueryExpander](https://docs.haystack.deepset.ai/docs/queryexpander) | Component | ✅ Available | +| [RecursiveDocumentSplitter](https://docs.haystack.deepset.ai/docs/recursivesplitter) | Preprocessor | ✅ Available | +| [RegexTextExtractor](https://docs.haystack.deepset.ai/docs/regextextextractor) | Extractor | ✅ Available | +| [RemoteWhisperTranscriber](https://docs.haystack.deepset.ai/docs/remotewhispertranscriber) | Audio | ✅ Available | +| [SearchApiWebSearch](https://docs.haystack.deepset.ai/docs/searchapiwebsearch) | Component | ✅ Available | +| [SentenceTransformersDiversityRanker](https://docs.haystack.deepset.ai/docs/sentencetransformersdiversityranker) | Ranker | ✅ Available | +| [SentenceTransformersDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder) | Embedder | ✅ Available | +| [SentenceTransformersDocumentImageEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentimageembedder) | Embedder | ✅ Available | +| [SentenceTransformersSimilarityRanker](https://docs.haystack.deepset.ai/docs/sentencetransformerssimilarityranker) | Ranker | ✅ Available | +| [SentenceTransformersSparseDocumentEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerssparsedocumentembedder) | Embedder | ✅ Available | +| [SentenceTransformersSparseTextEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerssparsetextembedder) | Embedder | ✅ Available | +| [SentenceTransformersTextEmbedder](https://docs.haystack.deepset.ai/docs/sentencetransformerstextembedder) | Embedder | ✅ Available | +| [SentenceWindowRetriever](https://docs.haystack.deepset.ai/docs/sentencewindowretriever) | Retriever | ✅ Available | +| [SerperDevWebSearch](https://docs.haystack.deepset.ai/docs/serperdevwebsearch) | Component | ✅ Available | +| [StringJoiner](https://docs.haystack.deepset.ai/docs/stringjoiner) | Joiner | ✅ Available | +| [TextCleaner](https://docs.haystack.deepset.ai/docs/textcleaner) | Preprocessor | ✅ Available | +| [TextEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/textembeddingretriever) | Retriever | ✅ Available | +| [TextFileToDocument](https://docs.haystack.deepset.ai/docs/textfiletodocument) | Converter | ✅ Available | +| [TextLanguageRouter](https://docs.haystack.deepset.ai/docs/textlanguagerouter) | Router | ✅ Available | +| [TikaDocumentConverter](https://docs.haystack.deepset.ai/docs/tikadocumentconverter) | Converter | ✅ Available | +| [ToolInvoker](https://docs.haystack.deepset.ai/docs/toolinvoker) | Component | ✅ Available | +| [TopPSampler](https://docs.haystack.deepset.ai/docs/toppsampler) | Sampler | ✅ Available | +| [TransformersSimilarityRanker](https://docs.haystack.deepset.ai/docs/transformerssimilarityranker) | Ranker | ✅ Available | +| [TransformersTextRouter](https://docs.haystack.deepset.ai/docs/transformerstextrouter) | Router | ✅ Available | +| [TransformersZeroShotDocumentClassifier](https://docs.haystack.deepset.ai/docs/transformerszeroshotdocumentclassifier) | Classifier | ✅ Available | +| [TransformersZeroShotTextRouter](https://docs.haystack.deepset.ai/docs/transformerszeroshottextrouter) | Router | ✅ Available | +| [XLSXToDocument](https://docs.haystack.deepset.ai/docs/xlsxtodocument) | Converter | ✅ Available | + +## AIML API + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AIMLAPIChatGenerator](https://docs.haystack.deepset.ai/docs/aimllapichatgenerator) | Generator | ✅ Available | + +## AlloyDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AlloyDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/alloydbembeddingretriever) | Retriever | ✅ Available | +| [AlloyDBKeywordRetriever](https://docs.haystack.deepset.ai/docs/alloydbkeywordretriever) | Retriever | ✅ Available | + +## Amazon Bedrock + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AmazonBedrockChatGenerator](https://docs.haystack.deepset.ai/docs/amazonbedrockchatgenerator) | Generator | ✅ Available | +| [AmazonBedrockDocumentEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrockdocumentembedder) | Embedder | ✅ Available | +| [AmazonBedrockDocumentImageEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrockdocumentimageembedder) | Embedder | ✅ Available | +| [AmazonBedrockRanker](https://docs.haystack.deepset.ai/docs/amazonbedrockranker) | Ranker | ✅ Available | +| [AmazonBedrockTextEmbedder](https://docs.haystack.deepset.ai/docs/amazonbedrocktextembedder) | Embedder | ✅ Available | + +## Amazon S3 + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [S3Downloader](https://docs.haystack.deepset.ai/docs/s3downloader) | Component | ✅ Available | + +## Amazon Textract + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AmazonTextractConverter](https://docs.haystack.deepset.ai/docs/amazontextractconverter) | Converter | ✅ Available | + +## Anthropic + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AnthropicChatGenerator](https://docs.haystack.deepset.ai/docs/anthropicchatgenerator) | Generator | ✅ Available | +| AnthropicFoundryChatGenerator | Generator | ✅ Available | +| [AnthropicVertexChatGenerator](https://docs.haystack.deepset.ai/docs/anthropicvertexchatgenerator) | Generator | ✅ Available | + +## ArcadeDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ArcadeDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/arcadedbembeddingretriever) | Retriever | ✅ Available | + +## Astra + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AstraEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/astraretriever) | Retriever | ✅ Available | + +## Azure AI Search + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AzureAISearchBM25Retriever](https://docs.haystack.deepset.ai/docs/azureaisearchbm25retriever) | Retriever | ✅ Available | +| [AzureAISearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/azureaisearchembeddingretriever) | Retriever | ✅ Available | +| [AzureAISearchHybridRetriever](https://docs.haystack.deepset.ai/docs/azureaisearchhybridretriever) | Retriever | ✅ Available | + +## Azure Document Intelligence + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [AzureDocumentIntelligenceConverter](https://docs.haystack.deepset.ai/docs/azuredocumentintelligenceconverter) | Converter | ✅ Available | + +## Brave + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [BraveWebSearch](https://docs.haystack.deepset.ai/docs/bravewebsearch) | Component | ✅ Available | + +## Chroma + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ChromaEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/chromaembeddingretriever) | Retriever | ✅ Available | +| [ChromaQueryTextRetriever](https://docs.haystack.deepset.ai/docs/chromaqueryretriever) | Retriever | ✅ Available | + +## Cohere + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [CohereChatGenerator](https://docs.haystack.deepset.ai/docs/coherechatgenerator) | Generator | ✅ Available | +| [CohereDocumentEmbedder](https://docs.haystack.deepset.ai/docs/coheredocumentembedder) | Embedder | ✅ Available | +| [CohereRanker](https://docs.haystack.deepset.ai/docs/cohereranker) | Ranker | ✅ Available | +| [CohereTextEmbedder](https://docs.haystack.deepset.ai/docs/coheretextembedder) | Embedder | ✅ Available | + +## Docling + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [DoclingConverter](https://docs.haystack.deepset.ai/docs/doclingconverter) | Converter | ✅ Available | +| [DoclingServeConverter](https://docs.haystack.deepset.ai/docs/doclingserveconverter) | Converter | ✅ Available | + +## Elasticsearch + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ElasticsearchBM25Retriever](https://docs.haystack.deepset.ai/docs/elasticsearchbm25retriever) | Retriever | ✅ Available | +| [ElasticsearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/elasticsearchembeddingretriever) | Retriever | ✅ Available | +| ElasticsearchHybridRetriever | Retriever | ✅ Available | +| [ElasticsearchSQLRetriever](https://docs.haystack.deepset.ai/docs/elasticsearchsqlretriever) | Retriever | ✅ Available | + +## FalkorDB + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FalkorDBCypherRetriever](https://docs.haystack.deepset.ai/docs/falkordbcypherretriever) | Retriever | ✅ Available | +| [FalkorDBEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/falkordbembeddingretriever) | Retriever | ✅ Available | + +## FastEmbed + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FastembedDocumentEmbedder](https://docs.haystack.deepset.ai/docs/fastembeddocumentembedder) | Embedder | ✅ Available | +| [FastembedRanker](https://docs.haystack.deepset.ai/docs/fastembedranker) | Ranker | ✅ Available | +| [FastembedSparseDocumentEmbedder](https://docs.haystack.deepset.ai/docs/fastembedsparsedocumentembedder) | Embedder | ✅ Available | +| [FastembedSparseTextEmbedder](https://docs.haystack.deepset.ai/docs/fastembedsparsetextembedder) | Embedder | ✅ Available | +| [FastembedTextEmbedder](https://docs.haystack.deepset.ai/docs/fastembedtextembedder) | Embedder | ✅ Available | + +## Firecrawl + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [FirecrawlCrawler](https://docs.haystack.deepset.ai/docs/firecrawlcrawler) | Fetcher | ✅ Available | +| [FirecrawlWebSearch](https://docs.haystack.deepset.ai/docs/firecrawlwebsearch) | Component | ✅ Available | + +## GitHub + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [GitHubFileEditor](https://docs.haystack.deepset.ai/docs/githubfileeditor) | Connector | ✅ Available | +| [GitHubIssueCommenter](https://docs.haystack.deepset.ai/docs/githubissuecommenter) | Connector | ✅ Available | +| [GitHubIssueViewer](https://docs.haystack.deepset.ai/docs/githubissueviewer) | Connector | ✅ Available | +| [GitHubPRCreator](https://docs.haystack.deepset.ai/docs/githubprcreator) | Connector | ✅ Available | +| [GitHubRepoForker](https://docs.haystack.deepset.ai/docs/githubrepoforker) | Connector | ✅ Available | +| [GitHubRepoViewer](https://docs.haystack.deepset.ai/docs/githubrepoviewer) | Connector | ✅ Available | + +## Google Generative AI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [GoogleGenAIChatGenerator](https://docs.haystack.deepset.ai/docs/googlegenaichatgenerator) | Generator | ✅ Available | +| [GoogleGenAIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/googlegenaidocumentembedder) | Embedder | ✅ Available | +| [GoogleGenAITextEmbedder](https://docs.haystack.deepset.ai/docs/googlegenaitextembedder) | Embedder | ✅ Available | + +## Hugging Face API + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [HuggingFaceAPIChatGenerator](https://docs.haystack.deepset.ai/docs/huggingfaceapichatgenerator) | Generator | ✅ Available | +| [HuggingFaceAPIDocumentEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapidocumentembedder) | Embedder | ✅ Available | +| [HuggingFaceAPITextEmbedder](https://docs.haystack.deepset.ai/docs/huggingfaceapitextembedder) | Embedder | ✅ Available | +| [HuggingFaceTEIRanker](https://docs.haystack.deepset.ai/docs/huggingfaceteiranker) | Ranker | ✅ Available | + +## Jina AI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [JinaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/jinadocumentembedder) | Embedder | ✅ Available | +| [JinaRanker](https://docs.haystack.deepset.ai/docs/jinaranker) | Ranker | ✅ Available | +| [JinaReaderConnector](https://docs.haystack.deepset.ai/docs/jinareaderconnector) | Connector | ✅ Available | +| [JinaTextEmbedder](https://docs.haystack.deepset.ai/docs/jinatextembedder) | Embedder | ✅ Available | + +## Langfuse + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LangfuseConnector](https://docs.haystack.deepset.ai/docs/langfuseconnector) | Connector | ✅ Available | + +## Lara + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LaraDocumentTranslator](https://docs.haystack.deepset.ai/docs/laradocumenttranslator) | Component | ✅ Available | + +## LiteLLM + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LiteLLMChatGenerator](https://docs.haystack.deepset.ai/docs/litellmchatgenerator) | Generator | ✅ Available | + +## Llama Stack + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LlamaStackChatGenerator](https://docs.haystack.deepset.ai/docs/llamastackchatgenerator) | Generator | ✅ Available | + +## Llama.cpp + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [LlamaCppChatGenerator](https://docs.haystack.deepset.ai/docs/llamacppchatgenerator) | Generator | ✅ Available | + +## MarkItDown + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MarkItDownConverter](https://docs.haystack.deepset.ai/docs/markitdownconverter) | Converter | ✅ Available | + +## Mem0 + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [Mem0MemoryRetriever](https://docs.haystack.deepset.ai/docs/mem0memoryretriever) | Retriever | ✅ Available | +| [Mem0MemoryWriter](https://docs.haystack.deepset.ai/docs/mem0memorywriter) | Writer | ✅ Available | + +## Meta Llama + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MetaLlamaChatGenerator](https://docs.haystack.deepset.ai/docs/metallamachatgenerator) | Generator | ✅ Available | + +## Mistral + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MistralChatGenerator](https://docs.haystack.deepset.ai/docs/mistralchatgenerator) | Generator | ✅ Available | +| [MistralDocumentEmbedder](https://docs.haystack.deepset.ai/docs/mistraldocumentembedder) | Embedder | ✅ Available | +| [MistralOCRDocumentConverter](https://docs.haystack.deepset.ai/docs/mistralocrdocumentconverter) | Converter | ✅ Available | +| [MistralTextEmbedder](https://docs.haystack.deepset.ai/docs/mistraltextembedder) | Embedder | ✅ Available | + +## MongoDB Atlas + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [MongoDBAtlasEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/mongodbatlasembeddingretriever) | Retriever | ✅ Available | +| [MongoDBAtlasFullTextRetriever](https://docs.haystack.deepset.ai/docs/mongodbatlasfulltextretriever) | Retriever | ✅ Available | + +## NVIDIA + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [NvidiaChatGenerator](https://docs.haystack.deepset.ai/docs/nvidiachatgenerator) | Generator | ✅ Available | +| [NvidiaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/nvidiadocumentembedder) | Embedder | ✅ Available | +| [NvidiaRanker](https://docs.haystack.deepset.ai/docs/nvidiaranker) | Ranker | ✅ Available | +| [NvidiaTextEmbedder](https://docs.haystack.deepset.ai/docs/nvidiatextembedder) | Embedder | ✅ Available | + +## Ollama + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OllamaChatGenerator](https://docs.haystack.deepset.ai/docs/ollamachatgenerator) | Generator | ✅ Available | +| [OllamaDocumentEmbedder](https://docs.haystack.deepset.ai/docs/ollamadocumentembedder) | Embedder | ✅ Available | +| [OllamaTextEmbedder](https://docs.haystack.deepset.ai/docs/ollamatextembedder) | Embedder | ✅ Available | + +## OpenRouter + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OpenRouterChatGenerator](https://docs.haystack.deepset.ai/docs/openrouterchatgenerator) | Generator | ✅ Available | + +## OpenSearch + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OpenSearchBM25Retriever](https://docs.haystack.deepset.ai/docs/opensearchbm25retriever) | Retriever | ✅ Available | +| [OpenSearchEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/opensearchembeddingretriever) | Retriever | ✅ Available | +| [OpenSearchHybridRetriever](https://docs.haystack.deepset.ai/docs/opensearchhybridretriever) | Retriever | ✅ Available | +| OpenSearchMetadataRetriever | Retriever | ✅ Available | +| [OpenSearchSQLRetriever](https://docs.haystack.deepset.ai/docs/opensearchsqlretriever) | Retriever | ✅ Available | + +## Oracle + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [OracleEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/oracleembeddingretriever) | Retriever | ✅ Available | +| [OracleKeywordRetriever](https://docs.haystack.deepset.ai/docs/oraclekeywordretriever) | Retriever | ✅ Available | + +## Perplexity + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PerplexityChatGenerator](https://docs.haystack.deepset.ai/docs/perplexitychatgenerator) | Generator | ✅ Available | +| [PerplexityDocumentEmbedder](https://docs.haystack.deepset.ai/docs/perplexitydocumentembedder) | Embedder | ✅ Available | +| [PerplexityTextEmbedder](https://docs.haystack.deepset.ai/docs/perplexitytextembedder) | Embedder | ✅ Available | +| [PerplexityWebSearch](https://docs.haystack.deepset.ai/docs/perplexitywebsearch) | Component | ✅ Available | + +## pgvector + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PgvectorEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/pgvectorembeddingretriever) | Retriever | ✅ Available | +| [PgvectorKeywordRetriever](https://docs.haystack.deepset.ai/docs/pgvectorkeywordretriever) | Retriever | ✅ Available | + +## Pinecone + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PineconeEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/pineconedenseretriever) | Retriever | ✅ Available | + +## Presidio + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PresidioDocumentCleaner](https://docs.haystack.deepset.ai/docs/presidiodocumentcleaner) | Preprocessor | ✅ Available | +| [PresidioEntityExtractor](https://docs.haystack.deepset.ai/docs/presidioentityextractor) | Extractor | ✅ Available | +| [PresidioTextCleaner](https://docs.haystack.deepset.ai/docs/presidiotextcleaner) | Preprocessor | ✅ Available | + +## Pyversity + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [PyversityRanker](https://docs.haystack.deepset.ai/docs/pyversityranker) | Ranker | ✅ Available | + +## Qdrant + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [QdrantEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/qdrantembeddingretriever) | Retriever | ✅ Available | +| [QdrantHybridRetriever](https://docs.haystack.deepset.ai/docs/qdranthybridretriever) | Retriever | ✅ Available | +| [QdrantSparseEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/qdrantsparseembeddingretriever) | Retriever | ✅ Available | + +## Snowflake + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [SnowflakeTableRetriever](https://docs.haystack.deepset.ai/docs/snowflaketableretriever) | Retriever | ✅ Available | + +## SQLAlchemy + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| SQLAlchemyTableRetriever | Retriever | ✅ Available | + +## STACKIT + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [STACKITChatGenerator](https://docs.haystack.deepset.ai/docs/stackitchatgenerator) | Generator | ✅ Available | +| [STACKITDocumentEmbedder](https://docs.haystack.deepset.ai/docs/stackitdocumentembedder) | Embedder | ✅ Available | +| [STACKITTextEmbedder](https://docs.haystack.deepset.ai/docs/stackittextembedder) | Embedder | ✅ Available | + +## Supabase + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [SupabasePgvectorEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/supabasepgvectorembeddingretriever) | Retriever | ✅ Available | +| [SupabasePgvectorKeywordRetriever](https://docs.haystack.deepset.ai/docs/supabasepgvectorkeywordretriever) | Retriever | ✅ Available | + +## Tavily + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [TavilyWebSearch](https://docs.haystack.deepset.ai/docs/tavilywebsearch) | Component | ✅ Available | + +## TogetherAI + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [TogetherAIChatGenerator](https://docs.haystack.deepset.ai/docs/togetheraichatgenerator) | Generator | ✅ Available | + +## Unstructured + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [UnstructuredFileConverter](https://docs.haystack.deepset.ai/docs/unstructuredfileconverter) | Converter | ✅ Available | + +## Valkey + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [ValkeyEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/valkeyembeddingretriever) | Retriever | ✅ Available | + +## Vespa + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [VespaEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/vespaembeddingretriever) | Retriever | ✅ Available | +| [VespaKeywordRetriever](https://docs.haystack.deepset.ai/docs/vespakeywordretriever) | Retriever | ✅ Available | + +## vLLM + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [VLLMChatGenerator](https://docs.haystack.deepset.ai/docs/vllmchatgenerator) | Generator | ✅ Available | +| [VLLMDocumentEmbedder](https://docs.haystack.deepset.ai/docs/vllmdocumentembedder) | Embedder | ✅ Available | +| [VLLMRanker](https://docs.haystack.deepset.ai/docs/vllmranker) | Ranker | ✅ Available | +| [VLLMTextEmbedder](https://docs.haystack.deepset.ai/docs/vllmtextembedder) | Embedder | ✅ Available | + +## Weaviate + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [WeaviateBM25Retriever](https://docs.haystack.deepset.ai/docs/weaviatebm25retriever) | Retriever | ✅ Available | +| [WeaviateEmbeddingRetriever](https://docs.haystack.deepset.ai/docs/weaviateembeddingretriever) | Retriever | ✅ Available | + +## Weights & Biases (Weave) + +| Component | Type | Haystack Enterprise Platform | +|-----------|------|------------------------------| +| [WeaveConnector](https://docs.haystack.deepset.ai/docs/weaveconnector) | Connector | ✅ Available | diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 4d99fa77807..9f618e510d2 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -21,6 +21,7 @@ export default { 'overview/breaking-change-policy', 'overview/migration', 'overview/migrating-from-langgraphlangchain-to-haystack', + 'overview/platform-components', ], }, { diff --git a/scripts/generate_platform_components_table.py b/scripts/generate_platform_components_table.py new file mode 100644 index 00000000000..e3ec95ca1ed --- /dev/null +++ b/scripts/generate_platform_components_table.py @@ -0,0 +1,403 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +""" +Generate a Haystack Enterprise Platform Components MDX table. + +Scans deepset-ai/haystack and deepset-ai/haystack-core-integrations for classes +decorated with @component, cross-references them against the platform component +schema (schema-full-component-list.json from deepset-ai/haystack-runtime), and +writes a formatted MDX page. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import logging +import sys +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + +# Maps schema `family` value / module path segment → display type name +TYPE_MAP: dict[str, str] = { + "generators": "Generator", + "retrievers": "Retriever", + "embedders": "Embedder", + "converters": "Converter", + "rankers": "Ranker", + "document_stores": "Document Store", + "preprocessors": "Preprocessor", + "readers": "Reader", + "routers": "Router", + "joiners": "Joiner", + "builders": "Builder", + "classifiers": "Classifier", + "extractors": "Extractor", + "samplers": "Sampler", + "writers": "Writer", + "connectors": "Connector", + "fetchers": "Fetcher", + "validators": "Validator", + "audio": "Audio", +} + +# Human-readable partner labels; snake_case partner keys → display name. +# Keys that map to the same value are intentionally merged into one section. +PARTNER_LABELS: dict[str, str] = { + "aimlapi": "AIML API", + "alloydb": "AlloyDB", + "amazon_bedrock": "Amazon Bedrock", + "amazon_sagemaker": "Amazon SageMaker", + "amazon_textract": "Amazon Textract", + "anthropic": "Anthropic", + "arcadedb": "ArcadeDB", + "astra": "Astra", + "azure_ai_search": "Azure AI Search", + "azure_doc_intelligence": "Azure Document Intelligence", + "brave": "Brave", + "chroma": "Chroma", + "cohere": "Cohere", + "docling": "Docling", + "docling_serve": "Docling", # merged with docling + "elasticsearch": "Elasticsearch", + "falkordb": "FalkorDB", + "fastembed": "FastEmbed", + "firecrawl": "Firecrawl", + "github": "GitHub", + "google_genai": "Google Generative AI", + "huggingface_api": "Hugging Face API", + "jina": "Jina AI", + "langfuse": "Langfuse", + "lara": "Lara", + "litellm": "LiteLLM", + "llama_cpp": "Llama.cpp", + "llama_stack": "Llama Stack", + "markitdown": "MarkItDown", + "mem0": "Mem0", + "meta_llama": "Meta Llama", + "mistral": "Mistral", + "mongodb_atlas": "MongoDB Atlas", + "nvidia": "NVIDIA", + "ollama": "Ollama", + "opensearch": "OpenSearch", + "openrouter": "OpenRouter", + "oracle": "Oracle", + "perplexity": "Perplexity", + "pgvector": "pgvector", + "pinecone": "Pinecone", + "presidio": "Presidio", + "pyversity": "Pyversity", + "qdrant": "Qdrant", + "s3": "Amazon S3", + "snowflake": "Snowflake", + "sqlalchemy": "SQLAlchemy", + "stackit": "STACKIT", + "supabase": "Supabase", + "tavily": "Tavily", + "togetherai": "TogetherAI", + "unstructured": "Unstructured", + "valkey": "Valkey", + "vespa": "Vespa", + "vllm": "vLLM", + "voyage": "Voyage AI", + "voyage_embedders": "Voyage AI", # merged with voyage + "weave": "Weights & Biases (Weave)", + "weaviate": "Weaviate", +} + +_HAYSTACK_DOCS_BASE = "https://docs.haystack.deepset.ai" +_HAYSTACK_DOCS_PREFIX = "/docs" # Docusaurus routeBasePath + +# Namespaces that must never appear in public documentation +_EXCLUDED_NAMESPACES = ("deepset_cloud_custom_nodes.", "studio_internal.", "haystack_experimental.", "deepl_haystack.") + +# Individual class names to exclude regardless of namespace +_EXCLUDED_CLASS_NAMES = { + "SuperComponent", # internal base class, not a user-facing component + "LLM", # alias not present in Haystack OS +} + +# Sets allow O(1) membership tests +_PACKAGE_ROOTS: set[str] = {"haystack_integrations", "haystack"} +_COMPONENT_DECORATORS: set[str] = {"component", "super_component"} + + +# ── Component type helpers ──────────────────────────────────────────────────── + + +def infer_type(family: str | None, fqn: str) -> str: + """Return a human-readable type string from the schema family or FQN path segments.""" + if family and family in TYPE_MAP: + return TYPE_MAP[family] + return next((TYPE_MAP[seg] for seg in fqn.split(".") if seg in TYPE_MAP), "Component") + + +def partner_label_for(fqn: str) -> str | None: + """Return the display-name partner label for a ``haystack_integrations.*`` FQN.""" + parts = fqn.split(".") + # haystack_integrations . components . . . … + if len(parts) < 4: + return None + key = parts[3] + return PARTNER_LABELS.get(key, key.replace("_", " ").title()) + + +# ── Docs-site link map ──────────────────────────────────────────────────────── + + +def _parse_frontmatter(text: str) -> dict[str, str]: + """Extract key-value pairs from YAML-style frontmatter delimited by ``---``.""" + if not text.startswith("---"): + return {} + try: + end = text.index("---", 3) + except ValueError: + return {} + result: dict[str, str] = {} + for line in text[3:end].splitlines(): + if ":" in line: + key, _, value = line.partition(":") + result[key.strip()] = value.strip().strip('"') + return result + + +def scan_docs_links(docs_src: Path) -> dict[str, str]: + """ + Scan docs-website MDX files and return ``{component_title: absolute_url}``. + + Reads only the first 1 KB of each file — enough to cover the frontmatter — + avoiding the cost of loading large MDX files in full. + """ + link_map: dict[str, str] = {} + for mdx_file in docs_src.rglob("*.mdx"): + try: + with mdx_file.open(encoding="utf-8") as fh: + head = fh.read(1024) + except OSError: + continue + fm = _parse_frontmatter(head) + title = fm.get("title", "") + slug = fm.get("slug", "") + if title and slug: + link_map[title] = f"{_HAYSTACK_DOCS_BASE}{_HAYSTACK_DOCS_PREFIX}{slug}" + return link_map + + +# ── Source-scanning helpers ─────────────────────────────────────────────────── + + +def has_component_decorator(node: ast.ClassDef) -> bool: + """Return True if *node* has a ``@component`` or ``@super_component`` decorator.""" + for dec in node.decorator_list: + if isinstance(dec, ast.Name) and dec.id in _COMPONENT_DECORATORS: + return True + if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Name) and dec.func.id in _COMPONENT_DECORATORS: + return True + return False + + +def compute_module_path(file_path: Path, repo_root: Path) -> str | None: + """Convert a ``.py`` file path to a dotted Python module path.""" + try: + parts = list(file_path.relative_to(repo_root).with_suffix("").parts) + except ValueError: + return None + if parts and parts[-1] == "__init__": + parts.pop() + start = next((i for i, p in enumerate(parts) if p in _PACKAGE_ROOTS), None) + return ".".join(parts[start:]) if start is not None else None + + +def scan_for_components(scan_dir: Path, repo_root: Path) -> set[str]: + """Scan *scan_dir* for Python files with ``@component``-decorated classes.""" + found: set[str] = set() + py_files = list(scan_dir.rglob("*.py")) + logger.info("Scanned %d Python files in %s", len(py_files), scan_dir) + + for py_file in py_files: + module_path = compute_module_path(py_file, repo_root) + if module_path is None: + continue + try: + source = py_file.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + # Fast pre-filter: skip files with no decorator mention at all + if "@component" not in source and "@super_component" not in source: + continue + try: + tree = ast.parse(source, filename=str(py_file)) + except SyntaxError as exc: + logger.warning("Could not parse %s: %s", py_file, exc) + continue + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and has_component_decorator(node): + found.add(f"{module_path}.{node.name}") + + return found + + +# ── Schema loading ──────────────────────────────────────────────────────────── + + +def load_platform_components(schema_path: Path) -> dict[str, dict]: + """Parse ``schema-full-component-list.json`` and return public components.""" + all_defs: dict = json.loads(schema_path.read_text(encoding="utf-8")).get("definitions", {}).get("Components", {}) + result: dict[str, dict] = {} + for fqn, defn in all_defs.items(): + if any(fqn.startswith(ns) for ns in _EXCLUDED_NAMESPACES): + continue + if not fqn.startswith(("haystack.", "haystack_integrations.")): + continue + if fqn.split(".")[-1] in _EXCLUDED_CLASS_NAMES: + continue + type_props = defn.get("properties", {}).get("type", {}) + result[fqn] = {"title": defn.get("title", fqn.split(".")[-1]), "family": type_props.get("family", "")} + return result + + +# ── MDX generation ──────────────────────────────────────────────────────────── + + +def build_mdx( + platform_components: dict[str, dict], source_components: set[str], docs_link_map: dict[str, str] | None = None +) -> str: + """Render the MDX page content.""" + _link_map = docs_link_map or {} + core_fqns: list[str] = [] + partner_components: dict[str, list[str]] = {} + visible_count = 0 + + for fqn in platform_components: + if fqn not in source_components: + logger.warning("Schema component not found in source scan: %s", fqn) + continue + cls = fqn.split(".")[-1] + if cls.endswith("Generator") and not cls.endswith("ChatGenerator"): + continue + visible_count += 1 + if fqn.startswith("haystack_integrations."): + partner_components.setdefault(partner_label_for(fqn) or "Other", []).append(fqn) + else: + core_fqns.append(fqn) + + def render_table(fqns: list[str]) -> list[str]: + rows = [ + "| Component | Type | Haystack Enterprise Platform |", + "|-----------|------|------------------------------|", + ] + for fqn in sorted(fqns, key=lambda x: x.split(".")[-1].lower()): + meta = platform_components[fqn] + name = meta["title"] + link = _link_map.get(name, "") + name_cell = f"[{name}]({link})" if link else name + rows.append(f"| {name_cell} | {infer_type(meta.get('family'), fqn)} | ✅ Available |") + return rows + + sections: list[str] = [ + "---", + 'title: "Haystack Enterprise Components"', + "id: platform-components", + 'slug: "/platform-components"', + 'description: "A complete list of Haystack components available on the Haystack Enterprise Platform,' + ' grouped by integration partner."', + "---", + "", + "# Haystack Enterprise Components", + "", + f"The Haystack Enterprise Platform currently supports **{visible_count} components**" + f" and **{len(partner_components)} integrations**." + " The following table lists them grouped by integration partner.", + "", + ] + + if core_fqns: + sections += ["## Core Components", "", *render_table(core_fqns), ""] + + for label in sorted(partner_components, key=str.lower): + sections += [f"## {label}", "", *render_table(partner_components[label]), ""] + + return "\n".join(sections) + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def main(argv: list[str] | None = None) -> None: + """Entry point: parse arguments, run the scan, and write the MDX file.""" + parser = argparse.ArgumentParser(description="Generate the Haystack Enterprise Platform Components MDX table.") + parser.add_argument( + "--haystack-src", required=True, type=Path, metavar="PATH", help="Root of the deepset-ai/haystack checkout." + ) + parser.add_argument( + "--integrations-src", + required=True, + type=Path, + metavar="PATH", + help="Root of the deepset-ai/haystack-core-integrations checkout.", + ) + parser.add_argument( + "--schema", required=True, type=Path, metavar="PATH", help="Path to schema JSON file in the platform repo." + ) + parser.add_argument( + "--output", + type=Path, + metavar="PATH", + default=Path("docs-website/docs/overview/platform-components.mdx"), + help="Destination .mdx file path (default: docs-website/docs/overview/platform-components.mdx).", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print generated content to stdout instead of writing the file." + ) + + args = parser.parse_args(argv) + + for path, flag in [ + (args.haystack_src, "--haystack-src"), + (args.integrations_src, "--integrations-src"), + (args.schema, "--schema"), + ]: + if not path.exists(): + print(f"ERROR: {flag} path does not exist: {path}", file=sys.stderr) + sys.exit(1) + + try: + platform_components = load_platform_components(args.schema) + except (json.JSONDecodeError, OSError) as exc: + print(f"ERROR: Cannot load schema file: {exc}", file=sys.stderr) + sys.exit(1) + + haystack_pkg = args.haystack_src / "haystack" + source_components = scan_for_components( + haystack_pkg if haystack_pkg.is_dir() else args.haystack_src, args.haystack_src + ) + integrations_pkg = args.integrations_src / "integrations" + source_components |= scan_for_components( + integrations_pkg if integrations_pkg.is_dir() else args.integrations_src, args.integrations_src + ) + + logger.info("Platform components in schema (public namespaces): %d", len(platform_components)) + logger.info("Components matched to source scan: %d", sum(1 for f in platform_components if f in source_components)) + + docs_src = args.haystack_src / "docs-website" / "docs" + if not docs_src.is_dir(): + logger.warning("docs-website/docs not found under --haystack-src, component links will be omitted") + docs_link_map = scan_docs_links(docs_src) if docs_src.is_dir() else {} + + mdx = build_mdx(platform_components, source_components, docs_link_map) + + if args.dry_run: + print(mdx) + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(mdx, encoding="utf-8") + logger.info("Written to %s", args.output) + + +if __name__ == "__main__": + main()