diff --git a/.github/utils/pydoc-markdown.sh b/.github/utils/pydoc-markdown.sh deleted file mode 100755 index a0af0906..00000000 --- a/.github/utils/pydoc-markdown.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Generate API documentation from pydoc-markdown config files. -# Run from repo root using hatch run docs - -set -e # Fails on any error in the following loop - -cd pydoc -rm -rf temp && mkdir temp -cd temp - -for file in ../*.yml ; do - echo "Converting $file..." - pydoc-markdown "$file" -done diff --git a/.github/workflows/CI_check_api_ref.yml b/.github/workflows/CI_check_api_ref.yml index 55e8bbe0..9531b036 100644 --- a/.github/workflows/CI_check_api_ref.yml +++ b/.github/workflows/CI_check_api_ref.yml @@ -29,7 +29,7 @@ jobs: node-version: "22" - name: Run Docusaurus md/mdx checker - working-directory: pydoc/temp + working-directory: tmp_api_reference run: | npx docusaurus-mdx-checker -v || { echo "" diff --git a/.github/workflows/docusaurus_sync.yml b/.github/workflows/docusaurus_sync.yml index 15fe817a..c1c838c3 100644 --- a/.github/workflows/docusaurus_sync.yml +++ b/.github/workflows/docusaurus_sync.yml @@ -37,7 +37,7 @@ jobs: uses: actions/upload-artifact@v6 with: name: experimental-api-reference - path: pydoc/temp + path: tmp_api_reference if-no-files-found: error retention-days: 1 overwrite: true @@ -64,7 +64,7 @@ jobs: uses: actions/download-artifact@v7 with: name: experimental-api-reference - path: temp_api_reference + path: tmp_api_reference - name: Sync API reference run: | @@ -72,7 +72,7 @@ jobs: sync_to_dest() { echo "Syncing to $1" mkdir -p "$1" - rsync -av --delete --exclude='.git/' "temp_api_reference/" "$1/" + rsync -av --delete --exclude='.git/' "tmp_api_reference/" "$1/" } # Sync to main reference diff --git a/.gitignore b/.gitignore index 58b66c1c..6a49c2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -74,8 +74,9 @@ instance/ # Scrapy stuff: .scrapy -# documentation +# API reference pydoc/temp/ +tmp_api_reference/ # PyBuilder target/ diff --git a/pydoc/README.md b/pydoc/README.md index 0d21fc23..43aa1da6 100644 --- a/pydoc/README.md +++ b/pydoc/README.md @@ -4,36 +4,12 @@ You can find Haystack's documentation at https://docs.haystack.deepset.ai/. # API Reference -We use Pydoc-Markdown to create Markdown files from the docstrings in our code. There is a Github Action that regenerates the API pages when tags are pushed or when manually triggered. +We use [haystack-pydoc-tools](https://github.com/deepset-ai/haystack-pydoc-tools) to create Markdown files from the docstrings in our code. There is a [Github workflow](https://github.com/deepset-ai/haystack-experimental/blob/main/.github/workflows/docusaurus_sync.yml) that regenerates the API reference at release time. -If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how Pydoc-Markdown will generate the page and commit it to main. +If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how haystack-pydoc-tools will generate the page and commit it to main. -All the updates to docstrings get pushed to documentation when a new version is released. +All the updates to API reference get pushed to documentation when a new version is released. ### Configuration -Pydoc will read the configuration from a `.yml` file which is located under `pydoc`. Our files contain three main sections: - -- **loaders**: A list of plugins that load API objects from python source files. - - **type**: Loader for python source files (we use `haystack_pydoc_tools.loaders.CustomPythonLoader`) - - **search_path**: Location of source files (relative to the config file) - - **modules**: Modules which are used for generating the markdown file - - **ignore_when_discovered**: Define which files should be ignored -- **processors**: A list of plugins that process API objects to modify their docstrings (e.g. to adapt them from a documentation format to Markdown or to remove items that should not be rendered into the documentation). - - **type: filter**: Filter for specific modules - - **documented_only**: Only documented API objects - - **do_not_filter_modules**: Do not filter module objects - - **skip_empty_modules**: Skip modules without content - - **type: smart**: Smart processor for docstring processing - - **type: crossref**: Cross-reference processor -- **renderer**: A plugin that produces the output files. We use a custom DocusaurusRenderer based on the Markdown renderer. It makes sure the Markdown files comply with Docusaurus requirements. - - **type**: Define the renderer which you want to use. We are using the DocusaurusRenderer to make sure the files display properly in Docusaurus. - - **description**: Add a short description of the page. It shows up right below the page title. - - **id**: This is the Docusaurus page ID to make sure the doc lands in the right section of Haystack docs. - - **title**: The title of the doc as it will appear on the website. Make sure you always add "API" at the end. - - **markdown**: - - **descriptive_class_title**: Remove the word "Object" from class titles. - - **descriptive_module_title**: Adding the word "Module" before the module name. - - **add_method_class_prefix**: Add the class name as a prefix to method names. - - **add_member_class_prefix**: Add the class name as a prefix to member names. - - **filename**: File name of the generated file, use underscores to separate each word. +For configuration details, see the [haystack-pydoc-tools documentation](https://github.com/deepset-ai/haystack-pydoc-tools/blob/main/README.md#configuration) or inspect the existing `.yml` files in `pydoc`. diff --git a/pydoc/agents_api.yml b/pydoc/agents_api.yml index 2c2cac37..aee7856b 100644 --- a/pydoc/agents_api.yml +++ b/pydoc/agents_api.yml @@ -1,31 +1,16 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.agents.agent - - haystack_experimental.components.agents.human_in_the_loop.breakpoint - - haystack_experimental.components.agents.human_in_the_loop.errors - - haystack_experimental.components.agents.human_in_the_loop.strategies - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.agents.agent + - haystack_experimental.components.agents.human_in_the_loop.breakpoint + - haystack_experimental.components.agents.human_in_the_loop.errors + - haystack_experimental.components.agents.human_in_the_loop.strategies processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Tool-using agents with provider-agnostic chat model support. - id: experimental-agents-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_agents_api.md title: Agents - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-agents-api + description: Tool-using agents with provider-agnostic chat model support. + filename: experimental_agents_api.md diff --git a/pydoc/chat_message_stores_api.yml b/pydoc/chat_message_stores_api.yml index 6d945777..907b1b35 100644 --- a/pydoc/chat_message_stores_api.yml +++ b/pydoc/chat_message_stores_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.chat_message_stores.in_memory - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.chat_message_stores.in_memory processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Storage for the chat messages. - id: experimental-chatmessage-store-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_chatmessage_store_api.md title: ChatMessage Store - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-chatmessage-store-api + description: Storage for the chat messages. + filename: experimental_chatmessage_store_api.md diff --git a/pydoc/generators_api.yml b/pydoc/generators_api.yml index 85bf1668..38e159bd 100644 --- a/pydoc/generators_api.yml +++ b/pydoc/generators_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.generators.chat.openai - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.generators.chat.openai processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Enables text generation using LLMs. - id: experimental-generators-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_generators_api.md title: Generators - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-generators-api + description: Enables text generation using LLMs. + filename: experimental_generators_api.md diff --git a/pydoc/memory_store_api.yml b/pydoc/memory_store_api.yml index 94011a3e..57ca1ba0 100644 --- a/pydoc/memory_store_api.yml +++ b/pydoc/memory_store_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.memory_stores.mem0.memory_store - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.memory_stores.mem0.memory_store processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Storage for the memories using Mem0 as the backend. - id: experimental-mem0-memory-store-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_mem0_memory_store_api.md title: Mem0 Memory Store - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-mem0-memory-store-api + description: Storage for the memories using Mem0 as the backend. + filename: experimental_mem0_memory_store_api.md diff --git a/pydoc/preprocessors_api.yml b/pydoc/preprocessors_api.yml index 53668402..cd2acd03 100644 --- a/pydoc/preprocessors_api.yml +++ b/pydoc/preprocessors_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.preprocessors.md_header_level_inferrer - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.preprocessors.md_header_level_inferrer processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Pipelines wrapped as components. - id: experimental-preprocessors-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_preprocessors_api.md title: Preprocessors - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-preprocessors-api + description: Pipelines wrapped as components. + filename: experimental_preprocessors_api.md diff --git a/pydoc/retrievers_api.yml b/pydoc/retrievers_api.yml index e3bf227f..5c25e052 100644 --- a/pydoc/retrievers_api.yml +++ b/pydoc/retrievers_api.yml @@ -1,29 +1,14 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.retrievers.chat_message_retriever - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.retrievers.chat_message_retriever processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: + title: Retrievers + id: experimental-retrievers-api description: Sweep through Document Stores and return a set of candidate documents that are relevant to the query. - id: experimental-retrievers-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_retrievers_api.md - title: Retrievers - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + filename: experimental_retrievers_api.md diff --git a/pydoc/summarizers_api.yml b/pydoc/summarizers_api.yml index a1b0c1d1..56a22775 100644 --- a/pydoc/summarizers_api.yml +++ b/pydoc/summarizers_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.summarizers.llm_summarizer - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.summarizers.llm_summarizer processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Components that summarize texts into concise versions. - id: experimental-summarizers-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_summarizer_api.md title: Summarizers - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-summarizers-api + description: Components that summarize texts into concise versions. + filename: experimental_summarizer_api.md diff --git a/pydoc/writers_api.yml b/pydoc/writers_api.yml index c6502362..f1ca720c 100644 --- a/pydoc/writers_api.yml +++ b/pydoc/writers_api.yml @@ -1,28 +1,13 @@ loaders: -- ignore_when_discovered: - - __init__ - modules: - - haystack_experimental.components.writers.chat_message_writer - search_path: - - ../ - type: haystack_pydoc_tools.loaders.CustomPythonLoader + - search_path: [../] + modules: + - haystack_experimental.components.writers.chat_message_writer processors: -- do_not_filter_modules: false - documented_only: true - expression: null - skip_empty_modules: true - type: filter -- type: smart -- type: crossref + - type: filter + documented_only: true + skip_empty_modules: true renderer: - description: Writers for Haystack. - id: experimental-writers-api - markdown: - add_member_class_prefix: false - add_method_class_prefix: true - classdef_code_block: false - descriptive_class_title: false - descriptive_module_title: true - filename: experimental_writers_api.md title: Writers - type: haystack_pydoc_tools.renderers.DocusaurusRenderer + id: experimental-writers-api + description: Writers for Haystack. + filename: experimental_writers_api.md diff --git a/pyproject.toml b/pyproject.toml index 41b2e409..985e90f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ [tool.hatch.envs.default.scripts] fmt = "ruff check --fix {args}; ruff format {args}" fmt-check = "ruff check {args} && ruff format --check {args}" -docs = "./.github/utils/pydoc-markdown.sh" +docs = "haystack-pydoc pydoc tmp_api_reference" [tool.hatch.envs.test] extra-dependencies = [