docs: simplify API reference config and remove script (#441)

Author: anakin87

.github/utils/pydoc-markdown.sh

@@ -29,7 +29,7 @@ jobs:
           node-version: "22"
 
       - name: Run Docusaurus md/mdx checker
-        working-directory: pydoc/temp
+        working-directory: tmp_api_reference
         run: |
           npx docusaurus-mdx-checker -v || {
               echo ""

.github/workflows/CI_check_api_ref.yml

@@ -37,7 +37,7 @@ jobs:
         uses: actions/upload-artifact@v6
         with:
           name: experimental-api-reference
-          path: pydoc/temp
+          path: tmp_api_reference
           if-no-files-found: error
           retention-days: 1
           overwrite: true
@@ -64,15 +64,15 @@ jobs:
         uses: actions/download-artifact@v7
         with:
           name: experimental-api-reference
-          path: temp_api_reference
+          path: tmp_api_reference
 
       - name: Sync API reference
         run: |
           # Function to sync generated API reference to a destination
           sync_to_dest() {
             echo "Syncing to $1"
             mkdir -p "$1"
-            rsync -av --delete --exclude='.git/' "temp_api_reference/" "$1/"
+            rsync -av --delete --exclude='.git/' "tmp_api_reference/" "$1/"
           }
 
           # Sync to main reference

.github/workflows/docusaurus_sync.yml

@@ -74,8 +74,9 @@ instance/
 # Scrapy stuff:
 .scrapy
 
-# documentation
+# API reference
 pydoc/temp/
+tmp_api_reference/
 
 # PyBuilder
 target/

.gitignore

@@ -4,36 +4,12 @@ You can find Haystack's documentation at https://docs.haystack.deepset.ai/.
 
 # API Reference
 
-We use Pydoc-Markdown to create Markdown files from the docstrings in our code. There is a Github Action that regenerates the API pages when tags are pushed or when manually triggered.
+We use [haystack-pydoc-tools](https://github.com/deepset-ai/haystack-pydoc-tools) to create Markdown files from the docstrings in our code. There is a [Github workflow](https://github.com/deepset-ai/haystack-experimental/blob/main/.github/workflows/docusaurus_sync.yml) that regenerates the API reference at release time.
 
-If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how Pydoc-Markdown will generate the page and commit it to main.
+If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how haystack-pydoc-tools will generate the page and commit it to main.
 
-All the updates to docstrings get pushed to documentation when a new version is released.
+All the updates to API reference get pushed to documentation when a new version is released.
 
 ### Configuration
 
-Pydoc will read the configuration from a `.yml` file which is located under `pydoc`. Our files contain three main sections:
-
-- **loaders**: A list of plugins that load API objects from python source files.
-    - **type**: Loader for python source files (we use `haystack_pydoc_tools.loaders.CustomPythonLoader`)
-    - **search_path**: Location of source files (relative to the config file)
-    - **modules**: Modules which are used for generating the markdown file
-    - **ignore_when_discovered**: Define which files should be ignored
-- **processors**: A list of plugins that process API objects to modify their docstrings (e.g. to adapt them from a documentation format to Markdown or to remove items that should not be rendered into the documentation).
-    - **type: filter**: Filter for specific modules
-    - **documented_only**: Only documented API objects
-    - **do_not_filter_modules**: Do not filter module objects
-    - **skip_empty_modules**: Skip modules without content
-    - **type: smart**: Smart processor for docstring processing
-    - **type: crossref**: Cross-reference processor
-- **renderer**: A plugin that produces the output files. We use a custom DocusaurusRenderer based on the Markdown renderer. It makes sure the Markdown files comply with Docusaurus requirements.
-    - **type**: Define the renderer which you want to use. We are using the DocusaurusRenderer to make sure the files display properly in Docusaurus.
-    - **description**: Add a short description of the page. It shows up right below the page title.
-    - **id**: This is the Docusaurus page ID to make sure the doc lands in the right section of Haystack docs.
-    - **title**: The title of the doc as it will appear on the website. Make sure you always add "API" at the end.
-    - **markdown**:
-        - **descriptive_class_title**: Remove the word "Object" from class titles.
-        - **descriptive_module_title**: Adding the word "Module" before the module name.
-        - **add_method_class_prefix**: Add the class name as a prefix to method names.
-        - **add_member_class_prefix**: Add the class name as a prefix to member names.
-        - **filename**: File name of the generated file, use underscores to separate each word.
+For configuration details, see the [haystack-pydoc-tools documentation](https://github.com/deepset-ai/haystack-pydoc-tools/blob/main/README.md#configuration) or inspect the existing `.yml` files in `pydoc`.

pydoc/README.md

@@ -1,31 +1,16 @@
 loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_experimental.components.agents.agent
-  - haystack_experimental.components.agents.human_in_the_loop.breakpoint
-  - haystack_experimental.components.agents.human_in_the_loop.errors
-  - haystack_experimental.components.agents.human_in_the_loop.strategies
-  search_path:
-  - ../
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+  - search_path: [../]
+    modules:
+      - haystack_experimental.components.agents.agent
+      - haystack_experimental.components.agents.human_in_the_loop.breakpoint
+      - haystack_experimental.components.agents.human_in_the_loop.errors
+      - haystack_experimental.components.agents.human_in_the_loop.strategies
 processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
 renderer:
-  description: Tool-using agents with provider-agnostic chat model support.
-  id: experimental-agents-api
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: experimental_agents_api.md
   title: Agents
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  id: experimental-agents-api
+  description: Tool-using agents with provider-agnostic chat model support.
+  filename: experimental_agents_api.md

pydoc/agents_api.yml

@@ -1,28 +1,13 @@
 loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_experimental.chat_message_stores.in_memory
-  search_path:
-  - ../
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+  - search_path: [../]
+    modules:
+      - haystack_experimental.chat_message_stores.in_memory
 processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
 renderer:
-  description: Storage for the chat messages.
-  id: experimental-chatmessage-store-api
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: experimental_chatmessage_store_api.md
   title: ChatMessage Store
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  id: experimental-chatmessage-store-api
+  description: Storage for the chat messages.
+  filename: experimental_chatmessage_store_api.md

pydoc/chat_message_stores_api.yml

@@ -1,28 +1,13 @@
 loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_experimental.components.generators.chat.openai
-  search_path:
-    - ../
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+  - search_path: [../]
+    modules:
+      - haystack_experimental.components.generators.chat.openai
 processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
 renderer:
-  description: Enables text generation using LLMs.
-  id: experimental-generators-api
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: experimental_generators_api.md
   title: Generators
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  id: experimental-generators-api
+  description: Enables text generation using LLMs.
+  filename: experimental_generators_api.md

pydoc/generators_api.yml

@@ -1,28 +1,13 @@
 loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_experimental.memory_stores.mem0.memory_store
-  search_path:
-  - ../
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+  - search_path: [../]
+    modules:
+      - haystack_experimental.memory_stores.mem0.memory_store
 processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
 renderer:
-  description: Storage for the memories using Mem0 as the backend.
-  id: experimental-mem0-memory-store-api
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: experimental_mem0_memory_store_api.md
   title: Mem0 Memory Store
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  id: experimental-mem0-memory-store-api
+  description: Storage for the memories using Mem0 as the backend.
+  filename: experimental_mem0_memory_store_api.md

pydoc/memory_store_api.yml

@@ -1,28 +1,13 @@
 loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_experimental.components.preprocessors.md_header_level_inferrer
-  search_path:
-  - ../
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+  - search_path: [../]
+    modules:
+      - haystack_experimental.components.preprocessors.md_header_level_inferrer
 processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
 renderer:
-  description: Pipelines wrapped as components.
-  id: experimental-preprocessors-api
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: experimental_preprocessors_api.md
   title: Preprocessors
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer
+  id: experimental-preprocessors-api
+  description: Pipelines wrapped as components.
+  filename: experimental_preprocessors_api.md