chore: update Confluence parameters documentation

Author: robodev-r2d2

libs/README.md

@@ -270,6 +270,8 @@ The following types of information can be extracted:
 - `TABLE`: data in tabular form found in the document
 - `IMAGE`: image found in the document
 
+For Confluence sources, provide the instance `url` and API `token` and include either a `space_key` or a `cql` filter (empty values are ignored). Optional flags such as `include_attachments`, `keep_markdown_format`, and `keep_newlines` mirror the parameters supported by LangChain's `ConfluenceLoader`.
+
 For sitemap sources, additional parameters can be provided, e.g.:
 
 - `web_path`: The URL of the XML sitemap to crawl

libs/extractor-api-lib/openapi.yaml

@@ -151,3 +151,47 @@ components:
                     title: type
                     description: Extractortype
                     type: string
+        confluence_parameters:
+            title: ConfluenceParameters
+            description: Parameters required to connect to and extract content from Confluence.
+            required:
+                - url
+                - token
+                - document_name
+            type: object
+            properties:
+                url:
+                    description: url of the confluence space.
+                    type: string
+                token:
+                    description: api key to access confluence.
+                    type: string
+                space_key:
+                    description: the space key of the confluence pages.
+                    type: string
+                cql:
+                    description: Optional Confluence Query Language (CQL) expression used to filter pages.
+                    type: string
+                include_attachments:
+                    description: >-
+                        whether to include file attachments (e.g., images, documents) in the parsed content. Default is `false`.
+                    type: boolean
+                    default: false
+                keep_markdown_format:
+                    description: whether to preserve markdown formatting in the output. Default is `true`.
+                    type: boolean
+                    default: true
+                keep_newlines:
+                    description: >-
+                        whether to retain newline characters in the output for better readability. Default is `true`.
+                    type: boolean
+                    default: true
+                document_name:
+                    description: >-
+                        The name that will be used to store the confluence db in the key value db and the vectordatabase (metadata.document).
+                    type: string
+                confluence_kwargs:
+                    description: Additional kwargs like verify_ssl
+                    type: array
+                    items:
+                        $ref: '#/components/schemas/key_value_pair'

libs/extractor-api-lib/src/extractor_api_lib/impl/extractors/confluence_extractor.py

@@ -54,9 +54,24 @@ async def aextract_content(
             A list of information pieces extracted from Confluence.
         """
         # Convert list of key value pairs to dict
-        confluence_loader_parameters = {
-            x.key: int(x.value) if x.value.isdigit() else x.value for x in extraction_parameters.kwargs
-        }
+        confluence_loader_parameters = {}
+        for key_value in extraction_parameters.kwargs or []:
+            if key_value is None or key_value.key is None:
+                continue
+
+            value = key_value.value
+            if isinstance(value, str):
+                value = value.strip()
+                if not value and key_value.key in {"space_key", "cql"}:
+                    # Skip empty optional parameters
+                    continue
+                if value.isdigit():
+                    value = int(value)
+
+            confluence_loader_parameters[key_value.key] = value
+
+        if "cql" not in confluence_loader_parameters and "space_key" not in confluence_loader_parameters:
+            raise ValueError("Either 'space_key' or 'cql' must be provided for Confluence extraction.")
         if not confluence_loader_parameters.get("max_pages") or isinstance(
             confluence_loader_parameters.get("max_pages"), str
         ):

libs/extractor-api-lib/src/extractor_api_lib/models/confluence_parameters.py

@@ -33,7 +33,11 @@ class ConfluenceParameters(BaseModel):
 
     url: StrictStr = Field(description="url of the confluence space.")
     token: StrictStr = Field(description="api key to access confluence.")
-    space_key: StrictStr = Field(description="the space key of the confluence pages.")
+    space_key: Optional[StrictStr] = Field(default=None, description="the space key of the confluence pages.")
+    cql: Optional[StrictStr] = Field(
+        default=None,
+        description="Optional Confluence Query Language (CQL) expression used to filter pages.",
+    )
     include_attachments: Optional[StrictBool] = Field(
         default=False,
         description="whether to include file attachments (e.g., images, documents) in the parsed content. Default is `false`.",
@@ -55,6 +59,7 @@ class ConfluenceParameters(BaseModel):
         "url",
         "token",
         "space_key",
+        "cql",
         "include_attachments",
         "keep_markdown_format",
         "keep_newlines",
@@ -120,6 +125,7 @@ def from_dict(cls, obj: Dict) -> Self:
                 "url": obj.get("url"),
                 "token": obj.get("token"),
                 "space_key": obj.get("space_key"),
+                "cql": obj.get("cql"),
                 "include_attachments": (
                     obj.get("include_attachments") if obj.get("include_attachments") is not None else False
                 ),

libs/extractor-api-lib/tests/confluence_extractor_test.py

@@ -0,0 +1,68 @@
+"""Tests for the ConfluenceExtractor."""
+
+import pytest
+from unittest.mock import MagicMock, patch
+from langchain_core.documents import Document as LangchainDocument
+
+from extractor_api_lib.impl.extractors.confluence_extractor import ConfluenceExtractor
+from extractor_api_lib.models.extraction_parameters import ExtractionParameters
+from extractor_api_lib.models.key_value_pair import KeyValuePair
+from extractor_api_lib.models.dataclasses.internal_information_piece import InternalInformationPiece
+from extractor_api_lib.impl.types.content_type import ContentType
+
+
+@pytest.fixture
+def confluence_mapper():
+    """Return a mapper mock that produces predictable information pieces."""
+    mapper = MagicMock()
+    mapper.map_document2informationpiece.return_value = InternalInformationPiece(
+        type=ContentType.TEXT,
+        metadata={"document": "doc", "id": "id", "related": []},
+        page_content="content",
+    )
+    return mapper
+
+
+@pytest.mark.asyncio
+@patch("extractor_api_lib.impl.extractors.confluence_extractor.ConfluenceLoader")
+async def test_aextract_content_supports_cql(mock_loader_cls, confluence_mapper):
+    """Ensure the extractor forwards the CQL parameter to the loader."""
+    extractor = ConfluenceExtractor(mapper=confluence_mapper)
+    extraction_parameters = ExtractionParameters(
+        document_name="confluence_doc",
+        source_type="confluence",
+        kwargs=[
+            KeyValuePair(key="url", value="https://example.atlassian.net"),
+            KeyValuePair(key="token", value="token"),
+            KeyValuePair(key="cql", value="type=page"),
+        ],
+    )
+
+    mock_loader_instance = MagicMock()
+    mock_loader_instance.load.return_value = [LangchainDocument(page_content="content", metadata={"title": "Doc"})]
+    mock_loader_cls.return_value = mock_loader_instance
+
+    results = await extractor.aextract_content(extraction_parameters)
+
+    assert len(results) == 1
+    confluence_mapper.map_document2informationpiece.assert_called_once()
+    loader_kwargs = mock_loader_cls.call_args.kwargs
+    assert loader_kwargs["cql"] == "type=page"
+    assert "space_key" not in loader_kwargs
+
+
+@pytest.mark.asyncio
+async def test_aextract_content_requires_space_key_or_cql(confluence_mapper):
+    """The extractor must receive either a space key or a CQL expression."""
+    extractor = ConfluenceExtractor(mapper=confluence_mapper)
+    extraction_parameters = ExtractionParameters(
+        document_name="confluence_doc",
+        source_type="confluence",
+        kwargs=[
+            KeyValuePair(key="url", value="https://example.atlassian.net"),
+            KeyValuePair(key="token", value="token"),
+        ],
+    )
+
+    with pytest.raises(ValueError):
+        await extractor.aextract_content(extraction_parameters)

services/frontend/libs/admin-app/data-access/document.api.ts

@@ -9,7 +9,8 @@ axios.defaults.auth = {
 
 // confluence configuration interface
 export interface ConfluenceConfig {
-  spaceKey: string;
+  spaceKey?: string;
+  cql?: string;
   token: string;
   url: string;
   maxPages?: number;
@@ -55,11 +56,20 @@ export class DocumentAPI {
     static async loadConfluence(config: ConfluenceConfig): Promise<void> {
         try {
             // convert config to list of key/value items for backend
-            const payload = [
-                { key: 'url', value: config.url },
+            const payload: { key: string; value: string }[] = [
+                { key: 'url', value: config.url.trim() },
                 { key: 'token', value: config.token },
-                { key: 'space_key', value: config.spaceKey },
-            ] as { key: string; value: string }[];
+            ];
+
+            const spaceKey = config.spaceKey?.trim();
+            if (spaceKey) {
+                payload.push({ key: 'space_key', value: spaceKey });
+            }
+
+            const cql = config.cql?.trim();
+            if (cql) {
+                payload.push({ key: 'cql', value: cql });
+            }
 
             if (typeof config.maxPages === 'number') {
                 payload.push({ key: 'max_pages', value: String(config.maxPages) });

services/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue

@@ -22,6 +22,7 @@ const spaceKey = ref('');
 const confluenceToken = ref('');
 const confluenceUrl = ref('');
 const maxPages = ref<number>();
+const confluenceCql = ref('');
 
 // sitemap configuration refs
 const sitemapName = ref('');
@@ -75,7 +76,8 @@ const handleConfluenceUpload = () => {
         spaceKey: spaceKey.value,
         token: confluenceToken.value,
         url: confluenceUrl.value,
-        maxPages: maxPages.value
+        maxPages: maxPages.value,
+        cql: confluenceCql.value,
     });
 }
 
@@ -182,13 +184,16 @@ const getErrorMessage = (errorType: string) => {
                       <label for="confluenceName" class="sr-only"> Confluence Name</label>
                       <input v-model="confluenceName" type="text" placeholder="Name" class="input input-bordered w-full" />
                       <label for="spaceKey" class="sr-only">Space key</label>
-                      <input v-model="spaceKey" type="text" placeholder="Space key" class="input input-bordered w-full" />
+                      <input v-model="spaceKey" type="text" placeholder="Space key (optional)" class="input input-bordered w-full" />
+                      <label for="confluenceCql" class="sr-only">CQL</label>
+                      <input v-model="confluenceCql" type="text" placeholder="CQL query (optional)" class="input input-bordered w-full" />
                       <label for="confluenceToken" class="sr-only">Token</label>
                       <input v-model="confluenceToken" type="password" placeholder="Token" class="input input-bordered w-full" />
                       <label for="maxPages" class="sr-only">Max pages</label>
                       <input v-model.number="maxPages" type="number" placeholder="Max number of pages" class="input input-bordered w-full" />
                     </div>
-                    <p class="text-xs opacity-50 mb-4">{{ t('documents.confluenceLoadDescription') }}</p>
+                    <p class="text-xs opacity-50">{{ t('documents.confluenceLoadDescription') }}</p>
+                    <p class="text-xs opacity-50 mb-4">{{ t('documents.confluenceQueryHint') }}</p>
                     <button class="btn btn-sm btn-accent" @click="handleConfluenceUpload">
                         {{ t('documents.loadConfluence') }}
                     </button>

services/frontend/libs/i18n/admin/de.json

@@ -10,8 +10,9 @@
     "uploadingDocument": "Wird hochgeladen...",
     "fileUpload": "Datei-Upload",
     "confluenceUpload": "Confluence",
-    "confluenceLoadTitle": "Confluence-Seiten laden",
-    "confluenceLoadDescription": "Klicken Sie auf den Button unten, um Seiten aus Confluence zu laden",
+    "confluenceLoadTitle": "Confluence-Inhalte laden",
+    "confluenceLoadDescription": "Geben Sie Ihre Confluence-Zugangsdaten an und wählen Sie einen Space-Key oder einen CQL-Filter",
+    "confluenceQueryHint": "Lassen Sie die Felder leer, um den gesamten Bereich zu laden, oder geben Sie einen Confluence Query Language (CQL) Ausdruck zum Filtern der Seiten ein",
     "loadConfluence": "Laden starten",
     "fileTypeNotAllowedTitle": "Dateityp nicht erlaubt",
     "fileTypeNotAllowedDescription": "Nur PDF-, DOCX-, PPTX- und XML-Dateien sind erlaubt",

services/frontend/libs/i18n/admin/en.json

@@ -12,8 +12,9 @@
     "fileTypeNotAllowedDescription": "Only PDF, DOCX, PPTX, and XML files are allowed",
     "fileUpload": "File Upload",
     "confluenceUpload": "Confluence",
-    "confluenceLoadTitle": "Load all Confluence pages from a space",
-    "confluenceLoadDescription": "Click the button below to load pages from Confluence",
+    "confluenceLoadTitle": "Load Confluence content",
+    "confluenceLoadDescription": "Provide your Confluence credentials and choose a space key or CQL filter",
+    "confluenceQueryHint": "Leave fields blank to load the whole space or supply a Confluence Query Language (CQL) expression to filter pages",
     "loadConfluence": "Load Confluence",
     "sitemapUpload": "Sitemap",
     "sitemapLoadTitle": "Load content from a sitemap",