feat: add support for multimodal indexing

.github/workflows/integration-test.yml

@@ -113,6 +113,8 @@ jobs:
           repository: nextcloud/context_chat
           path: apps/context_chat
           persist-credentials: false
+          # todo: remove later
+          ref: feat/noid/ocr-support
 
       - name: Checkout backend
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -151,29 +153,44 @@ jobs:
           ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
           composer run serve &
 
-      - name: Enable context_chat, app_api and testing
-        run: ./occ app:enable -vvv -f context_chat app_api testing
+      - name: Enable context_chat and app_api
+        run: ./occ app:enable -vvv -f context_chat app_api
 
       - name: Checkout documentation
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           repository: nextcloud/documentation
-          path: data/admin/files/documentation
+          path: documentation
           persist-credentials: false
 
       - name: Prepare docs
         run: |
-          cd data/admin/files
-          mv documentation/admin_manual .
-          cp -R documentation/developer_manual .
-          cd developer_manual
-          find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \;
-          cd ..
-          cp -R documentation/developer_manual ./developer_manual2
-          cd developer_manual2
-          find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \;
-          cd ..
-          rm -rf documentation
+          # images and audio test files
+          cp -r context_chat_backend/tests/res data/admin/files/
+          # admin_manual - text files only, rename .rst -> .md
+          find documentation/admin_manual -type f \( -name '*.rst' -o -name '*.md' -o -name '*.txt' \) \
+            | while IFS= read -r src; do
+                rel="${src#documentation/admin_manual/}"
+                dest="data/admin/files/admin_manual/${rel%.rst}.md"
+                mkdir -p "$(dirname "$dest")"
+                cp "$src" "$dest"
+              done
+          # developer_manual - text files only, rename .rst -> .txt
+          find documentation/developer_manual -type f \( -name '*.rst' -o -name '*.md' -o -name '*.txt' \) \
+            | while IFS= read -r src; do
+                rel="${src#documentation/developer_manual/}"
+                dest="data/admin/files/developer_manual/${rel%.rst}.txt"
+                mkdir -p "$(dirname "$dest")"
+                cp "$src" "$dest"
+              done
+          # user_manual - text files only, no rename
+          find documentation/user_manual -type f \( -name '*.rst' -o -name '*.md' -o -name '*.txt' \) \
+            | while IFS= read -r src; do
+                rel="${src#documentation/user_manual/}"
+                dest="data/admin/files/user_manual/${rel}"
+                mkdir -p "$(dirname "$dest")"
+                cp "$src" "$dest"
+              done
 
       - name: Run files scan
         run: |
@@ -202,12 +219,34 @@ jobs:
           echo $! > ../pid.txt          # Save the process ID (PID)
           sleep 60 # Wait for the backend to get ready
 
-      - name: Register backend
+      - name: Setup the fake OCR and STT provider
+        env:
+          APP_ID: ccb_test_providers
+          APP_DISPLAY_NAME: CCB Test Providers
+          APP_VERSION: '1.0.0'
+          APP_SECRET: '12345'
+          APP_HOST: '0.0.0.0'
+          APP_PORT: '10020'
+          APP_PERSISTENT_STORAGE: persistent_storage
+          NEXTCLOUD_URL: http://localhost:8080
+        run: |
+          cd context_chat_backend/tests/ex_app
+          python3 -u lib/main.py >& ../../ocr_stt_exapp_logs &
+
+      - name: Register backend and the OCR/STT provider
         run: |
           timeout 10 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          timeout 120 ./occ app_api:app:register ccb_test_providers manual_install --json-info "{\"appid\":\"ccb_test_providers\",\"name\":\"CCB Test Providers\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":10020}" --wait-finish
+          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034}" --wait-finish
           ls -la context_chat_backend/persistent_storage/*
 
+      - name: Enable multimodal indexing
+        run: |
+          ./occ context_chat:queue-multimodal-files
+
+      - name: Enable the testing app late so it does not register for the OCR/STT first
+        run: ./occ app:enable testing
+
       - name: Initial memory usage check
         run: |
           ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
@@ -233,7 +272,7 @@ jobs:
         run: |
           success=0
           echo "::group::Checking stats periodically for 15 minutes to allow the backend to index the files"
-          for i in {1..90}; do
+          for i in {1..30}; do
             echo "Checking stats, attempt $i..."
 
             stats_err=$(mktemp)
@@ -251,7 +290,7 @@ jobs:
             # Check for critical errors in output
             if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then
               echo "Backend connection error detected (exit=$stats_exit), retrying..."
-              sleep 10
+              sleep 30
               continue
             fi
 
@@ -260,9 +299,11 @@ jobs:
 
             # Extract indexed documents count (files__default)
             indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "")
+            queued_count=$(echo "$stats" | jq '.queued_documents_counts.files__default // empty' || echo "")
 
             echo "Total eligible files: $total_eligible_files"
             echo "Indexed documents (files__default): $indexed_count"
+            echo "Queued documents (files__default): $queued_count"
 
             diff=$((total_eligible_files - indexed_count))
             threshold=$((total_eligible_files * 3 / 100))
@@ -277,14 +318,20 @@ jobs:
               echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
             fi
 
+            # Exit early if no queued documents remain since nothing more will be indexed
+            if [ "$queued_count" = "0" ]; then
+              echo "No queued documents remaining but indexing outside tolerance (diff=$diff, threshold=$threshold). Possible locked/failed documents."
+              break
+            fi
+
             # Check if backend is still alive
             ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
             if [ "$ccb_alive" -eq 0 ]; then
               echo "Error: Context Chat Backend process is not running. Exiting."
               exit 1
             fi
 
-            sleep 10
+            sleep 30
           done
 
           echo "::endgroup::"
@@ -299,17 +346,40 @@ jobs:
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker1_logs 2>&1 &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker2_logs 2>&1 &
 
-          echo ::group::English prompt
-          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
-          echo "$OUT1"
-          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
-          echo ::endgroup::
+          # Each entry: "Label;prompt text;grep pattern;grep flags"
+          checks=(
+            "Text - Ethical AI Rating (EN);Which factors are taken into account for the Ethical AI Rating?;If all of these points are met, we give a Green label.;"
+            "Text - Ethical AI Rating (DE);Welche Faktoren beeinflussen das Ethical AI Rating?;If all of these points are met, we give a Green label.;"
+            "Text - occ maintenance mode;How do I enable maintenance mode in Nextcloud using the command line?;maintenance:mode --on;-iE"
+            "Text - background job configuration;What are the available background job execution methods in Nextcloud?;cron|webcron|ajax;-iE"
+            "STT - Amazon oxygen fraction;What fraction of the world's oxygen supply does the Amazon rainforest produce?;twenty percent;-i"
+            "STT - quantum computing;How does quantum computing perform calculations?;superposition|entanglement;-iE"
+            "STT - Jungle Book;Who raised Mowgli in the jungle and which villain does he face?;wolves|Shere Khan;-iE"
+            "OCR - invoice total;What is the total amount on invoice number 10042?;299;-E"
+            "OCR - train to Amsterdam;What time does the express train to Amsterdam depart from Platform 3?;08:14;"
+          )
+
+          failed=0
+          for entry in "${checks[@]}"; do
+            IFS=';' read -r label prompt pattern flags <<< "$entry"
+            OUT=$(./occ context_chat:prompt admin "$prompt")
+            # shellcheck disable=SC2086
+            if echo "$OUT" | grep -q $flags "$pattern"; then
+              status="PASS"
+            else
+              status="FAIL"
+              failed=1
+            fi
+            echo "::group::[$status] $label"
+            echo "$OUT"
+            if [ "$status" = "FAIL" ]; then
+              echo "FAIL: expected to find '$pattern' in output"
+            fi
+            echo "::endgroup::"
+            sleep 10
+          done
 
-          echo ::group::German prompt
-          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          echo "$OUT2"
-          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
-          echo ::endgroup::
+          [ $failed -eq 0 ] || exit 1
 
       - name: Check python memory usage
         run: |
@@ -349,6 +419,11 @@ jobs:
         run: |
           tail -v -n +1 worker?_logs || echo "No worker logs"
 
+      - name: Show ccb_test_providers app logs
+        if: always()
+        run: |
+          cat context_chat_backend/ocr_stt_exapp_logs || echo "No ccb_test_providers app logs"
+
       - name: Show main app logs
         if: always()
         run: |

REUSE.toml

@@ -10,3 +10,9 @@ path = ["requirements.txt", "requirements_dev.txt"]
 precedence = "aggregate"
 SPDX-FileCopyrightText = "2024 Nextcloud GmbH and Nextcloud contributors"
 SPDX-License-Identifier = "AGPL-3.0-or-later"
+
+[[annotations]]
+path = ["tests/res/**"]
+precedence = "aggregate"
+SPDX-FileCopyrightText = "2026 Nextcloud GmbH and Nextcloud contributors"
+SPDX-License-Identifier = "CC0-1.0"