integration-test.yml

# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
# SPDX-License-Identifier: AGPL-3.0-or-later

name: Integration test

on:
  pull_request:
  push:
    branches:
      - master
      - stable*

permissions:
  contents: read

concurrency:
  group: integration-test-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true


jobs:
  changes:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read

    outputs:
      src: ${{ steps.changes.outputs.src}}

    steps:
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: changes
        continue-on-error: true
        with:
          filters: |
            src:
              - 'main.py'
              - 'main_em.py'
              - 'config.cpu.yaml'
              - 'config.gpu.yaml'
              - 'context_chat_backend/**'
              - 'appinfo/**'
              - 'example.env'
              - 'hwdetect.sh'
              - 'persistent_storage/**'
              - 'project.toml'
              - 'requirements.txt'
              - 'logger_config.yaml'
              - 'logger_config_em.yaml'
              - 'supervisord.conf'
              - '.github/workflows/integration-test.yml'

  integration:
    runs-on: ubuntu-24.04

    needs: changes
    if: needs.changes.outputs.src != 'false'

    strategy:
      # do not stop on another job's failure
      fail-fast: false
      matrix:
        php-versions: [ '8.2' ]
        databases: [ 'pgsql' ]
        server-versions: [ 'stable32', 'stable33', 'master' ]

    name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}

    env:
      MYSQL_PORT: 4444
      PGSQL_PORT: 4445
      # use the same db for ccb and nextcloud
      CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud

    services:
      mysql:
        image: mariadb:10.5
        ports:
          - 4444:3306/tcp
        env:
          MYSQL_ROOT_PASSWORD: rootpassword
        options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
      postgres:
        image: pgvector/pgvector:pg17
        ports:
          - 4445:5432/tcp
        env:
          POSTGRES_USER: root
          POSTGRES_PASSWORD: rootpassword
          POSTGRES_DB: nextcloud
        options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 --name postgres --hostname postgres

    steps:
      - name: Checkout server
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          repository: nextcloud/server
          ref: ${{ matrix.server-versions }}
          submodules: 'recursive'
          persist-credentials: false

      - name: Set up php ${{ matrix.php-versions }}
        uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2
        with:
          php-version: ${{ matrix.php-versions }}
          tools: phpunit
          extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip

      - name: Checkout context_chat php app
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          repository: nextcloud/context_chat
          path: apps/context_chat
          persist-credentials: false
          # todo: remove later
          ref: feat/reverse-content-flow

      - name: Checkout backend
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          path: context_chat_backend/
          persist-credentials: false

      - name: Get app version
        id: appinfo
        uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
        with:
          filename: context_chat_backend/appinfo/info.xml
          expression: "/info/version/text()"

      - name: Set up Nextcloud MYSQL
        if: ${{ matrix.databases != 'pgsql'}}
        run: |
          sleep 25
          mkdir data
          ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
          composer run serve &

      - name: Set up Nextcloud PGSQL
        if: ${{ matrix.databases == 'pgsql'}}
        run: |
          sleep 25
          mkdir data
          ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
          composer run serve &

      - name: Enable context_chat, app_api and testing
        run: ./occ app:enable -vvv -f context_chat app_api testing

      - name: Checkout documentation
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
        with:
          repository: nextcloud/documentation
          path: data/admin/files/documentation
          persist-credentials: false

      - name: Prepare docs
        run: |
          cd data/admin/files
          mv documentation/admin_manual .
          cp -R documentation/developer_manual .
          cd developer_manual
          find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \;
          cd ..
          cp -R documentation/developer_manual ./developer_manual2
          cd developer_manual2
          find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \;
          cd ..
          rm -rf documentation

      - name: Run files scan
        run: |
          ./occ files:scan --all

      - name: Setup python 3.11
        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5
        with:
          python-version: '3.11'
          cache: 'pip'
          cache-dependency-path: |
            requirements.txt

      - name: Install and init backend
        run: |
          cd context_chat_backend
          pip install --upgrade pip setuptools wheel
          # use the cpu version of torch to not run out of space
          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
          sed -i '/torch(vision)?/d' requirements.txt
          pip install -r requirements.txt
          cp example.env .env
          echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
          python3 -u ./main_em.py > em_backend_logs 2>&1 &
          python3 -u ./main.py > backend_logs 2>&1 &
          echo $! > ../pid.txt          # Save the process ID (PID)
          sleep 60 # Wait for the backend to get ready

      - name: Register backend
        run: |
          timeout 10 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
          ls -la context_chat_backend/persistent_storage/*

      - name: Initial memory usage check
        run: |
          ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
          ps -p $(cat pid.txt) -o %mem --no-headers > initial_mem.txt

      - name: Run cron jobs
        run: |
          # every 10 seconds indefinitely
          while true; do
            php cron.php
            sleep 10
          done &
          sleep 30
          # list all the bg jobs
          ./occ background-job:list

      - name: Initial dump of DB with context_chat_queue populated
        run: |
          docker exec postgres pg_dump nextcloud > /tmp/0_pgdump_nextcloud

      - name: Periodically check context_chat stats for 15 minutes to allow the backend to index the files
        run: |
          success=0
          echo "::group::Checking stats periodically for 15 minutes to allow the backend to index the files"
          for i in {1..90}; do
            echo "Checking stats, attempt $i..."

            stats_err=$(mktemp)
            stats=$(timeout 5 ./occ context_chat:stats --json 2>"$stats_err")
            stats_exit=$?
            echo "Stats output:"
            echo "$stats"
            if [ -s "$stats_err" ]; then
              echo "Stderr:"
              cat "$stats_err"
            fi
            echo "---"
            rm -f "$stats_err"

            # Check for critical errors in output
            if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then
              echo "Backend connection error detected (exit=$stats_exit), retrying..."
              sleep 10
              continue
            fi

            # Extract total eligible files
            total_eligible_files=$(echo "$stats" | jq '.eligible_files_count' || echo "")

            # Extract indexed documents count (files__default)
            indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "")

            echo "Total eligible files: $total_eligible_files"
            echo "Indexed documents (files__default): $indexed_count"

            diff=$((total_eligible_files - indexed_count))
            threshold=$((total_eligible_files * 3 / 100))

            # Check if difference is within tolerance
            if [ $diff -le $threshold ]; then
              echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)"
              success=1
              break
            else
              progress=$((diff * 100 / total_eligible_files))
              echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
            fi

            # Check if backend is still alive
            ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
            if [ "$ccb_alive" -eq 0 ]; then
              echo "Error: Context Chat Backend process is not running. Exiting."
              exit 1
            fi

            sleep 10
          done

          echo "::endgroup::"

          ./occ context_chat:stats
          ./occ context_chat:stats --json

          if [ $success -ne 1 ]; then
            echo "Max attempts reached"
            exit 1
          fi

      - name: Run the prompts
        run: |
          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker1_logs 2>&1 &
          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker2_logs 2>&1 &

          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
          echo "$OUT1"
          echo '--------------------------------------------------'
          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
          echo "$OUT2"

          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1

      - name: Check python memory usage
        run: |
          ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
          ps -p $(cat pid.txt) -o %mem --no-headers > after_prompt_mem.txt

      - name: Compare memory usage and detect leak
        run: |
          initial_mem=$(cat initial_mem.txt | tr -d ' ')
          final_mem=$(cat after_scan_mem.txt | tr -d ' ')
          echo "Initial Memory Usage: $initial_mem%"
          echo "Memory Usage after scan: $final_mem%"

          if (( $(echo "$final_mem > $initial_mem" | bc -l) )); then
            echo "Memory usage has increased during scan. Possible memory leak detected!"
          else
            echo "Memory usage during scan is stable. No memory leak detected."
          fi

      - name: Final dump of DB with vectordb populated
        run: |
          docker exec postgres pg_dump nextcloud > /tmp/1_pgdump_nextcloud

      - name: Show server logs
        if: always()
        run: |
          cat data/nextcloud.log

      - name: Show context_chat specific logs
        if: always()
        run: |
          cat data/context_chat.log

      - name: Show task processing worker logs
        if: always()
        run: |
          tail -v -n +1 worker?_logs || echo "No worker logs"

      - name: Show main app logs
        if: always()
        run: |
          cat context_chat_backend/backend_logs || echo "No main backend logs"

      - name: Show main app JSON logs
        if: always()
        run: |
          tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"

      - name: Show embedding server logs
        if: always()
        run: |
          cat context_chat_backend/em_backend_logs || echo "No main backend logs"

      - name: Show embedding server JSON logs
        if: always()
        run: |
          tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"

      - name: Upload database dumps
        uses: actions/upload-artifact@v4
        with:
          name: database-dumps
          path: |
            /tmp/0_pgdump_nextcloud
            /tmp/1_pgdump_nextcloud

  summary:
    permissions:
      contents: none
    runs-on: ubuntu-latest-low
    needs: [changes, integration]

    if: always()

    # This is the summary, we just avoid to rename it so that branch protection rules still match
    name: integration-test

    steps:
      - name: Summary status
        run: if ${{ needs.changes.outputs.src != 'false' && needs.integration.result != 'success' }}; then exit 1; fi