feat: add kubernetes support #798
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors | |
| # SPDX-License-Identifier: AGPL-3.0-or-later | |
| name: Integration test | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| - stable* | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: integration-test-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| src: ${{ steps.changes.outputs.src}} | |
| steps: | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 | |
| id: changes | |
| continue-on-error: true | |
| with: | |
| filters: | | |
| src: | |
| - 'main.py' | |
| - 'main_em.py' | |
| - 'config.cpu.yaml' | |
| - 'config.gpu.yaml' | |
| - 'context_chat_backend/**' | |
| - 'appinfo/**' | |
| - 'example.env' | |
| - 'hwdetect.sh' | |
| - 'persistent_storage/**' | |
| - 'project.toml' | |
| - 'requirements.txt' | |
| - 'logger_config.yaml' | |
| - 'logger_config_em.yaml' | |
| - 'supervisord.conf' | |
| - '.github/workflows/integration-test.yml' | |
| integration: | |
| runs-on: ubuntu-24.04 | |
| needs: changes | |
| if: needs.changes.outputs.src != 'false' | |
| strategy: | |
| # do not stop on another job's failure | |
| fail-fast: false | |
| matrix: | |
| php-versions: [ '8.2' ] | |
| databases: [ 'pgsql' ] | |
| server-versions: [ 'stable32', 'stable33', 'master' ] | |
| name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} | |
| env: | |
| MYSQL_PORT: 4444 | |
| PGSQL_PORT: 4445 | |
| # use the same db for ccb and nextcloud | |
| CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud | |
| services: | |
| mysql: | |
| image: mariadb:10.5 | |
| ports: | |
| - 4444:3306/tcp | |
| env: | |
| MYSQL_ROOT_PASSWORD: rootpassword | |
| options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 | |
| postgres: | |
| image: pgvector/pgvector:pg17 | |
| ports: | |
| - 4445:5432/tcp | |
| env: | |
| POSTGRES_USER: root | |
| POSTGRES_PASSWORD: rootpassword | |
| POSTGRES_DB: nextcloud | |
| options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 | |
| steps: | |
| - name: Checkout server | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| repository: nextcloud/server | |
| ref: ${{ matrix.server-versions }} | |
| submodules: 'recursive' | |
| persist-credentials: false | |
| - name: Set up php ${{ matrix.php-versions }} | |
| uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2 | |
| with: | |
| php-version: ${{ matrix.php-versions }} | |
| tools: phpunit | |
| extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip | |
| - name: Checkout context_chat php app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| repository: nextcloud/context_chat | |
| path: apps/context_chat | |
| persist-credentials: false | |
| # todo: remove later | |
| ref: feat/reverse-content-flow | |
| - name: Checkout backend | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| path: context_chat_backend/ | |
| persist-credentials: false | |
| - name: Get app version | |
| id: appinfo | |
| uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master | |
| with: | |
| filename: context_chat_backend/appinfo/info.xml | |
| expression: "/info/version/text()" | |
| - name: Set up Nextcloud MYSQL | |
| if: ${{ matrix.databases != 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| composer run serve & | |
| - name: Set up Nextcloud PGSQL | |
| if: ${{ matrix.databases == 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| composer run serve & | |
| - name: Enable context_chat, app_api and testing | |
| run: ./occ app:enable -vvv -f context_chat app_api testing | |
| - name: Checkout documentation | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | |
| with: | |
| repository: nextcloud/documentation | |
| path: data/admin/files/documentation | |
| persist-credentials: false | |
| - name: Prepare docs | |
| run: | | |
| cd data/admin/files | |
| mv documentation/admin_manual . | |
| cp -R documentation/developer_manual . | |
| cd developer_manual | |
| find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \; | |
| cd .. | |
| cp -R documentation/developer_manual ./developer_manual2 | |
| cd developer_manual2 | |
| find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \; | |
| cd .. | |
| rm -rf documentation | |
| - name: Run files scan | |
| run: | | |
| ./occ files:scan --all | |
| - name: Setup python 3.11 | |
| uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements.txt | |
| - name: Install and init backend | |
| run: | | |
| cd context_chat_backend | |
| pip install --upgrade pip setuptools wheel | |
| # use the cpu version of torch to not run out of space | |
| pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu | |
| sed -i '/torch(vision)?/d' requirements.txt | |
| pip install -r requirements.txt | |
| cp example.env .env | |
| echo "NEXTCLOUD_URL=http://localhost:8080" >> .env | |
| python3 -u ./main_em.py > em_backend_logs 2>&1 & | |
| python3 -u ./main.py > backend_logs 2>&1 & | |
| echo $! > ../pid.txt # Save the process ID (PID) | |
| sleep 60 # Wait for the backend to get ready | |
| - name: Register backend | |
| run: | | |
| timeout 10 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 | |
| timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish | |
| ls -la context_chat_backend/persistent_storage/* | |
| - name: Initial memory usage check | |
| run: | | |
| ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem | |
| ps -p $(cat pid.txt) -o %mem --no-headers > initial_mem.txt | |
| - name: Run cron jobs | |
| run: | | |
| # every 10 seconds indefinitely | |
| while true; do | |
| php cron.php | |
| sleep 10 | |
| done & | |
| - name: Periodically check context_chat stats for 15 minutes to allow the backend to index the files | |
| run: | | |
| success=0 | |
| echo "::group::Checking stats periodically for 15 minutes to allow the backend to index the files" | |
| for i in {1..90}; do | |
| echo "Checking stats, attempt $i..." | |
| stats_err=$(mktemp) | |
| stats=$(timeout 5 ./occ context_chat:stats --json 2>"$stats_err") | |
| stats_exit=$? | |
| echo "Stats output:" | |
| echo "$stats" | |
| if [ -s "$stats_err" ]; then | |
| echo "Stderr:" | |
| cat "$stats_err" | |
| fi | |
| echo "---" | |
| rm -f "$stats_err" | |
| # Check for critical errors in output | |
| if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then | |
| echo "Backend connection error detected (exit=$stats_exit), retrying..." | |
| sleep 10 | |
| continue | |
| fi | |
| # Extract total queued files | |
| total_files=$(echo "$stats" | jq '.queued_documents_counts.files__default' || echo "") | |
| # Extract indexed documents count (files__default) | |
| indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "") | |
| echo "Total queued files: $total_files" | |
| echo "Indexed documents (files__default): $indexed_count" | |
| diff=$((total_files - indexed_count)) | |
| threshold=$((total_files * 3 / 100)) | |
| # Check if difference is within tolerance | |
| if [ $diff -le $threshold ]; then | |
| echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)" | |
| success=1 | |
| break | |
| else | |
| progress=$((diff * 100 / total_files)) | |
| echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold" | |
| fi | |
| # Check if backend is still alive | |
| ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0") | |
| if [ "$ccb_alive" -eq 0 ]; then | |
| echo "Error: Context Chat Backend process is not running. Exiting." | |
| exit 1 | |
| fi | |
| sleep 10 | |
| done | |
| echo "::endgroup::" | |
| ./occ context_chat:stats | |
| ./occ context_chat:stats --json | |
| if [ $success -ne 1 ]; then | |
| echo "Max attempts reached" | |
| exit 1 | |
| fi | |
| - name: Run the prompts | |
| run: | | |
| ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker1_logs 2>&1 & | |
| ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker2_logs 2>&1 & | |
| OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?") | |
| echo "$OUT1" | |
| echo '--------------------------------------------------' | |
| OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?") | |
| echo "$OUT2" | |
| echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1 | |
| echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1 | |
| - name: Check python memory usage | |
| run: | | |
| ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem | |
| ps -p $(cat pid.txt) -o %mem --no-headers > after_prompt_mem.txt | |
| - name: Compare memory usage and detect leak | |
| run: | | |
| initial_mem=$(cat initial_mem.txt | tr -d ' ') | |
| final_mem=$(cat after_scan_mem.txt | tr -d ' ') | |
| echo "Initial Memory Usage: $initial_mem%" | |
| echo "Memory Usage after scan: $final_mem%" | |
| if (( $(echo "$final_mem > $initial_mem" | bc -l) )); then | |
| echo "Memory usage has increased during scan. Possible memory leak detected!" | |
| else | |
| echo "Memory usage during scan is stable. No memory leak detected." | |
| fi | |
| - name: Show server logs | |
| if: always() | |
| run: | | |
| cat data/nextcloud.log | |
| - name: Show context_chat specific logs | |
| if: always() | |
| run: | | |
| cat data/context_chat.log | |
| - name: Show task processing worker logs | |
| if: always() | |
| run: | | |
| tail -v -n +1 worker?_logs || echo "No worker logs" | |
| - name: Show main app logs | |
| if: always() | |
| run: | | |
| cat context_chat_backend/backend_logs || echo "No main backend logs" | |
| - name: Show main app JSON logs | |
| if: always() | |
| run: | | |
| tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" | |
| - name: Show embedding server logs | |
| if: always() | |
| run: | | |
| cat context_chat_backend/em_backend_logs || echo "No main backend logs" | |
| - name: Show embedding server JSON logs | |
| if: always() | |
| run: | | |
| tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" | |
| summary: | |
| permissions: | |
| contents: none | |
| runs-on: ubuntu-latest-low | |
| needs: [changes, integration] | |
| if: always() | |
| # This is the summary, we just avoid to rename it so that branch protection rules still match | |
| name: integration-test | |
| steps: | |
| - name: Summary status | |
| run: if ${{ needs.changes.outputs.src != 'false' && needs.integration.result != 'success' }}; then exit 1; fi |