From 98c1ca30eddbf3a2fcd970385f14ea2d80d61bd3 Mon Sep 17 00:00:00 2001 From: earayu Date: Wed, 2 Jul 2025 18:25:00 +0800 Subject: [PATCH 01/24] feat: update default version --- Makefile | 10 ++++------ deploy/aperag/values.yaml | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 2a38d7496..d02108835 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,9 @@ # Configuration variables -VERSION ?= v0.5.0-alpha.29 +VERSION ?= nightly VERSION_FILE ?= aperag/version/__init__.py BUILDX_PLATFORM ?= linux/amd64,linux/arm64 BUILDX_ARGS ?= --sbom=false --provenance=false REGISTRY ?= registry.cn-hangzhou.aliyuncs.com -DOCRAY_VERSION ?= v0.1.1 # Image names APERAG_IMAGE = apecloud/aperag @@ -105,13 +104,13 @@ endif .PHONY: compose-up compose-down compose-logs compose-up: - VERSION=$(VERSION) REGISTRY=$(REGISTRY) DOCRAY_VERSION=$(DOCRAY_VERSION) $(_EXTRA_ENVS) docker-compose $(_PROFILES_TO_ACTIVATE) -f docker-compose.yml up -d + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 $(_EXTRA_ENVS) docker-compose $(_PROFILES_TO_ACTIVATE) -f docker-compose.yml up -d compose-down: - VERSION=$(VERSION) REGISTRY=$(REGISTRY) DOCRAY_VERSION=$(DOCRAY_VERSION) docker-compose --profile docray --profile docray-gpu -f docker-compose.yml down $(_COMPOSE_DOWN_FLAGS) + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose --profile docray --profile docray-gpu -f docker-compose.yml down $(_COMPOSE_DOWN_FLAGS) compose-logs: - VERSION=$(VERSION) REGISTRY=$(REGISTRY) DOCRAY_VERSION=$(DOCRAY_VERSION) docker-compose -f docker-compose.yml logs -f + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose -f docker-compose.yml logs -f # Environment cleanup .PHONY: clean @@ -310,7 +309,6 @@ build-aperag-frontend-local: setup-builder .PHONY: info info: @echo "VERSION: $(VERSION)" - @echo "DOCRAY_VERSION: $(DOCRAY_VERSION)" @echo "BUILDX_PLATFORM: $(BUILDX_PLATFORM)" @echo "LOCAL_PLATFORM: $(LOCAL_PLATFORM)" @echo "REGISTRY: $(REGISTRY)" diff --git a/deploy/aperag/values.yaml b/deploy/aperag/values.yaml index 8bbc4167a..f7afd9459 100644 --- a/deploy/aperag/values.yaml +++ b/deploy/aperag/values.yaml @@ -1,6 +1,6 @@ image: repository: "docker.io/apecloud/aperag" # Full image name including registry - tag: "v0.5.0-alpha.28" + tag: "v0.5.0-alpha.30" pullPolicy: IfNotPresent nameOverride: "" @@ -255,7 +255,7 @@ frontend: replicaCount: 1 image: repository: "docker.io/apecloud/aperag-frontend" # Full image name including registry - tag: "v0.5.0-alpha.28" + tag: "v0.5.0-alpha.30" pullPolicy: IfNotPresent resources: {} # We usually recommend not to specify default resources and to leave this as a conscious From beb6350c788e3071eb27901b772d136363930b16 Mon Sep 17 00:00:00 2001 From: earayu Date: Wed, 2 Jul 2025 23:06:12 +0800 Subject: [PATCH 02/24] feat: update makefile & readme --- Makefile | 37 +++++++----- README.md | 147 +++++++++++++++++++++++++++++++++------------ docker-compose.yml | 33 ++++++++++ 3 files changed, 164 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index d02108835..4895bf923 100644 --- a/Makefile +++ b/Makefile @@ -77,16 +77,24 @@ run-db: # Variables for compose command based on environment flags # Usage examples: -# make compose-up -# make compose-up WITH_DOCRAY=1 -# make compose-up WITH_DOCRAY=1 WITH_GPU=1 -# make compose-down -# make compose-down REMOVE_VOLUMES=1 +# make compose-up # Full application +# make compose-up WITH_NEO4J=1 # Full application + Neo4j +# make compose-up WITH_DOCRAY=1 # Full application + DocRay +# make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 # Full application + Neo4j + DocRay +# make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 # All features +# make compose-infra # Infrastructure only (databases) +# make compose-infra WITH_NEO4J=1 # Infrastructure + Neo4j +# make compose-down # Stop all services +# make compose-down REMOVE_VOLUMES=1 # Stop and remove volumes _PROFILES_TO_ACTIVATE := _EXTRA_ENVS := _COMPOSE_DOWN_FLAGS := -# Determine which docray profile to use for 'compose-up' +# Determine which additional profiles to activate +ifeq ($(WITH_NEO4J),1) + _PROFILES_TO_ACTIVATE += --profile neo4j +endif + ifeq ($(WITH_DOCRAY),1) ifeq ($(WITH_GPU),1) _PROFILES_TO_ACTIVATE += --profile docray-gpu @@ -102,12 +110,17 @@ ifeq ($(REMOVE_VOLUMES),1) _COMPOSE_DOWN_FLAGS += -v endif -.PHONY: compose-up compose-down compose-logs +.PHONY: compose-up compose-down compose-logs compose-infra +# Full application startup compose-up: - VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 $(_EXTRA_ENVS) docker-compose $(_PROFILES_TO_ACTIVATE) -f docker-compose.yml up -d + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 $(_EXTRA_ENVS) docker-compose --profile app $(_PROFILES_TO_ACTIVATE) -f docker-compose.yml up -d + +# Infrastructure only (databases + supporting services) +compose-infra: + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose $(_PROFILES_TO_ACTIVATE) -f docker-compose.yml up -d compose-down: - VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose --profile docray --profile docray-gpu -f docker-compose.yml down $(_COMPOSE_DOWN_FLAGS) + VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose --profile app --profile docray --profile docray-gpu --profile neo4j -f docker-compose.yml down $(_COMPOSE_DOWN_FLAGS) compose-logs: VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose -f docker-compose.yml logs -f @@ -419,9 +432,3 @@ load-images-to-kind: kind load docker-image $(APERAG_IMAGE):$(VERSION) --name $(KIND_CLUSTER_NAME) kind load docker-image $(APERAG_FRONTEND_IMG):$(VERSION) --name $(KIND_CLUSTER_NAME) @echo "Already Load Image To KinD" - -# Compatibility aliases -.PHONY: image celery flower -image: build -celery: run-celery -flower: run-flower diff --git a/README.md b/README.md index f32a5fd05..18bb1c8db 100644 --- a/README.md +++ b/README.md @@ -199,48 +199,119 @@ For detailed development workflows, see the [Development Guide](./docs/DEVELOPME ### Getting Started with Docker Compose -To get started with ApeRAG using Docker Compose, follow these steps: +Docker Compose provides the fastest way to get ApeRAG running. We support two deployment modes and flexible service combinations to meet different needs. -1. **Prerequisites**: - * Docker & Docker Compose - * Git +#### Prerequisites +* Docker & Docker Compose +* Git -2. **Environment Setup**: - Configure environment variables by copying the template files: - ```bash - cp envs/env.template .env - cp frontend/deploy/env.local.template frontend/.env - ``` - Then, **edit the `.env` file** to configure your AI service settings and other necessary configurations according to your needs. +#### Environment Setup +Configure environment variables by copying the template files: +```bash +cp envs/env.template .env +cp frontend/deploy/env.local.template frontend/.env +``` +Then, **edit the `.env` file** to configure your AI service settings and other configurations according to your needs. -3. **Start Services**: - You can start all ApeRAG services using the following `make` command: - ```bash - # Optional: Use Aliyun registry if in China - # export REGISTRY=apecloud-registry.cn-zhangjiakou.cr.aliyuncs.com +#### Deployment Modes - # Start ApeRAG services - make compose-up - ``` - If you need to use the `doc-ray` service for advanced document parsing (recommended for complex documents, tables, or formulas), you can start it along with other services: - ```bash - make compose-up WITH_DOCRAY=1 - ``` - If your environment has GPUs, you can enable GPU support for `doc-ray` for better performance: - ```bash - make compose-up WITH_DOCRAY=1 WITH_GPU=1 - ``` - > **About the doc-ray parsing service** - > - > ApeRAG includes a basic built-in parser for extracting text from documents like PDFs and DOCX files for RAG indexing. However, this parser may not optimally handle complex document structures, tables, or formulas. - > - > For enhanced document parsing capabilities and more accurate content extraction, we recommend deploying the [doc-ray](https://github.com/apecloud/doc-ray) service. `doc-ray` leverages **MinerU** for advanced document analysis. - > - > * When `WITH_GPU=1` is not specified, `doc-ray` will run using only the CPU. In this case, it is recommended to allocate at least 4 CPU cores and 8GB+ of RAM for it. - > * When `WITH_GPU=1` is specified, `doc-ray` will run using the GPU. It requires approximately 6GB of VRAM, along with 2 CPU cores and 8GB of RAM. - -4. **Access ApeRAG**: - Once the services are up and running, open your browser and navigate to: http://localhost:3000/web/ +**πŸ—οΈ Infrastructure Mode (Recommended for Development)** + +Start only the essential database services. Perfect for developers who want to run the application code locally for debugging and development. + +```bash +# Core databases: PostgreSQL, Redis, Qdrant, Elasticsearch +make compose-infra + +# Add graph database capabilities +make compose-infra WITH_NEO4J=1 +``` + +After starting infrastructure, run your app locally: +```bash +make run-backend # Start API server at localhost:8000 +make run-frontend # Start frontend at localhost:3000 (optional) +``` + +**πŸš€ Full Application Mode (Production Ready)** + +Launch the complete ApeRAG platform with all services containerized. + +```bash +# Complete system (API + Frontend + Background workers + Databases) +make compose-up + +# Add graph knowledge capabilities with Neo4j +make compose-up WITH_NEO4J=1 + +# Add advanced document parsing with DocRay +make compose-up WITH_DOCRAY=1 + +# Combine multiple optional services +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 + +# Full-featured deployment with GPU acceleration +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +``` + +#### Optional Services + +**Neo4j Graph Database** (`WITH_NEO4J=1`) +- Enables graph-based knowledge extraction and querying +- Powers advanced relational search capabilities +- Accessible at http://localhost:7474 (Web UI) + +**DocRay Advanced Document Parsing** (`WITH_DOCRAY=1`) +- Enhanced parsing for complex documents, tables, and formulas +- Powered by [MinerU](https://github.com/opendatalab/MinerU) technology +- CPU mode: Requires 4+ CPU cores, 8GB+ RAM +- GPU mode (`WITH_GPU=1`): Requires ~6GB VRAM, 2 CPU cores, 8GB RAM +- Service endpoint: http://localhost:8639 + +#### Access Your Deployment + +Once services are running: +- **Web Interface**: http://localhost:3000/web/ +- **API Documentation**: http://localhost:8000/docs +- **Flower (Task Monitor)**: http://localhost:5555 +- **Neo4j Browser**: http://localhost:7474 (if enabled) + +#### Service Management + +```bash +# View running services +docker-compose ps + +# View logs +make compose-logs + +# Stop all services +make compose-down + +# Stop services and remove data volumes +make compose-down REMOVE_VOLUMES=1 +``` + +#### Example Workflows + +**For Open Source Users (Quick Start)**: +```bash +make compose-up +# Access http://localhost:3000/web/ and start exploring! +``` + +**For Developers**: +```bash +make compose-infra WITH_NEO4J=1 # Start databases +make run-backend # Run API in development mode +# Code with hot reload and debugging +``` + +**For Production Deployment**: +```bash +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +# Full-featured deployment with all capabilities +``` ## Acknowledgments diff --git a/docker-compose.yml b/docker-compose.yml index 664be6a84..0757f3a34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,9 +3,13 @@ volumes: aperag-qdrant-data: {} aperag-redis-data: {} aperag-es-data: {} + aperag-neo4j-data: {} aperag-shared-data: {} services: + # ============================================== + # Application Services (Default startup, excluded from infra) + # ============================================== api: &api build: context: . @@ -33,6 +37,7 @@ services: ports: - "8000:8000" command: ["/app/scripts/entrypoint.sh", "/app/scripts/start-api.sh"] + profiles: ["app"] frontend: build: @@ -49,6 +54,7 @@ services: - APERAG_CONSOLE_SERVICE_PORT=8000 ports: - "3000:3000" + profiles: ["app"] celeryworker: image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} @@ -73,6 +79,7 @@ services: - NODE_IP=aperag-celeryworker - DOCRAY_HOST=${DOCRAY_HOST} command: ["/app/scripts/entrypoint.sh", "/app/scripts/start-celery-worker.sh"] + profiles: ["app"] celerybeat: image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} @@ -89,6 +96,7 @@ services: environment: - NODE_IP=aperag-celerybeat command: ["/app/scripts/entrypoint.sh", "/app/scripts/start-celery-beat.sh"] + profiles: ["app"] flower: <<: *api @@ -105,7 +113,11 @@ services: environment: - NODE_IP=aperag-flower command: ["/app/scripts/entrypoint.sh", "/app/scripts/start-celery-flower.sh"] + profiles: ["app"] + # ============================================== + # Infrastructure Services (always available) + # ============================================== postgres: image: pgvector/pgvector:pg16 container_name: aperag-postgres @@ -155,6 +167,27 @@ services: command: bash /usr/share/elasticsearch/bin/init-es.sh restart: on-failure + # ============================================== + # Optional Services + # ============================================== + neo4j: + image: neo4j:5.26.5-enterprise + container_name: aperag-neo4j + ports: + - "7474:7474" # HTTP + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=neo4j/password + - NEO4J_PLUGINS=["apoc"] + - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes + - NEO4J_apoc_export_file_enabled=true + volumes: + - aperag-neo4j-data:/data + profiles: ["neo4j"] + + # ============================================== + # DocRay Services (existing profiles) + # ============================================== docray: image: ${REGISTRY:-docker.io}/apecloud/doc-ray:${DOCRAY_VERSION:-v0.1.1} container_name: aperag-docray From 711cecaae6bb5bd262abf5624d8a69af352850bc Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:10:46 +0800 Subject: [PATCH 03/24] feat: update makefile & readme --- README.md | 101 ------------------------------------ docker-compose.yml | 10 ++-- docs/development-guide.md | 104 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 18bb1c8db..4e15ee53e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ - [Getting Started](#getting-started) - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes) - - [Getting Started with Source Code](#getting-started-with-source-code) - [Getting Started with Docker Compose](#getting-started-with-docker-compose) - [Development](./docs/development-guide.md) - [Build Docker Image](./docs/build-docker-image.md) @@ -95,107 +94,7 @@ With databases running: For KubeBlocks details (credentials, uninstall), see `deploy/databases/README.md`. -### Getting Started with Source Code -This guide is for developers looking to contribute to ApeRAG or run it locally for development. Follow these steps to get ApeRAG running from the source code: - -**1. Clone the Repository** - -First, get the source code: -```bash -git clone https://github.com/apecloud/ApeRAG.git -cd ApeRAG -``` - -**2. System Prerequisites** - -Before you begin, ensure your system has: - -* **Python 3.11**: The project uses Python 3.11. If it's not your system default, `uv` (see below) will attempt to use it when creating the virtual environment if available. -* **Node.js**: Version 20 or higher is recommended for frontend development. -* **`uv`**: This is a fast Python package installer and virtual environment manager. - * If you don't have `uv`, the `make install` command (Step 3) will try to install it via `pip`. -* **Docker**: (Recommended for local databases) If you plan to run dependent services like PostgreSQL, Redis, etc., locally, Docker is the easiest way. The `make run-db` command uses Docker Compose. - -**3. Install Dependencies & Setup Virtual Environment** - -This crucial `make` command automates several setup tasks: - -```bash -make install -``` - -This command will: -* Verify or install `uv`. -* Create a Python 3.11 virtual environment (located in `.venv/`) using `uv`. -* Install all Python backend dependencies (including development tools) from `pyproject.toml` into the virtual environment. -* Install frontend Node.js dependencies using `yarn`. - -**4. Configure Environment Variables** - -ApeRAG uses `.env` files for configuration. - -* **Backend (`.env`)**: Copy the template and customize it for your setup. - ```bash - cp envs/env.template .env - ``` - Then, edit the newly created `.env` file. - - **Note**: If you start the required database services using the `make run-db` command (see Step 5), the default connection settings in the `.env` file (copied from `envs/env.template`) are pre-configured to work with these services, and you typically won't need to change them. You would only need to modify these if you are connecting to externally managed databases or have custom configurations. - -* **Frontend (`frontend/.env`)** (Optional - if you are developing the frontend): - ```bash - cp frontend/deploy/env.local.template frontend/.env - ``` - Edit `frontend/.env` if you need to change frontend-specific settings, such as the backend API URL (though defaults usually work for local development). - -**5. Start Databases & Apply Migrations** - -* **Start Database Services**: - If you're using Docker for local databases, the `Makefile` provides a convenient command: - ```bash - make run-db - ``` - -* **Apply Database Migrations**: - Once your databases are running and configured in `.env`, set up the database schema: - ```bash - make migrate - ``` - -**6. Run ApeRAG Backend Services** - -These should typically be run in separate terminal windows/tabs. The `make` commands will automatically use the correct Python virtual environment. - -* **FastAPI Development Server**: - ```bash - make run-backend - ``` - This starts the main backend application. It will typically be accessible at `http://localhost:8000` and features auto-reload on code changes. - -* **Celery Worker & Beat**: - ```bash - make run-celery - ``` - This starts the Celery worker for processing asynchronous background tasks. - -**7. Run Frontend Development Server (Optional)** - -If you need to work on or view the frontend: -```bash -make run-frontend -``` -This will start the frontend development server, usually available at `http://localhost:3000`. It's configured to proxy API requests to the backend running on port 8000. - -**8. Access ApeRAG** - -With the backend (and optionally frontend) services running: -* Access the **Frontend UI** at `http://localhost:3000` (if started). -* The **Backend API** is available at `http://localhost:8000`. - -Now you have ApeRAG running locally from the source code, ready for development or testing! - -For detailed development workflows, see the [Development Guide](./docs/DEVELOPMENT.md). ### Getting Started with Docker Compose diff --git a/docker-compose.yml b/docker-compose.yml index 0757f3a34..bad297891 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: build: context: . dockerfile: ./Dockerfile - image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} + image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-v0.5.0-alpha.30} container_name: aperag-api depends_on: redis: @@ -43,7 +43,7 @@ services: build: context: ./frontend dockerfile: ./Dockerfile - image: ${REGISTRY:-docker.io}/apecloud/aperag-frontend:${VERSION:-latest} + image: ${REGISTRY:-docker.io}/apecloud/aperag-frontend:${VERSION:-v0.5.0-alpha.30} container_name: aperag-frontend depends_on: - api @@ -57,7 +57,7 @@ services: profiles: ["app"] celeryworker: - image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} + image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-v0.5.0-alpha.30} build: context: . dockerfile: ./Dockerfile @@ -82,7 +82,7 @@ services: profiles: ["app"] celerybeat: - image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} + image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-v0.5.0-alpha.30} build: context: . dockerfile: ./Dockerfile @@ -100,7 +100,7 @@ services: flower: <<: *api - image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-latest} + image: ${REGISTRY:-docker.io}/apecloud/aperag:${VERSION:-v0.5.0-alpha.30} build: context: . dockerfile: ./Dockerfile diff --git a/docs/development-guide.md b/docs/development-guide.md index 2649720ee..74ddcc775 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -1,10 +1,106 @@ # Development Guide -This section focuses on the development workflow and tools provided for ApeRAG. +This guide focuses on setting up a development environment and the development workflow for ApeRAG. This is designed for developers looking to contribute to ApeRAG or run it locally for development purposes. -## Development Environment +## Development Environment Setup -It's recommended to use the "Getting Started with Source Code" approach for setting up a development environment. Ensure all prerequisites are met and dependencies are installed using `make install`. +Follow these steps to set up ApeRAG from source code for development: + +### 1. Clone the Repository + +First, get the source code: +```bash +git clone https://github.com/apecloud/ApeRAG.git +cd ApeRAG +``` + +### 2. System Prerequisites + +Before you begin, ensure your system has: + +* **Python 3.11**: The project uses Python 3.11. If it's not your system default, `uv` (see below) will attempt to use it when creating the virtual environment if available. +* **Node.js**: Version 20 or higher is recommended for frontend development. +* **`uv`**: This is a fast Python package installer and virtual environment manager. + * If you don't have `uv`, the `make install` command (Step 3) will try to install it via `pip`. +* **Docker**: (Recommended for local databases) If you plan to run dependent services like PostgreSQL, Redis, etc., locally, Docker is the easiest way. The `make run-db` command uses Docker Compose. + +### 3. Install Dependencies & Setup Virtual Environment + +This crucial `make` command automates several setup tasks: + +```bash +make install +``` + +This command will: +* Verify or install `uv`. +* Create a Python 3.11 virtual environment (located in `.venv/`) using `uv`. +* Install all Python backend dependencies (including development tools) from `pyproject.toml` into the virtual environment. +* Install frontend Node.js dependencies using `yarn`. + +### 4. Configure Environment Variables + +ApeRAG uses `.env` files for configuration. + +* **Backend (`.env`)**: Copy the template and customize it for your setup. + ```bash + cp envs/env.template .env + ``` + Then, edit the newly created `.env` file. + + **Note**: If you start the required database services using the `make run-db` command (see Step 5), the default connection settings in the `.env` file (copied from `envs/env.template`) are pre-configured to work with these services, and you typically won't need to change them. You would only need to modify these if you are connecting to externally managed databases or have custom configurations. + +* **Frontend (`frontend/.env`)** (Optional - if you are developing the frontend): + ```bash + cp frontend/deploy/env.local.template frontend/.env + ``` + Edit `frontend/.env` if you need to change frontend-specific settings, such as the backend API URL (though defaults usually work for local development). + +### 5. Start Databases & Apply Migrations + +* **Start Database Services**: + If you're using Docker for local databases, the `Makefile` provides a convenient command: + ```bash + make run-db + ``` + +* **Apply Database Migrations**: + Once your databases are running and configured in `.env`, set up the database schema: + ```bash + make migrate + ``` + +### 6. Run ApeRAG Backend Services + +These should typically be run in separate terminal windows/tabs. The `make` commands will automatically use the correct Python virtual environment. + +* **FastAPI Development Server**: + ```bash + make run-backend + ``` + This starts the main backend application. It will typically be accessible at `http://localhost:8000` and features auto-reload on code changes. + +* **Celery Worker & Beat**: + ```bash + make run-celery + ``` + This starts the Celery worker for processing asynchronous background tasks. + +### 7. Run Frontend Development Server (Optional) + +If you need to work on or view the frontend: +```bash +make run-frontend +``` +This will start the frontend development server, usually available at `http://localhost:3000`. It's configured to proxy API requests to the backend running on port 8000. + +### 8. Access ApeRAG + +With the backend (and optionally frontend) services running: +* Access the **Frontend UI** at `http://localhost:3000` (if started). +* The **Backend API** is available at `http://localhost:8000`. + +Now you have ApeRAG running locally from the source code, ready for development or testing! ## Key `make` Commands for Development @@ -52,7 +148,7 @@ Contributing to ApeRAG involves the following typical workflow. Before starting * Fork the official ApeRAG repository to your GitHub account. * Create a new branch from `main` for your feature or bug fix. Use a descriptive branch name (e.g., `feat/add-new-parser` or `fix/login-bug`). -2. **Environment Setup**: Ensure your development environment is set up as described in [Development Environment](#development-environment) and [Getting Started with Source Code](../README.md#getting-started-with-source-code) (dependencies installed, databases running/accessible). +2. **Environment Setup**: Ensure your development environment is set up as described in [Development Environment Setup](#development-environment-setup) (dependencies installed, databases running/accessible). 3. **Code Implementation**: * Make your code changes in the backend (`aperag/`) or frontend (`frontend/src/`) directories. From c46723213f0c27770367b202d65e50580c075b75 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:14:13 +0800 Subject: [PATCH 04/24] feat: update makefile & readme --- README.md | 122 +++++++++++++++++++++++++++--------------------------- 1 file changed, 60 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 4e15ee53e..f1cba5f15 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ ## Table of Contents - [Getting Started](#getting-started) - - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes) - [Getting Started with Docker Compose](#getting-started-with-docker-compose) + - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes) - [Development](./docs/development-guide.md) - [Build Docker Image](./docs/build-docker-image.md) - [Acknowledgments](#acknowledgments) @@ -35,67 +35,6 @@ Key features include: This section will guide you through setting up ApeRAG using different methods. -### Getting Started with Kubernetes - -This guide covers deploying ApeRAG to Kubernetes using the provided Helm chart. It involves two main phases: setting up databases (optional if you have them) and deploying the ApeRAG application. - -**Phase 1: Deploy Databases with KubeBlocks (Optional)** - -ApeRAG needs PostgreSQL, Redis, Qdrant, and Elasticsearch. If you don't have these, use the KubeBlocks scripts in `deploy/databases/`. - -*Skip this phase if your databases are already available in your Kubernetes cluster.* - -1. **Prerequisites**: - * Kubernetes cluster. - * `kubectl` configured. - * Helm v3+. - -2. **Database Configuration (`deploy/databases/00-config.sh`)**: - This script controls database deployment (defaults: PostgreSQL, Redis, Qdrant, Elasticsearch in the `default` namespace). **Defaults are usually fine; no changes needed for a standard setup.** Edit only for advanced cases (e.g., changing namespace, enabling optional databases like Neo4j). - -3. **Run Database Deployment Scripts**: - ```bash - cd deploy/databases/ - bash ./01-prepare.sh # Prepares KubeBlocks environment. - bash ./02-install-database.sh # Deploys database clusters. - cd ../.. # Back to project root. - ``` - Monitor pods in the `default` namespace (or your custom one) until ready: - ```bash - kubectl get pods -n default - ``` - -**Phase 2: Deploy ApeRAG Application** - -With databases running: - -1. **Helm Chart Configuration (`deploy/aperag/values.yaml`)**: - * **Using KubeBlocks (Phase 1 in `default` namespace)?** Database connections in `values.yaml` are likely pre-configured. **No changes usually needed.** - * **Using your own databases?** You MUST update `values.yaml` with your database connection details. - * By default, this Helm chart deploys the [`doc-ray`](https://github.com/apecloud/doc-ray) service for advanced document parsing, which requires at least 4 CPU cores and 8GB of memory. If your Kubernetes cluster has insufficient resources, you can disable the `doc-ray` deployment by setting `docray.enabled` to `false`. In this case, a basic document parser will be used. - * Optionally, review other settings (images, resources, Ingress, etc.). - -2. **Deploy ApeRAG with Helm**: - This installs ApeRAG to the `default` namespace: - ```bash - helm install aperag ./deploy/aperag --namespace default --create-namespace - ``` - Monitor ApeRAG pods until `Running`: - ```bash - kubectl get pods -n default -l app.kubernetes.io/instance=aperag - ``` - -3. **Access ApeRAG UI**: - Use `kubectl port-forward` for quick access: - ```bash - kubectl port-forward svc/aperag-frontend 3000:3000 -n default - ``` - Open `http://localhost:3000` in your browser. - -For KubeBlocks details (credentials, uninstall), see `deploy/databases/README.md`. - - - ### Getting Started with Docker Compose Docker Compose provides the fastest way to get ApeRAG running. We support two deployment modes and flexible service combinations to meet different needs. @@ -212,6 +151,65 @@ make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 # Full-featured deployment with all capabilities ``` +### Getting Started with Kubernetes (Recommend for Production) + +This guide covers deploying ApeRAG to Kubernetes using the provided Helm chart. It involves two main phases: setting up databases (optional if you have them) and deploying the ApeRAG application. + +**Phase 1: Deploy Databases with KubeBlocks (Optional)** + +ApeRAG needs PostgreSQL, Redis, Qdrant, and Elasticsearch. If you don't have these, use the KubeBlocks scripts in `deploy/databases/`. + +*Skip this phase if your databases are already available in your Kubernetes cluster.* + +1. **Prerequisites**: + * Kubernetes cluster. + * `kubectl` configured. + * Helm v3+. + +2. **Database Configuration (`deploy/databases/00-config.sh`)**: + This script controls database deployment (defaults: PostgreSQL, Redis, Qdrant, Elasticsearch in the `default` namespace). **Defaults are usually fine; no changes needed for a standard setup.** Edit only for advanced cases (e.g., changing namespace, enabling optional databases like Neo4j). + +3. **Run Database Deployment Scripts**: + ```bash + cd deploy/databases/ + bash ./01-prepare.sh # Prepares KubeBlocks environment. + bash ./02-install-database.sh # Deploys database clusters. + cd ../.. # Back to project root. + ``` + Monitor pods in the `default` namespace (or your custom one) until ready: + ```bash + kubectl get pods -n default + ``` + +**Phase 2: Deploy ApeRAG Application** + +With databases running: + +1. **Helm Chart Configuration (`deploy/aperag/values.yaml`)**: + * **Using KubeBlocks (Phase 1 in `default` namespace)?** Database connections in `values.yaml` are likely pre-configured. **No changes usually needed.** + * **Using your own databases?** You MUST update `values.yaml` with your database connection details. + * By default, this Helm chart deploys the [`doc-ray`](https://github.com/apecloud/doc-ray) service for advanced document parsing, which requires at least 4 CPU cores and 8GB of memory. If your Kubernetes cluster has insufficient resources, you can disable the `doc-ray` deployment by setting `docray.enabled` to `false`. In this case, a basic document parser will be used. + * Optionally, review other settings (images, resources, Ingress, etc.). + +2. **Deploy ApeRAG with Helm**: + This installs ApeRAG to the `default` namespace: + ```bash + helm install aperag ./deploy/aperag --namespace default --create-namespace + ``` + Monitor ApeRAG pods until `Running`: + ```bash + kubectl get pods -n default -l app.kubernetes.io/instance=aperag + ``` + +3. **Access ApeRAG UI**: + Use `kubectl port-forward` for quick access: + ```bash + kubectl port-forward svc/aperag-frontend 3000:3000 -n default + ``` + Open `http://localhost:3000` in your browser. + +For KubeBlocks details (credentials, uninstall), see `deploy/databases/README.md`. + ## Acknowledgments ApeRAG integrates and builds upon several excellent open-source projects: From c4b24be6c0ed91d30405b6db0843bde863773bb4 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:19:33 +0800 Subject: [PATCH 05/24] feat: update makefile & readme --- README.md | 152 +++++++++----------------------------- docs/development-guide.md | 94 +++++++++++++++++++++++ 2 files changed, 128 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index f1cba5f15..8a9d9d0f5 100644 --- a/README.md +++ b/README.md @@ -6,150 +6,66 @@ ## Table of Contents -- [Getting Started](#getting-started) - - [Getting Started with Docker Compose](#getting-started-with-docker-compose) - - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes) +- [Quick Start](#quick-start) +- [Key Features](#key-features) +- [Advanced Setup](#advanced-setup) + - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes-recommend-for-production) - [Development](./docs/development-guide.md) - [Build Docker Image](./docs/build-docker-image.md) - [Acknowledgments](#acknowledgments) - [License](#license) -ApeRAG is a production-ready, comprehensive RAG (Retrieval-Augmented Generation) platform designed for building advanced, enterprise-grade AI applications. It empowers developers to create sophisticated **Agentic RAG** systems with a powerful, hybrid retrieval engine. +ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction powered by **[LightRAG](https://github.com/HKUDS/LightRAG)**. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. -Key features include: +## Quick Start -* **Advanced Hybrid Retrieval**: Go beyond simple vector search. ApeRAG integrates three powerful indexing strategies: - * **Vector Index**: For semantic similarity search. - * **Full-Text Index**: For precise keyword-based retrieval. - * **Graph Knowledge Index**: Powered by an integrated and enhanced version of **[LightRAG](https://github.com/HKUDS/LightRAG)**, enabling deep relational and contextual queries. +> Before installing ApeRAG, make sure your machine meets the following minimum system requirements: +> +> - CPU >= 2 Core +> - RAM >= 4 GiB +> - Docker & Docker Compose -* **Multimodal Document Processing**: Ingest and understand a wide array of document formats, extracting not just text but also tables, images, and complex structures from files like PDFs and DOCX. +The easiest way to start ApeRAG is through Docker Compose. Before running the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine: -* **Enterprise-Grade Management**: ApeRAG is built for production environments with a suite of essential features: - * **Audit Logging**: Track all critical system and user activities. - * **LLM Model Management**: Easily configure and switch between various Large Language Models. - * **Graph Visualization**: Visually explore and understand the knowledge graph. - * **Comprehensive Document Management**: A user-friendly interface to manage document collections, track processing status, and inspect content. - -## Getting Started - -This section will guide you through setting up ApeRAG using different methods. - -### Getting Started with Docker Compose - -Docker Compose provides the fastest way to get ApeRAG running. We support two deployment modes and flexible service combinations to meet different needs. - -#### Prerequisites -* Docker & Docker Compose -* Git - -#### Environment Setup -Configure environment variables by copying the template files: ```bash +git clone https://github.com/apecloud/ApeRAG.git +cd ApeRAG cp envs/env.template .env cp frontend/deploy/env.local.template frontend/.env -``` -Then, **edit the `.env` file** to configure your AI service settings and other configurations according to your needs. - -#### Deployment Modes - -**πŸ—οΈ Infrastructure Mode (Recommended for Development)** - -Start only the essential database services. Perfect for developers who want to run the application code locally for debugging and development. - -```bash -# Core databases: PostgreSQL, Redis, Qdrant, Elasticsearch -make compose-infra - -# Add graph database capabilities -make compose-infra WITH_NEO4J=1 -``` - -After starting infrastructure, run your app locally: -```bash -make run-backend # Start API server at localhost:8000 -make run-frontend # Start frontend at localhost:3000 (optional) -``` - -**πŸš€ Full Application Mode (Production Ready)** - -Launch the complete ApeRAG platform with all services containerized. - -```bash -# Complete system (API + Frontend + Background workers + Databases) make compose-up - -# Add graph knowledge capabilities with Neo4j -make compose-up WITH_NEO4J=1 - -# Add advanced document parsing with DocRay -make compose-up WITH_DOCRAY=1 - -# Combine multiple optional services -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 - -# Full-featured deployment with GPU acceleration -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 ``` -#### Optional Services - -**Neo4j Graph Database** (`WITH_NEO4J=1`) -- Enables graph-based knowledge extraction and querying -- Powers advanced relational search capabilities -- Accessible at http://localhost:7474 (Web UI) - -**DocRay Advanced Document Parsing** (`WITH_DOCRAY=1`) -- Enhanced parsing for complex documents, tables, and formulas -- Powered by [MinerU](https://github.com/opendatalab/MinerU) technology -- CPU mode: Requires 4+ CPU cores, 8GB+ RAM -- GPU mode (`WITH_GPU=1`): Requires ~6GB VRAM, 2 CPU cores, 8GB RAM -- Service endpoint: http://localhost:8639 - -#### Access Your Deployment - -Once services are running: +After running, you can access ApeRAG in your browser at: - **Web Interface**: http://localhost:3000/web/ - **API Documentation**: http://localhost:8000/docs -- **Flower (Task Monitor)**: http://localhost:5555 -- **Neo4j Browser**: http://localhost:7474 (if enabled) -#### Service Management +#### Seeking help -```bash -# View running services -docker-compose ps +Please refer to our [Development Guide](./docs/development-guide.md) for advanced configurations, development setup, and troubleshooting. Reach out to [the community](#acknowledgments) if you are still having issues. -# View logs -make compose-logs +> If you'd like to contribute to ApeRAG or do additional development, refer to our [Development Guide](./docs/development-guide.md) -# Stop all services -make compose-down +## Key Features -# Stop services and remove data volumes -make compose-down REMOVE_VOLUMES=1 -``` +**1. Hybrid Retrieval Engine**: +Combines vector search, full-text search, and graph knowledge extraction for comprehensive document understanding and retrieval. -#### Example Workflows +**2. LightRAG Integration**: +Enhanced version of LightRAG for advanced graph-based knowledge extraction, enabling deep relational and contextual queries. -**For Open Source Users (Quick Start)**: -```bash -make compose-up -# Access http://localhost:3000/web/ and start exploring! -``` +**3. Multimodal Document Processing**: +Supports various document formats (PDF, DOCX, etc.) with advanced parsing capabilities for text, tables, and images. -**For Developers**: -```bash -make compose-infra WITH_NEO4J=1 # Start databases -make run-backend # Run API in development mode -# Code with hot reload and debugging -``` +**4. Enterprise Management**: +Built-in audit logging, LLM model management, graph visualization, and comprehensive document management interface. -**For Production Deployment**: -```bash -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 -# Full-featured deployment with all capabilities -``` +**5. Production Ready**: +Kubernetes support, Docker deployment, async task processing with Celery, and comprehensive API documentation. + +**6. Developer Friendly**: +FastAPI backend, React frontend, extensive testing, and detailed development guides for easy contribution and customization. + +## Advanced Setup ### Getting Started with Kubernetes (Recommend for Production) diff --git a/docs/development-guide.md b/docs/development-guide.md index 74ddcc775..ad6bb1496 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -102,6 +102,100 @@ With the backend (and optionally frontend) services running: Now you have ApeRAG running locally from the source code, ready for development or testing! +## Docker Compose Development Options + +For developers who prefer containerized development, Docker Compose offers flexible deployment modes: + +### Infrastructure Mode (Recommended for Development) + +Start only the essential database services. Perfect for developers who want to run the application code locally for debugging and development. + +```bash +# Core databases: PostgreSQL, Redis, Qdrant, Elasticsearch +make compose-infra + +# Add graph database capabilities +make compose-infra WITH_NEO4J=1 +``` + +After starting infrastructure, run your app locally: +```bash +make run-backend # Start API server at localhost:8000 +make run-frontend # Start frontend at localhost:3000 (optional) +``` + +### Full Application Mode + +Launch the complete ApeRAG platform with all services containerized. + +```bash +# Complete system (API + Frontend + Background workers + Databases) +make compose-up + +# Add graph knowledge capabilities with Neo4j +make compose-up WITH_NEO4J=1 + +# Add advanced document parsing with DocRay +make compose-up WITH_DOCRAY=1 + +# Combine multiple optional services +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 + +# Full-featured deployment with GPU acceleration +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +``` + +### Optional Services + +**Neo4j Graph Database** (`WITH_NEO4J=1`) +- Enables graph-based knowledge extraction and querying +- Powers advanced relational search capabilities +- Accessible at http://localhost:7474 (Web UI) + +**DocRay Advanced Document Parsing** (`WITH_DOCRAY=1`) +- Enhanced parsing for complex documents, tables, and formulas +- Powered by [MinerU](https://github.com/opendatalab/MinerU) technology +- CPU mode: Requires 4+ CPU cores, 8GB+ RAM +- GPU mode (`WITH_GPU=1`): Requires ~6GB VRAM, 2 CPU cores, 8GB RAM +- Service endpoint: http://localhost:8639 + +### Service Management + +```bash +# View running services +docker-compose ps + +# View logs +make compose-logs + +# Stop all services +make compose-down + +# Stop services and remove data volumes +make compose-down REMOVE_VOLUMES=1 +``` + +### Example Development Workflows + +**For Quick Testing**: +```bash +make compose-up +# Access http://localhost:3000/web/ and start exploring! +``` + +**For Active Development**: +```bash +make compose-infra WITH_NEO4J=1 # Start databases +make run-backend # Run API in development mode +# Code with hot reload and debugging +``` + +**For Production Testing**: +```bash +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +# Full-featured deployment with all capabilities +``` + ## Key `make` Commands for Development The `Makefile` at the root of the project provides several helpful commands to streamline development: From 480366122ee02c6bd49d71914e0cce8a7071637c Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:26:51 +0800 Subject: [PATCH 06/24] feat: update makefile & readme --- README.md | 2 +- docs/development-guide.md | 163 ++++++++++++++++++++++---------------- 2 files changed, 95 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 8a9d9d0f5..8e82a9aaa 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ - [Acknowledgments](#acknowledgments) - [License](#license) -ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction powered by **[LightRAG](https://github.com/HKUDS/LightRAG)**. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. +ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction inspired by **[LightRAG](https://github.com/HKUDS/LightRAG)**. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. ## Quick Start diff --git a/docs/development-guide.md b/docs/development-guide.md index ad6bb1496..b25edcc53 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -6,101 +6,125 @@ This guide focuses on setting up a development environment and the development w Follow these steps to set up ApeRAG from source code for development: -### 1. Clone the Repository +### 1. Clone the Repository and Setup Environment + +First, get the source code and configure environment variables: -First, get the source code: ```bash git clone https://github.com/apecloud/ApeRAG.git cd ApeRAG +cp envs/env.template .env +cp frontend/deploy/env.local.template frontend/.env ``` +Edit the `.env` file to configure your AI service settings if needed. The default settings work with the local database services started in the next step. + ### 2. System Prerequisites Before you begin, ensure your system has: -* **Python 3.11**: The project uses Python 3.11. If it's not your system default, `uv` (see below) will attempt to use it when creating the virtual environment if available. +* **Python 3.11**: The project uses Python 3.11. * **Node.js**: Version 20 or higher is recommended for frontend development. -* **`uv`**: This is a fast Python package installer and virtual environment manager. - * If you don't have `uv`, the `make install` command (Step 3) will try to install it via `pip`. -* **Docker**: (Recommended for local databases) If you plan to run dependent services like PostgreSQL, Redis, etc., locally, Docker is the easiest way. The `make run-db` command uses Docker Compose. +* **Docker & Docker Compose**: Required for running database services locally. -### 3. Install Dependencies & Setup Virtual Environment +### 3. Start Database Services -This crucial `make` command automates several setup tasks: +Use Docker Compose to start the essential database services: ```bash -make install +# Start core databases: PostgreSQL, Redis, Qdrant, Elasticsearch +make compose-infra + +# Optional: Add graph database capabilities +# make compose-infra WITH_NEO4J=1 ``` -This command will: -* Verify or install `uv`. -* Create a Python 3.11 virtual environment (located in `.venv/`) using `uv`. -* Install all Python backend dependencies (including development tools) from `pyproject.toml` into the virtual environment. -* Install frontend Node.js dependencies using `yarn`. +This will start all required database services in the background. The default connection settings in your `.env` file are pre-configured to work with these services. -### 4. Configure Environment Variables +### 4. Setup Development Environment -ApeRAG uses `.env` files for configuration. +Create Python virtual environment and setup development tools: -* **Backend (`.env`)**: Copy the template and customize it for your setup. - ```bash - cp envs/env.template .env - ``` - Then, edit the newly created `.env` file. +```bash +make dev +``` - **Note**: If you start the required database services using the `make run-db` command (see Step 5), the default connection settings in the `.env` file (copied from `envs/env.template`) are pre-configured to work with these services, and you typically won't need to change them. You would only need to modify these if you are connecting to externally managed databases or have custom configurations. +This command will: +* Install `uv` if not already available +* Create a Python 3.11 virtual environment (located in `.venv/`) +* Install pre-commit hooks for code quality -* **Frontend (`frontend/.env`)** (Optional - if you are developing the frontend): - ```bash - cp frontend/deploy/env.local.template frontend/.env - ``` - Edit `frontend/.env` if you need to change frontend-specific settings, such as the backend API URL (though defaults usually work for local development). +### 5. Install Dependencies -### 5. Start Databases & Apply Migrations +Install all backend and frontend dependencies: -* **Start Database Services**: - If you're using Docker for local databases, the `Makefile` provides a convenient command: - ```bash - make run-db - ``` +```bash +make install +``` + +This command will: +* Install all Python backend dependencies from `pyproject.toml` into the virtual environment +* Install frontend Node.js dependencies using `yarn` + +### 6. Apply Database Migrations -* **Apply Database Migrations**: - Once your databases are running and configured in `.env`, set up the database schema: - ```bash - make migrate - ``` +Setup the database schema: -### 6. Run ApeRAG Backend Services +```bash +make migrate +``` -These should typically be run in separate terminal windows/tabs. The `make` commands will automatically use the correct Python virtual environment. +### 7. Start Development Services -* **FastAPI Development Server**: - ```bash - make run-backend - ``` - This starts the main backend application. It will typically be accessible at `http://localhost:8000` and features auto-reload on code changes. +Now you can start the development services. Open separate terminal windows/tabs for each service: -* **Celery Worker & Beat**: - ```bash - make run-celery - ``` - This starts the Celery worker for processing asynchronous background tasks. +**Terminal 1 - Backend API Server:** +```bash +make run-backend +``` +This starts the FastAPI development server at `http://localhost:8000` with auto-reload on code changes. -### 7. Run Frontend Development Server (Optional) +**Terminal 2 - Celery Worker:** +```bash +make run-celery +``` +This starts the Celery worker for processing asynchronous background tasks. -If you need to work on or view the frontend: +**Terminal 3 - Frontend (Optional):** ```bash make run-frontend ``` -This will start the frontend development server, usually available at `http://localhost:3000`. It's configured to proxy API requests to the backend running on port 8000. +This starts the frontend development server at `http://localhost:3000` with hot reload. ### 8. Access ApeRAG -With the backend (and optionally frontend) services running: -* Access the **Frontend UI** at `http://localhost:3000` (if started). -* The **Backend API** is available at `http://localhost:8000`. +With the services running, you can access: +* **Backend API**: http://localhost:8000 +* **API Documentation**: http://localhost:8000/docs +* **Frontend UI**: http://localhost:3000 (if started) + +### 9. Development Workflow -Now you have ApeRAG running locally from the source code, ready for development or testing! +Your typical development cycle: + +1. **Code Changes**: Edit files in `aperag/` (backend) or `frontend/src/` (frontend) +2. **Auto-reload**: Backend and frontend automatically reload on file changes +3. **Testing**: Run `make test` to execute tests +4. **Code Quality**: Run `make format` and `make lint` before committing +5. **Database Changes**: If you modify models, run `make makemigration` then `make migrate` + +### 10. Stopping Services + +To stop the development environment: + +```bash +# Stop database services +make compose-down + +# Stop development services (Ctrl+C in each terminal) +``` + +Now you have ApeRAG running locally from source code, ready for development! ## Docker Compose Development Options @@ -201,15 +225,18 @@ make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 The `Makefile` at the root of the project provides several helpful commands to streamline development: * **Environment & Dependencies**: - * `make install`: Installs all necessary backend (Python) and frontend (Node.js) dependencies. It sets up a Python 3.11 virtual environment using `uv`. - * `make dev`: Installs development tools like pre-commit hooks to ensure code quality before commits. + * `make dev`: Sets up the development environment by creating Python virtual environment, installing `uv`, and setting up pre-commit hooks. + * `make install`: Installs all necessary backend (Python) and frontend (Node.js) dependencies into the virtual environment. + +* **Database Services**: + * `make compose-infra`: Starts essential database services (PostgreSQL, Redis, Qdrant, Elasticsearch) using Docker Compose. + * `make compose-infra WITH_NEO4J=1`: Starts database services including Neo4j for graph knowledge capabilities. + * `make compose-down`: Stops all database services. -* **Running Services**: - * `make run-db`: (Uses Docker Compose) Starts all required database services (PostgreSQL, Redis, Qdrant, etc.) as defined in `docker-compose.yml`. Useful if you don't have these services running elsewhere. - * `make run-backend`: Starts the FastAPI development server. - * `make run-frontend`: Starts the UmiJS frontend development server. - * `make run-celery`: Starts a Celery worker for processing background tasks (includes Celery Beat). - * `make run-celery-beat`: (Note: `make run-celery` usually includes Beat due to the `-B` flag. This target might be redundant or for specific scenarios. Check Makefile if explicitly needed separate from worker). +* **Development Services**: + * `make run-backend`: Starts the FastAPI development server with auto-reload. + * `make run-frontend`: Starts the UmiJS frontend development server with hot reload. + * `make run-celery`: Starts a Celery worker for processing background tasks. * **Code Quality & Testing**: * `make format`: Formats Python code using Ruff and frontend code using Prettier. @@ -218,17 +245,15 @@ The `Makefile` at the root of the project provides several helpful commands to s * `make test`: Runs all automated tests (Python unit tests, integration tests). * **Database Management**: - * `make makemigration`: Creates new database migration files based on changes to SQLAlchemy models. * `make migrate`: Applies pending database migrations to your connected database. - * `make connect-metadb`: Provides a command to connect to the primary PostgreSQL database (usually for inspection, if run via `make run-db`). + * `make makemigration`: Creates new database migration files based on changes to SQLAlchemy models. * **Generators**: * `make generate-models`: Generates Pydantic models from the OpenAPI schema. * `make generate-frontend-sdk`: Generates the frontend API client/SDK from the OpenAPI specification. **Run this command whenever backend API definitions change.** -* **Docker Compose (for local full-stack testing)**: +* **Docker Compose (for full application testing)**: * `make compose-up`: Starts all services (backend, frontend, databases, Celery) using Docker Compose. - * `make compose-down`: Stops all services started with `make compose-up`. * `make compose-logs`: Tails the logs from all services running under Docker Compose. * **Cleanup**: From 68295165e3804a71847abcec1f70f68860021785 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:31:11 +0800 Subject: [PATCH 07/24] feat: update makefile & readme --- README.md | 132 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 8e82a9aaa..a3a118530 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,10 @@ # ApeRAG -[ι˜…θ―»δΈ­ζ–‡ζ–‡ζ‘£](./README_zh.md) - -![collection-page.png](docs%2Fimages%2Fcollection-page.png) - ## Table of Contents - [Quick Start](#quick-start) - [Key Features](#key-features) -- [Advanced Setup](#advanced-setup) - - [Getting Started with Kubernetes (Recommend for Production)](#getting-started-with-kubernetes-recommend-for-production) +- [Kubernetes Deployment](#kubernetes-deployment) - [Development](./docs/development-guide.md) - [Build Docker Image](./docs/build-docker-image.md) - [Acknowledgments](#acknowledgments) @@ -65,66 +60,99 @@ Kubernetes support, Docker deployment, async task processing with Celery, and co **6. Developer Friendly**: FastAPI backend, React frontend, extensive testing, and detailed development guides for easy contribution and customization. -## Advanced Setup +## Kubernetes Deployment + +> **Recommended for Production Environment** + +Deploy ApeRAG to Kubernetes using our provided Helm chart. This approach offers high availability, scalability, and production-grade management capabilities. + +### Prerequisites + +* Kubernetes cluster (v1.20+) +* `kubectl` configured and connected to your cluster +* Helm v3+ installed + +### Option 1: Quick Deployment (Recommended) + +If you already have PostgreSQL, Redis, Qdrant, and Elasticsearch running in your cluster, you can deploy ApeRAG directly: + +```bash +# Clone the repository +git clone https://github.com/apecloud/ApeRAG.git +cd ApeRAG + +# Update database connections in values.yaml for your existing databases +# Then deploy ApeRAG +helm install aperag ./deploy/aperag --namespace aperag --create-namespace +``` + +### Option 2: Full Deployment with KubeBlocks + +If you need to deploy databases as well, use our KubeBlocks integration: -### Getting Started with Kubernetes (Recommend for Production) +#### Step 1: Deploy Database Services -This guide covers deploying ApeRAG to Kubernetes using the provided Helm chart. It involves two main phases: setting up databases (optional if you have them) and deploying the ApeRAG application. +```bash +cd deploy/databases/ + +# (Optional) Review configuration - defaults work for most cases +# edit 00-config.sh + +# Install KubeBlocks and deploy databases +bash ./01-prepare.sh # Installs KubeBlocks +bash ./02-install-database.sh # Deploys PostgreSQL, Redis, Qdrant, Elasticsearch + +# Monitor database deployment +kubectl get pods -n default +``` + +Wait for all database pods to be in `Running` status before proceeding. + +#### Step 2: Deploy ApeRAG Application + +```bash +cd ../../ # Back to project root + +# Deploy ApeRAG (database connections pre-configured for KubeBlocks) +helm install aperag ./deploy/aperag --namespace default --create-namespace + +# Monitor ApeRAG deployment +kubectl get pods -n default -l app.kubernetes.io/instance=aperag +``` -**Phase 1: Deploy Databases with KubeBlocks (Optional)** +### Configuration Options -ApeRAG needs PostgreSQL, Redis, Qdrant, and Elasticsearch. If you don't have these, use the KubeBlocks scripts in `deploy/databases/`. +**Database Connections**: Edit `deploy/aperag/values.yaml` to configure database connections for your environment. -*Skip this phase if your databases are already available in your Kubernetes cluster.* +**Resource Requirements**: By default, includes [`doc-ray`](https://github.com/apecloud/doc-ray) service (requires 4+ CPU cores, 8GB+ RAM). To disable: set `docray.enabled: false` in `values.yaml`. -1. **Prerequisites**: - * Kubernetes cluster. - * `kubectl` configured. - * Helm v3+. +**Advanced Settings**: Review `values.yaml` for additional configuration options including images, resources, and Ingress settings. -2. **Database Configuration (`deploy/databases/00-config.sh`)**: - This script controls database deployment (defaults: PostgreSQL, Redis, Qdrant, Elasticsearch in the `default` namespace). **Defaults are usually fine; no changes needed for a standard setup.** Edit only for advanced cases (e.g., changing namespace, enabling optional databases like Neo4j). +### Access Your Deployment -3. **Run Database Deployment Scripts**: - ```bash - cd deploy/databases/ - bash ./01-prepare.sh # Prepares KubeBlocks environment. - bash ./02-install-database.sh # Deploys database clusters. - cd ../.. # Back to project root. - ``` - Monitor pods in the `default` namespace (or your custom one) until ready: - ```bash - kubectl get pods -n default - ``` +Once deployed, access ApeRAG using port forwarding: -**Phase 2: Deploy ApeRAG Application** +```bash +# Forward ports for quick access +kubectl port-forward svc/aperag-frontend 3000:3000 -n default +kubectl port-forward svc/aperag-api 8000:8000 -n default -With databases running: +# Access in browser +# Web Interface: http://localhost:3000 +# API Documentation: http://localhost:8000/docs +``` -1. **Helm Chart Configuration (`deploy/aperag/values.yaml`)**: - * **Using KubeBlocks (Phase 1 in `default` namespace)?** Database connections in `values.yaml` are likely pre-configured. **No changes usually needed.** - * **Using your own databases?** You MUST update `values.yaml` with your database connection details. - * By default, this Helm chart deploys the [`doc-ray`](https://github.com/apecloud/doc-ray) service for advanced document parsing, which requires at least 4 CPU cores and 8GB of memory. If your Kubernetes cluster has insufficient resources, you can disable the `doc-ray` deployment by setting `docray.enabled` to `false`. In this case, a basic document parser will be used. - * Optionally, review other settings (images, resources, Ingress, etc.). +For production environments, configure Ingress in `values.yaml` for external access. -2. **Deploy ApeRAG with Helm**: - This installs ApeRAG to the `default` namespace: - ```bash - helm install aperag ./deploy/aperag --namespace default --create-namespace - ``` - Monitor ApeRAG pods until `Running`: - ```bash - kubectl get pods -n default -l app.kubernetes.io/instance=aperag - ``` +### Troubleshooting -3. **Access ApeRAG UI**: - Use `kubectl port-forward` for quick access: - ```bash - kubectl port-forward svc/aperag-frontend 3000:3000 -n default - ``` - Open `http://localhost:3000` in your browser. +**Database Issues**: See `deploy/databases/README.md` for KubeBlocks management, credentials, and uninstall procedures. -For KubeBlocks details (credentials, uninstall), see `deploy/databases/README.md`. +**Pod Status**: Check pod logs for any deployment issues: +```bash +kubectl logs -f deployment/aperag-api -n default +kubectl logs -f deployment/aperag-frontend -n default +``` ## Acknowledgments From f2a7e1c3eca3432cd4f09cc8f46b70e07afedd9e Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:35:02 +0800 Subject: [PATCH 08/24] feat: update makefile & readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a3a118530..67076fe1c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # ApeRAG -## Table of Contents - - [Quick Start](#quick-start) - [Key Features](#key-features) - [Kubernetes Deployment](#kubernetes-deployment) @@ -34,9 +32,13 @@ After running, you can access ApeRAG in your browser at: - **Web Interface**: http://localhost:3000/web/ - **API Documentation**: http://localhost:8000/docs +#### Advanced Options + +For enhanced document parsing capabilities, ApeRAG supports **DocRay** integration powered by [MinerU](https://github.com/opendatalab/MinerU) technology, which provides superior parsing for complex documents, tables, and formulas. See our [Development Guide](./docs/development-guide.md) for DocRay setup and other advanced deployment options. + #### Seeking help -Please refer to our [Development Guide](./docs/development-guide.md) for advanced configurations, development setup, and troubleshooting. Reach out to [the community](#acknowledgments) if you are still having issues. +Please refer to our [Development Guide](./docs/development-guide.md) for advanced configurations, development setup, and troubleshooting. > If you'd like to contribute to ApeRAG or do additional development, refer to our [Development Guide](./docs/development-guide.md) From 819ea49deb13dcbeb0c442ca799cca96c7944c73 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:38:54 +0800 Subject: [PATCH 09/24] feat: update makefile & readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 67076fe1c..5e1a15b92 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - [Acknowledgments](#acknowledgments) - [License](#license) -ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction inspired by **[LightRAG](https://github.com/HKUDS/LightRAG)**. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. +ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction inspired by LightRAG. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. ## Quick Start @@ -34,13 +34,13 @@ After running, you can access ApeRAG in your browser at: #### Advanced Options -For enhanced document parsing capabilities, ApeRAG supports **DocRay** integration powered by [MinerU](https://github.com/opendatalab/MinerU) technology, which provides superior parsing for complex documents, tables, and formulas. See our [Development Guide](./docs/development-guide.md) for DocRay setup and other advanced deployment options. +For enhanced document parsing capabilities, ApeRAG supports an **advanced document parsing service** powered by MinerU, which provides superior parsing for complex documents, tables, and formulas. See our [Development Guide](./docs/development-guide.md) for setup and other advanced deployment options. #### Seeking help -Please refer to our [Development Guide](./docs/development-guide.md) for advanced configurations, development setup, and troubleshooting. +Please refer to our Development Guide for advanced configurations, development setup, and troubleshooting. -> If you'd like to contribute to ApeRAG or do additional development, refer to our [Development Guide](./docs/development-guide.md) +> If you'd like to contribute to ApeRAG or do additional development, check out the Development Guide above. ## Key Features From f8b110c9c3b77b1a4bbcf19a0d7a707580204681 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:41:01 +0800 Subject: [PATCH 10/24] feat: update makefile & readme --- README.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e1a15b92..4dd842aca 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,28 @@ After running, you can access ApeRAG in your browser at: #### Advanced Options -For enhanced document parsing capabilities, ApeRAG supports an **advanced document parsing service** powered by MinerU, which provides superior parsing for complex documents, tables, and formulas. See our [Development Guide](./docs/development-guide.md) for setup and other advanced deployment options. +For enhanced document parsing capabilities, ApeRAG supports an **advanced document parsing service** powered by MinerU, which provides superior parsing for complex documents, tables, and formulas. + +
+Advanced Deployment Commands + +```bash +# Add graph knowledge capabilities with Neo4j +make compose-up WITH_NEO4J=1 + +# Add advanced document parsing service +make compose-up WITH_DOCRAY=1 + +# Combine both advanced features +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 + +# Full-featured deployment with GPU acceleration +make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +``` + +
+ +See our [Development Guide](./docs/development-guide.md) for detailed setup instructions and other advanced deployment options. #### Seeking help From 5329dc2f13c9b9e691e5910479ffff7a1641404b Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:43:49 +0800 Subject: [PATCH 11/24] feat: update makefile & readme --- README.md | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4dd842aca..4fef16e53 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - [Acknowledgments](#acknowledgments) - [License](#license) -ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction inspired by LightRAG. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. +ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. ## Quick Start @@ -32,36 +32,26 @@ After running, you can access ApeRAG in your browser at: - **Web Interface**: http://localhost:3000/web/ - **API Documentation**: http://localhost:8000/docs -#### Advanced Options +#### Enhanced Document Parsing For enhanced document parsing capabilities, ApeRAG supports an **advanced document parsing service** powered by MinerU, which provides superior parsing for complex documents, tables, and formulas.
-Advanced Deployment Commands +Enhanced Document Parsing Commands ```bash -# Add graph knowledge capabilities with Neo4j -make compose-up WITH_NEO4J=1 - -# Add advanced document parsing service +# Enable advanced document parsing service make compose-up WITH_DOCRAY=1 -# Combine both advanced features -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 - -# Full-featured deployment with GPU acceleration -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +# Enable advanced parsing with GPU acceleration (recommended) +make compose-up WITH_DOCRAY=1 WITH_GPU=1 ```
-See our [Development Guide](./docs/development-guide.md) for detailed setup instructions and other advanced deployment options. - -#### Seeking help - -Please refer to our Development Guide for advanced configurations, development setup, and troubleshooting. +#### Development & Contributing -> If you'd like to contribute to ApeRAG or do additional development, check out the Development Guide above. +For developers interested in source code development, advanced configurations, or contributing to ApeRAG, please refer to our [Development Guide](./docs/development-guide.md) for detailed setup instructions. ## Key Features From d6087f8689e41073e5b684727c294fbc200ec8c7 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:50:41 +0800 Subject: [PATCH 12/24] feat: update makefile & readme --- README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4fef16e53..391899f72 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ - [Quick Start](#quick-start) - [Key Features](#key-features) -- [Kubernetes Deployment](#kubernetes-deployment) +- [Kubernetes Deployment (Recommended for Production)](#kubernetes-deployment-recommended-for-production) - [Development](./docs/development-guide.md) - [Build Docker Image](./docs/build-docker-image.md) - [Acknowledgments](#acknowledgments) - [License](#license) -ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines vector search, full-text search, and graph knowledge extraction. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. +ApeRAG is a production-ready RAG (Retrieval-Augmented Generation) platform that combines Graph RAG, vector search, and full-text search. Build sophisticated AI applications with hybrid retrieval, multimodal document processing, and enterprise-grade management features. ## Quick Start @@ -56,26 +56,29 @@ For developers interested in source code development, advanced configurations, o ## Key Features **1. Hybrid Retrieval Engine**: -Combines vector search, full-text search, and graph knowledge extraction for comprehensive document understanding and retrieval. +Combines Graph RAG, vector search, and full-text search for comprehensive document understanding and retrieval. -**2. LightRAG Integration**: +**2. Graph RAG with LightRAG**: Enhanced version of LightRAG for advanced graph-based knowledge extraction, enabling deep relational and contextual queries. -**3. Multimodal Document Processing**: -Supports various document formats (PDF, DOCX, etc.) with advanced parsing capabilities for text, tables, and images. +**3. MinerU Integration**: +Advanced document parsing service powered by MinerU technology, providing superior parsing for complex documents, tables, formulas, and scientific content with optional GPU acceleration. -**4. Enterprise Management**: -Built-in audit logging, LLM model management, graph visualization, and comprehensive document management interface. +**4. Production-Grade Deployment**: +Full Kubernetes support with Helm charts and KubeBlocks integration for simplified deployment of production-grade databases (PostgreSQL, Redis, Qdrant, Elasticsearch, Neo4j). + +**5. Multimodal Document Processing**: +Supports various document formats (PDF, DOCX, etc.) with intelligent content extraction and structure recognition. -**5. Production Ready**: -Kubernetes support, Docker deployment, async task processing with Celery, and comprehensive API documentation. +**6. Enterprise Management**: +Built-in audit logging, LLM model management, graph visualization, and comprehensive document management interface. -**6. Developer Friendly**: -FastAPI backend, React frontend, extensive testing, and detailed development guides for easy contribution and customization. +**7. Developer Friendly**: +FastAPI backend, React frontend, async task processing with Celery, extensive testing, and comprehensive development guides for easy contribution and customization. -## Kubernetes Deployment +## Kubernetes Deployment (Recommended for Production) -> **Recommended for Production Environment** +> **Enterprise-grade deployment with high availability and scalability** Deploy ApeRAG to Kubernetes using our provided Helm chart. This approach offers high availability, scalability, and production-grade management capabilities. From 1123598cb0d3aa6e0d2163d8542189d2e147a4a5 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 11:59:06 +0800 Subject: [PATCH 13/24] feat: update makefile & readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 391899f72..4a87213d9 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,11 @@ Deploy ApeRAG to Kubernetes using our provided Helm chart. This approach offers ### Prerequisites -* Kubernetes cluster (v1.20+) -* `kubectl` configured and connected to your cluster -* Helm v3+ installed +* [Kubernetes cluster](https://kubernetes.io/docs/setup/) (v1.20+) +* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) configured and connected to your cluster +* [Helm v3+](https://helm.sh/docs/intro/install/) installed -### Option 1: Quick Deployment (Recommended) +### Option 1: Quick Deployment If you already have PostgreSQL, Redis, Qdrant, and Elasticsearch running in your cluster, you can deploy ApeRAG directly: @@ -102,7 +102,7 @@ cd ApeRAG helm install aperag ./deploy/aperag --namespace aperag --create-namespace ``` -### Option 2: Full Deployment with KubeBlocks +### Option 2: Full Deployment with KubeBlocks (Recommended) If you need to deploy databases as well, use our KubeBlocks integration: From 6666b181d758e734b5015e8b0e3963fb7dd70057 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:13:20 +0800 Subject: [PATCH 14/24] feat: update makefile & readme --- README.md | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4a87213d9..0bb8daf6a 100644 --- a/README.md +++ b/README.md @@ -88,25 +88,13 @@ Deploy ApeRAG to Kubernetes using our provided Helm chart. This approach offers * [`kubectl`](https://kubernetes.io/docs/tasks/tools/) configured and connected to your cluster * [Helm v3+](https://helm.sh/docs/intro/install/) installed -### Option 1: Quick Deployment +### Step 1: Deploy Database Services -If you already have PostgreSQL, Redis, Qdrant, and Elasticsearch running in your cluster, you can deploy ApeRAG directly: +ApeRAG requires PostgreSQL, Redis, Qdrant, and Elasticsearch. You have two options: -```bash -# Clone the repository -git clone https://github.com/apecloud/ApeRAG.git -cd ApeRAG - -# Update database connections in values.yaml for your existing databases -# Then deploy ApeRAG -helm install aperag ./deploy/aperag --namespace aperag --create-namespace -``` +**Option A: Use existing databases** - If you already have these databases running in your cluster, skip to Step 2 and update `deploy/aperag/values.yaml` with your database connection details. -### Option 2: Full Deployment with KubeBlocks (Recommended) - -If you need to deploy databases as well, use our KubeBlocks integration: - -#### Step 1: Deploy Database Services +**Option B: Deploy databases with KubeBlocks** - Use our automated database deployment: ```bash cd deploy/databases/ @@ -124,12 +112,17 @@ kubectl get pods -n default Wait for all database pods to be in `Running` status before proceeding. -#### Step 2: Deploy ApeRAG Application +### Step 2: Deploy ApeRAG Application ```bash -cd ../../ # Back to project root +# Clone the repository (if not already done) +git clone https://github.com/apecloud/ApeRAG.git +cd ApeRAG + +# If you deployed databases with KubeBlocks in Step 1, database connections are pre-configured +# If you're using existing databases, edit deploy/aperag/values.yaml with your connection details -# Deploy ApeRAG (database connections pre-configured for KubeBlocks) +# Deploy ApeRAG helm install aperag ./deploy/aperag --namespace default --create-namespace # Monitor ApeRAG deployment @@ -138,7 +131,7 @@ kubectl get pods -n default -l app.kubernetes.io/instance=aperag ### Configuration Options -**Database Connections**: Edit `deploy/aperag/values.yaml` to configure database connections for your environment. +**Database Connections**: If using existing databases (Option A in Step 1), edit `deploy/aperag/values.yaml` to configure your database connection details. KubeBlocks deployments (Option B) are pre-configured. **Resource Requirements**: By default, includes [`doc-ray`](https://github.com/apecloud/doc-ray) service (requires 4+ CPU cores, 8GB+ RAM). To disable: set `docray.enabled: false` in `values.yaml`. From a36c6a4a5ae1a4d2e652efe8334e8193953f4ae6 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:17:18 +0800 Subject: [PATCH 15/24] feat: update makefile & readme --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0bb8daf6a..3b55c125a 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,15 @@ Deploy ApeRAG to Kubernetes using our provided Helm chart. This approach offers * [`kubectl`](https://kubernetes.io/docs/tasks/tools/) configured and connected to your cluster * [Helm v3+](https://helm.sh/docs/intro/install/) installed +### Clone the Repository + +First, clone the ApeRAG repository to get the deployment files: + +```bash +git clone https://github.com/apecloud/ApeRAG.git +cd ApeRAG +``` + ### Step 1: Deploy Database Services ApeRAG requires PostgreSQL, Redis, Qdrant, and Elasticsearch. You have two options: @@ -97,6 +106,7 @@ ApeRAG requires PostgreSQL, Redis, Qdrant, and Elasticsearch. You have two optio **Option B: Deploy databases with KubeBlocks** - Use our automated database deployment: ```bash +# Navigate to database deployment scripts cd deploy/databases/ # (Optional) Review configuration - defaults work for most cases @@ -108,6 +118,9 @@ bash ./02-install-database.sh # Deploys PostgreSQL, Redis, Qdrant, Elasticsearch # Monitor database deployment kubectl get pods -n default + +# Return to project root for Step 2 +cd ../../ ``` Wait for all database pods to be in `Running` status before proceeding. @@ -115,10 +128,6 @@ Wait for all database pods to be in `Running` status before proceeding. ### Step 2: Deploy ApeRAG Application ```bash -# Clone the repository (if not already done) -git clone https://github.com/apecloud/ApeRAG.git -cd ApeRAG - # If you deployed databases with KubeBlocks in Step 1, database connections are pre-configured # If you're using existing databases, edit deploy/aperag/values.yaml with your connection details From cdf7130338833128f02b079f616457fd5a90f3c7 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:19:09 +0800 Subject: [PATCH 16/24] feat: update makefile & readme --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3b55c125a..272effff2 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,9 @@ cd ApeRAG ApeRAG requires PostgreSQL, Redis, Qdrant, and Elasticsearch. You have two options: -**Option A: Use existing databases** - If you already have these databases running in your cluster, skip to Step 2 and update `deploy/aperag/values.yaml` with your database connection details. +**Option A: Use existing databases** - If you already have these databases running in your cluster, edit `deploy/aperag/values.yaml` to configure your database connection details, then skip to Step 2. -**Option B: Deploy databases with KubeBlocks** - Use our automated database deployment: +**Option B: Deploy databases with KubeBlocks** - Use our automated database deployment (database connections are pre-configured): ```bash # Navigate to database deployment scripts @@ -140,8 +140,6 @@ kubectl get pods -n default -l app.kubernetes.io/instance=aperag ### Configuration Options -**Database Connections**: If using existing databases (Option A in Step 1), edit `deploy/aperag/values.yaml` to configure your database connection details. KubeBlocks deployments (Option B) are pre-configured. - **Resource Requirements**: By default, includes [`doc-ray`](https://github.com/apecloud/doc-ray) service (requires 4+ CPU cores, 8GB+ RAM). To disable: set `docray.enabled: false` in `values.yaml`. **Advanced Settings**: Review `values.yaml` for additional configuration options including images, resources, and Ingress settings. From 517d8d1b630a4e79d5d810d7c864a364cc131d27 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:25:31 +0800 Subject: [PATCH 17/24] feat: update makefile & readme --- docs/development-guide.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/development-guide.md b/docs/development-guide.md index b25edcc53..5fa736b26 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -23,10 +23,11 @@ Edit the `.env` file to configure your AI service settings if needed. The defaul Before you begin, ensure your system has: -* **Python 3.11**: The project uses Python 3.11. * **Node.js**: Version 20 or higher is recommended for frontend development. * **Docker & Docker Compose**: Required for running database services locally. +**Note**: Python 3.11 is required but will be automatically managed by `uv` in the next steps. + ### 3. Start Database Services Use Docker Compose to start the essential database services: @@ -35,7 +36,7 @@ Use Docker Compose to start the essential database services: # Start core databases: PostgreSQL, Redis, Qdrant, Elasticsearch make compose-infra -# Optional: Add graph database capabilities +# Optional: Use Neo4j instead of PostgreSQL for graph storage # make compose-infra WITH_NEO4J=1 ``` @@ -138,7 +139,7 @@ Start only the essential database services. Perfect for developers who want to r # Core databases: PostgreSQL, Redis, Qdrant, Elasticsearch make compose-infra -# Add graph database capabilities +# Use Neo4j instead of PostgreSQL for graph storage make compose-infra WITH_NEO4J=1 ``` @@ -156,7 +157,7 @@ Launch the complete ApeRAG platform with all services containerized. # Complete system (API + Frontend + Background workers + Databases) make compose-up -# Add graph knowledge capabilities with Neo4j +# Use Neo4j instead of PostgreSQL for graph storage make compose-up WITH_NEO4J=1 # Add advanced document parsing with DocRay @@ -172,8 +173,8 @@ make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 ### Optional Services **Neo4j Graph Database** (`WITH_NEO4J=1`) -- Enables graph-based knowledge extraction and querying -- Powers advanced relational search capabilities +- Uses Neo4j instead of PostgreSQL for graph storage backend +- Provides native graph database capabilities for better performance - Accessible at http://localhost:7474 (Web UI) **DocRay Advanced Document Parsing** (`WITH_DOCRAY=1`) @@ -209,7 +210,7 @@ make compose-up **For Active Development**: ```bash -make compose-infra WITH_NEO4J=1 # Start databases +make compose-infra WITH_NEO4J=1 # Start databases (with Neo4j for graph storage) make run-backend # Run API in development mode # Code with hot reload and debugging ``` @@ -230,7 +231,7 @@ The `Makefile` at the root of the project provides several helpful commands to s * **Database Services**: * `make compose-infra`: Starts essential database services (PostgreSQL, Redis, Qdrant, Elasticsearch) using Docker Compose. - * `make compose-infra WITH_NEO4J=1`: Starts database services including Neo4j for graph knowledge capabilities. + * `make compose-infra WITH_NEO4J=1`: Starts database services using Neo4j instead of PostgreSQL for graph storage. * `make compose-down`: Stops all database services. * **Development Services**: From e759efc38cd72f4f1caf1af2033bf291f7fc3e86 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:31:04 +0800 Subject: [PATCH 18/24] feat: update makefile & readme --- Makefile | 10 +++++++++- docs/development-guide.md | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4895bf923..03ee51868 100644 --- a/Makefile +++ b/Makefile @@ -142,11 +142,19 @@ clean: # Development tools installation .PHONY: dev install-hooks -dev: install-uv install-addlicense install-hooks +dev: install-uv venv install-addlicense install-hooks @echo "Installing development tools..." @command -v redocly >/dev/null || npm install @redocly/cli -g @command -v openapi-generator-cli >/dev/null || npm install @openapitools/openapi-generator-cli -g @command -v datamodel-codegen >/dev/null || uv tool install datamodel-code-generator + @echo "" + @echo "βœ… Development environment ready!" + @echo "πŸ“ Next steps:" + @echo " 1. Activate virtual environment: source .venv/bin/activate" + @echo " 2. Install dependencies: make install" + @echo " 3. Start databases: make compose-infra" + @echo " 4. Apply migrations: make migrate" + @echo " 5. Run services: make run-backend, make run-celery" # Code quality checks .PHONY: format lint static-check test unit-test e2e-test diff --git a/docs/development-guide.md b/docs/development-guide.md index 5fa736b26..deaaf2098 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -53,7 +53,16 @@ make dev This command will: * Install `uv` if not already available * Create a Python 3.11 virtual environment (located in `.venv/`) +* Install development tools (redocly, openapi-generator-cli, etc.) * Install pre-commit hooks for code quality +* Install addlicense tool for license management + +**Activate the virtual environment:** +```bash +source .venv/bin/activate +``` + +You'll know it's active when you see `(.venv)` in your terminal prompt. ### 5. Install Dependencies From fed8c091e570f8fdadad2b65d0982c2c5e028dd0 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:34:23 +0800 Subject: [PATCH 19/24] feat: update makefile & readme --- docs/development-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development-guide.md b/docs/development-guide.md index deaaf2098..066b92127 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -109,9 +109,9 @@ This starts the frontend development server at `http://localhost:3000` with hot ### 8. Access ApeRAG With the services running, you can access: +* **Frontend UI**: http://localhost:3000 (if started) * **Backend API**: http://localhost:8000 * **API Documentation**: http://localhost:8000/docs -* **Frontend UI**: http://localhost:3000 (if started) ### 9. Development Workflow From 8b31424c5d39cf4ffecdbaa192db9e500b80494a Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:54:46 +0800 Subject: [PATCH 20/24] feat: update makefile & readme --- docs/development-guide.md | 365 +++++++++++++++++++++----------------- 1 file changed, 201 insertions(+), 164 deletions(-) diff --git a/docs/development-guide.md b/docs/development-guide.md index 066b92127..cf3e4af03 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -113,191 +113,228 @@ With the services running, you can access: * **Backend API**: http://localhost:8000 * **API Documentation**: http://localhost:8000/docs -### 9. Development Workflow - -Your typical development cycle: - -1. **Code Changes**: Edit files in `aperag/` (backend) or `frontend/src/` (frontend) -2. **Auto-reload**: Backend and frontend automatically reload on file changes -3. **Testing**: Run `make test` to execute tests -4. **Code Quality**: Run `make format` and `make lint` before committing -5. **Database Changes**: If you modify models, run `make makemigration` then `make migrate` - -### 10. Stopping Services +### 9. Stopping Services To stop the development environment: +**Stop Database Services:** ```bash -# Stop database services +# Stop database services (data preserved) make compose-down -# Stop development services (Ctrl+C in each terminal) -``` - -Now you have ApeRAG running locally from source code, ready for development! - -## Docker Compose Development Options - -For developers who prefer containerized development, Docker Compose offers flexible deployment modes: - -### Infrastructure Mode (Recommended for Development) - -Start only the essential database services. Perfect for developers who want to run the application code locally for debugging and development. - -```bash -# Core databases: PostgreSQL, Redis, Qdrant, Elasticsearch -make compose-infra - -# Use Neo4j instead of PostgreSQL for graph storage -make compose-infra WITH_NEO4J=1 -``` - -After starting infrastructure, run your app locally: -```bash -make run-backend # Start API server at localhost:8000 -make run-frontend # Start frontend at localhost:3000 (optional) +# Stop services and remove all data volumes +make compose-down REMOVE_VOLUMES=1 ``` -### Full Application Mode +**Stop Development Services:** +- Backend API Server: Press `Ctrl+C` in the terminal running `make run-backend` +- Celery Worker: Press `Ctrl+C` in the terminal running `make run-celery` +- Frontend Server: Press `Ctrl+C` in the terminal running `make run-frontend` -Launch the complete ApeRAG platform with all services containerized. +**Data Management:** +- `make compose-down` - Stops services but preserves all data (PostgreSQL, Redis, Qdrant, etc.) +- `make compose-down REMOVE_VOLUMES=1` - Stops services and **permanently deletes all data** +- You can run `make compose-down REMOVE_VOLUMES=1` even after already running `make compose-down` +**Verify Data Removal:** ```bash -# Complete system (API + Frontend + Background workers + Databases) -make compose-up - -# Use Neo4j instead of PostgreSQL for graph storage -make compose-up WITH_NEO4J=1 - -# Add advanced document parsing with DocRay -make compose-up WITH_DOCRAY=1 +# Check if volumes still exist +docker volume ls | grep aperag -# Combine multiple optional services -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 - -# Full-featured deployment with GPU acceleration -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 +# Should return no results after REMOVE_VOLUMES=1 ``` -### Optional Services +Now you have ApeRAG running locally from source code, ready for development! -**Neo4j Graph Database** (`WITH_NEO4J=1`) -- Uses Neo4j instead of PostgreSQL for graph storage backend -- Provides native graph database capabilities for better performance -- Accessible at http://localhost:7474 (Web UI) -**DocRay Advanced Document Parsing** (`WITH_DOCRAY=1`) -- Enhanced parsing for complex documents, tables, and formulas -- Powered by [MinerU](https://github.com/opendatalab/MinerU) technology -- CPU mode: Requires 4+ CPU cores, 8GB+ RAM -- GPU mode (`WITH_GPU=1`): Requires ~6GB VRAM, 2 CPU cores, 8GB RAM -- Service endpoint: http://localhost:8639 -### Service Management +## Key `make` Commands for Development +The `Makefile` provides essential commands for development workflow: + +### Environment Setup +* `make dev` - Sets up complete development environment (creates virtual environment, installs development tools, git hooks) +* `make install` - Installs all Python and Node.js dependencies + +### Database Services +* `make compose-infra` - Start core databases (PostgreSQL, Redis, Qdrant, Elasticsearch) +* `make compose-infra WITH_NEO4J=1` - Start databases with Neo4j for graph storage +* `make compose-down` - Stop database services (preserves data) +* `make compose-down REMOVE_VOLUMES=1` - Stop services and delete all data + +### Development Services +* `make run-backend` - Start FastAPI development server with auto-reload +* `make run-celery` - Start Celery worker for background tasks +* `make run-frontend` - Start React frontend development server + +### Database Management +* `make migrate` - Apply database migrations +* `make makemigration` - Create new migration files for model changes + +### Code Quality +* `make format` - Format code (Python with Ruff, frontend with Prettier) +* `make lint` - Lint code and check style +* `make test` - Run all tests (unit + integration) + +### Code Generation +* `make generate-models` - Generate Pydantic models from OpenAPI schema +* `make generate-frontend-sdk` - Generate frontend API client (**run after API changes**) + +### Docker Compose Testing +* `make compose-up` - Start complete application stack for testing +* `make compose-logs` - View logs from all services + +## Common Development Tasks + +### Q: How do I add or modify a REST API endpoint? + +**Steps:** +1. Edit the OpenAPI specification: `aperag/api/paths/[endpoint-name].yaml` +2. Update backend view: `aperag/views/[module].py` +3. Generate models and frontend SDK: + ```bash + make generate-models + make generate-frontend-sdk + ``` +4. Test the API: + ```bash + make test + # Check API docs at http://localhost:8000/docs + ``` + +### Q: How do I modify database models/schema? + +**Steps:** +1. Edit SQLModel classes in `aperag/db/models.py` +2. Generate and apply migration: + ```bash + make makemigration + make migrate + ``` +3. Update related code (repositories, services) +4. Test database changes: + ```bash + make test + ``` + +### Q: How do I add a new feature from scratch? + +**Complete workflow:** +1. Create feature branch: `git checkout -b feat/my-feature` +2. Start development environment: + ```bash + make compose-infra + make run-backend + make run-celery + ``` +3. Implement backend logic in `aperag/[module]/` +4. Add database models and migrate: + ```bash + make makemigration + make migrate + ``` +5. Create API endpoints and update OpenAPI spec +6. Generate frontend code: + ```bash + make generate-models + make generate-frontend-sdk + ``` +7. Implement frontend (optional): `frontend/src/` +8. Code quality check: + ```bash + make format + make lint + make test + ``` + +### Q: How do I debug a failing test? + +**Debugging workflow:** +1. Run specific test: + ```bash + # Unit tests + make unit-test + + # Specific test file + uv run pytest tests/unit_test/test_specific.py -v + + # Integration tests + uv run pytest tests/e2e_test/test_specific.py -v + ``` +2. Check services are running: + ```bash + make compose-infra + make run-backend + ``` +3. Check logs and debug with breakpoints +4. Fix code and retest: + ```bash + make format + make test + ``` + +### Q: How do I update dependencies? + +**For Python dependencies:** +1. Edit `pyproject.toml` +2. Update environment: + ```bash + make install + make test + ``` + +**For frontend dependencies:** +1. Edit `frontend/package.json` +2. Update and test: + ```bash + cd frontend && yarn install + make run-frontend + ``` + +### Q: How do I prepare code for pull request? + +**Before submitting:** +1. Ensure all tests pass: + ```bash + make test + ``` +2. Code quality checks: + ```bash + make format + make lint + ``` +3. If API changed, update frontend SDK: + ```bash + make generate-frontend-sdk + ``` +4. If models changed, ensure migrations are created: + ```bash + make makemigration + ``` +5. Test complete application: + ```bash + make compose-up + # Test at http://localhost:3000/web/ + make compose-down + ``` + +### Q: How do I reset my development environment? + +**Complete reset:** ```bash -# View running services -docker-compose ps - -# View logs -make compose-logs - -# Stop all services -make compose-down - -# Stop services and remove data volumes +# Stop all services and delete data make compose-down REMOVE_VOLUMES=1 -``` - -### Example Development Workflows - -**For Quick Testing**: -```bash -make compose-up -# Access http://localhost:3000/web/ and start exploring! -``` -**For Active Development**: -```bash -make compose-infra WITH_NEO4J=1 # Start databases (with Neo4j for graph storage) -make run-backend # Run API in development mode -# Code with hot reload and debugging +# Restart fresh environment +make compose-infra +make migrate +make run-backend +make run-celery ``` -**For Production Testing**: +**Partial reset (keep data):** ```bash -make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 WITH_GPU=1 -# Full-featured deployment with all capabilities -``` - -## Key `make` Commands for Development - -The `Makefile` at the root of the project provides several helpful commands to streamline development: - -* **Environment & Dependencies**: - * `make dev`: Sets up the development environment by creating Python virtual environment, installing `uv`, and setting up pre-commit hooks. - * `make install`: Installs all necessary backend (Python) and frontend (Node.js) dependencies into the virtual environment. - -* **Database Services**: - * `make compose-infra`: Starts essential database services (PostgreSQL, Redis, Qdrant, Elasticsearch) using Docker Compose. - * `make compose-infra WITH_NEO4J=1`: Starts database services using Neo4j instead of PostgreSQL for graph storage. - * `make compose-down`: Stops all database services. - -* **Development Services**: - * `make run-backend`: Starts the FastAPI development server with auto-reload. - * `make run-frontend`: Starts the UmiJS frontend development server with hot reload. - * `make run-celery`: Starts a Celery worker for processing background tasks. - -* **Code Quality & Testing**: - * `make format`: Formats Python code using Ruff and frontend code using Prettier. - * `make lint`: Lints Python code with Ruff and frontend code. - * `make static-check`: Performs static type checking for Python code using Mypy (if configured). - * `make test`: Runs all automated tests (Python unit tests, integration tests). - -* **Database Management**: - * `make migrate`: Applies pending database migrations to your connected database. - * `make makemigration`: Creates new database migration files based on changes to SQLAlchemy models. - -* **Generators**: - * `make generate-models`: Generates Pydantic models from the OpenAPI schema. - * `make generate-frontend-sdk`: Generates the frontend API client/SDK from the OpenAPI specification. **Run this command whenever backend API definitions change.** - -* **Docker Compose (for full application testing)**: - * `make compose-up`: Starts all services (backend, frontend, databases, Celery) using Docker Compose. - * `make compose-logs`: Tails the logs from all services running under Docker Compose. - -* **Cleanup**: - * `make clean`: Removes temporary files, build artifacts, and caches from the development environment. - -## Typical Development Workflow - -Contributing to ApeRAG involves the following typical workflow. Before starting significant work, it's a good idea to open an issue to discuss your proposed changes with the maintainers. - -1. **Fork and Branch**: - * Fork the official ApeRAG repository to your GitHub account. - * Create a new branch from `main` for your feature or bug fix. Use a descriptive branch name (e.g., `feat/add-new-parser` or `fix/login-bug`). - -2. **Environment Setup**: Ensure your development environment is set up as described in [Development Environment Setup](#development-environment-setup) (dependencies installed, databases running/accessible). - -3. **Code Implementation**: - * Make your code changes in the backend (`aperag/`) or frontend (`frontend/src/`) directories. - * **Follow Code Style**: Adhere to PEP 8 for Python and standard practices for TypeScript/React. Use English for all code, comments, and documentation. - * Regularly use `make format` and `make lint` to ensure code consistency and quality. - -4. **Handle API and Model Changes**: - * If you change backend API endpoints (add, remove, modify parameters/responses): Update the OpenAPI specification (usually in `aperag/api/openapi.yaml`) and then run `make generate-frontend-sdk` to update the frontend client. Also, run `make generate-models` if schema components are affected. - * If you change SQLAlchemy models: Run `make makemigration` to create migration files, and then `make migrate` to apply changes to your development database. - -5. **Testing**: Add unit tests for new backend logic and integration tests for API changes. Ensure all existing tests pass by running `make test`. - -6. **Documentation**: If your changes affect API specifications, user guides, or deployment processes, update the relevant documentation (e.g., OpenAPI specs, this README, files in `docs/`). - -7. **Commit and Push**: - * Make clear and concise commit messages. - * Push your branch to your fork on GitHub. - -8. **Submit a Pull Request (PR)**: - * Submit a PR from your branch to the `main` branch of the official ApeRAG repository. - * Provide a clear description of your changes in the PR and link any relevant issues. - -9. **Code Review**: Your PR will be reviewed by maintainers. Be prepared to address feedback and make further changes if necessary. \ No newline at end of file +# Just restart services +make compose-down +make compose-infra +``` \ No newline at end of file From e07c51e8c28c81adb73d1f2d218c1f42a2a865f9 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 12:59:48 +0800 Subject: [PATCH 21/24] feat: update makefile & readme --- Makefile | 97 +---------- docs/development-guide.md | 333 +++++++++++++++++++++++++------------- 2 files changed, 223 insertions(+), 207 deletions(-) diff --git a/Makefile b/Makefile index 03ee51868..ed447e957 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ migrate: @alembic -c aperag/alembic.ini upgrade head # Local services -.PHONY: run-backend run-frontend run-db run-celery run-flower +.PHONY: run-backend run-frontend run-celery run-flower run-backend: migrate uvicorn aperag.app:app --host 0.0.0.0 --reload --log-config scripts/uvicorn-log-config.yaml @@ -69,10 +69,6 @@ run-frontend: cp ./frontend/deploy/env.local.template ./frontend/.env cd ./frontend && yarn dev -run-db: - @echo "Starting all database services..." - @$(MAKE) run-redis run-postgres run-qdrant run-es run-minio - # Docker Compose deployment # Variables for compose command based on environment flags @@ -130,11 +126,7 @@ compose-logs: clean: @echo "Cleaning development environment..." @rm -f db.sqlite3 - @docker rm -fv aperag-postgres-dev aperag-redis-dev aperag-qdrant-dev aperag-es-dev aperag-minio-dev aperag-neo4j-dev 2>/dev/null || true - @if [ -f "nebula-docker-compose.yml" ]; then \ - echo "Stopping NebulaGraph containers..."; \ - docker-compose -f nebula-docker-compose.yml down 2>/dev/null || true; \ - fi + @echo "Use 'make compose-down REMOVE_VOLUMES=1' to clean Docker Compose services and data" ################################################## # Developers - Code Quality and Tools @@ -335,92 +327,7 @@ info: @echo "REGISTRY: $(REGISTRY)" @echo "HOST ARCH: $(UNAME_M)" -# Database connection tools -.PHONY: connect-metadb -connect-metadb: - @docker exec -it aperag-postgres-dev psql -p 5432 -U postgres - -# Individual service startup (for advanced users) -.PHONY: run-redis run-postgres run-qdrant run-es run-minio run-neo4j run-nebula stop-nebula -run-redis: - @docker inspect aperag-redis-dev >/dev/null 2>&1 || docker run -d --name aperag-redis-dev -p 6379:6379 redis:latest - @docker start aperag-redis-dev - -run-postgres: - @docker inspect aperag-postgres-dev >/dev/null 2>&1 || \ - docker run -d --name aperag-postgres-dev -p 5432:5432 -e POSTGRES_PASSWORD=postgres pgvector/pgvector:pg16 - @docker start aperag-postgres-dev - @sleep 3 - @docker exec aperag-postgres-dev psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS vector;" 2>/dev/null || true - -run-qdrant: - @docker inspect aperag-qdrant-dev >/dev/null 2>&1 || docker run -d --name aperag-qdrant-dev -p 6333:6333 qdrant/qdrant - @docker start aperag-qdrant-dev - -run-es: - @echo "Starting Elasticsearch (development mode)" - @docker inspect aperag-es-dev > /dev/null 2>&1 || \ - docker run -d \ - --name aperag-es-dev \ - -p 9200:9200 \ - -e discovery.type=single-node \ - -e ES_JAVA_OPTS="-Xms1g -Xmx1g" \ - -e xpack.security.enabled=false \ - -v esdata:/usr/share/elasticsearch/data \ - apecloud/elasticsearch:8.8.2 - @docker start aperag-es-dev || true - @echo "Checking if IK Analyzer is installed..." - @docker exec aperag-es-dev bash -c \ - "if [ ! -d plugins/analysis-ik ]; then \ - echo 'Installing IK Analyzer from get.infini.cloud...'; \ - curl -L --output /tmp/analysis-ik.zip https://get.infini.cloud/elasticsearch/analysis-ik/8.8.2; \ - echo 'y' | bin/elasticsearch-plugin install file:///tmp/analysis-ik.zip; \ - echo 'Restarting Elasticsearch to apply changes...'; \ - else \ - echo 'IK Analyzer is already installed.'; \ - fi" - @docker restart aperag-es-dev > /dev/null - @echo "Elasticsearch is ready with IK Analyzer!" - -run-minio: - @docker inspect aperag-minio-dev >/dev/null 2>&1 || \ - docker run -d --name aperag-minio-dev -p 9000:9000 -p 9001:9001 \ - quay.io/minio/minio server /data --console-address ":9001" - @docker start aperag-minio-dev - -run-neo4j: - @docker inspect aperag-neo4j-dev >/dev/null 2>&1 || \ - docker run -d --name aperag-neo4j-dev -p 7474:7474 -p 7687:7687 \ - -e NEO4J_AUTH=neo4j/password \ - -e NEO4J_PLUGINS=\[\"apoc\"\] \ - -e NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - -e NEO4J_apoc_export_file_enabled=true \ - neo4j:5.26.5-enterprise - @docker start aperag-neo4j-dev - -run-nebula: - @echo "Setting up NebulaGraph with docker-compose (no persistence)..." - @TZ=UTC docker-compose -f nebula-docker-compose.yml up -d - @echo "NebulaGraph is starting up..." - @echo "" - @echo "βœ… Graph service available at: localhost:9669" - @echo "" - @echo "🌐 Studio Web UI: http://localhost:7001" - @echo " πŸ“ Connection Info:" - @echo " β€’ Graphd IP address: graphd" - @echo " β€’ Port: 9669" - @echo " β€’ Username: root" - @echo " β€’ Password: nebula (or any password)" - @echo "" - @echo "πŸ’» Console: docker run --rm -ti --network host vesoft/nebula-console:nightly -addr 127.0.0.1 -port 9669 -u root -p nebula" - @echo "" - @echo "πŸ” Check status: docker-compose -f nebula-docker-compose.yml ps" - @echo "πŸ›‘ Stop: make stop-nebula" -stop-nebula: - @echo "Stopping NebulaGraph..." - @docker-compose -f nebula-docker-compose.yml down - @echo "NebulaGraph stopped." .PHONY: load-images-to-minikube diff --git a/docs/development-guide.md b/docs/development-guide.md index cf3e4af03..e1daa62f3 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -23,8 +23,8 @@ Edit the `.env` file to configure your AI service settings if needed. The defaul Before you begin, ensure your system has: -* **Node.js**: Version 20 or higher is recommended for frontend development. -* **Docker & Docker Compose**: Required for running database services locally. +* **Node.js**: Version 20 or higher is recommended for frontend development. [Download Node.js](https://nodejs.org/) +* **Docker & Docker Compose**: Required for running database services locally. [Download Docker](https://docs.docker.com/get-docker/) **Note**: Python 3.11 is required but will be automatically managed by `uv` in the next steps. @@ -150,191 +150,300 @@ Now you have ApeRAG running locally from source code, ready for development! ## Key `make` Commands for Development -The `Makefile` provides essential commands for development workflow: +Based on the Makefile structure, here are the essential commands organized by workflow: -### Environment Setup -* `make dev` - Sets up complete development environment (creates virtual environment, installs development tools, git hooks) -* `make install` - Installs all Python and Node.js dependencies +### πŸš€ Environment Setup +```bash +make dev # Complete development setup (virtual env, tools, git hooks) +make install # Install all Python and Node.js dependencies +make venv # Create Python virtual environment only +make install-uv # Install uv package manager +``` -### Database Services -* `make compose-infra` - Start core databases (PostgreSQL, Redis, Qdrant, Elasticsearch) -* `make compose-infra WITH_NEO4J=1` - Start databases with Neo4j for graph storage -* `make compose-down` - Stop database services (preserves data) -* `make compose-down REMOVE_VOLUMES=1` - Stop services and delete all data +### πŸ—„οΈ Database & Infrastructure +```bash +# Start databases only (recommended for development) +make compose-infra # PostgreSQL + Redis + Qdrant + Elasticsearch +make compose-infra WITH_NEO4J=1 # Add Neo4j for graph storage -### Development Services -* `make run-backend` - Start FastAPI development server with auto-reload -* `make run-celery` - Start Celery worker for background tasks -* `make run-frontend` - Start React frontend development server +# Database schema management +make migrate # Apply pending migrations +make makemigration # Generate new migration files -### Database Management -* `make migrate` - Apply database migrations -* `make makemigration` - Create new migration files for model changes +# Stop services +make compose-down # Stop services (keep data) +make compose-down REMOVE_VOLUMES=1 # Stop services and DELETE ALL DATA +``` -### Code Quality -* `make format` - Format code (Python with Ruff, frontend with Prettier) -* `make lint` - Lint code and check style -* `make test` - Run all tests (unit + integration) +### ⚑ Development Services +```bash +# Backend services (run in separate terminals) +make run-backend # FastAPI server with auto-reload (includes migration) +make run-celery # Celery worker + beat scheduler (--pool=threads --concurrency=16) +make run-beat # Celery beat scheduler only +make run-flower # Celery monitoring web UI + +# Frontend service +make run-frontend # React development server (copies env template) +``` -### Code Generation -* `make generate-models` - Generate Pydantic models from OpenAPI schema -* `make generate-frontend-sdk` - Generate frontend API client (**run after API changes**) +### 🐳 Docker Compose (Full Stack) +```bash +# Complete application testing +make compose-up # Full stack (API + Frontend + Workers + Databases) +make compose-up WITH_NEO4J=1 # Add Neo4j +make compose-up WITH_DOCRAY=1 # Add advanced document parsing (CPU) +make compose-up WITH_DOCRAY=1 WITH_GPU=1 # Add GPU-accelerated document parsing +make compose-logs # View all service logs +``` -### Docker Compose Testing -* `make compose-up` - Start complete application stack for testing -* `make compose-logs` - View logs from all services +### πŸ§ͺ Testing & Quality +```bash +# Testing +make test # All tests (unit + e2e) +make unit-test # Unit tests only +make e2e-test # End-to-end tests (--benchmark-disable) +make e2e-performance-test # Performance benchmarks + +# Code quality +make format # Auto-fix code style (Ruff for Python) +make lint # Check code style (no auto-fix) +make static-check # Type checking with MyPy +``` + +### πŸ”§ Code Generation & API +```bash +# When you modify APIs +make generate-models # Regenerate Pydantic models from OpenAPI spec +make generate-frontend-sdk # Regenerate frontend TypeScript API client + +# OpenAPI workflow +make merge-openapi # Bundle OpenAPI spec files + +# LLM configuration +make llm_provider # Generate LLM provider configurations +``` + +### πŸ“Š Evaluation & Analysis +```bash +make evaluate # Run RAG evaluation suite +make clean # Clean temporary files +``` ## Common Development Tasks ### Q: How do I add or modify a REST API endpoint? -**Steps:** -1. Edit the OpenAPI specification: `aperag/api/paths/[endpoint-name].yaml` -2. Update backend view: `aperag/views/[module].py` -3. Generate models and frontend SDK: +**Complete workflow:** +1. Edit OpenAPI specification: `aperag/api/paths/[endpoint-name].yaml` +2. Regenerate backend models: ```bash - make generate-models - make generate-frontend-sdk + make generate-models # This runs merge-openapi internally ``` -4. Test the API: +3. Implement backend view: `aperag/views/[module].py` +4. Generate frontend TypeScript client: + ```bash + make generate-frontend-sdk # Updates frontend/src/api/ + ``` +5. Test the API: ```bash make test - # Check API docs at http://localhost:8000/docs + # Check live docs: http://localhost:8000/docs ``` ### Q: How do I modify database models/schema? -**Steps:** +**Database migration workflow:** 1. Edit SQLModel classes in `aperag/db/models.py` -2. Generate and apply migration: +2. Generate migration file: ```bash - make makemigration - make migrate + make makemigration # Creates new migration in migration/versions/ + ``` +3. Apply migration to database: + ```bash + make migrate # Updates database schema ``` -3. Update related code (repositories, services) -4. Test database changes: +4. Update related code (repositories in `aperag/db/repositories/`, services in `aperag/service/`) +5. Verify changes: ```bash make test ``` -### Q: How do I add a new feature from scratch? +### Q: How do I add a new feature with background processing? -**Complete workflow:** -1. Create feature branch: `git checkout -b feat/my-feature` -2. Start development environment: +**Full-stack feature development:** +1. Setup development environment: ```bash - make compose-infra - make run-backend - make run-celery + make dev # One-time setup + source .venv/bin/activate # Activate environment + make compose-infra # Start databases ``` -3. Implement backend logic in `aperag/[module]/` -4. Add database models and migrate: +2. Start development services (3 terminals): ```bash - make makemigration - make migrate + # Terminal 1: Backend API + make run-backend # Includes auto-migration + + # Terminal 2: Background tasks + make run-celery # Worker + scheduler (threads=16) + + # Terminal 3: Task monitoring (optional) + make run-flower # Web UI at http://localhost:5555 ``` -5. Create API endpoints and update OpenAPI spec -6. Generate frontend code: +3. Implement feature: + - Backend logic: `aperag/[module]/` + - Async tasks: `aperag/tasks/` + - Database models: `aperag/db/models.py` +4. Update API and generate code: ```bash + make makemigration + make migrate make generate-models make generate-frontend-sdk ``` -7. Implement frontend (optional): `frontend/src/` -8. Code quality check: +5. Quality assurance: ```bash - make format - make lint - make test + make format && make lint && make test ``` -### Q: How do I debug a failing test? +### Q: How do I test with different database backends? -**Debugging workflow:** -1. Run specific test: +**Testing with PostgreSQL (default):** +```bash +make compose-infra # PostgreSQL + Redis + Qdrant + Elasticsearch +make run-backend +``` + +**Testing with Neo4j for graph storage:** +```bash +make compose-infra WITH_NEO4J=1 # Neo4j replaces PostgreSQL for graphs +make run-backend +# Access Neo4j UI: http://localhost:7474 +``` + +**Testing complete stack with advanced parsing:** +```bash +# CPU-based document parsing +make compose-up WITH_DOCRAY=1 + +# GPU-accelerated parsing (requires ~6GB VRAM) +make compose-up WITH_DOCRAY=1 WITH_GPU=1 +``` + +### Q: How do I debug failing tests? + +**Test debugging workflow:** +1. Run specific test categories: ```bash - # Unit tests - make unit-test - - # Specific test file - uv run pytest tests/unit_test/test_specific.py -v + make unit-test # Fast unit tests only + make e2e-test # Integration tests (--benchmark-disable) - # Integration tests - uv run pytest tests/e2e_test/test_specific.py -v + # Specific test file with verbose output + uv run pytest tests/unit_test/test_specific.py -v -s ``` -2. Check services are running: +2. Debug with services running: ```bash - make compose-infra - make run-backend + make compose-infra # Ensure databases are up + make run-backend # API server for integration tests ``` -3. Check logs and debug with breakpoints -4. Fix code and retest: +3. Performance testing: ```bash - make format - make test + make e2e-performance-test # Benchmarks with --benchmark-enable ``` +4. Fix and verify: + ```bash + make format # Auto-fix style issues + make lint # Check remaining issues + make test # Full test suite + ``` + +### Q: How do I run RAG evaluation and analysis? + +**Evaluation workflow:** +```bash +# Ensure environment is ready +make compose-infra WITH_NEO4J=1 # Use Neo4j for better graph performance +make run-backend +make run-celery + +# Run comprehensive RAG evaluation +make evaluate # Runs aperag.evaluation.run module + +# Check evaluation reports in tests/report/ +``` -### Q: How do I update dependencies? +### Q: How do I update dependencies safely? -**For Python dependencies:** -1. Edit `pyproject.toml` -2. Update environment: +**Python dependencies:** +1. Edit `pyproject.toml` (add/update packages) +2. Update virtual environment: ```bash - make install - make test + make install # Syncs all groups and extras with uv + make test # Verify compatibility ``` -**For frontend dependencies:** +**Frontend dependencies:** 1. Edit `frontend/package.json` 2. Update and test: ```bash cd frontend && yarn install - make run-frontend + make run-frontend # Test frontend compilation + make generate-frontend-sdk # Ensure API client still works ``` -### Q: How do I prepare code for pull request? +### Q: How do I prepare code for production deployment? -**Before submitting:** -1. Ensure all tests pass: +**Pre-deployment checklist:** +1. Code quality validation: ```bash - make test + make format # Auto-fix all style issues + make lint # Verify no style violations + make static-check # MyPy type checking ``` -2. Code quality checks: +2. Comprehensive testing: ```bash - make format - make lint + make test # All unit + e2e tests + make e2e-performance-test # Performance benchmarks ``` -3. If API changed, update frontend SDK: +3. API consistency: ```bash - make generate-frontend-sdk + make generate-models # Ensure models match OpenAPI spec + make generate-frontend-sdk # Update frontend client ``` -4. If models changed, ensure migrations are created: +4. Database migrations: ```bash - make makemigration + make makemigration # Generate any pending migrations ``` -5. Test complete application: +5. Full-stack integration test: ```bash - make compose-up - # Test at http://localhost:3000/web/ + make compose-up WITH_NEO4J=1 WITH_DOCRAY=1 # Production-like setup + # Manual testing at http://localhost:3000/web/ make compose-down ``` -### Q: How do I reset my development environment? +### Q: How do I completely reset my development environment? -**Complete reset:** +**Nuclear reset (destroys all data):** ```bash -# Stop all services and delete data -make compose-down REMOVE_VOLUMES=1 +make compose-down REMOVE_VOLUMES=1 # Stop services + delete ALL data +make clean # Clean temporary files + +# Restart fresh +make compose-infra # Fresh databases +make migrate # Apply all migrations +make run-backend # Start API server +make run-celery # Start background workers +``` -# Restart fresh environment -make compose-infra -make migrate -make run-backend -make run-celery +**Soft reset (preserve data):** +```bash +make compose-down # Stop services, keep data +make compose-infra # Restart databases +make migrate # Apply any new migrations ``` -**Partial reset (keep data):** +**Reset just Python environment:** ```bash -# Just restart services -make compose-down -make compose-infra +rm -rf .venv/ # Remove virtual environment +make dev # Recreate everything +source .venv/bin/activate # Reactivate ``` \ No newline at end of file From 75e0c4bb849f15014fd1ba41788d04a535251cfd Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 13:10:17 +0800 Subject: [PATCH 22/24] feat: update makefile & readme --- Makefile | 251 ++++++++++++++++++++------------------ docs/development-guide.md | 221 +++++++++++++-------------------- 2 files changed, 214 insertions(+), 258 deletions(-) diff --git a/Makefile b/Makefile index ed447e957..9ad454939 100644 --- a/Makefile +++ b/Makefile @@ -22,11 +22,11 @@ else endif ################################################## -# Users - Local Development and Deployment +# Environment & Dependencies ################################################## -# Environment setup -.PHONY: install-uv venv install +# Python environment setup +.PHONY: install-uv venv install clean install-uv: @if [ -z "$$(which uv)" ]; then \ echo "Installing uv..."; \ @@ -43,7 +43,37 @@ install: venv @echo "Installing Python dependencies..." uv sync --all-groups --all-extras -# Database management +# Development environment setup +.PHONY: dev install-hooks +dev: install-uv venv install-addlicense install-hooks + @echo "Installing development tools..." + @command -v redocly >/dev/null || npm install @redocly/cli -g + @command -v openapi-generator-cli >/dev/null || npm install @openapitools/openapi-generator-cli -g + @command -v datamodel-codegen >/dev/null || uv tool install datamodel-code-generator + @echo "" + @echo "βœ… Development environment ready!" + @echo "πŸ“ Next steps:" + @echo " 1. Activate virtual environment: source .venv/bin/activate" + @echo " 2. Install dependencies: make install" + @echo " 3. Start databases: make compose-infra" + @echo " 4. Apply migrations: make migrate" + @echo " 5. Run services: make run-backend, make run-celery" + +install-hooks: + @echo "Installing git hooks..." + @./scripts/install-hooks.sh + +# Environment cleanup +clean: + @echo "Cleaning development environment..." + @rm -f db.sqlite3 + @$(MAKE) compose-down REMOVE_VOLUMES=1 + +################################################## +# Database & Infrastructure +################################################## + +# Database schema management .PHONY: makemigration migrate makemigration: @alembic -c aperag/alembic.ini revision --autogenerate @@ -51,25 +81,7 @@ makemigration: migrate: @alembic -c aperag/alembic.ini upgrade head -# Local services -.PHONY: run-backend run-frontend run-celery run-flower -run-backend: migrate - uvicorn aperag.app:app --host 0.0.0.0 --reload --log-config scripts/uvicorn-log-config.yaml - -run-celery: - celery -A config.celery worker -B -l INFO --pool=threads --concurrency=16 - -run-beat: - celery -A config.celery beat -l INFO - -run-flower: - celery -A config.celery flower --conf/flowerconfig.py - -run-frontend: - cp ./frontend/deploy/env.local.template ./frontend/.env - cd ./frontend && yarn dev - -# Docker Compose deployment +# Docker Compose infrastructure # Variables for compose command based on environment flags # Usage examples: @@ -121,35 +133,34 @@ compose-down: compose-logs: VERSION=v0.5.0-alpha.30 DOCRAY_VERSION=v0.1.1 docker-compose -f docker-compose.yml logs -f -# Environment cleanup -.PHONY: clean -clean: - @echo "Cleaning development environment..." - @rm -f db.sqlite3 - @echo "Use 'make compose-down REMOVE_VOLUMES=1' to clean Docker Compose services and data" - ################################################## -# Developers - Code Quality and Tools +# Development Services ################################################## -# Development tools installation -.PHONY: dev install-hooks -dev: install-uv venv install-addlicense install-hooks - @echo "Installing development tools..." - @command -v redocly >/dev/null || npm install @redocly/cli -g - @command -v openapi-generator-cli >/dev/null || npm install @openapitools/openapi-generator-cli -g - @command -v datamodel-codegen >/dev/null || uv tool install datamodel-code-generator - @echo "" - @echo "βœ… Development environment ready!" - @echo "πŸ“ Next steps:" - @echo " 1. Activate virtual environment: source .venv/bin/activate" - @echo " 2. Install dependencies: make install" - @echo " 3. Start databases: make compose-infra" - @echo " 4. Apply migrations: make migrate" - @echo " 5. Run services: make run-backend, make run-celery" +# Local development services +.PHONY: run-backend run-frontend run-celery run-flower run-beat +run-backend: migrate + uvicorn aperag.app:app --host 0.0.0.0 --reload --log-config scripts/uvicorn-log-config.yaml + +run-celery: + celery -A config.celery worker -B -l INFO --pool=threads --concurrency=16 + +run-beat: + celery -A config.celery beat -l INFO + +run-flower: + celery -A config.celery flower --conf/flowerconfig.py + +run-frontend: + cp ./frontend/deploy/env.local.template ./frontend/.env + cd ./frontend && yarn dev + +################################################## +# Code Quality & Testing +################################################## # Code quality checks -.PHONY: format lint static-check test unit-test e2e-test +.PHONY: format lint static-check format: uvx ruff check --fix ./aperag ./tests uvx ruff format ./aperag ./tests @@ -161,6 +172,8 @@ lint: static-check: uvx mypy ./aperag +# Testing suite +.PHONY: test unit-test e2e-test e2e-performance-test test: uv run pytest tests/ -v @@ -181,14 +194,18 @@ e2e-performance-test: --benchmark-save=benchmark-result-$$(date +%Y%m%d%H%M%S) \ tests/e2e_test/ -# Evaluation +# RAG evaluation .PHONY: evaluate evaluate: @echo "Running RAG evaluation..." @python -m aperag.evaluation.run -# Code generation -.PHONY: merge-openapi generate-models generate-frontend-sdk llm_provider +################################################## +# Code Generation & API +################################################## + +# OpenAPI and model generation +.PHONY: merge-openapi generate-models generate-frontend-sdk merge-openapi: @cd aperag && redocly bundle ./api/openapi.yaml > ./api/openapi.merged.yaml @@ -207,10 +224,12 @@ generate-models: merge-openapi generate-frontend-sdk: cd ./frontend && yarn api:build +# LLM configuration generation +.PHONY: llm_provider llm_provider: python ./models/generate_model_configs.py -# Version management and licensing +# Version management .PHONY: version version: @git rev-parse HEAD | cut -c1-7 > commit_id.txt @@ -218,56 +237,8 @@ version: @echo "GIT_COMMIT_ID = \"$$(cat commit_id.txt)\"" >> $(VERSION_FILE) @rm commit_id.txt -.PHONY: add-license -add-license: install-addlicense - ./downloads/addlicense -c "ApeCloud, Inc." -y 2025 -l apache \ - -ignore "aperag/readers/**" \ - -ignore "aperag/vectorstore/**" \ - aperag/**/*.py - -.PHONY: check-license -check-license: install-addlicense - ./downloads/addlicense -check \ - -c "ApeCloud, Inc." -y 2025 -l apache \ - -ignore "aperag/readers/**" \ - -ignore "aperag/vectorstore/**" \ - aperag/**/*.py - -.PHONY: install-addlicense -install-addlicense: - @mkdir -p ./downloads - @if [ ! -f ./downloads/addlicense ]; then \ - echo "Installing addlicense..."; \ - OS=$$(uname -s); \ - ARCH=$$(uname -m); \ - case $$OS in \ - Darwin) OS=macOS ;; \ - Linux) OS=Linux ;; \ - MINGW*|CYGWIN*) OS=Windows ;; \ - esac; \ - case $$ARCH in \ - x86_64) ARCH=x86_64 ;; \ - aarch64) ARCH=arm64 ;; \ - arm64) ARCH=arm64 ;; \ - esac; \ - echo "Detected platform: $$OS/$$ARCH"; \ - if [ "$$OS" = "Windows" ]; then \ - curl -L https://github.com/google/addlicense/releases/download/v1.1.1/addlicense_1.1.1_$${OS}_$${ARCH}.zip -o /tmp/addlicense.zip; \ - unzip -j /tmp/addlicense.zip -d ./downloads; \ - rm /tmp/addlicense.zip; \ - else \ - curl -L https://github.com/google/addlicense/releases/download/v1.1.1/addlicense_1.1.1_$${OS}_$${ARCH}.tar.gz | tar -xz -C ./downloads; \ - fi; \ - chmod +x ./downloads/addlicense; \ - echo "addlicense installed to ./downloads/addlicense"; \ - fi - -install-hooks: - @echo "Installing git hooks..." - @./scripts/install-hooks.sh - ################################################## -# Build and CI/CD +# Build & Deploy ################################################## # Docker builder setup @@ -284,7 +255,7 @@ clean-builder: docker buildx rm multi-platform; \ fi -# Image builds - multi-platform +# Production builds (multi-platform with registry push) .PHONY: build build-aperag build-aperag-frontend build: build-aperag build-aperag-frontend @@ -299,7 +270,7 @@ build-aperag-frontend: setup-builder --platform=$(BUILDX_PLATFORM) -f Dockerfile.prod --push \ -t $(REGISTRY)/$(APERAG_FRONTEND_IMG):$(VERSION) . -# Image builds - local platform +# Local builds (single platform for testing) .PHONY: build-local build-aperag-local build-aperag-frontend-local build-local: build-aperag-local build-aperag-frontend-local @@ -314,23 +285,8 @@ build-aperag-frontend-local: setup-builder --platform=$(LOCAL_PLATFORM) -f Dockerfile.prod --load \ -t $(APERAG_FRONTEND_IMG):$(VERSION) . -################################################## -# Utilities and Information -################################################## - -# Configuration info -.PHONY: info -info: - @echo "VERSION: $(VERSION)" - @echo "BUILDX_PLATFORM: $(BUILDX_PLATFORM)" - @echo "LOCAL_PLATFORM: $(LOCAL_PLATFORM)" - @echo "REGISTRY: $(REGISTRY)" - @echo "HOST ARCH: $(UNAME_M)" - - - - -.PHONY: load-images-to-minikube +# Kubernetes deployment helpers +.PHONY: load-images-to-minikube load-images-to-kind load-images-to-minikube: @echo "Start To Load Image To Minikube" docker save $(APERAG_IMAGE):$(VERSION) -o aperag.tar @@ -341,9 +297,64 @@ load-images-to-minikube: rm aperag-frontend.tar @echo "Already Load Image To Minikube" -.PHONY: load-images-to-kind load-images-to-kind: @echo "Start To Load Image To KinD" kind load docker-image $(APERAG_IMAGE):$(VERSION) --name $(KIND_CLUSTER_NAME) kind load docker-image $(APERAG_FRONTEND_IMG):$(VERSION) --name $(KIND_CLUSTER_NAME) @echo "Already Load Image To KinD" + +################################################## +# Utilities & Tools +################################################## + +# System information +.PHONY: info +info: + @echo "VERSION: $(VERSION)" + @echo "BUILDX_PLATFORM: $(BUILDX_PLATFORM)" + @echo "LOCAL_PLATFORM: $(LOCAL_PLATFORM)" + @echo "REGISTRY: $(REGISTRY)" + @echo "HOST ARCH: $(UNAME_M)" + +# License management +.PHONY: add-license check-license install-addlicense +add-license: install-addlicense + ./downloads/addlicense -c "ApeCloud, Inc." -y 2025 -l apache \ + -ignore "aperag/readers/**" \ + -ignore "aperag/vectorstore/**" \ + aperag/**/*.py + +check-license: install-addlicense + ./downloads/addlicense -check \ + -c "ApeCloud, Inc." -y 2025 -l apache \ + -ignore "aperag/readers/**" \ + -ignore "aperag/vectorstore/**" \ + aperag/**/*.py + +install-addlicense: + @mkdir -p ./downloads + @if [ ! -f ./downloads/addlicense ]; then \ + echo "Installing addlicense..."; \ + OS=$$(uname -s); \ + ARCH=$$(uname -m); \ + case $$OS in \ + Darwin) OS=macOS ;; \ + Linux) OS=Linux ;; \ + MINGW*|CYGWIN*) OS=Windows ;; \ + esac; \ + case $$ARCH in \ + x86_64) ARCH=x86_64 ;; \ + aarch64) ARCH=arm64 ;; \ + arm64) ARCH=arm64 ;; \ + esac; \ + echo "Detected platform: $$OS/$$ARCH"; \ + if [ "$$OS" = "Windows" ]; then \ + curl -L https://github.com/google/addlicense/releases/download/v1.1.1/addlicense_1.1.1_$${OS}_$${ARCH}.zip -o /tmp/addlicense.zip; \ + unzip -j /tmp/addlicense.zip -d ./downloads; \ + rm /tmp/addlicense.zip; \ + else \ + curl -L https://github.com/google/addlicense/releases/download/v1.1.1/addlicense_1.1.1_$${OS}_$${ARCH}.tar.gz | tar -xz -C ./downloads; \ + fi; \ + chmod +x ./downloads/addlicense; \ + echo "addlicense installed to ./downloads/addlicense"; \ + fi diff --git a/docs/development-guide.md b/docs/development-guide.md index e1daa62f3..8b1a224f9 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -35,13 +35,31 @@ Use Docker Compose to start the essential database services: ```bash # Start core databases: PostgreSQL, Redis, Qdrant, Elasticsearch make compose-infra - -# Optional: Use Neo4j instead of PostgreSQL for graph storage -# make compose-infra WITH_NEO4J=1 ``` This will start all required database services in the background. The default connection settings in your `.env` file are pre-configured to work with these services. +
+Advanced Database Options + +```bash +# Use Neo4j instead of PostgreSQL for graph storage +make compose-infra WITH_NEO4J=1 + +# Add advanced document parsing service (DocRay) +make compose-infra WITH_DOCRAY=1 + +# Combine multiple options +make compose-infra WITH_NEO4J=1 WITH_DOCRAY=1 + +# GPU-accelerated document parsing (requires ~6GB VRAM) +make compose-infra WITH_DOCRAY=1 WITH_GPU=1 +``` + +**Note**: DocRay provides enhanced document parsing for complex PDFs, tables, and formulas. CPU mode requires 4+ cores and 8GB+ RAM. + +
+ ### 4. Setup Development Environment Create Python virtual environment and setup development tools: @@ -146,90 +164,6 @@ docker volume ls | grep aperag Now you have ApeRAG running locally from source code, ready for development! - - -## Key `make` Commands for Development - -Based on the Makefile structure, here are the essential commands organized by workflow: - -### πŸš€ Environment Setup -```bash -make dev # Complete development setup (virtual env, tools, git hooks) -make install # Install all Python and Node.js dependencies -make venv # Create Python virtual environment only -make install-uv # Install uv package manager -``` - -### πŸ—„οΈ Database & Infrastructure -```bash -# Start databases only (recommended for development) -make compose-infra # PostgreSQL + Redis + Qdrant + Elasticsearch -make compose-infra WITH_NEO4J=1 # Add Neo4j for graph storage - -# Database schema management -make migrate # Apply pending migrations -make makemigration # Generate new migration files - -# Stop services -make compose-down # Stop services (keep data) -make compose-down REMOVE_VOLUMES=1 # Stop services and DELETE ALL DATA -``` - -### ⚑ Development Services -```bash -# Backend services (run in separate terminals) -make run-backend # FastAPI server with auto-reload (includes migration) -make run-celery # Celery worker + beat scheduler (--pool=threads --concurrency=16) -make run-beat # Celery beat scheduler only -make run-flower # Celery monitoring web UI - -# Frontend service -make run-frontend # React development server (copies env template) -``` - -### 🐳 Docker Compose (Full Stack) -```bash -# Complete application testing -make compose-up # Full stack (API + Frontend + Workers + Databases) -make compose-up WITH_NEO4J=1 # Add Neo4j -make compose-up WITH_DOCRAY=1 # Add advanced document parsing (CPU) -make compose-up WITH_DOCRAY=1 WITH_GPU=1 # Add GPU-accelerated document parsing -make compose-logs # View all service logs -``` - -### πŸ§ͺ Testing & Quality -```bash -# Testing -make test # All tests (unit + e2e) -make unit-test # Unit tests only -make e2e-test # End-to-end tests (--benchmark-disable) -make e2e-performance-test # Performance benchmarks - -# Code quality -make format # Auto-fix code style (Ruff for Python) -make lint # Check code style (no auto-fix) -make static-check # Type checking with MyPy -``` - -### πŸ”§ Code Generation & API -```bash -# When you modify APIs -make generate-models # Regenerate Pydantic models from OpenAPI spec -make generate-frontend-sdk # Regenerate frontend TypeScript API client - -# OpenAPI workflow -make merge-openapi # Bundle OpenAPI spec files - -# LLM configuration -make llm_provider # Generate LLM provider configurations -``` - -### πŸ“Š Evaluation & Analysis -```bash -make evaluate # Run RAG evaluation suite -make clean # Clean temporary files -``` - ## Common Development Tasks ### Q: How do I add or modify a REST API endpoint? @@ -271,89 +205,100 @@ make clean # Clean temporary files ### Q: How do I add a new feature with background processing? -**Full-stack feature development:** -1. Setup development environment: - ```bash - make dev # One-time setup - source .venv/bin/activate # Activate environment - make compose-infra # Start databases - ``` -2. Start development services (3 terminals): - ```bash - # Terminal 1: Backend API - make run-backend # Includes auto-migration - - # Terminal 2: Background tasks - make run-celery # Worker + scheduler (threads=16) - - # Terminal 3: Task monitoring (optional) - make run-flower # Web UI at http://localhost:5555 - ``` -3. Implement feature: +**Feature implementation workflow:** +1. Implement feature components: - Backend logic: `aperag/[module]/` - Async tasks: `aperag/tasks/` - Database models: `aperag/db/models.py` -4. Update API and generate code: +2. Update API and generate code: ```bash - make makemigration - make migrate - make generate-models - make generate-frontend-sdk + make makemigration # Generate migration files + make migrate # Apply database changes + make generate-models # Update Pydantic models + make generate-frontend-sdk # Update TypeScript client ``` -5. Quality assurance: +3. Quality assurance: ```bash make format && make lint && make test ``` -### Q: How do I test with different database backends? +### Q: How do I run unit tests and e2e tests? -**Testing with PostgreSQL (default):** +**Unit Tests (Fast, No External Dependencies):** ```bash -make compose-infra # PostgreSQL + Redis + Qdrant + Elasticsearch -make run-backend +# Run all unit tests +make unit-test + +# Run specific test file +uv run pytest tests/unit_test/test_model_service.py -v + +# Run specific test class or function +uv run pytest tests/unit_test/test_model_service.py::TestModelService::test_get_models -v + +# Run tests with coverage +uv run pytest tests/unit_test/ --cov=aperag --cov-report=html ``` -**Testing with Neo4j for graph storage:** +**E2E Tests (Require Running Services):** ```bash -make compose-infra WITH_NEO4J=1 # Neo4j replaces PostgreSQL for graphs -make run-backend -# Access Neo4j UI: http://localhost:7474 +# Setup: Start required services first +make compose-infra # Start databases +make run-backend # Start API server (separate terminal) + +# Run all e2e tests +make e2e-test + +# Run specific e2e test modules +uv run pytest tests/e2e_test/test_chat/ -v +uv run pytest tests/e2e_test/graphstorage/ -v + +# Run with detailed output and no capture +uv run pytest tests/e2e_test/test_specific.py -v -s + +# Performance benchmarks (with timing) +make e2e-performance-test ``` -**Testing complete stack with advanced parsing:** +**Complete Test Suite:** ```bash -# CPU-based document parsing -make compose-up WITH_DOCRAY=1 +# Run everything (unit + e2e) +make test -# GPU-accelerated parsing (requires ~6GB VRAM) -make compose-up WITH_DOCRAY=1 WITH_GPU=1 +# Test with different configurations +make compose-infra WITH_NEO4J=1 # Test with Neo4j instead of PostgreSQL +make test ``` ### Q: How do I debug failing tests? -**Test debugging workflow:** -1. Run specific test categories: +**Debugging workflow:** +1. Run failing test in isolation: ```bash - make unit-test # Fast unit tests only - make e2e-test # Integration tests (--benchmark-disable) + # Single test with full output + uv run pytest tests/unit_test/test_failing.py::test_specific_function -v -s - # Specific test file with verbose output - uv run pytest tests/unit_test/test_specific.py -v -s + # Stop on first failure + uv run pytest tests/unit_test/ -x --tb=short ``` -2. Debug with services running: +2. For e2e test failures, ensure services are running: ```bash - make compose-infra # Ensure databases are up - make run-backend # API server for integration tests + make compose-infra # Database services + make run-backend # API server + make run-celery # Background workers (if testing async tasks) ``` -3. Performance testing: +3. Use debugging tools: ```bash - make e2e-performance-test # Benchmarks with --benchmark-enable + # Run with pdb debugger + uv run pytest tests/unit_test/test_failing.py --pdb + + # Capture logs during test + uv run pytest tests/e2e_test/test_failing.py --log-cli-level=DEBUG ``` -4. Fix and verify: +4. Fix and retest: ```bash make format # Auto-fix style issues make lint # Check remaining issues - make test # Full test suite + uv run pytest tests/path/to/fixed_test.py -v # Verify fix ``` ### Q: How do I run RAG evaluation and analysis? From aadf8f66894c0b454a9a8561a895848380d85fe3 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 13:15:27 +0800 Subject: [PATCH 23/24] feat: update makefile & readme --- docs/development-guide.md | 84 +++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/development-guide.md b/docs/development-guide.md index 8b1a224f9..85184ad48 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -1,12 +1,12 @@ -# Development Guide +# πŸ› οΈ Development Guide This guide focuses on setting up a development environment and the development workflow for ApeRAG. This is designed for developers looking to contribute to ApeRAG or run it locally for development purposes. -## Development Environment Setup +## πŸš€ Development Environment Setup Follow these steps to set up ApeRAG from source code for development: -### 1. Clone the Repository and Setup Environment +### 1. πŸ“‚ Clone the Repository and Setup Environment First, get the source code and configure environment variables: @@ -19,7 +19,7 @@ cp frontend/deploy/env.local.template frontend/.env Edit the `.env` file to configure your AI service settings if needed. The default settings work with the local database services started in the next step. -### 2. System Prerequisites +### 2. πŸ“‹ System Prerequisites Before you begin, ensure your system has: @@ -28,7 +28,7 @@ Before you begin, ensure your system has: **Note**: Python 3.11 is required but will be automatically managed by `uv` in the next steps. -### 3. Start Database Services +### 3. πŸ—„οΈ Start Database Services Use Docker Compose to start the essential database services: @@ -60,7 +60,7 @@ make compose-infra WITH_DOCRAY=1 WITH_GPU=1 -### 4. Setup Development Environment +### 4. βš™οΈ Setup Development Environment Create Python virtual environment and setup development tools: @@ -82,7 +82,7 @@ source .venv/bin/activate You'll know it's active when you see `(.venv)` in your terminal prompt. -### 5. Install Dependencies +### 5. πŸ“¦ Install Dependencies Install all backend and frontend dependencies: @@ -94,7 +94,7 @@ This command will: * Install all Python backend dependencies from `pyproject.toml` into the virtual environment * Install frontend Node.js dependencies using `yarn` -### 6. Apply Database Migrations +### 6. πŸ”„ Apply Database Migrations Setup the database schema: @@ -102,7 +102,7 @@ Setup the database schema: make migrate ``` -### 7. Start Development Services +### 7. ▢️ Start Development Services Now you can start the development services. Open separate terminal windows/tabs for each service: @@ -124,14 +124,14 @@ make run-frontend ``` This starts the frontend development server at `http://localhost:3000` with hot reload. -### 8. Access ApeRAG +### 8. 🌐 Access ApeRAG With the services running, you can access: * **Frontend UI**: http://localhost:3000 (if started) * **Backend API**: http://localhost:8000 * **API Documentation**: http://localhost:8000/docs -### 9. Stopping Services +### 9. ⏹️ Stopping Services To stop the development environment: @@ -151,7 +151,7 @@ make compose-down REMOVE_VOLUMES=1 **Data Management:** - `make compose-down` - Stops services but preserves all data (PostgreSQL, Redis, Qdrant, etc.) -- `make compose-down REMOVE_VOLUMES=1` - Stops services and **permanently deletes all data** +- `make compose-down REMOVE_VOLUMES=1` - Stops services and **⚠️ permanently deletes all data** - You can run `make compose-down REMOVE_VOLUMES=1` even after already running `make compose-down` **Verify Data Removal:** @@ -162,11 +162,11 @@ docker volume ls | grep aperag # Should return no results after REMOVE_VOLUMES=1 ``` -Now you have ApeRAG running locally from source code, ready for development! +Now you have ApeRAG running locally from source code, ready for development! πŸŽ‰ -## Common Development Tasks +## ❓ Common Development Tasks -### Q: How do I add or modify a REST API endpoint? +### Q: πŸ”§ How do I add or modify a REST API endpoint? **Complete workflow:** 1. Edit OpenAPI specification: `aperag/api/paths/[endpoint-name].yaml` @@ -182,10 +182,10 @@ Now you have ApeRAG running locally from source code, ready for development! 5. Test the API: ```bash make test - # Check live docs: http://localhost:8000/docs + # βœ… Check live docs: http://localhost:8000/docs ``` -### Q: How do I modify database models/schema? +### Q: πŸ—ƒοΈ How do I modify database models/schema? **Database migration workflow:** 1. Edit SQLModel classes in `aperag/db/models.py` @@ -200,10 +200,10 @@ Now you have ApeRAG running locally from source code, ready for development! 4. Update related code (repositories in `aperag/db/repositories/`, services in `aperag/service/`) 5. Verify changes: ```bash - make test + make test # βœ… Ensure everything works ``` -### Q: How do I add a new feature with background processing? +### Q: ⚑ How do I add a new feature with background processing? **Feature implementation workflow:** 1. Implement feature components: @@ -222,7 +222,7 @@ Now you have ApeRAG running locally from source code, ready for development! make format && make lint && make test ``` -### Q: How do I run unit tests and e2e tests? +### Q: πŸ§ͺ How do I run unit tests and e2e tests? **Unit Tests (Fast, No External Dependencies):** ```bash @@ -241,9 +241,9 @@ uv run pytest tests/unit_test/ --cov=aperag --cov-report=html **E2E Tests (Require Running Services):** ```bash -# Setup: Start required services first -make compose-infra # Start databases -make run-backend # Start API server (separate terminal) +# Setup: Start required services first +make compose-infra # πŸ—„οΈ Start databases +make run-backend # πŸš€ Start API server (separate terminal) # Run all e2e tests make e2e-test @@ -269,7 +269,7 @@ make compose-infra WITH_NEO4J=1 # Test with Neo4j instead of PostgreSQL make test ``` -### Q: How do I debug failing tests? +### Q: πŸ› How do I debug failing tests? **Debugging workflow:** 1. Run failing test in isolation: @@ -301,7 +301,7 @@ make test uv run pytest tests/path/to/fixed_test.py -v # Verify fix ``` -### Q: How do I run RAG evaluation and analysis? +### Q: πŸ“Š How do I run RAG evaluation and analysis? **Evaluation workflow:** ```bash @@ -311,12 +311,12 @@ make run-backend make run-celery # Run comprehensive RAG evaluation -make evaluate # Runs aperag.evaluation.run module +make evaluate # πŸ“Š Runs aperag.evaluation.run module -# Check evaluation reports in tests/report/ +# πŸ“ˆ Check evaluation reports in tests/report/ ``` -### Q: How do I update dependencies safely? +### Q: πŸ“¦ How do I update dependencies safely? **Python dependencies:** 1. Edit `pyproject.toml` (add/update packages) @@ -335,7 +335,7 @@ make evaluate # Runs aperag.evaluation.run module make generate-frontend-sdk # Ensure API client still works ``` -### Q: How do I prepare code for production deployment? +### Q: πŸš€ How do I prepare code for production deployment? **Pre-deployment checklist:** 1. Code quality validation: @@ -365,30 +365,30 @@ make evaluate # Runs aperag.evaluation.run module make compose-down ``` -### Q: How do I completely reset my development environment? +### Q: πŸ”„ How do I completely reset my development environment? **Nuclear reset (destroys all data):** ```bash -make compose-down REMOVE_VOLUMES=1 # Stop services + delete ALL data -make clean # Clean temporary files +make compose-down REMOVE_VOLUMES=1 # ⚠️ Stop services + delete ALL data +make clean # 🧹 Clean temporary files # Restart fresh -make compose-infra # Fresh databases -make migrate # Apply all migrations -make run-backend # Start API server -make run-celery # Start background workers +make compose-infra # πŸ—„οΈ Fresh databases +make migrate # πŸ”„ Apply all migrations +make run-backend # πŸš€ Start API server +make run-celery # ⚑ Start background workers ``` **Soft reset (preserve data):** ```bash -make compose-down # Stop services, keep data -make compose-infra # Restart databases -make migrate # Apply any new migrations +make compose-down # ⏹️ Stop services, keep data +make compose-infra # πŸ—„οΈ Restart databases +make migrate # πŸ”„ Apply any new migrations ``` **Reset just Python environment:** ```bash -rm -rf .venv/ # Remove virtual environment -make dev # Recreate everything -source .venv/bin/activate # Reactivate +rm -rf .venv/ # πŸ—‘οΈ Remove virtual environment +make dev # βš™οΈ Recreate everything +source .venv/bin/activate # βœ… Reactivate ``` \ No newline at end of file From bb2cbb984cecf4ce26d7b86c7555c90ed5f47a92 Mon Sep 17 00:00:00 2001 From: earayu Date: Thu, 3 Jul 2025 13:36:48 +0800 Subject: [PATCH 24/24] feat: update makefile & readme --- Makefile | 4 +- aperag/migration/sql/extensions_init.sql | 15 +++ .../versions/20250630123213-495840dd6ff9.py | 107 ------------------ .../versions/20250630151946-97d64d3fe985.py | 46 -------- .../versions/20250703133046-db9c88848f52.py | 33 ++++++ ...c08f.py => 20250703133208-0b274fcc91e2.py} | 96 +++++++++------- ...c84a.py => 20250703133304-b598e645b2ba.py} | 17 +-- 7 files changed, 117 insertions(+), 201 deletions(-) create mode 100644 aperag/migration/sql/extensions_init.sql delete mode 100644 aperag/migration/versions/20250630123213-495840dd6ff9.py delete mode 100644 aperag/migration/versions/20250630151946-97d64d3fe985.py create mode 100644 aperag/migration/versions/20250703133046-db9c88848f52.py rename aperag/migration/versions/{20250624132425-850b2c5dc08f.py => 20250703133208-0b274fcc91e2.py} (83%) rename aperag/migration/versions/{20250624132601-66b96592c84a.py => 20250703133304-b598e645b2ba.py} (65%) diff --git a/Makefile b/Makefile index 9ad454939..3abbae657 100644 --- a/Makefile +++ b/Makefile @@ -76,10 +76,10 @@ clean: # Database schema management .PHONY: makemigration migrate makemigration: - @alembic -c aperag/alembic.ini revision --autogenerate + @uv run alembic -c aperag/alembic.ini revision --autogenerate migrate: - @alembic -c aperag/alembic.ini upgrade head + @uv run alembic -c aperag/alembic.ini upgrade head # Docker Compose infrastructure diff --git a/aperag/migration/sql/extensions_init.sql b/aperag/migration/sql/extensions_init.sql new file mode 100644 index 000000000..199fd123e --- /dev/null +++ b/aperag/migration/sql/extensions_init.sql @@ -0,0 +1,15 @@ +-- PostgreSQL Extensions Initialization +-- This script creates necessary extensions for ApeRAG +-- Extensions must be created before schema tables that use them + +-- Create pgvector extension for vector operations +-- Used by LightRAG tables: lightrag_doc_chunks, lightrag_vdb_entity, lightrag_vdb_relation +CREATE EXTENSION IF NOT EXISTS vector; + +-- Optional: Create other useful extensions +-- Uncomment as needed based on project requirements + +-- CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -- For UUID generation +-- CREATE EXTENSION IF NOT EXISTS pg_trgm; -- For trigram text search +-- CREATE EXTENSION IF NOT EXISTS btree_gin; -- For GIN indexes on btree data +-- CREATE EXTENSION IF NOT EXISTS btree_gist; -- For GIST indexes on btree data \ No newline at end of file diff --git a/aperag/migration/versions/20250630123213-495840dd6ff9.py b/aperag/migration/versions/20250630123213-495840dd6ff9.py deleted file mode 100644 index 9110c266b..000000000 --- a/aperag/migration/versions/20250630123213-495840dd6ff9.py +++ /dev/null @@ -1,107 +0,0 @@ -"""empty message - -Revision ID: 495840dd6ff9 -Revises: 66b96592c84a -Create Date: 2025-06-30 12:32:13.869581 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = '495840dd6ff9' -down_revision: Union[str, None] = '66b96592c84a' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('lightrag_graph_edges', - sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), - sa.Column('source_entity_id', sa.String(length=255), nullable=False), - sa.Column('target_entity_id', sa.String(length=255), nullable=False), - sa.Column('weight', sa.Numeric(precision=10, scale=6), nullable=False), - sa.Column('keywords', sa.Text(), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('source_id', sa.Text(), nullable=True), - sa.Column('file_path', sa.Text(), nullable=True), - sa.Column('workspace', sa.String(length=255), nullable=False), - sa.Column('createtime', sa.DateTime(timezone=True), nullable=False), - sa.Column('updatetime', sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('workspace', 'source_entity_id', 'target_entity_id', name='uq_lightrag_graph_edges_workspace_source_target') - ) - op.create_index('idx_lightrag_edges_weight', 'lightrag_graph_edges', ['workspace', 'weight'], unique=False) - op.create_index('idx_lightrag_edges_workspace_source', 'lightrag_graph_edges', ['workspace', 'source_entity_id'], unique=False) - op.create_index('idx_lightrag_edges_workspace_target', 'lightrag_graph_edges', ['workspace', 'target_entity_id'], unique=False) - op.create_table('lightrag_graph_nodes', - sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), - sa.Column('entity_id', sa.String(length=256), nullable=False), - sa.Column('entity_name', sa.String(length=255), nullable=True), - sa.Column('entity_type', sa.String(length=255), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('source_id', sa.Text(), nullable=True), - sa.Column('file_path', sa.Text(), nullable=True), - sa.Column('workspace', sa.String(length=255), nullable=False), - sa.Column('createtime', sa.DateTime(timezone=True), nullable=False), - sa.Column('updatetime', sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('workspace', 'entity_id', name='uq_lightrag_graph_nodes_workspace_entity') - ) - op.create_index('idx_lightrag_nodes_entity_name', 'lightrag_graph_nodes', ['workspace', 'entity_name'], unique=False) - op.create_index('idx_lightrag_nodes_entity_type', 'lightrag_graph_nodes', ['workspace', 'entity_type'], unique=False) - op.drop_table('lightrag_doc_status') - op.drop_table('lightrag_doc_full') - op.drop_table('lightrag_llm_cache') - # ### end Alembic commands ### - - -def downgrade() -> None: - """Downgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('lightrag_llm_cache', - sa.Column('workspace', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('id', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('mode', sa.VARCHAR(length=32), autoincrement=False, nullable=False), - sa.Column('original_prompt', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('return_value', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('create_time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), - sa.Column('update_time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint('workspace', 'id', 'mode', name=op.f('lightrag_llm_cache_pkey')) - ) - op.create_table('lightrag_doc_full', - sa.Column('id', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('workspace', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('doc_name', sa.VARCHAR(length=1024), autoincrement=False, nullable=True), - sa.Column('content', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('meta', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), - sa.Column('create_time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), - sa.Column('update_time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), - sa.PrimaryKeyConstraint('id', 'workspace', name=op.f('lightrag_doc_full_pkey')) - ) - op.create_table('lightrag_doc_status', - sa.Column('workspace', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('id', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('content', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('content_summary', sa.VARCHAR(length=255), autoincrement=False, nullable=True), - sa.Column('content_length', sa.INTEGER(), autoincrement=False, nullable=True), - sa.Column('chunks_count', sa.INTEGER(), autoincrement=False, nullable=True), - sa.Column('status', postgresql.ENUM('pending', 'processing', 'processed', 'failed', name='lightragdocstatus'), autoincrement=False, nullable=True), - sa.Column('file_path', sa.VARCHAR(length=512), autoincrement=False, nullable=True), - sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), - sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False), - sa.PrimaryKeyConstraint('workspace', 'id', name=op.f('uq_lightrag_doc_status_workspace_id')) - ) - op.drop_index('idx_lightrag_nodes_entity_type', table_name='lightrag_graph_nodes') - op.drop_index('idx_lightrag_nodes_entity_name', table_name='lightrag_graph_nodes') - op.drop_table('lightrag_graph_nodes') - op.drop_index('idx_lightrag_edges_workspace_target', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_workspace_source', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_weight', table_name='lightrag_graph_edges') - op.drop_table('lightrag_graph_edges') - # ### end Alembic commands ### diff --git a/aperag/migration/versions/20250630151946-97d64d3fe985.py b/aperag/migration/versions/20250630151946-97d64d3fe985.py deleted file mode 100644 index 18589926c..000000000 --- a/aperag/migration/versions/20250630151946-97d64d3fe985.py +++ /dev/null @@ -1,46 +0,0 @@ -"""empty message - -Revision ID: 97d64d3fe985 -Revises: 495840dd6ff9 -Create Date: 2025-06-30 15:19:46.211896 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = '97d64d3fe985' -down_revision: Union[str, None] = '495840dd6ff9' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Upgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.create_index('idx_lightrag_edges_degree_calc', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id', 'weight'], unique=False) - op.create_index('idx_lightrag_edges_metadata', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id', 'weight', 'keywords'], unique=False) - op.create_index('idx_lightrag_edges_workspace_createtime', 'lightrag_graph_edges', ['workspace', 'createtime'], unique=False) - op.create_index('idx_lightrag_edges_workspace_source_target', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id'], unique=False) - op.create_index('idx_lightrag_edges_workspace_target_source', 'lightrag_graph_edges', ['workspace', 'target_entity_id', 'source_entity_id'], unique=False) - op.create_index('idx_lightrag_nodes_entity_type_createtime', 'lightrag_graph_nodes', ['workspace', 'entity_type', 'createtime'], unique=False) - op.create_index('idx_lightrag_nodes_workspace_createtime', 'lightrag_graph_nodes', ['workspace', 'createtime'], unique=False) - op.create_index('idx_lightrag_nodes_workspace_type_id', 'lightrag_graph_nodes', ['workspace', 'entity_type', 'entity_id'], unique=False) - # ### end Alembic commands ### - - -def downgrade() -> None: - """Downgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index('idx_lightrag_nodes_workspace_type_id', table_name='lightrag_graph_nodes') - op.drop_index('idx_lightrag_nodes_workspace_createtime', table_name='lightrag_graph_nodes') - op.drop_index('idx_lightrag_nodes_entity_type_createtime', table_name='lightrag_graph_nodes') - op.drop_index('idx_lightrag_edges_workspace_target_source', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_workspace_source_target', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_workspace_createtime', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_metadata', table_name='lightrag_graph_edges') - op.drop_index('idx_lightrag_edges_degree_calc', table_name='lightrag_graph_edges') - # ### end Alembic commands ### diff --git a/aperag/migration/versions/20250703133046-db9c88848f52.py b/aperag/migration/versions/20250703133046-db9c88848f52.py new file mode 100644 index 000000000..613ce95fa --- /dev/null +++ b/aperag/migration/versions/20250703133046-db9c88848f52.py @@ -0,0 +1,33 @@ +"""Create PostgreSQL extensions (pgvector) + +Revision ID: db9c88848f52 +Revises: +Create Date: 2025-07-03 13:30:46.635272 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +from aperag.migration.utils import execute_sql_file + + +# revision identifiers, used by Alembic. +revision: str = 'db9c88848f52' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create PostgreSQL extensions.""" + # Execute extensions initialization SQL + execute_sql_file("extensions_init.sql") + + +def downgrade() -> None: + """Drop PostgreSQL extensions.""" + # Note: Dropping extensions should be done carefully in production + # as it may affect existing data and other databases + op.execute(sa.text("DROP EXTENSION IF EXISTS vector CASCADE")) diff --git a/aperag/migration/versions/20250624132425-850b2c5dc08f.py b/aperag/migration/versions/20250703133208-0b274fcc91e2.py similarity index 83% rename from aperag/migration/versions/20250624132425-850b2c5dc08f.py rename to aperag/migration/versions/20250703133208-0b274fcc91e2.py index 14861723c..16686c9d0 100644 --- a/aperag/migration/versions/20250624132425-850b2c5dc08f.py +++ b/aperag/migration/versions/20250703133208-0b274fcc91e2.py @@ -1,8 +1,8 @@ """empty message -Revision ID: 850b2c5dc08f -Revises: -Create Date: 2025-06-24 13:24:25.714734 +Revision ID: 0b274fcc91e2 +Revises: db9c88848f52 +Create Date: 2025-07-03 13:32:08.830672 """ from typing import Sequence, Union @@ -12,8 +12,8 @@ from pgvector.sqlalchemy import Vector # revision identifiers, used by Alembic. -revision: str = '850b2c5dc08f' -down_revision: Union[str, None] = None +revision: str = '0b274fcc91e2' +down_revision: Union[str, None] = 'db9c88848f52' branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -197,40 +197,48 @@ def upgrade() -> None: sa.Column('update_time', sa.DateTime(timezone=True), nullable=False), sa.PrimaryKeyConstraint('id', 'workspace') ) - op.create_table('lightrag_doc_full', - sa.Column('id', sa.String(length=255), nullable=False), - sa.Column('workspace', sa.String(length=255), nullable=False), - sa.Column('doc_name', sa.String(length=1024), nullable=True), - sa.Column('content', sa.Text(), nullable=True), - sa.Column('meta', sa.JSON(), nullable=True), - sa.Column('create_time', sa.DateTime(timezone=True), nullable=False), - sa.Column('update_time', sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint('id', 'workspace') - ) - op.create_table('lightrag_doc_status', + op.create_table('lightrag_graph_edges', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('source_entity_id', sa.String(length=255), nullable=False), + sa.Column('target_entity_id', sa.String(length=255), nullable=False), + sa.Column('weight', sa.Numeric(precision=10, scale=6), nullable=False), + sa.Column('keywords', sa.Text(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('source_id', sa.Text(), nullable=True), + sa.Column('file_path', sa.Text(), nullable=True), sa.Column('workspace', sa.String(length=255), nullable=False), - sa.Column('id', sa.String(length=255), nullable=False), - sa.Column('content', sa.Text(), nullable=True), - sa.Column('content_summary', sa.String(length=255), nullable=True), - sa.Column('content_length', sa.Integer(), nullable=True), - sa.Column('chunks_count', sa.Integer(), nullable=True), - sa.Column('status', sa.Enum('pending', 'processing', 'processed', 'failed', name='lightragdocstatus'), nullable=True), - sa.Column('file_path', sa.String(length=512), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint('workspace', 'id'), - sa.UniqueConstraint('workspace', 'id', name='uq_lightrag_doc_status_workspace_id') + sa.Column('createtime', sa.DateTime(timezone=True), nullable=False), + sa.Column('updatetime', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('workspace', 'source_entity_id', 'target_entity_id', name='uq_lightrag_graph_edges_workspace_source_target') ) - op.create_table('lightrag_llm_cache', + op.create_index('idx_lightrag_edges_degree_calc', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id', 'weight'], unique=False) + op.create_index('idx_lightrag_edges_metadata', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id', 'weight', 'keywords'], unique=False) + op.create_index('idx_lightrag_edges_weight', 'lightrag_graph_edges', ['workspace', 'weight'], unique=False) + op.create_index('idx_lightrag_edges_workspace_createtime', 'lightrag_graph_edges', ['workspace', 'createtime'], unique=False) + op.create_index('idx_lightrag_edges_workspace_source', 'lightrag_graph_edges', ['workspace', 'source_entity_id'], unique=False) + op.create_index('idx_lightrag_edges_workspace_source_target', 'lightrag_graph_edges', ['workspace', 'source_entity_id', 'target_entity_id'], unique=False) + op.create_index('idx_lightrag_edges_workspace_target', 'lightrag_graph_edges', ['workspace', 'target_entity_id'], unique=False) + op.create_index('idx_lightrag_edges_workspace_target_source', 'lightrag_graph_edges', ['workspace', 'target_entity_id', 'source_entity_id'], unique=False) + op.create_table('lightrag_graph_nodes', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('entity_id', sa.String(length=256), nullable=False), + sa.Column('entity_name', sa.String(length=255), nullable=True), + sa.Column('entity_type', sa.String(length=255), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('source_id', sa.Text(), nullable=True), + sa.Column('file_path', sa.Text(), nullable=True), sa.Column('workspace', sa.String(length=255), nullable=False), - sa.Column('id', sa.String(length=255), nullable=False), - sa.Column('mode', sa.String(length=32), nullable=False), - sa.Column('original_prompt', sa.Text(), nullable=True), - sa.Column('return_value', sa.Text(), nullable=True), - sa.Column('create_time', sa.DateTime(timezone=True), nullable=False), - sa.Column('update_time', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('workspace', 'id', 'mode') + sa.Column('createtime', sa.DateTime(timezone=True), nullable=False), + sa.Column('updatetime', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('workspace', 'entity_id', name='uq_lightrag_graph_nodes_workspace_entity') ) + op.create_index('idx_lightrag_nodes_entity_name', 'lightrag_graph_nodes', ['workspace', 'entity_name'], unique=False) + op.create_index('idx_lightrag_nodes_entity_type', 'lightrag_graph_nodes', ['workspace', 'entity_type'], unique=False) + op.create_index('idx_lightrag_nodes_entity_type_createtime', 'lightrag_graph_nodes', ['workspace', 'entity_type', 'createtime'], unique=False) + op.create_index('idx_lightrag_nodes_workspace_createtime', 'lightrag_graph_nodes', ['workspace', 'createtime'], unique=False) + op.create_index('idx_lightrag_nodes_workspace_type_id', 'lightrag_graph_nodes', ['workspace', 'entity_type', 'entity_id'], unique=False) op.create_table('lightrag_vdb_entity', sa.Column('id', sa.String(length=255), nullable=False), sa.Column('workspace', sa.String(length=255), nullable=False), @@ -391,9 +399,21 @@ def downgrade() -> None: op.drop_table('llm_provider') op.drop_table('lightrag_vdb_relation') op.drop_table('lightrag_vdb_entity') - op.drop_table('lightrag_llm_cache') - op.drop_table('lightrag_doc_status') - op.drop_table('lightrag_doc_full') + op.drop_index('idx_lightrag_nodes_workspace_type_id', table_name='lightrag_graph_nodes') + op.drop_index('idx_lightrag_nodes_workspace_createtime', table_name='lightrag_graph_nodes') + op.drop_index('idx_lightrag_nodes_entity_type_createtime', table_name='lightrag_graph_nodes') + op.drop_index('idx_lightrag_nodes_entity_type', table_name='lightrag_graph_nodes') + op.drop_index('idx_lightrag_nodes_entity_name', table_name='lightrag_graph_nodes') + op.drop_table('lightrag_graph_nodes') + op.drop_index('idx_lightrag_edges_workspace_target_source', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_workspace_target', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_workspace_source_target', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_workspace_source', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_workspace_createtime', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_weight', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_metadata', table_name='lightrag_graph_edges') + op.drop_index('idx_lightrag_edges_degree_calc', table_name='lightrag_graph_edges') + op.drop_table('lightrag_graph_edges') op.drop_table('lightrag_doc_chunks') op.drop_table('invitation') op.drop_index(op.f('ix_document_index_status'), table_name='document_index') diff --git a/aperag/migration/versions/20250624132601-66b96592c84a.py b/aperag/migration/versions/20250703133304-b598e645b2ba.py similarity index 65% rename from aperag/migration/versions/20250624132601-66b96592c84a.py rename to aperag/migration/versions/20250703133304-b598e645b2ba.py index 0c6c1c551..ad004be7b 100644 --- a/aperag/migration/versions/20250624132601-66b96592c84a.py +++ b/aperag/migration/versions/20250703133304-b598e645b2ba.py @@ -1,8 +1,8 @@ -"""empty message +"""Initialize LLM model configurations -Revision ID: 66b96592c84a -Revises: 850b2c5dc08f -Create Date: 2025-06-24 13:26:01.031627 +Revision ID: b598e645b2ba +Revises: 0b274fcc91e2 +Create Date: 2025-07-03 13:33:04.315167 """ from typing import Sequence, Union @@ -14,19 +14,20 @@ # revision identifiers, used by Alembic. -revision: str = '66b96592c84a' -down_revision: Union[str, None] = '850b2c5dc08f' +revision: str = 'b598e645b2ba' +down_revision: Union[str, None] = '0b274fcc91e2' branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None + def upgrade() -> None: - """Initialize model configurations data.""" + """Initialize LLM model configurations data.""" # Execute model configurations initialization SQL execute_sql_file("model_configs_init.sql") def downgrade() -> None: - """Remove model configurations data.""" + """Remove LLM model configurations data.""" # Clean up model configurations data op.execute(sa.text("DELETE FROM llm_provider_models")) op.execute(sa.text("DELETE FROM llm_provider"))