From 96941b443ead75c66f631e5cbf0f98eba3a6ead3 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Fri, 20 Jun 2025 17:19:57 +0800 Subject: [PATCH 01/19] feat: support auditing --- aperag/api/components/schemas/audit.yaml | 89 ++++ aperag/api/openapi.yaml | 6 + aperag/api/paths/audit.yaml | 111 +++++ aperag/app.py | 6 + aperag/db/models.py | 59 +++ aperag/middleware/audit_middleware.py | 188 +++++++ .../20250617113449-audit_log_table.py | 81 +++ aperag/schema/view_models.py | 76 ++- aperag/service/audit_service.py | 274 +++++++++++ aperag/views/api_key.py | 8 +- aperag/views/audit.py | 167 +++++++ aperag/views/auth.py | 18 +- aperag/views/chat_completion.py | 2 +- aperag/views/config.py | 2 +- aperag/views/flow.py | 4 +- aperag/views/llm.py | 4 +- aperag/views/main.py | 78 +-- frontend/src/api/api.ts | 1 + frontend/src/api/apis/audit-api.ts | 406 +++++++++++++++ frontend/src/api/apis/default-api.ts | 20 +- frontend/src/api/models/audit-log-list.ts | 33 ++ frontend/src/api/models/audit-log.ts | 155 ++++++ frontend/src/api/models/index.ts | 2 + frontend/src/api/openapi.merged.yaml | 233 ++++++++- frontend/src/locales/zh-CN.ts | 154 ++++++ frontend/src/pages/settings/_navbar.tsx | 4 + frontend/src/pages/settings/auditLogs.tsx | 462 ++++++++++++++++++ 27 files changed, 2550 insertions(+), 93 deletions(-) create mode 100644 aperag/api/components/schemas/audit.yaml create mode 100644 aperag/api/paths/audit.yaml create mode 100644 aperag/middleware/audit_middleware.py create mode 100644 aperag/migration/versions/20250617113449-audit_log_table.py create mode 100644 aperag/service/audit_service.py create mode 100644 aperag/views/audit.py create mode 100644 frontend/src/api/apis/audit-api.ts create mode 100644 frontend/src/api/models/audit-log-list.ts create mode 100644 frontend/src/api/models/audit-log.ts create mode 100644 frontend/src/pages/settings/auditLogs.tsx diff --git a/aperag/api/components/schemas/audit.yaml b/aperag/api/components/schemas/audit.yaml new file mode 100644 index 000000000..51d082fb9 --- /dev/null +++ b/aperag/api/components/schemas/audit.yaml @@ -0,0 +1,89 @@ +auditLog: + type: object + description: Audit log entry + properties: + id: + type: string + description: Audit log ID + user_id: + type: string + nullable: true + description: User ID who performed the action + username: + type: string + nullable: true + description: Username for display + resource_type: + type: string + enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config] + nullable: true + description: Type of resource + resource_id: + type: string + nullable: true + description: ID of the resource (extracted at query time) + api_name: + type: string + description: API operation name + http_method: + type: string + description: HTTP method (POST, PUT, DELETE) + path: + type: string + description: API path + status_code: + type: integer + nullable: true + description: HTTP status code + start_time: + type: integer + format: int64 + description: Request start time (milliseconds since epoch) + end_time: + type: integer + format: int64 + nullable: true + description: Request end time (milliseconds since epoch) + duration_ms: + type: integer + nullable: true + description: Request duration in milliseconds (calculated) + request_data: + type: string + nullable: true + description: Request data (JSON string) + response_data: + type: string + nullable: true + description: Response data (JSON string) + error_message: + type: string + nullable: true + description: Error message if failed + ip_address: + type: string + nullable: true + description: Client IP address + user_agent: + type: string + nullable: true + description: User agent string + request_id: + type: string + description: Request ID for tracking + created: + type: string + format: date-time + description: Created timestamp + +auditLogList: + type: object + description: List of audit logs + properties: + items: + type: array + description: Audit log entries + items: + $ref: '#/auditLog' + + \ No newline at end of file diff --git a/aperag/api/openapi.yaml b/aperag/api/openapi.yaml index 453930cb7..43f2c119d 100644 --- a/aperag/api/openapi.yaml +++ b/aperag/api/openapi.yaml @@ -88,6 +88,12 @@ paths: /prompt-templates: $ref: './paths/prompt_templates.yaml#/promptTemplates' + # audit + /audit-logs: + $ref: './paths/audit.yaml#/audit_logs' + /audit-logs/{audit_id}: + $ref: './paths/audit.yaml#/audit_log_detail' + # users /invite: $ref: './paths/auth.yaml#/invite' diff --git a/aperag/api/paths/audit.yaml b/aperag/api/paths/audit.yaml new file mode 100644 index 000000000..44b808e7d --- /dev/null +++ b/aperag/api/paths/audit.yaml @@ -0,0 +1,111 @@ +audit_logs: + get: + tags: + - audit + summary: List audit logs + description: List audit logs with filtering options + operationId: list_audit_logs + parameters: + - name: user_id + in: query + required: false + schema: + type: string + description: Filter by user ID + - name: username + in: query + required: false + schema: + type: string + description: Filter by username + - name: resource_type + in: query + required: false + schema: + type: string + enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config] + description: Filter by resource type + - name: resource_id + in: query + required: false + schema: + type: string + description: Filter by resource ID + - name: api_name + in: query + required: false + schema: + type: string + description: Filter by API name + - name: http_method + in: query + required: false + schema: + type: string + enum: [POST, PUT, DELETE] + description: Filter by HTTP method + - name: status_code + in: query + required: false + schema: + type: integer + description: Filter by status code + - name: start_date + in: query + required: false + schema: + type: string + format: date-time + description: Filter by start date + - name: end_date + in: query + required: false + schema: + type: string + format: date-time + description: Filter by end date + - name: limit + in: query + required: false + schema: + type: integer + maximum: 5000 + default: 1000 + description: Maximum number of records + responses: + "200": + description: Audit logs retrieved successfully + content: + application/json: + schema: + $ref: "../components/schemas/audit.yaml#/auditLogList" + "403": + description: Admin access required + +audit_log_detail: + get: + tags: + - audit + summary: Get audit log detail + description: Get a specific audit log by ID + operationId: get_audit_log + parameters: + - name: audit_id + in: path + required: true + schema: + type: string + description: Audit log ID + responses: + "200": + description: Audit log retrieved successfully + content: + application/json: + schema: + $ref: "../components/schemas/audit.yaml#/auditLog" + "403": + description: Admin access required + "404": + description: Audit log not found + + \ No newline at end of file diff --git a/aperag/app.py b/aperag/app.py index 572d14bf7..05d5bec45 100644 --- a/aperag/app.py +++ b/aperag/app.py @@ -16,7 +16,9 @@ from aperag.exception_handlers import register_exception_handlers from aperag.llm.litellm_track import register_opik_llm_track +from aperag.middleware.audit_middleware import AuditMiddleware from aperag.views.api_key import router as api_key_router +from aperag.views.audit import router as audit_router from aperag.views.auth import router as auth_router from aperag.views.chat_completion import router as chat_completion_router from aperag.views.config import router as config_router @@ -35,9 +37,13 @@ register_opik_llm_track() +# Add audit middleware - should be added before other middlewares/routers +app.add_middleware(AuditMiddleware, enabled=True) + app.include_router(auth_router, prefix="/api/v1") app.include_router(main_router, prefix="/api/v1") app.include_router(api_key_router, prefix="/api/v1") +app.include_router(audit_router, prefix="/api/v1") # Add audit router app.include_router(flow_router, prefix="/api/v1") app.include_router(llm_router, prefix="/api/v1") app.include_router(chat_completion_router, prefix="/v1") diff --git a/aperag/db/models.py b/aperag/db/models.py index 90e12c185..5097dfb42 100644 --- a/aperag/db/models.py +++ b/aperag/db/models.py @@ -29,6 +29,9 @@ Text, UniqueConstraint, select, + Index, + Float, + func, ) from sqlalchemy import Enum as SQLEnum from sqlalchemy.ext.declarative import declarative_base @@ -746,3 +749,59 @@ def update_spec(self, desired_state: IndexDesiredState = None, created_by: str = self.created_by = created_by self.version += 1 self.gmt_updated = utc_now() + + +class AuditResource(str, Enum): + """Audit resource types""" + COLLECTION = "collection" + DOCUMENT = "document" + BOT = "bot" + CHAT = "chat" + MESSAGE = "message" + API_KEY = "api_key" + LLM_PROVIDER = "llm_provider" + LLM_PROVIDER_MODEL = "llm_provider_model" + MODEL_SERVICE_PROVIDER = "model_service_provider" + USER = "user" + CONFIG = "config" + + +class AuditLog(Base): + """Audit log model to track all system operations""" + + __tablename__ = "audit_log" + + id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + user_id = Column(String(36), nullable=True, comment="User ID") + username = Column(String(255), nullable=True, comment="Username") + resource_type = Column(Enum(AuditResource), nullable=True, comment="Resource type") + resource_id = Column(String(255), nullable=True, comment="Resource ID (extracted at query time)") + api_name = Column(String(255), nullable=False, comment="API operation name") + http_method = Column(String(10), nullable=False, comment="HTTP method (POST, PUT, DELETE)") + path = Column(String(512), nullable=False, comment="API path") + status_code = Column(Integer, nullable=True, comment="HTTP status code") + request_data = Column(Text, nullable=True, comment="Request data (JSON)") + response_data = Column(Text, nullable=True, comment="Response data (JSON)") + error_message = Column(Text, nullable=True, comment="Error message if failed") + ip_address = Column(String(45), nullable=True, comment="Client IP address") + user_agent = Column(String(500), nullable=True, comment="User agent string") + request_id = Column(String(255), nullable=False, comment="Request ID for tracking") + start_time = Column(BigInteger, nullable=False, comment="Request start time (milliseconds since epoch)") + end_time = Column(BigInteger, nullable=True, comment="Request end time (milliseconds since epoch)") + gmt_created = Column(DateTime(timezone=True), nullable=False, default=func.now(), comment="Created time") + + # Index for better query performance + __table_args__ = ( + Index("idx_audit_user_id", "user_id"), + Index("idx_audit_resource_type", "resource_type"), + Index("idx_audit_api_name", "api_name"), + Index("idx_audit_http_method", "http_method"), + Index("idx_audit_status_code", "status_code"), + Index("idx_audit_gmt_created", "gmt_created"), + Index("idx_audit_resource_id", "resource_id"), + Index("idx_audit_request_id", "request_id"), + Index("idx_audit_start_time", "start_time"), + ) + + def __repr__(self): + return f"" diff --git a/aperag/middleware/audit_middleware.py b/aperag/middleware/audit_middleware.py new file mode 100644 index 000000000..ded8b7137 --- /dev/null +++ b/aperag/middleware/audit_middleware.py @@ -0,0 +1,188 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import time +from typing import Any, Dict, Optional, Tuple + +from fastapi import Request, Response +from starlette.middleware.base import BaseHTTPMiddleware + +from aperag.service.audit_service import audit_service + +logger = logging.getLogger(__name__) + + +class AuditMiddleware(BaseHTTPMiddleware): + """Middleware for automatic audit logging""" + + def __init__(self, app, enabled: bool = True): + super().__init__(app) + self.enabled = enabled + self.exclude_paths = [ + "/docs", "/openapi.json", "/redoc", "/static", + "/health", "/favicon.ico", "/metrics" + ] + + def _should_audit(self, path: str, method: str) -> bool: + """Check if the request should be audited""" + if not self.enabled: + return False + + # Skip GET requests - only audit change operations + if method.upper() == "GET": + return False + + # Skip excluded paths + for exclude_path in self.exclude_paths: + if path.startswith(exclude_path): + return False + + # Only audit API endpoints + if not path.startswith("/api/"): + return False + + return True + + def _get_audit_info_from_route(self, request: Request) -> Tuple[Optional[str], Optional[str]]: + """Get API name and resource type from route name and tags""" + try: + if hasattr(request, 'scope') and 'route' in request.scope: + route = request.scope['route'] + + # Get API name from route name + api_name = None + if hasattr(route, 'name') and route.name: + api_name = route.name + elif hasattr(route, 'endpoint') and hasattr(route.endpoint, '__name__'): + api_name = route.endpoint.__name__ + + # Get resource type from tags + resource_type = None + if hasattr(route, 'tags') and route.tags: + resource_type = audit_service.get_resource_type_from_tags(route.tags) + + # Both API name and resource type are required + if api_name and resource_type: + return api_name, resource_type + + except Exception as e: + logger.warning(f"Failed to get audit info from route: {e}") + + return None, None + + async def _extract_request_data(self, request: Request) -> Optional[Dict[str, Any]]: + """Extract request data safely""" + try: + # Get JSON body if available + if request.headers.get("content-type", "").startswith("application/json"): + body = await request.body() + if body: + return json.loads(body.decode()) + + # Get form data if available + elif request.headers.get("content-type", "").startswith("application/x-www-form-urlencoded"): + form_data = await request.form() + return dict(form_data) + + # Get query parameters + if request.query_params: + return dict(request.query_params) + + except Exception as e: + logger.warning(f"Failed to extract request data: {e}") + + return None + + async def dispatch(self, request: Request, call_next): + # Check if audit is needed + if not self._should_audit(request.url.path, request.method): + return await call_next(request) + + # Get audit info from route + api_name, resource_type = self._get_audit_info_from_route(request) + + if not api_name or not resource_type: + # No matching operation found, skip audit + return await call_next(request) + + # Record start time in milliseconds + start_time_ms = int(time.time() * 1000) + request_data = None + response_data = None + error_message = None + status_code = 200 + end_time_ms = None + + try: + # Extract request data + request_data = await self._extract_request_data(request) + + # Call the actual endpoint + response = await call_next(request) + status_code = response.status_code + + # Record end time + end_time_ms = int(time.time() * 1000) + + # Try to extract response data for non-streaming responses + if hasattr(response, 'body'): + try: + response_body = response.body.decode() if response.body else None + if response_body: + response_data = json.loads(response_body) + except: + pass + + except Exception as e: + error_message = str(e) + status_code = 500 + end_time_ms = int(time.time() * 1000) + # Re-raise for normal error handling + raise + finally: + # Log audit asynchronously + try: + # Get user info from request state (set by auth middleware) + user_id = getattr(request.state, 'user_id', None) + username = getattr(request.state, 'username', None) + + # Extract client info + ip_address, user_agent = audit_service._extract_client_info(request) + + # Log audit in background (don't await to avoid blocking) + import asyncio + asyncio.create_task( + audit_service.log_audit( + user_id=user_id, + username=username, + resource_type=resource_type, + api_name=api_name, + http_method=request.method, + path=request.url.path, + status_code=status_code, + start_time=start_time_ms, + end_time=end_time_ms, + request_data=request_data, + response_data=response_data, + error_message=error_message, + ip_address=ip_address, + user_agent=user_agent + ) + ) + except Exception as audit_error: + logger.error(f"Failed to log audit: {audit_error}") + + return response \ No newline at end of file diff --git a/aperag/migration/versions/20250617113449-audit_log_table.py b/aperag/migration/versions/20250617113449-audit_log_table.py new file mode 100644 index 000000000..ba3dc5286 --- /dev/null +++ b/aperag/migration/versions/20250617113449-audit_log_table.py @@ -0,0 +1,81 @@ +"""audit_log_table + +Revision ID: 20250617113449 +Revises: 12ea6d2bf365 +Create Date: 2025-06-17 11:34:49.123456 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '20250617113449' +down_revision = '12ea6d2bf365' +branch_labels = None +depends_on = None + + +def upgrade(): + # Create ENUM types + audit_resource_enum = postgresql.ENUM( + 'COLLECTION', 'DOCUMENT', 'BOT', 'CHAT', 'MESSAGE', + 'API_KEY', 'LLM_PROVIDER', 'LLM_PROVIDER_MODEL', + 'MODEL_SERVICE_PROVIDER', 'USER', 'CONFIG', + name='auditresource', + create_type=False + ) + audit_resource_enum.create(op.get_bind(), checkfirst=True) + + # Create audit_log table + op.create_table( + 'audit_log', + sa.Column('id', sa.String(36), primary_key=True), + sa.Column('user_id', sa.String(36), nullable=True, comment='User ID'), + sa.Column('username', sa.String(255), nullable=True, comment='Username'), + sa.Column('resource_type', audit_resource_enum, nullable=True, comment='Resource type'), + sa.Column('resource_id', sa.String(255), nullable=True, comment='Resource ID (extracted at query time)'), + sa.Column('api_name', sa.String(255), nullable=False, comment='API operation name'), + sa.Column('http_method', sa.String(10), nullable=False, comment='HTTP method (POST, PUT, DELETE)'), + sa.Column('path', sa.String(512), nullable=False, comment='API path'), + sa.Column('status_code', sa.Integer, nullable=True, comment='HTTP status code'), + sa.Column('start_time', sa.BigInteger, nullable=False, comment='Request start time (milliseconds since epoch)'), + sa.Column('end_time', sa.BigInteger, nullable=True, comment='Request end time (milliseconds since epoch)'), + sa.Column('request_data', sa.Text, nullable=True, comment='Request data (JSON)'), + sa.Column('response_data', sa.Text, nullable=True, comment='Response data (JSON)'), + sa.Column('error_message', sa.Text, nullable=True, comment='Error message if failed'), + sa.Column('ip_address', sa.String(45), nullable=True, comment='Client IP address'), + sa.Column('user_agent', sa.String(500), nullable=True, comment='User agent string'), + sa.Column('request_id', sa.String(255), nullable=False, comment='Request ID for tracking'), + sa.Column('gmt_created', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), comment='Created time'), + ) + + # Create indexes for better query performance + op.create_index('idx_audit_user_id', 'audit_log', ['user_id']) + op.create_index('idx_audit_resource_type', 'audit_log', ['resource_type']) + op.create_index('idx_audit_api_name', 'audit_log', ['api_name']) + op.create_index('idx_audit_http_method', 'audit_log', ['http_method']) + op.create_index('idx_audit_status_code', 'audit_log', ['status_code']) + op.create_index('idx_audit_start_time', 'audit_log', ['start_time']) + op.create_index('idx_audit_gmt_created', 'audit_log', ['gmt_created']) + op.create_index('idx_audit_resource_id', 'audit_log', ['resource_id']) + op.create_index('idx_audit_request_id', 'audit_log', ['request_id']) + + +def downgrade(): + # Drop indexes + op.drop_index('idx_audit_request_id', 'audit_log') + op.drop_index('idx_audit_resource_id', 'audit_log') + op.drop_index('idx_audit_gmt_created', 'audit_log') + op.drop_index('idx_audit_start_time', 'audit_log') + op.drop_index('idx_audit_status_code', 'audit_log') + op.drop_index('idx_audit_http_method', 'audit_log') + op.drop_index('idx_audit_api_name', 'audit_log') + op.drop_index('idx_audit_resource_type', 'audit_log') + op.drop_index('idx_audit_user_id', 'audit_log') + + # Drop table + op.drop_table('audit_log') + + # Drop ENUM types + postgresql.ENUM(name='auditresource').drop(op.get_bind(), checkfirst=True) \ No newline at end of file diff --git a/aperag/schema/view_models.py b/aperag/schema/view_models.py index 97de5a975..56a6f62e1 100644 --- a/aperag/schema/view_models.py +++ b/aperag/schema/view_models.py @@ -1,20 +1,6 @@ -# Copyright 2025 ApeCloud, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # generated by datamodel-codegen: # filename: openapi.merged.yaml -# timestamp: 2025-06-16T12:13:33+00:00 +# timestamp: 2025-06-20T08:44:29+00:00 from __future__ import annotations @@ -1063,6 +1049,66 @@ class PromptTemplateList(BaseModel): pageResult: Optional[PageResult] = None +class AuditLog(BaseModel): + """ + Audit log entry + """ + + id: Optional[str] = Field(None, description='Audit log ID') + user_id: Optional[str] = Field(None, description='User ID who performed the action') + username: Optional[str] = Field(None, description='Username for display') + resource_type: Optional[ + Literal[ + 'collection', + 'document', + 'bot', + 'chat', + 'message', + 'api_key', + 'llm_provider', + 'llm_provider_model', + 'model_service_provider', + 'user', + 'config', + ] + ] = Field(None, description='Type of resource') + resource_id: Optional[str] = Field( + None, description='ID of the resource (extracted at query time)' + ) + api_name: Optional[str] = Field(None, description='API operation name') + http_method: Optional[str] = Field( + None, description='HTTP method (POST, PUT, DELETE)' + ) + path: Optional[str] = Field(None, description='API path') + status_code: Optional[int] = Field(None, description='HTTP status code') + start_time: Optional[int] = Field( + None, description='Request start time (milliseconds since epoch)' + ) + end_time: Optional[int] = Field( + None, description='Request end time (milliseconds since epoch)' + ) + duration_ms: Optional[int] = Field( + None, description='Request duration in milliseconds (calculated)' + ) + request_data: Optional[str] = Field(None, description='Request data (JSON string)') + response_data: Optional[str] = Field( + None, description='Response data (JSON string)' + ) + error_message: Optional[str] = Field(None, description='Error message if failed') + ip_address: Optional[str] = Field(None, description='Client IP address') + user_agent: Optional[str] = Field(None, description='User agent string') + request_id: Optional[str] = Field(None, description='Request ID for tracking') + created: Optional[datetime] = Field(None, description='Created timestamp') + + +class AuditLogList(BaseModel): + """ + List of audit logs + """ + + items: Optional[list[AuditLog]] = Field(None, description='Audit log entries') + + class InvitationCreate(BaseModel): username: Optional[str] = Field(None, description='The username of the user') email: Optional[str] = Field(None, description='The email of the user') diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py new file mode 100644 index 000000000..539d57c0a --- /dev/null +++ b/aperag/service/audit_service.py @@ -0,0 +1,274 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import re +import time +import uuid +from typing import Any, Dict, List, Optional + +from sqlalchemy import and_, desc, select + +from aperag.config import get_async_session +from aperag.db.models import AuditLog, AuditResource + +logger = logging.getLogger(__name__) + + +class AuditService: + """Service for handling audit logs""" + + def __init__(self): + self.enabled = True + # Sensitive fields that should be filtered from logs + self.sensitive_fields = { + "password", "token", "api_key", "secret", "authorization", + "access_token", "refresh_token", "private_key", "credential" + } + + # Map FastAPI tags to audit resources + self.tag_resource_map = { + "collections": AuditResource.COLLECTION, + "documents": AuditResource.DOCUMENT, + "bots": AuditResource.BOT, + "chats": AuditResource.CHAT, + "messages": AuditResource.MESSAGE, + "apikeys": AuditResource.API_KEY, + "llm_providers": AuditResource.LLM_PROVIDER, + "llm_provider_models": AuditResource.LLM_PROVIDER_MODEL, + "users": AuditResource.USER, + "config": AuditResource.CONFIG, + } + + def _filter_sensitive_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Filter sensitive information from data""" + if not isinstance(data, dict): + return data + + filtered = {} + for key, value in data.items(): + lower_key = key.lower() + if any(sensitive in lower_key for sensitive in self.sensitive_fields): + filtered[key] = "***FILTERED***" + elif isinstance(value, dict): + filtered[key] = self._filter_sensitive_data(value) + elif isinstance(value, list): + filtered[key] = [ + self._filter_sensitive_data(item) if isinstance(item, dict) else item + for item in value + ] + else: + filtered[key] = value + return filtered + + def _safe_json_serialize(self, data: Any) -> str: + """Safely serialize data to JSON string""" + if data is None: + return None + + try: + # Filter sensitive data first + if isinstance(data, dict): + data = self._filter_sensitive_data(data) + + # Handle special types that aren't JSON serializable + def json_serializer(obj): + if hasattr(obj, 'dict'): # Pydantic models + return obj.dict() + elif hasattr(obj, '__dict__'): # Regular objects + return obj.__dict__ + else: + return str(obj) + + return json.dumps(data, default=json_serializer, ensure_ascii=False) + except Exception as e: + logger.warning(f"Failed to serialize data: {e}") + return str(data) + + def _extract_client_info(self, request) -> tuple[Optional[str], Optional[str]]: + """Extract client IP and User-Agent from request""" + try: + # Get IP address + ip_address = None + if hasattr(request, 'client') and request.client: + ip_address = request.client.host + + # Check for forwarded headers + if hasattr(request, 'headers'): + forwarded_for = request.headers.get('X-Forwarded-For') + if forwarded_for: + ip_address = forwarded_for.split(',')[0].strip() + elif request.headers.get('X-Real-IP'): + ip_address = request.headers.get('X-Real-IP') + + # Get User-Agent + user_agent = None + if hasattr(request, 'headers'): + user_agent = request.headers.get('User-Agent') + + return ip_address, user_agent + except Exception as e: + logger.warning(f"Failed to extract client info: {e}") + return None, None + + def get_resource_type_from_tags(self, tags: List[str]) -> Optional[AuditResource]: + """Get resource type from FastAPI tags""" + if not tags: + return None + + # Find the first tag that matches our resource mapping + for tag in tags: + if tag in self.tag_resource_map: + return self.tag_resource_map[tag] + + return None + + def extract_resource_id_from_path(self, path: str, resource_type: AuditResource) -> Optional[str]: + """Extract resource ID from path - called during query time""" + try: + # Define ID extraction patterns for different resource types + id_patterns = { + AuditResource.MESSAGE: r'/messages/([^/]+)', + AuditResource.CHAT: r'/chats/([^/]+)', + AuditResource.DOCUMENT: r'/documents/([^/]+)', + AuditResource.BOT: r'/bots/([^/]+)', + AuditResource.COLLECTION: r'/collections/([^/]+)', + AuditResource.API_KEY: r'/apikeys/([^/]+)', + AuditResource.LLM_PROVIDER: r'/llm_providers/([^/]+)', + AuditResource.LLM_PROVIDER_MODEL: r'/models/([^/]+/[^/]+)', + AuditResource.USER: r'/users/([^/]+)', + } + + pattern = id_patterns.get(resource_type) + if pattern: + match = re.search(pattern, path) + if match: + return match.group(1) + + except Exception as e: + logger.warning(f"Failed to extract resource ID: {e}") + + return None + + async def log_audit( + self, + user_id: Optional[str], + username: Optional[str], + resource_type: AuditResource, + api_name: str, + http_method: str, + path: str, + status_code: int, + start_time: int, + end_time: Optional[int] = None, + request_data: Optional[Dict[str, Any]] = None, + response_data: Optional[Dict[str, Any]] = None, + error_message: Optional[str] = None, + ip_address: Optional[str] = None, + user_agent: Optional[str] = None, + request_id: Optional[str] = None + ): + """Log an audit entry""" + if not self.enabled: + return + + try: + # Create audit log entry + audit_log = AuditLog( + id=str(uuid.uuid4()), + user_id=user_id, + username=username, + resource_type=resource_type, + api_name=api_name, + http_method=http_method, + path=path, + status_code=status_code, + start_time=start_time, + end_time=end_time, + request_data=self._safe_json_serialize(request_data), + response_data=self._safe_json_serialize(response_data), + error_message=error_message, + ip_address=ip_address, + user_agent=user_agent, + request_id=request_id or str(uuid.uuid4()) + ) + + # Save to database asynchronously + async with get_async_session() as session: + session.add(audit_log) + await session.commit() + + except Exception as e: + logger.error(f"Failed to log audit: {e}") + + async def list_audit_logs( + self, + user_id: Optional[str] = None, + resource_type: Optional[AuditResource] = None, + api_name: Optional[str] = None, + http_method: Optional[str] = None, + status_code: Optional[int] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + limit: int = 1000 + ) -> List[AuditLog]: + """List audit logs with filtering""" + async with get_async_session() as session: + # Build query + stmt = select(AuditLog) + + # Add filters + conditions = [] + if user_id: + conditions.append(AuditLog.user_id == user_id) + if resource_type: + conditions.append(AuditLog.resource_type == resource_type) + if api_name: + conditions.append(AuditLog.api_name.like(f"%{api_name}%")) + if http_method: + conditions.append(AuditLog.http_method == http_method) + if status_code: + conditions.append(AuditLog.status_code == status_code) + if start_date: + conditions.append(AuditLog.gmt_created >= start_date) + if end_date: + conditions.append(AuditLog.gmt_created <= end_date) + + if conditions: + stmt = stmt.where(and_(*conditions)) + + # Order by creation time (newest first) and limit + stmt = stmt.order_by(desc(AuditLog.gmt_created)).limit(limit) + + # Execute query + result = await session.execute(stmt) + audit_logs = result.scalars().all() + + # Extract resource_id for each log during query time + for log in audit_logs: + if log.resource_type and log.path: + log.resource_id = self.extract_resource_id_from_path(log.path, log.resource_type) + + # Calculate duration if both times are available + if log.start_time and log.end_time: + log.duration_ms = log.end_time - log.start_time + else: + log.duration_ms = None + + return audit_logs + + +# Global audit service instance +audit_service = AuditService() \ No newline at end of file diff --git a/aperag/views/api_key.py b/aperag/views/api_key.py index 6fc2fed55..b5d4821dc 100644 --- a/aperag/views/api_key.py +++ b/aperag/views/api_key.py @@ -23,13 +23,13 @@ router = APIRouter() -@router.get("/apikeys") +@router.get("/apikeys", tags=["apikey"], name="ListApiKeys") async def list_api_keys_view(request: Request, user: User = Depends(current_user)) -> ApiKeyList: """List all API keys for the current user""" return await api_key_service.list_api_keys(str(user.id)) -@router.post("/apikeys") +@router.post("/apikeys", tags=["apikey"], name="CreateApiKey") async def create_api_key_view( request: Request, api_key_create: ApiKeyCreate, @@ -39,13 +39,13 @@ async def create_api_key_view( return await api_key_service.create_api_key(str(user.id), api_key_create) -@router.delete("/apikeys/{apikey_id}") +@router.delete("/apikeys/{apikey_id}", tags=["apikey"], name="DeleteApiKey") async def delete_api_key_view(request: Request, apikey_id: str, user: User = Depends(current_user)): """Delete an API key""" return await api_key_service.delete_api_key(str(user.id), apikey_id) -@router.put("/apikeys/{apikey_id}") +@router.put("/apikeys/{apikey_id}", tags=["apikey"], name="UpdateApiKey") async def update_api_key_view( request: Request, apikey_id: str, diff --git a/aperag/views/audit.py b/aperag/views/audit.py new file mode 100644 index 000000000..f5e486591 --- /dev/null +++ b/aperag/views/audit.py @@ -0,0 +1,167 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import desc, select + +from aperag.db.models import AuditLog, AuditResource, User +from aperag.config import get_async_session +from aperag.schema import view_models +from aperag.service.audit_service import audit_service +from aperag.views.auth import get_current_active_user, get_current_admin + +router = APIRouter() + + +async def verify_admin_user(current_user: User = Depends(get_current_active_user)) -> User: + """Verify that the current user is an admin""" + if not current_user.is_superuser: + raise HTTPException(status_code=403, detail="Admin access required") + return current_user + + +@router.get("/audit-logs", tags=["audit"], name="ListAuditLogs", response_model=view_models.AuditLogList) +async def list_audit_logs( + user_id: Optional[str] = Query(None, description="Filter by user ID"), + username: Optional[str] = Query(None, description="Filter by username"), + resource_type: Optional[str] = Query(None, description="Filter by resource type"), + resource_id: Optional[str] = Query(None, description="Filter by resource ID"), + api_name: Optional[str] = Query(None, description="Filter by API name"), + http_method: Optional[str] = Query(None, description="Filter by HTTP method"), + status_code: Optional[int] = Query(None, description="Filter by status code"), + start_date: Optional[datetime] = Query(None, description="Filter by start date"), + end_date: Optional[datetime] = Query(None, description="Filter by end date"), + limit: int = Query(1000, le=5000, description="Maximum number of records"), + current_user: User = Depends(verify_admin_user) +): + """List audit logs with filtering""" + + # Convert string enums to actual enum values + audit_resource = None + + if resource_type: + try: + audit_resource = AuditResource(resource_type) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid resource_type: {resource_type}") + + # Get audit logs + audit_logs = await audit_service.list_audit_logs( + user_id=user_id, + resource_type=audit_resource, + api_name=api_name, + http_method=http_method, + status_code=status_code, + start_date=start_date.isoformat() if start_date else None, + end_date=end_date.isoformat() if end_date else None, + limit=limit + ) + + # Convert to view models + items = [] + for log in audit_logs: + items.append(view_models.AuditLog( + id=str(log.id), + user_id=log.user_id, + username=log.username, + resource_type=log.resource_type.value if log.resource_type else None, + resource_id=getattr(log, 'resource_id', None), # This is set during query + api_name=log.api_name, + http_method=log.http_method, + path=log.path, + status_code=log.status_code, + start_time=log.start_time, + end_time=log.end_time, + duration_ms=getattr(log, 'duration_ms', None), # Calculated during query + request_data=log.request_data, + response_data=log.response_data, + error_message=log.error_message, + ip_address=log.ip_address, + user_agent=log.user_agent, + request_id=log.request_id, + created=log.gmt_created + )) + + return view_models.AuditLogList(items=items) + + +@router.get("/audit-logs/{audit_id}", tags=["audit"], name="GetAuditLog", response_model=view_models.AuditLog) +async def get_audit_log( + audit_id: str, + current_user: User = Depends(verify_admin_user) +): + """Get a specific audit log by ID""" + + async with get_async_session() as session: + stmt = select(AuditLog).where(AuditLog.id == audit_id) + result = await session.execute(stmt) + audit_log = result.scalar_one_or_none() + + if not audit_log: + raise HTTPException(status_code=404, detail="Audit log not found") + + # Extract resource_id for this specific log + resource_id = None + if audit_log.resource_type and audit_log.path: + resource_id = audit_service.extract_resource_id_from_path(audit_log.path, audit_log.resource_type) + + # Calculate duration if both times are available + duration_ms = None + if audit_log.start_time and audit_log.end_time: + duration_ms = audit_log.end_time - audit_log.start_time + + return view_models.AuditLog( + id=str(audit_log.id), + user_id=audit_log.user_id, + username=audit_log.username, + resource_type=audit_log.resource_type.value if audit_log.resource_type else None, + resource_id=resource_id, + api_name=audit_log.api_name, + http_method=audit_log.http_method, + path=audit_log.path, + status_code=audit_log.status_code, + start_time=audit_log.start_time, + end_time=audit_log.end_time, + duration_ms=duration_ms, + request_data=audit_log.request_data, + response_data=audit_log.response_data, + error_message=audit_log.error_message, + ip_address=audit_log.ip_address, + user_agent=audit_log.user_agent, + request_id=audit_log.request_id, + created=audit_log.gmt_created + ) + + +@router.get("/audit/logs", tags=["audit"], name="ListAuditLogs") +async def list_audit_logs_view( + page: int = Query(1, ge=1, description="Page number"), + limit: int = Query(20, ge=1, le=100, description="Items per page"), + resource_type: Optional[str] = Query(None, description="Filter by resource type"), + api_name: Optional[str] = Query(None, description="Filter by API name"), + user: User = Depends(get_current_admin), +) -> view_models.AuditLogList: + """List audit logs with filtering and pagination""" + return await audit_service.list_audit_logs( + page=page, + limit=limit, + resource_type=resource_type, + api_name=api_name, + ) + + + \ No newline at end of file diff --git a/aperag/views/auth.py b/aperag/views/auth.py index e7cbd8e9f..49c0f9728 100644 --- a/aperag/views/auth.py +++ b/aperag/views/auth.py @@ -238,7 +238,7 @@ async def get_current_admin(session: AsyncSessionDep, user: User = Depends(get_c # --- API Implementation --- -@router.post("/invite") +@router.post("/invite", tags=["invitation"], name="CreateInvitation") async def create_invitation_view( data: view_models.InvitationCreate, session: AsyncSessionDep, user: User = Depends(get_current_admin) ) -> view_models.Invitation: @@ -272,7 +272,7 @@ async def create_invitation_view( ) -@router.get("/invitations") +@router.get("/invitations", tags=["invitation"], name="ListInvitations") async def list_invitations_view( session: AsyncSessionDep, user: User = Depends(get_current_admin) ) -> view_models.InvitationList: @@ -296,7 +296,7 @@ async def list_invitations_view( return view_models.InvitationList(items=invitations) -@router.post("/register") +@router.post("/register", tags=["auth"], name="Register") async def register_view( data: view_models.Register, session: AsyncSessionDep, user_manager: UserManager = Depends(get_user_manager) ) -> view_models.User: @@ -352,7 +352,7 @@ async def register_view( ) -@router.post("/login") +@router.post("/login", tags=["auth"], name="Login") async def login_view( request: Request, response: Response, @@ -395,14 +395,14 @@ async def login_view( ) -@router.post("/logout") +@router.post("/logout", tags=["auth"], name="Logout") async def logout_view(response: Response): # Clear authentication cookie response.delete_cookie(key="session") return {"success": True} -@router.get("/user") +@router.get("/user", tags=["user"], name="GetCurrentUser") async def get_user_view(request: Request, session: AsyncSessionDep, user: Optional[User] = Depends(current_user)): """Get user info, return 401 if not authenticated""" if not user: @@ -418,7 +418,7 @@ async def get_user_view(request: Request, session: AsyncSessionDep, user: Option ) -@router.get("/users") +@router.get("/users", tags=["user"], name="ListUsers") async def list_users_view(session: AsyncSessionDep, user: User = Depends(get_current_admin)) -> view_models.UserList: from sqlalchemy import select @@ -437,7 +437,7 @@ async def list_users_view(session: AsyncSessionDep, user: User = Depends(get_cur return view_models.UserList(items=users) -@router.post("/change-password") +@router.post("/change-password", tags=["user"], name="ChangePassword") async def change_password_view( data: view_models.ChangePassword, session: AsyncSessionDep, @@ -467,7 +467,7 @@ async def change_password_view( ) -@router.delete("/users/{user_id}") +@router.delete("/users/{user_id}", tags=["user"], name="DeleteUser") async def delete_user_view(user_id: str, session: AsyncSessionDep, user: User = Depends(get_current_admin)): from sqlalchemy import select diff --git a/aperag/views/chat_completion.py b/aperag/views/chat_completion.py index f023353b8..57469694a 100644 --- a/aperag/views/chat_completion.py +++ b/aperag/views/chat_completion.py @@ -26,7 +26,7 @@ router = APIRouter() -@router.post("/chat/completions") +@router.post("/chat/completions", tags=["chat_completion"], name="OpenAIChatCompletions") async def openai_chat_completions_view(request: Request, user: User = Depends(current_user)): try: body_data = await request.json() diff --git a/aperag/views/config.py b/aperag/views/config.py index 8f3f529c1..b1f4dc65b 100644 --- a/aperag/views/config.py +++ b/aperag/views/config.py @@ -21,7 +21,7 @@ router = APIRouter() -@router.get("") +@router.get("", tags=["config"], name="GetConfig") async def config_view() -> Config: auth = Auth( type=settings.auth_type, diff --git a/aperag/views/flow.py b/aperag/views/flow.py index 88650a80a..a0057a75f 100644 --- a/aperag/views/flow.py +++ b/aperag/views/flow.py @@ -24,14 +24,14 @@ router = APIRouter() -@router.get("/bots/{bot_id}/flow") +@router.get("/bots/{bot_id}/flow", tags=["flow"], name="GetFlow") async def get_flow_view( request: Request, bot_id: str, user: User = Depends(current_user) ) -> Union[WorkflowDefinition, dict]: return await flow_service_global.get_flow(str(user.id), bot_id) -@router.put("/bots/{bot_id}/flow") +@router.put("/bots/{bot_id}/flow", tags=["flow"], name="UpdateFlow") async def update_flow_view( request: Request, bot_id: str, diff --git a/aperag/views/llm.py b/aperag/views/llm.py index 4611776ec..930523088 100644 --- a/aperag/views/llm.py +++ b/aperag/views/llm.py @@ -47,7 +47,7 @@ router = APIRouter() -@router.post("/embeddings", response_model=EmbeddingResponse) +@router.post("/embeddings", response_model=EmbeddingResponse, tags=["llm"], name="CreateEmbeddings") async def create_embeddings(request: EmbeddingRequest, user: User = Depends(current_user)): """ Create embeddings for the given input text(s). @@ -156,7 +156,7 @@ async def _get_provider_info(provider: str, model: str, user_id: str, api_type: ) from e -@router.post("/rerank", response_model=RerankResponse) +@router.post("/rerank", response_model=RerankResponse, tags=["llm"], name="CreateRerank") async def create_rerank(request: RerankRequest, user: User = Depends(current_user)): """ Rerank documents based on relevance to a query. diff --git a/aperag/views/main.py b/aperag/views/main.py index c6f213570..494ea0d7f 100644 --- a/aperag/views/main.py +++ b/aperag/views/main.py @@ -46,7 +46,7 @@ router = APIRouter() -@router.get("/prompt-templates") +@router.get("/prompt-templates", tags=["prompt_template"], name="ListPromptTemplates") async def list_prompt_templates_view( request: Request, user: User = Depends(current_user) ) -> view_models.PromptTemplateList: @@ -54,7 +54,7 @@ async def list_prompt_templates_view( return list_prompt_templates(language) -@router.post("/collections") +@router.post("/collections", tags=["collection"], name="CreateCollection") async def create_collection_view( request: Request, collection: view_models.CollectionCreate, @@ -63,19 +63,19 @@ async def create_collection_view( return await collection_service.create_collection(str(user.id), collection) -@router.get("/collections") +@router.get("/collections", tags=["collection"], name="ListCollections") async def list_collections_view(request: Request, user: User = Depends(current_user)) -> view_models.CollectionList: return await collection_service.list_collections(str(user.id)) -@router.get("/collections/{collection_id}") +@router.get("/collections/{collection_id}", tags=["collection"], name="GetCollection") async def get_collection_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.Collection: return await collection_service.get_collection(str(user.id), collection_id) -@router.put("/collections/{collection_id}") +@router.put("/collections/{collection_id}", tags=["collection"], name="UpdateCollection") async def update_collection_view( request: Request, collection_id: str, @@ -85,14 +85,14 @@ async def update_collection_view( return await collection_service.update_collection(str(user.id), collection_id, collection) -@router.delete("/collections/{collection_id}") +@router.delete("/collections/{collection_id}", tags=["collection"], name="DeleteCollection") async def delete_collection_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.Collection: return await collection_service.delete_collection(str(user.id), collection_id) -@router.post("/collections/{collection_id}/documents") +@router.post("/collections/{collection_id}/documents", tags=["document"], name="CreateDocuments") async def create_documents_view( request: Request, collection_id: str, @@ -102,14 +102,14 @@ async def create_documents_view( return await document_service.create_documents(str(user.id), collection_id, files) -@router.get("/collections/{collection_id}/documents") +@router.get("/collections/{collection_id}/documents", tags=["document"], name="ListDocuments") async def list_documents_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.DocumentList: return await document_service.list_documents(str(user.id), collection_id) -@router.get("/collections/{collection_id}/documents/{document_id}") +@router.get("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="GetDocument") async def get_document_view( request: Request, collection_id: str, @@ -119,7 +119,7 @@ async def get_document_view( return await document_service.get_document(str(user.id), collection_id, document_id) -@router.put("/collections/{collection_id}/documents/{document_id}") +@router.put("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="UpdateDocument") async def update_document_view( request: Request, collection_id: str, @@ -130,7 +130,7 @@ async def update_document_view( return await document_service.update_document(str(user.id), collection_id, document_id, document) -@router.delete("/collections/{collection_id}/documents/{document_id}") +@router.delete("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="DeleteDocument") async def delete_document_view( request: Request, collection_id: str, @@ -140,7 +140,7 @@ async def delete_document_view( return await document_service.delete_document(str(user.id), collection_id, document_id) -@router.delete("/collections/{collection_id}/documents") +@router.delete("/collections/{collection_id}/documents", tags=["document"], name="DeleteDocuments") async def delete_documents_view( request: Request, collection_id: str, @@ -150,24 +150,24 @@ async def delete_documents_view( return await document_service.delete_documents(str(user.id), collection_id, document_ids) -@router.post("/bots/{bot_id}/chats") +@router.post("/bots/{bot_id}/chats", tags=["chat"], name="CreateChat") async def create_chat_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.Chat: return await chat_service_global.create_chat(str(user.id), bot_id) -@router.get("/bots/{bot_id}/chats") +@router.get("/bots/{bot_id}/chats", tags=["chat"], name="ListChats") async def list_chats_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.ChatList: return await chat_service_global.list_chats(str(user.id), bot_id) -@router.get("/bots/{bot_id}/chats/{chat_id}") +@router.get("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="GetChat") async def get_chat_view( request: Request, bot_id: str, chat_id: str, user: User = Depends(current_user) ) -> view_models.ChatDetails: return await chat_service_global.get_chat(str(user.id), bot_id, chat_id) -@router.put("/bots/{bot_id}/chats/{chat_id}") +@router.put("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="UpdateChat") async def update_chat_view( request: Request, bot_id: str, @@ -178,7 +178,7 @@ async def update_chat_view( return await chat_service_global.update_chat(str(user.id), bot_id, chat_id, chat_in) -@router.post("/bots/{bot_id}/chats/{chat_id}/messages/{message_id}") +@router.post("/bots/{bot_id}/chats/{chat_id}/messages/{message_id}", tags=["message"], name="FeedbackMessage") async def feedback_message_view( request: Request, bot_id: str, @@ -192,13 +192,13 @@ async def feedback_message_view( ) -@router.delete("/bots/{bot_id}/chats/{chat_id}") +@router.delete("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="DeleteChat") async def delete_chat_view(request: Request, bot_id: str, chat_id: str, user: User = Depends(current_user)): await chat_service_global.delete_chat(str(user.id), bot_id, chat_id) return Response(status_code=204) -@router.post("/bots") +@router.post("/bots", tags=["bot"], name="CreateBot") async def create_bot_view( request: Request, bot_in: view_models.BotCreate, @@ -207,17 +207,17 @@ async def create_bot_view( return await bot_service.create_bot(str(user.id), bot_in) -@router.get("/bots") +@router.get("/bots", tags=["bot"], name="ListBots") async def list_bots_view(request: Request, user: User = Depends(current_user)) -> view_models.BotList: return await bot_service.list_bots(str(user.id)) -@router.get("/bots/{bot_id}") +@router.get("/bots/{bot_id}", tags=["bot"], name="GetBot") async def get_bot_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.Bot: return await bot_service.get_bot(str(user.id), bot_id) -@router.put("/bots/{bot_id}") +@router.put("/bots/{bot_id}", tags=["bot"], name="UpdateBot") async def update_bot_view( request: Request, bot_id: str, @@ -227,13 +227,13 @@ async def update_bot_view( return await bot_service.update_bot(str(user.id), bot_id, bot_in) -@router.delete("/bots/{bot_id}") +@router.delete("/bots/{bot_id}", tags=["bot"], name="DeleteBot") async def delete_bot_view(request: Request, bot_id: str, user: User = Depends(current_user)): await bot_service.delete_bot(str(user.id), bot_id) return Response(status_code=204) -@router.post("/available_models") +@router.post("/available_models", tags=["model"], name="ListAvailableModels") async def get_available_models_view( request: Request, tag_filter_request: Optional[view_models.TagFilterRequest] = Body(None), @@ -247,7 +247,7 @@ async def get_available_models_view( return await llm_available_model_service.get_available_models(str(user.id), tag_filter_request) -@router.post("/chat/completions/frontend") +@router.post("/chat/completions/frontend", tags=["chat_completion"], name="FrontendChatCompletions") async def frontend_chat_completions_view(request: Request, user: User = Depends(current_user)): body = await request.body() message = body.decode("utf-8") @@ -259,7 +259,7 @@ async def frontend_chat_completions_view(request: Request, user: User = Depends( return await chat_service_global.frontend_chat_completions(str(user.id), message, stream, bot_id, chat_id, msg_id) -@router.post("/collections/{collection_id}/searchTests") +@router.post("/collections/{collection_id}/searchTests", tags=["search_test"], name="CreateSearchTest") async def create_search_test_view( request: Request, collection_id: str, @@ -269,7 +269,7 @@ async def create_search_test_view( return await collection_service.create_search_test(str(user.id), collection_id, data) -@router.delete("/collections/{collection_id}/searchTests/{search_test_id}") +@router.delete("/collections/{collection_id}/searchTests/{search_test_id}", tags=["search_test"], name="DeleteSearchTest") async def delete_search_test_view( request: Request, collection_id: str, @@ -279,14 +279,14 @@ async def delete_search_test_view( return await collection_service.delete_search_test(str(user.id), collection_id, search_test_id) -@router.get("/collections/{collection_id}/searchTests") +@router.get("/collections/{collection_id}/searchTests", tags=["search_test"], name="ListSearchTests") async def list_search_tests_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.SearchTestResultList: return await collection_service.list_search_tests(str(user.id), collection_id) -@router.post("/bots/{bot_id}/flow/debug") +@router.post("/bots/{bot_id}/flow/debug", tags=["bot"], name="DebugBotFlow") async def debug_flow_stream_view( request: Request, bot_id: str, @@ -313,7 +313,7 @@ async def websocket_chat_endpoint( # LLM Configuration API endpoints -@router.get("/llm_configuration") +@router.get("/llm_configuration", tags=["llm_provider"], name="GetLLMConfiguration") async def get_llm_configuration_view(request: Request, user: User = Depends(current_user)): """Get complete LLM configuration including providers and models""" from aperag.db.models import Role @@ -322,7 +322,7 @@ async def get_llm_configuration_view(request: Request, user: User = Depends(curr return await get_llm_configuration(str(user.id), is_admin) -@router.post("/llm_providers") +@router.post("/llm_providers", tags=["llm_provider"], name="CreateLLMProvider") async def create_llm_provider_view( request: Request, provider_data: view_models.LlmProviderCreateWithApiKey, @@ -335,7 +335,7 @@ async def create_llm_provider_view( return await create_llm_provider(provider_data.model_dump(), str(user.id), is_admin) -@router.get("/llm_providers/{provider_name}") +@router.get("/llm_providers/{provider_name}", tags=["llm_provider"], name="GetLLMProvider") async def get_llm_provider_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Get a specific LLM provider""" from aperag.db.models import Role @@ -344,7 +344,7 @@ async def get_llm_provider_view(request: Request, provider_name: str, user: User return await get_llm_provider(provider_name, str(user.id), is_admin) -@router.put("/llm_providers/{provider_name}") +@router.put("/llm_providers/{provider_name}", tags=["llm_provider"], name="UpdateLLMProvider") async def update_llm_provider_view( request: Request, provider_name: str, @@ -358,7 +358,7 @@ async def update_llm_provider_view( return await update_llm_provider(provider_name, provider_data.model_dump(), str(user.id), is_admin) -@router.delete("/llm_providers/{provider_name}") +@router.delete("/llm_providers/{provider_name}", tags=["llm_provider"], name="DeleteLLMProvider") async def delete_llm_provider_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Delete an LLM provider""" from aperag.db.models import Role @@ -367,7 +367,7 @@ async def delete_llm_provider_view(request: Request, provider_name: str, user: U return await delete_llm_provider(provider_name, str(user.id), is_admin) -@router.get("/llm_provider_models") +@router.get("/llm_provider_models", tags=["llm_provider_model"], name="ListAllLLMProviderModels") async def list_llm_provider_models_view( request: Request, provider_name: str = None, user: User = Depends(current_user) ): @@ -378,7 +378,7 @@ async def list_llm_provider_models_view( return await list_llm_provider_models(provider_name, str(user.id), is_admin) -@router.get("/llm_providers/{provider_name}/models") +@router.get("/llm_providers/{provider_name}/models", tags=["llm_provider_model"], name="ListProviderModels") async def get_provider_models_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Get all models for a specific provider""" from aperag.db.models import Role @@ -387,7 +387,7 @@ async def get_provider_models_view(request: Request, provider_name: str, user: U return await list_llm_provider_models(provider_name=provider_name, user_id=str(user.id), is_admin=is_admin) -@router.post("/llm_providers/{provider_name}/models") +@router.post("/llm_providers/{provider_name}/models", tags=["llm_provider_model"], name="CreateProviderModel") async def create_provider_model_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Create a new model for a specific provider""" import json @@ -400,7 +400,7 @@ async def create_provider_model_view(request: Request, provider_name: str, user: return await create_llm_provider_model(provider_name, data, str(user.id), is_admin) -@router.put("/llm_providers/{provider_name}/models/{api}/{model}") +@router.put("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="UpdateProviderModel") async def update_provider_model_view( request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user) ): @@ -415,7 +415,7 @@ async def update_provider_model_view( return await update_llm_provider_model(provider_name, api, model, data, str(user.id), is_admin) -@router.delete("/llm_providers/{provider_name}/models/{api}/{model}") +@router.delete("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="DeleteProviderModel") async def delete_provider_model_view( request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user) ): diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts index 01311bd26..646e009f2 100644 --- a/frontend/src/api/api.ts +++ b/frontend/src/api/api.ts @@ -14,6 +14,7 @@ +export * from './apis/audit-api'; export * from './apis/default-api'; export * from './apis/llmapi'; diff --git a/frontend/src/api/apis/audit-api.ts b/frontend/src/api/apis/audit-api.ts new file mode 100644 index 000000000..4c1c42a4d --- /dev/null +++ b/frontend/src/api/apis/audit-api.ts @@ -0,0 +1,406 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +import type { Configuration } from '../configuration'; +import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios'; +import globalAxios from 'axios'; +// Some imports not used depending on template conditions +// @ts-ignore +import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from '../common'; +// @ts-ignore +import { BASE_PATH, COLLECTION_FORMATS, type RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base'; +// @ts-ignore +import type { AuditLog } from '../models'; +// @ts-ignore +import type { AuditLogList } from '../models'; +/** + * AuditApi - axios parameter creator + * @export + */ +export const AuditApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * Get a specific audit log by ID + * @summary Get audit log detail + * @param {string} auditId Audit log ID + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditLog: async (auditId: string, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'auditId' is not null or undefined + assertParamExists('getAuditLog', 'auditId', auditId) + const localVarPath = `/audit-logs/{audit_id}` + .replace(`{${"audit_id"}}`, encodeURIComponent(String(auditId))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * List audit logs with filtering options + * @summary List audit logs + * @param {string} [userId] Filter by user ID + * @param {string} [username] Filter by username + * @param {ListAuditLogsResourceTypeEnum} [resourceType] Filter by resource type + * @param {string} [resourceId] Filter by resource ID + * @param {string} [apiName] Filter by API name + * @param {ListAuditLogsHttpMethodEnum} [httpMethod] Filter by HTTP method + * @param {number} [statusCode] Filter by status code + * @param {string} [startDate] Filter by start date + * @param {string} [endDate] Filter by end date + * @param {number} [limit] Maximum number of records + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + listAuditLogs: async (userId?: string, username?: string, resourceType?: ListAuditLogsResourceTypeEnum, resourceId?: string, apiName?: string, httpMethod?: ListAuditLogsHttpMethodEnum, statusCode?: number, startDate?: string, endDate?: string, limit?: number, options: RawAxiosRequestConfig = {}): Promise => { + const localVarPath = `/audit-logs`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + if (userId !== undefined) { + localVarQueryParameter['user_id'] = userId; + } + + if (username !== undefined) { + localVarQueryParameter['username'] = username; + } + + if (resourceType !== undefined) { + localVarQueryParameter['resource_type'] = resourceType; + } + + if (resourceId !== undefined) { + localVarQueryParameter['resource_id'] = resourceId; + } + + if (apiName !== undefined) { + localVarQueryParameter['api_name'] = apiName; + } + + if (httpMethod !== undefined) { + localVarQueryParameter['http_method'] = httpMethod; + } + + if (statusCode !== undefined) { + localVarQueryParameter['status_code'] = statusCode; + } + + if (startDate !== undefined) { + localVarQueryParameter['start_date'] = (startDate as any instanceof Date) ? + (startDate as any).toISOString() : + startDate; + } + + if (endDate !== undefined) { + localVarQueryParameter['end_date'] = (endDate as any instanceof Date) ? + (endDate as any).toISOString() : + endDate; + } + + if (limit !== undefined) { + localVarQueryParameter['limit'] = limit; + } + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * AuditApi - functional programming interface + * @export + */ +export const AuditApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = AuditApiAxiosParamCreator(configuration) + return { + /** + * Get a specific audit log by ID + * @summary Get audit log detail + * @param {string} auditId Audit log ID + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getAuditLog(auditId: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAuditLog(auditId, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['AuditApi.getAuditLog']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + /** + * List audit logs with filtering options + * @summary List audit logs + * @param {string} [userId] Filter by user ID + * @param {string} [username] Filter by username + * @param {ListAuditLogsResourceTypeEnum} [resourceType] Filter by resource type + * @param {string} [resourceId] Filter by resource ID + * @param {string} [apiName] Filter by API name + * @param {ListAuditLogsHttpMethodEnum} [httpMethod] Filter by HTTP method + * @param {number} [statusCode] Filter by status code + * @param {string} [startDate] Filter by start date + * @param {string} [endDate] Filter by end date + * @param {number} [limit] Maximum number of records + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async listAuditLogs(userId?: string, username?: string, resourceType?: ListAuditLogsResourceTypeEnum, resourceId?: string, apiName?: string, httpMethod?: ListAuditLogsHttpMethodEnum, statusCode?: number, startDate?: string, endDate?: string, limit?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listAuditLogs(userId, username, resourceType, resourceId, apiName, httpMethod, statusCode, startDate, endDate, limit, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['AuditApi.listAuditLogs']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + } +}; + +/** + * AuditApi - factory interface + * @export + */ +export const AuditApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = AuditApiFp(configuration) + return { + /** + * Get a specific audit log by ID + * @summary Get audit log detail + * @param {AuditApiGetAuditLogRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getAuditLog(requestParameters: AuditApiGetAuditLogRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.getAuditLog(requestParameters.auditId, options).then((request) => request(axios, basePath)); + }, + /** + * List audit logs with filtering options + * @summary List audit logs + * @param {AuditApiListAuditLogsRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + listAuditLogs(requestParameters: AuditApiListAuditLogsRequest = {}, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.listAuditLogs(requestParameters.userId, requestParameters.username, requestParameters.resourceType, requestParameters.resourceId, requestParameters.apiName, requestParameters.httpMethod, requestParameters.statusCode, requestParameters.startDate, requestParameters.endDate, requestParameters.limit, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * AuditApi - interface + * @export + * @interface AuditApi + */ +export interface AuditApiInterface { + /** + * Get a specific audit log by ID + * @summary Get audit log detail + * @param {AuditApiGetAuditLogRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApiInterface + */ + getAuditLog(requestParameters: AuditApiGetAuditLogRequest, options?: RawAxiosRequestConfig): AxiosPromise; + + /** + * List audit logs with filtering options + * @summary List audit logs + * @param {AuditApiListAuditLogsRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApiInterface + */ + listAuditLogs(requestParameters?: AuditApiListAuditLogsRequest, options?: RawAxiosRequestConfig): AxiosPromise; + +} + +/** + * Request parameters for getAuditLog operation in AuditApi. + * @export + * @interface AuditApiGetAuditLogRequest + */ +export interface AuditApiGetAuditLogRequest { + /** + * Audit log ID + * @type {string} + * @memberof AuditApiGetAuditLog + */ + readonly auditId: string +} + +/** + * Request parameters for listAuditLogs operation in AuditApi. + * @export + * @interface AuditApiListAuditLogsRequest + */ +export interface AuditApiListAuditLogsRequest { + /** + * Filter by user ID + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly userId?: string + + /** + * Filter by username + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly username?: string + + /** + * Filter by resource type + * @type {'collection' | 'document' | 'bot' | 'chat' | 'message' | 'api_key' | 'llm_provider' | 'llm_provider_model' | 'model_service_provider' | 'user' | 'config'} + * @memberof AuditApiListAuditLogs + */ + readonly resourceType?: ListAuditLogsResourceTypeEnum + + /** + * Filter by resource ID + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly resourceId?: string + + /** + * Filter by API name + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly apiName?: string + + /** + * Filter by HTTP method + * @type {'POST' | 'PUT' | 'DELETE'} + * @memberof AuditApiListAuditLogs + */ + readonly httpMethod?: ListAuditLogsHttpMethodEnum + + /** + * Filter by status code + * @type {number} + * @memberof AuditApiListAuditLogs + */ + readonly statusCode?: number + + /** + * Filter by start date + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly startDate?: string + + /** + * Filter by end date + * @type {string} + * @memberof AuditApiListAuditLogs + */ + readonly endDate?: string + + /** + * Maximum number of records + * @type {number} + * @memberof AuditApiListAuditLogs + */ + readonly limit?: number +} + +/** + * AuditApi - object-oriented interface + * @export + * @class AuditApi + * @extends {BaseAPI} + */ +export class AuditApi extends BaseAPI implements AuditApiInterface { + /** + * Get a specific audit log by ID + * @summary Get audit log detail + * @param {AuditApiGetAuditLogRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApi + */ + public getAuditLog(requestParameters: AuditApiGetAuditLogRequest, options?: RawAxiosRequestConfig) { + return AuditApiFp(this.configuration).getAuditLog(requestParameters.auditId, options).then((request) => request(this.axios, this.basePath)); + } + + /** + * List audit logs with filtering options + * @summary List audit logs + * @param {AuditApiListAuditLogsRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AuditApi + */ + public listAuditLogs(requestParameters: AuditApiListAuditLogsRequest = {}, options?: RawAxiosRequestConfig) { + return AuditApiFp(this.configuration).listAuditLogs(requestParameters.userId, requestParameters.username, requestParameters.resourceType, requestParameters.resourceId, requestParameters.apiName, requestParameters.httpMethod, requestParameters.statusCode, requestParameters.startDate, requestParameters.endDate, requestParameters.limit, options).then((request) => request(this.axios, this.basePath)); + } +} + +/** + * @export + */ +export const ListAuditLogsResourceTypeEnum = { + collection: 'collection', + document: 'document', + bot: 'bot', + chat: 'chat', + message: 'message', + api_key: 'api_key', + llm_provider: 'llm_provider', + llm_provider_model: 'llm_provider_model', + model_service_provider: 'model_service_provider', + user: 'user', + config: 'config' +} as const; +export type ListAuditLogsResourceTypeEnum = typeof ListAuditLogsResourceTypeEnum[keyof typeof ListAuditLogsResourceTypeEnum]; +/** + * @export + */ +export const ListAuditLogsHttpMethodEnum = { + POST: 'POST', + PUT: 'PUT', + DELETE: 'DELETE' +} as const; +export type ListAuditLogsHttpMethodEnum = typeof ListAuditLogsHttpMethodEnum[keyof typeof ListAuditLogsHttpMethodEnum]; diff --git a/frontend/src/api/apis/default-api.ts b/frontend/src/api/apis/default-api.ts index 3644963da..5462a8334 100644 --- a/frontend/src/api/apis/default-api.ts +++ b/frontend/src/api/apis/default-api.ts @@ -310,7 +310,7 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati }; }, /** - * Delete a chat + * Delete a chat (idempotent operation) * @summary Delete a chat * @param {string} botId * @param {string} chatId @@ -582,7 +582,7 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati }; }, /** - * Delete a bot + * Delete a bot (idempotent operation) * @summary Delete a bot * @param {string} botId * @param {*} [options] Override http request option. @@ -2266,7 +2266,7 @@ export const DefaultApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Delete a chat + * Delete a chat (idempotent operation) * @summary Delete a chat * @param {string} botId * @param {string} chatId @@ -2354,7 +2354,7 @@ export const DefaultApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Delete a bot + * Delete a bot (idempotent operation) * @summary Delete a bot * @param {string} botId * @param {*} [options] Override http request option. @@ -2974,7 +2974,7 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa return localVarFp.availableModelsPost(requestParameters.tagFilterRequest, options).then((request) => request(axios, basePath)); }, /** - * Delete a chat + * Delete a chat (idempotent operation) * @summary Delete a chat * @param {DefaultApiBotsBotIdChatsChatIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. @@ -3034,7 +3034,7 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa return localVarFp.botsBotIdChatsPost(requestParameters.botId, requestParameters.chatCreate, options).then((request) => request(axios, basePath)); }, /** - * Delete a bot + * Delete a bot (idempotent operation) * @summary Delete a bot * @param {DefaultApiBotsBotIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. @@ -3505,7 +3505,7 @@ export interface DefaultApiInterface { availableModelsPost(requestParameters?: DefaultApiAvailableModelsPostRequest, options?: RawAxiosRequestConfig): AxiosPromise; /** - * Delete a chat + * Delete a chat (idempotent operation) * @summary Delete a chat * @param {DefaultApiBotsBotIdChatsChatIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. @@ -3565,7 +3565,7 @@ export interface DefaultApiInterface { botsBotIdChatsPost(requestParameters: DefaultApiBotsBotIdChatsPostRequest, options?: RawAxiosRequestConfig): AxiosPromise; /** - * Delete a bot + * Delete a bot (idempotent operation) * @summary Delete a bot * @param {DefaultApiBotsBotIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. @@ -4907,7 +4907,7 @@ export class DefaultApi extends BaseAPI implements DefaultApiInterface { } /** - * Delete a chat + * Delete a chat (idempotent operation) * @summary Delete a chat * @param {DefaultApiBotsBotIdChatsChatIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. @@ -4979,7 +4979,7 @@ export class DefaultApi extends BaseAPI implements DefaultApiInterface { } /** - * Delete a bot + * Delete a bot (idempotent operation) * @summary Delete a bot * @param {DefaultApiBotsBotIdDeleteRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. diff --git a/frontend/src/api/models/audit-log-list.ts b/frontend/src/api/models/audit-log-list.ts new file mode 100644 index 000000000..08f6c8645 --- /dev/null +++ b/frontend/src/api/models/audit-log-list.ts @@ -0,0 +1,33 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { AuditLog } from './audit-log'; + +/** + * List of audit logs + * @export + * @interface AuditLogList + */ +export interface AuditLogList { + /** + * Audit log entries + * @type {Array} + * @memberof AuditLogList + */ + 'items'?: Array; +} + diff --git a/frontend/src/api/models/audit-log.ts b/frontend/src/api/models/audit-log.ts new file mode 100644 index 000000000..21673bf2c --- /dev/null +++ b/frontend/src/api/models/audit-log.ts @@ -0,0 +1,155 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * Audit log entry + * @export + * @interface AuditLog + */ +export interface AuditLog { + /** + * Audit log ID + * @type {string} + * @memberof AuditLog + */ + 'id'?: string; + /** + * User ID who performed the action + * @type {string} + * @memberof AuditLog + */ + 'user_id'?: string | null; + /** + * Username for display + * @type {string} + * @memberof AuditLog + */ + 'username'?: string | null; + /** + * Type of resource + * @type {string} + * @memberof AuditLog + */ + 'resource_type'?: AuditLogResourceTypeEnum | null; + /** + * ID of the resource (extracted at query time) + * @type {string} + * @memberof AuditLog + */ + 'resource_id'?: string | null; + /** + * API operation name + * @type {string} + * @memberof AuditLog + */ + 'api_name'?: string; + /** + * HTTP method (POST, PUT, DELETE) + * @type {string} + * @memberof AuditLog + */ + 'http_method'?: string; + /** + * API path + * @type {string} + * @memberof AuditLog + */ + 'path'?: string; + /** + * HTTP status code + * @type {number} + * @memberof AuditLog + */ + 'status_code'?: number | null; + /** + * Request start time (milliseconds since epoch) + * @type {number} + * @memberof AuditLog + */ + 'start_time'?: number; + /** + * Request end time (milliseconds since epoch) + * @type {number} + * @memberof AuditLog + */ + 'end_time'?: number | null; + /** + * Request duration in milliseconds (calculated) + * @type {number} + * @memberof AuditLog + */ + 'duration_ms'?: number | null; + /** + * Request data (JSON string) + * @type {string} + * @memberof AuditLog + */ + 'request_data'?: string | null; + /** + * Response data (JSON string) + * @type {string} + * @memberof AuditLog + */ + 'response_data'?: string | null; + /** + * Error message if failed + * @type {string} + * @memberof AuditLog + */ + 'error_message'?: string | null; + /** + * Client IP address + * @type {string} + * @memberof AuditLog + */ + 'ip_address'?: string | null; + /** + * User agent string + * @type {string} + * @memberof AuditLog + */ + 'user_agent'?: string | null; + /** + * Request ID for tracking + * @type {string} + * @memberof AuditLog + */ + 'request_id'?: string; + /** + * Created timestamp + * @type {string} + * @memberof AuditLog + */ + 'created'?: string; +} + +export const AuditLogResourceTypeEnum = { + collection: 'collection', + document: 'document', + bot: 'bot', + chat: 'chat', + message: 'message', + api_key: 'api_key', + llm_provider: 'llm_provider', + llm_provider_model: 'llm_provider_model', + model_service_provider: 'model_service_provider', + user: 'user', + config: 'config' +} as const; + +export type AuditLogResourceTypeEnum = typeof AuditLogResourceTypeEnum[keyof typeof AuditLogResourceTypeEnum]; + + diff --git a/frontend/src/api/models/index.ts b/frontend/src/api/models/index.ts index a45663215..16c6dea67 100644 --- a/frontend/src/api/models/index.ts +++ b/frontend/src/api/models/index.ts @@ -2,6 +2,8 @@ export * from './api-key'; export * from './api-key-create'; export * from './api-key-list'; export * from './api-key-update'; +export * from './audit-log'; +export * from './audit-log-list'; export * from './bot'; export * from './bot-create'; export * from './bot-list'; diff --git a/frontend/src/api/openapi.merged.yaml b/frontend/src/api/openapi.merged.yaml index ddc78cd56..f32c4bb50 100644 --- a/frontend/src/api/openapi.merged.yaml +++ b/frontend/src/api/openapi.merged.yaml @@ -94,7 +94,7 @@ paths: $ref: '#/components/schemas/failResponse' delete: summary: Delete a bot - description: Delete a bot + description: Delete a bot (idempotent operation) security: - BearerAuth: [] parameters: @@ -105,7 +105,7 @@ paths: type: string responses: '204': - description: Bot deleted successfully + description: Bot deleted successfully (or already deleted) put: summary: Update a bot description: Update a bot @@ -344,7 +344,7 @@ paths: $ref: '#/components/schemas/failResponse' delete: summary: Delete a chat - description: Delete a chat + description: Delete a chat (idempotent operation) security: - BearerAuth: [] parameters: @@ -360,19 +360,13 @@ paths: type: string responses: '204': - description: Chat deleted successfully + description: Chat deleted successfully (or already deleted) '401': description: Unauthorized content: application/json: schema: $ref: '#/components/schemas/failResponse' - '404': - description: Chat not found - content: - application/json: - schema: - $ref: '#/components/schemas/failResponse' /bots/{bot_id}/chats/{chat_id}/messages/{message_id}: post: summary: Feedback a message @@ -1667,6 +1661,128 @@ paths: application/json: schema: $ref: '#/components/schemas/failResponse' + /audit-logs: + get: + tags: + - audit + summary: List audit logs + description: List audit logs with filtering options + operationId: list_audit_logs + parameters: + - name: user_id + in: query + required: false + schema: + type: string + description: Filter by user ID + - name: username + in: query + required: false + schema: + type: string + description: Filter by username + - name: resource_type + in: query + required: false + schema: + type: string + enum: + - collection + - document + - bot + - chat + - message + - api_key + - llm_provider + - llm_provider_model + - model_service_provider + - user + - config + description: Filter by resource type + - name: resource_id + in: query + required: false + schema: + type: string + description: Filter by resource ID + - name: api_name + in: query + required: false + schema: + type: string + description: Filter by API name + - name: http_method + in: query + required: false + schema: + type: string + enum: + - POST + - PUT + - DELETE + description: Filter by HTTP method + - name: status_code + in: query + required: false + schema: + type: integer + description: Filter by status code + - name: start_date + in: query + required: false + schema: + type: string + format: date-time + description: Filter by start date + - name: end_date + in: query + required: false + schema: + type: string + format: date-time + description: Filter by end date + - name: limit + in: query + required: false + schema: + type: integer + maximum: 5000 + default: 1000 + description: Maximum number of records + responses: + '200': + description: Audit logs retrieved successfully + content: + application/json: + schema: + $ref: '#/components/schemas/auditLogList' + '403': + description: Admin access required + /audit-logs/{audit_id}: + get: + tags: + - audit + summary: Get audit log detail + description: Get a specific audit log by ID + operationId: get_audit_log + parameters: + - name: audit_id + in: path + required: true + schema: + type: string + description: Audit log ID + responses: + '200': + description: Audit log retrieved successfully + content: + application/json: + schema: + $ref: '#/components/schemas/auditLog' + '403': + description: Admin access required + '404': + description: Audit log not found /invite: post: summary: Create an invitation @@ -3461,6 +3577,103 @@ components: $ref: '#/components/schemas/promptTemplate' pageResult: $ref: '#/components/schemas/pageResult' + auditLog: + type: object + description: Audit log entry + properties: + id: + type: string + description: Audit log ID + user_id: + type: string + nullable: true + description: User ID who performed the action + username: + type: string + nullable: true + description: Username for display + resource_type: + type: string + enum: + - collection + - document + - bot + - chat + - message + - api_key + - llm_provider + - llm_provider_model + - model_service_provider + - user + - config + nullable: true + description: Type of resource + resource_id: + type: string + nullable: true + description: ID of the resource (extracted at query time) + api_name: + type: string + description: API operation name + http_method: + type: string + description: HTTP method (POST, PUT, DELETE) + path: + type: string + description: API path + status_code: + type: integer + nullable: true + description: HTTP status code + start_time: + type: integer + format: int64 + description: Request start time (milliseconds since epoch) + end_time: + type: integer + format: int64 + nullable: true + description: Request end time (milliseconds since epoch) + duration_ms: + type: integer + nullable: true + description: Request duration in milliseconds (calculated) + request_data: + type: string + nullable: true + description: Request data (JSON string) + response_data: + type: string + nullable: true + description: Response data (JSON string) + error_message: + type: string + nullable: true + description: Error message if failed + ip_address: + type: string + nullable: true + description: Client IP address + user_agent: + type: string + nullable: true + description: User agent string + request_id: + type: string + description: Request ID for tracking + created: + type: string + format: date-time + description: Created timestamp + auditLogList: + type: object + description: List of audit logs + properties: + items: + type: array + description: Audit log entries + items: + $ref: '#/components/schemas/auditLog' invitationCreate: type: object properties: diff --git a/frontend/src/locales/zh-CN.ts b/frontend/src/locales/zh-CN.ts index c667cdc48..84b26a803 100644 --- a/frontend/src/locales/zh-CN.ts +++ b/frontend/src/locales/zh-CN.ts @@ -274,4 +274,158 @@ export default { 'searchTest.history_question': '历史问题', 'searchTest.confirmDeleteHistory': '确定要删除这条历史记录吗？', 'searchTest.similarity': '相似度', + + audit: '---------------', + 'audit.title': '审计日志', + 'audit.description': '系统会自动记录所有重要操作的审计日志，帮助管理员监控和审计系统使用情况。', + 'audit.username': '用户名', + 'audit.userId': '用户ID', + 'audit.apiName': 'API名称', + 'audit.httpMethod': 'HTTP方法', + 'audit.path': 'API路径', + 'audit.resourceType': '资源类型', + 'audit.resourceId': '资源ID', + 'audit.statusCode': '状态码', + 'audit.duration': '耗时', + 'audit.ipAddress': 'IP地址', + 'audit.userAgent': 'User Agent', + 'audit.requestId': '请求ID', + 'audit.startTime': '开始时间', + 'audit.endTime': '结束时间', + 'audit.created': '创建时间', + 'audit.errorMessage': '错误信息', + 'audit.requestData': '请求数据', + 'audit.responseData': '响应数据', + 'audit.actions': '操作', + 'audit.search': '搜索', + 'audit.reset': '重置', + 'audit.refresh': '刷新', + 'audit.dateRange': '时间范围', + 'audit.startDate': '开始时间', + 'audit.endDate': '结束时间', + 'audit.enterUsername': '请输入用户名', + 'audit.enterApiName': '请输入API名称', + 'audit.selectResourceType': '请选择资源类型', + 'audit.selectHttpMethod': '请选择HTTP方法', + 'audit.detailTitle': '审计日志详情', + 'audit.id': 'ID', + 'audit.pagination': '第 {start}-{end} 条，共 {total} 条', + 'audit.fetchError': '获取审计日志失败', + 'audit.fetchDetailError': '获取审计日志详情失败', + + 'menu.welcome': '欢迎', + 'menu.more-blocks': '更多区块', + 'menu.home': '首页', + 'menu.admin': '管理页', + 'menu.admin.sub-page': '二级管理页', + 'menu.login': '登录', + 'menu.register': '注册', + 'menu.register-result': '注册结果', + 'menu.dashboard': '仪表板', + 'menu.dashboard.analysis': '分析页', + 'menu.dashboard.monitor': '监控页', + 'menu.dashboard.workplace': '工作台', + 'menu.exception.403': '403', + 'menu.exception.404': '404', + 'menu.exception.500': '500', + 'menu.form': '表单页', + 'menu.form.basic-form': '基础表单', + 'menu.form.step-form': '分步表单', + 'menu.form.step-form.info': '分步表单（填写转账信息）', + 'menu.form.step-form.confirm': '分步表单（确认转账信息）', + 'menu.form.step-form.result': '分步表单（完成）', + 'menu.form.advanced-form': '高级表单', + 'menu.list': '列表页', + 'menu.list.table-list': '查询表格', + 'menu.list.basic-list': '标准列表', + 'menu.list.card-list': '卡片列表', + 'menu.list.search-list': '搜索列表', + 'menu.list.search-list.articles': '搜索列表（文章）', + 'menu.list.search-list.projects': '搜索列表（项目）', + 'menu.list.search-list.applications': '搜索列表（应用）', + 'menu.profile': '详情页', + 'menu.profile.basic': '基础详情页', + 'menu.profile.advanced': '高级详情页', + 'menu.result': '结果页', + 'menu.result.success': '成功页', + 'menu.result.fail': '失败页', + 'menu.exception': '异常页', + 'menu.exception.not-permission': '403', + 'menu.exception.not-find': '404', + 'menu.exception.server-error': '500', + 'menu.exception.trigger': '触发错误', + 'menu.account': '个人页', + 'menu.account.center': '个人中心', + 'menu.account.settings': '个人设置', + 'menu.account.trigger': '触发报错', + 'menu.account.logout': '退出登录', + 'app.copyright.produced': '蚂蚁集团体验技术部出品', + 'app.preview.down.block': '下载此页面到本地项目', + 'app.welcome.link.fetch-blocks': '获取全部区块', + 'app.welcome.link.block-list': '基于 block 开发，快速构建标准页面', + + // 设置相关 + 'settings.title': '设置', + 'settings.profile': '个人资料', + 'settings.apiKeys': 'API 密钥', + 'settings.auditLogs': '审计日志', + 'settings.models': '模型管理', + 'settings.invitations': '邀请管理', + + // API Keys 相关 + 'apiKeys.title': 'API 密钥', + 'apiKeys.description': 'API 密钥用于访问 ApeRAG API。请妥善保管您的密钥，不要与他人分享。', + 'apiKeys.create': '创建 API 密钥', + 'apiKeys.name': '名称', + 'apiKeys.key': '密钥', + 'apiKeys.created': '创建时间', + 'apiKeys.lastUsed': '最后使用', + 'apiKeys.status': '状态', + 'apiKeys.actions': '操作', + 'apiKeys.edit': '编辑', + 'apiKeys.delete': '删除', + 'apiKeys.copy': '复制', + 'apiKeys.copied': '已复制到剪贴板', + 'apiKeys.active': '活跃', + 'apiKeys.inactive': '停用', + 'apiKeys.createTitle': '创建 API 密钥', + 'apiKeys.editTitle': '编辑 API 密钥', + 'apiKeys.nameLabel': '密钥名称', + 'apiKeys.namePlaceholder': '请输入密钥名称', + 'apiKeys.descriptionLabel': '描述', + 'apiKeys.descriptionPlaceholder': '请输入描述（可选）', + 'apiKeys.expiresAtLabel': '过期时间', + 'apiKeys.expiresAtPlaceholder': '选择过期时间（可选）', + 'apiKeys.isActiveLabel': '状态', + 'apiKeys.confirm': '确定', + 'apiKeys.cancel': '取消', + 'apiKeys.deleteConfirm': '确定要删除这个 API 密钥吗？', + 'apiKeys.deleteSuccess': 'API 密钥删除成功', + 'apiKeys.createSuccess': 'API 密钥创建成功', + 'apiKeys.updateSuccess': 'API 密钥更新成功', + 'apiKeys.fetchError': '获取 API 密钥列表失败', + 'apiKeys.neverUsed': '从未使用', + 'apiKeys.expired': '已过期', + + // 通用 + 'common.success': '成功', + 'common.error': '错误', + 'common.warning': '警告', + 'common.info': '信息', + 'common.confirm': '确认', + 'common.cancel': '取消', + 'common.save': '保存', + 'common.edit': '编辑', + 'common.delete': '删除', + 'common.create': '创建', + 'common.update': '更新', + 'common.view': '查看', + 'common.loading': '加载中...', + 'common.noData': '暂无数据', + 'common.operation': '操作', + 'common.status': '状态', + 'common.name': '名称', + 'common.description': '描述', + 'common.createdAt': '创建时间', + 'common.updatedAt': '更新时间', }; diff --git a/frontend/src/pages/settings/_navbar.tsx b/frontend/src/pages/settings/_navbar.tsx index ed4c9eb5b..34c734e6b 100644 --- a/frontend/src/pages/settings/_navbar.tsx +++ b/frontend/src/pages/settings/_navbar.tsx @@ -22,6 +22,10 @@ export const NavbarSettings = () => { label: , key: `/settings/apiKeys`, }, + { + label: , + key: `/settings/auditLogs`, + }, ], [], ); diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx new file mode 100644 index 000000000..4704c2e66 --- /dev/null +++ b/frontend/src/pages/settings/auditLogs.tsx @@ -0,0 +1,462 @@ +import React, { useState, useEffect } from 'react'; +import { + Table, + Card, + Form, + Input, + Button, + Select, + DatePicker, + Space, + Tag, + Modal, + message, + Typography, + Descriptions, + InputNumber, + Tooltip, + Alert, +} from 'antd'; +import { SearchOutlined, ReloadOutlined, EyeOutlined } from '@ant-design/icons'; +import { useIntl } from '@umijs/max'; +import type { ColumnsType } from 'antd/es/table'; +import dayjs from 'dayjs'; +import { AuditApi } from '@/api'; +import type { AuditLog } from '@/api/models'; + +const { RangePicker } = DatePicker; +const { Text } = Typography; +const { Option } = Select; + +const AuditLogsPage: React.FC = () => { + const intl = useIntl(); + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + const [data, setData] = useState([]); + const [selectedRecord, setSelectedRecord] = useState(null); + const [detailModalVisible, setDetailModalVisible] = useState(false); + + const resourceTypes = [ + 'collection', 'document', 'bot', 'chat', 'message', + 'api_key', 'llm_provider', 'llm_provider_model', + 'model_service_provider', 'user', 'config' + ]; + + const httpMethodOptions = [ + { value: 'POST', label: 'POST' }, + { value: 'PUT', label: 'PUT' }, + { value: 'DELETE', label: 'DELETE' }, + ]; + + // Format duration + const formatDuration = (ms?: number): string => { + if (!ms) return '-'; + if (ms < 1000) return `${ms.toFixed(0)}ms`; + return `${(ms / 1000).toFixed(2)}s`; + }; + + // Format JSON data + const formatJsonData = (data?: string): string => { + if (!data) return ''; + try { + return JSON.stringify(JSON.parse(data), null, 2); + } catch { + return data; + } + }; + + // Get status color + const getStatusColor = (statusCode?: number): string => { + if (!statusCode) return 'default'; + if (statusCode >= 200 && statusCode < 300) return 'green'; + if (statusCode >= 400 && statusCode < 500) return 'orange'; + if (statusCode >= 500) return 'red'; + return 'default'; + }; + + // Get HTTP method color + const getHttpMethodColor = (method?: string): string => { + switch (method) { + case 'POST': return 'blue'; + case 'PUT': return 'orange'; + case 'DELETE': return 'red'; + default: return 'default'; + } + }; + + // Fetch audit logs + const fetchData = async (params?: any) => { + setLoading(true); + try { + const api = new AuditApi(); + const response = await api.listAuditLogs(params); + setData(response.data.items || []); + } catch (error) { + console.error('Failed to fetch audit logs:', error); + message.error(intl.formatMessage({ id: 'audit.fetchError' })); + } finally { + setLoading(false); + } + }; + + // Initial load + useEffect(() => { + fetchData({ limit: 1000 }); + }, []); + + // Handle search + const handleSearch = () => { + const values = form.getFieldsValue(); + const params: any = { ...values, limit: 1000 }; + + // Handle date range + if (values.dateRange) { + params.startDate = values.dateRange[0]?.toISOString(); + params.endDate = values.dateRange[1]?.toISOString(); + delete params.dateRange; + } + + fetchData(params); + }; + + // Handle reset + const handleReset = () => { + form.resetFields(); + fetchData({ limit: 1000 }); + }; + + // Handle view details + const handleViewDetails = async (record: AuditLog) => { + try { + const api = new AuditApi(); + const log = await api.getAuditLog({ auditId: record.id! }); + setSelectedRecord(log.data); + setDetailModalVisible(true); + } catch (error) { + message.error(intl.formatMessage({ id: 'audit.fetchDetailError' })); + } + }; + + // Table columns + const columns: ColumnsType = [ + { + title: intl.formatMessage({ id: 'audit.username' }), + dataIndex: 'username', + key: 'username', + width: 120, + render: (text?: string) => text || '-', + }, + { + title: intl.formatMessage({ id: 'audit.apiName' }), + dataIndex: 'api_name', + key: 'api_name', + width: 150, + render: (text?: string) => ( + + + {text && text.length > 20 ? `${text.substring(0, 20)}...` : text || '-'} + + + ), + }, + { + title: intl.formatMessage({ id: 'audit.httpMethod' }), + dataIndex: 'http_method', + key: 'http_method', + width: 100, + render: (method?: string) => ( + {method || '-'} + ), + }, + { + title: intl.formatMessage({ id: 'audit.resourceType' }), + dataIndex: 'resource_type', + key: 'resource_type', + width: 120, + render: (type?: string) => { + return type ? {type} : '-'; + }, + }, + { + title: intl.formatMessage({ id: 'audit.resourceId' }), + dataIndex: 'resource_id', + key: 'resource_id', + width: 120, + render: (id?: string) => ( + id ? ( + + + {id.length > 15 ? `${id.substring(0, 15)}...` : id} + + + ) : '-' + ), + }, + { + title: intl.formatMessage({ id: 'audit.statusCode' }), + dataIndex: 'status_code', + key: 'status_code', + width: 100, + render: (code?: number) => { + if (!code) return '-'; + return {code}; + }, + }, + { + title: intl.formatMessage({ id: 'audit.duration' }), + dataIndex: 'duration_ms', + key: 'duration_ms', + width: 100, + render: (ms?: number) => formatDuration(ms), + }, + { + title: intl.formatMessage({ id: 'audit.ipAddress' }), + dataIndex: 'ip_address', + key: 'ip_address', + width: 120, + render: (ip?: string) => ip || '-', + }, + { + title: intl.formatMessage({ id: 'audit.created' }), + dataIndex: 'created', + key: 'created', + width: 150, + render: (text: string) => dayjs(text).format('YYYY-MM-DD HH:mm:ss'), + }, + { + title: intl.formatMessage({ id: 'audit.actions' }), + key: 'actions', + width: 80, + render: (_, record: AuditLog) => ( + + + + + + + + + intl.formatMessage( + { id: 'audit.pagination' }, + { start: range[0], end: range[1], total } + ), + }} + scroll={{ x: 1200 }} + size="small" + /> + + + + setDetailModalVisible(false)} + footer={null} + width={800} + > + {selectedRecord && ( + + + {selectedRecord.id} + + + {selectedRecord.username || '-'} + + + {selectedRecord.user_id || '-'} + + + {selectedRecord.api_name || '-'} + + + + {selectedRecord.http_method} + + + + {selectedRecord.path} + + + {selectedRecord.resource_type ? ( + {selectedRecord.resource_type} + ) : '-'} + + + + {selectedRecord.resource_id || '-'} + + + + + {selectedRecord.status_code} + + + + {selectedRecord.start_time ? + dayjs(selectedRecord.start_time).format('YYYY-MM-DD HH:mm:ss') : '-' + } + + + {selectedRecord.end_time ? + dayjs(selectedRecord.end_time).format('YYYY-MM-DD HH:mm:ss') : '-' + } + + + {formatDuration(selectedRecord.duration_ms)} + + + {selectedRecord.ip_address || '-'} + + + + {selectedRecord.user_agent || '-'} + + + + + {selectedRecord.request_id || '-'} + + + + {selectedRecord.created ? + dayjs(selectedRecord.created).format('YYYY-MM-DD HH:mm:ss') : '-' + } + + {selectedRecord.error_message && ( + + {selectedRecord.error_message} + + )} + {selectedRecord.request_data && ( + + 
+                  {formatJsonData(selectedRecord.request_data)}
+
+ + )} + {selectedRecord.response_data && ( + + 
+                  {formatJsonData(selectedRecord.response_data)}
+
+ + )} + + )} + + + ); +}; + +export default AuditLogsPage; \ No newline at end of file From 97e9a04812e7526a81ff7ad5830e2d1329c50be7 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Fri, 20 Jun 2025 18:34:19 +0800 Subject: [PATCH 02/19] chore: tidy up --- aperag/db/models.py | 2 +- aperag/middleware/audit_middleware.py | 131 ++++-- .../20250617113449-audit_log_table.py | 81 ---- .../versions/20250620172242-3369b50dc810.py | 70 +++ aperag/service/audit_service.py | 19 +- aperag/views/audit.py | 21 +- frontend/src/locales/zh-CN.ts | 2 +- frontend/src/pages/settings/auditLogs.tsx | 438 ++++++++++-------- 8 files changed, 419 insertions(+), 345 deletions(-) delete mode 100644 aperag/migration/versions/20250617113449-audit_log_table.py create mode 100644 aperag/migration/versions/20250620172242-3369b50dc810.py diff --git a/aperag/db/models.py b/aperag/db/models.py index 5097dfb42..2016056b0 100644 --- a/aperag/db/models.py +++ b/aperag/db/models.py @@ -774,7 +774,7 @@ class AuditLog(Base): id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) user_id = Column(String(36), nullable=True, comment="User ID") username = Column(String(255), nullable=True, comment="Username") - resource_type = Column(Enum(AuditResource), nullable=True, comment="Resource type") + resource_type = Column(EnumColumn(AuditResource), nullable=True, comment="Resource type") resource_id = Column(String(255), nullable=True, comment="Resource ID (extracted at query time)") api_name = Column(String(255), nullable=False, comment="API operation name") http_method = Column(String(10), nullable=False, comment="HTTP method (POST, PUT, DELETE)") diff --git a/aperag/middleware/audit_middleware.py b/aperag/middleware/audit_middleware.py index ded8b7137..f216f156a 100644 --- a/aperag/middleware/audit_middleware.py +++ b/aperag/middleware/audit_middleware.py @@ -74,9 +74,15 @@ def _get_audit_info_from_route(self, request: Request) -> Tuple[Optional[str], O if hasattr(route, 'tags') and route.tags: resource_type = audit_service.get_resource_type_from_tags(route.tags) + # Debug logging + logger.debug(f"Route info - Path: {request.url.path}, API name: {api_name}, Tags: {getattr(route, 'tags', None)}, Resource type: {resource_type}") + # Both API name and resource type are required if api_name and resource_type: return api_name, resource_type + elif api_name or resource_type: + # Log when we have partial info to help debugging + logger.warning(f"Partial audit info - Path: {request.url.path}, API name: {api_name}, Resource type: {resource_type}, Tags: {getattr(route, 'tags', None)}") except Exception as e: logger.warning(f"Failed to get audit info from route: {e}") @@ -111,13 +117,6 @@ async def dispatch(self, request: Request, call_next): if not self._should_audit(request.url.path, request.method): return await call_next(request) - # Get audit info from route - api_name, resource_type = self._get_audit_info_from_route(request) - - if not api_name or not resource_type: - # No matching operation found, skip audit - return await call_next(request) - # Record start time in milliseconds start_time_ms = int(time.time() * 1000) request_data = None @@ -127,7 +126,7 @@ async def dispatch(self, request: Request, call_next): end_time_ms = None try: - # Extract request data + # Extract request data before calling next request_data = await self._extract_request_data(request) # Call the actual endpoint @@ -137,52 +136,92 @@ async def dispatch(self, request: Request, call_next): # Record end time end_time_ms = int(time.time() * 1000) - # Try to extract response data for non-streaming responses - if hasattr(response, 'body'): + # Get audit info from route (now available after call_next) + api_name, resource_type = self._get_audit_info_from_route(request) + + # Only proceed with audit if we have both api_name and resource_type + if api_name and resource_type: + # Try to extract response data for non-streaming responses + if hasattr(response, 'body'): + try: + response_body = response.body.decode() if response.body else None + if response_body: + response_data = json.loads(response_body) + except: + pass + + # Log audit asynchronously try: - response_body = response.body.decode() if response.body else None - if response_body: - response_data = json.loads(response_body) - except: - pass + # Get user info from request state (set by auth middleware) + user_id = getattr(request.state, 'user_id', None) + username = getattr(request.state, 'username', None) + + # Extract client info + ip_address, user_agent = audit_service._extract_client_info(request) + + # Log audit in background (don't await to avoid blocking) + import asyncio + asyncio.create_task( + audit_service.log_audit( + user_id=user_id, + username=username, + resource_type=resource_type, + api_name=api_name, + http_method=request.method, + path=request.url.path, + status_code=status_code, + start_time=start_time_ms, + end_time=end_time_ms, + request_data=request_data, + response_data=response_data, + error_message=error_message, + ip_address=ip_address, + user_agent=user_agent + ) + ) + except Exception as audit_error: + logger.error(f"Failed to log audit: {audit_error}") except Exception as e: error_message = str(e) status_code = 500 end_time_ms = int(time.time() * 1000) - # Re-raise for normal error handling - raise - finally: - # Log audit asynchronously + + # Still try to log the error audit if route info is available try: - # Get user info from request state (set by auth middleware) - user_id = getattr(request.state, 'user_id', None) - username = getattr(request.state, 'username', None) - - # Extract client info - ip_address, user_agent = audit_service._extract_client_info(request) - - # Log audit in background (don't await to avoid blocking) - import asyncio - asyncio.create_task( - audit_service.log_audit( - user_id=user_id, - username=username, - resource_type=resource_type, - api_name=api_name, - http_method=request.method, - path=request.url.path, - status_code=status_code, - start_time=start_time_ms, - end_time=end_time_ms, - request_data=request_data, - response_data=response_data, - error_message=error_message, - ip_address=ip_address, - user_agent=user_agent + api_name, resource_type = self._get_audit_info_from_route(request) + if api_name and resource_type: + # Get user info from request state (set by auth middleware) + user_id = getattr(request.state, 'user_id', None) + username = getattr(request.state, 'username', None) + + # Extract client info + ip_address, user_agent = audit_service._extract_client_info(request) + + # Log audit in background (don't await to avoid blocking) + import asyncio + asyncio.create_task( + audit_service.log_audit( + user_id=user_id, + username=username, + resource_type=resource_type, + api_name=api_name, + http_method=request.method, + path=request.url.path, + status_code=status_code, + start_time=start_time_ms, + end_time=end_time_ms, + request_data=request_data, + response_data=response_data, + error_message=error_message, + ip_address=ip_address, + user_agent=user_agent + ) ) - ) except Exception as audit_error: - logger.error(f"Failed to log audit: {audit_error}") + logger.error(f"Failed to log error audit: {audit_error}") + + # Re-raise for normal error handling + raise return response \ No newline at end of file diff --git a/aperag/migration/versions/20250617113449-audit_log_table.py b/aperag/migration/versions/20250617113449-audit_log_table.py deleted file mode 100644 index ba3dc5286..000000000 --- a/aperag/migration/versions/20250617113449-audit_log_table.py +++ /dev/null @@ -1,81 +0,0 @@ -"""audit_log_table - -Revision ID: 20250617113449 -Revises: 12ea6d2bf365 -Create Date: 2025-06-17 11:34:49.123456 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '20250617113449' -down_revision = '12ea6d2bf365' -branch_labels = None -depends_on = None - - -def upgrade(): - # Create ENUM types - audit_resource_enum = postgresql.ENUM( - 'COLLECTION', 'DOCUMENT', 'BOT', 'CHAT', 'MESSAGE', - 'API_KEY', 'LLM_PROVIDER', 'LLM_PROVIDER_MODEL', - 'MODEL_SERVICE_PROVIDER', 'USER', 'CONFIG', - name='auditresource', - create_type=False - ) - audit_resource_enum.create(op.get_bind(), checkfirst=True) - - # Create audit_log table - op.create_table( - 'audit_log', - sa.Column('id', sa.String(36), primary_key=True), - sa.Column('user_id', sa.String(36), nullable=True, comment='User ID'), - sa.Column('username', sa.String(255), nullable=True, comment='Username'), - sa.Column('resource_type', audit_resource_enum, nullable=True, comment='Resource type'), - sa.Column('resource_id', sa.String(255), nullable=True, comment='Resource ID (extracted at query time)'), - sa.Column('api_name', sa.String(255), nullable=False, comment='API operation name'), - sa.Column('http_method', sa.String(10), nullable=False, comment='HTTP method (POST, PUT, DELETE)'), - sa.Column('path', sa.String(512), nullable=False, comment='API path'), - sa.Column('status_code', sa.Integer, nullable=True, comment='HTTP status code'), - sa.Column('start_time', sa.BigInteger, nullable=False, comment='Request start time (milliseconds since epoch)'), - sa.Column('end_time', sa.BigInteger, nullable=True, comment='Request end time (milliseconds since epoch)'), - sa.Column('request_data', sa.Text, nullable=True, comment='Request data (JSON)'), - sa.Column('response_data', sa.Text, nullable=True, comment='Response data (JSON)'), - sa.Column('error_message', sa.Text, nullable=True, comment='Error message if failed'), - sa.Column('ip_address', sa.String(45), nullable=True, comment='Client IP address'), - sa.Column('user_agent', sa.String(500), nullable=True, comment='User agent string'), - sa.Column('request_id', sa.String(255), nullable=False, comment='Request ID for tracking'), - sa.Column('gmt_created', sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), comment='Created time'), - ) - - # Create indexes for better query performance - op.create_index('idx_audit_user_id', 'audit_log', ['user_id']) - op.create_index('idx_audit_resource_type', 'audit_log', ['resource_type']) - op.create_index('idx_audit_api_name', 'audit_log', ['api_name']) - op.create_index('idx_audit_http_method', 'audit_log', ['http_method']) - op.create_index('idx_audit_status_code', 'audit_log', ['status_code']) - op.create_index('idx_audit_start_time', 'audit_log', ['start_time']) - op.create_index('idx_audit_gmt_created', 'audit_log', ['gmt_created']) - op.create_index('idx_audit_resource_id', 'audit_log', ['resource_id']) - op.create_index('idx_audit_request_id', 'audit_log', ['request_id']) - - -def downgrade(): - # Drop indexes - op.drop_index('idx_audit_request_id', 'audit_log') - op.drop_index('idx_audit_resource_id', 'audit_log') - op.drop_index('idx_audit_gmt_created', 'audit_log') - op.drop_index('idx_audit_start_time', 'audit_log') - op.drop_index('idx_audit_status_code', 'audit_log') - op.drop_index('idx_audit_http_method', 'audit_log') - op.drop_index('idx_audit_api_name', 'audit_log') - op.drop_index('idx_audit_resource_type', 'audit_log') - op.drop_index('idx_audit_user_id', 'audit_log') - - # Drop table - op.drop_table('audit_log') - - # Drop ENUM types - postgresql.ENUM(name='auditresource').drop(op.get_bind(), checkfirst=True) \ No newline at end of file diff --git a/aperag/migration/versions/20250620172242-3369b50dc810.py b/aperag/migration/versions/20250620172242-3369b50dc810.py new file mode 100644 index 000000000..72bd701d8 --- /dev/null +++ b/aperag/migration/versions/20250620172242-3369b50dc810.py @@ -0,0 +1,70 @@ +"""empty message + +Revision ID: 3369b50dc810 +Revises: 12ea6d2bf365 +Create Date: 2025-06-20 17:22:42.655570 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '3369b50dc810' +down_revision: Union[str, None] = '12ea6d2bf365' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('audit_log', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('user_id', sa.String(length=36), nullable=True, comment='User ID'), + sa.Column('username', sa.String(length=255), nullable=True, comment='Username'), + sa.Column('resource_type', sa.Enum('collection', 'document', 'bot', 'chat', 'message', 'api_key', 'llm_provider', 'llm_provider_model', 'model_service_provider', 'user', 'config', name='auditresource'), nullable=True, comment='Resource type'), + sa.Column('resource_id', sa.String(length=255), nullable=True, comment='Resource ID (extracted at query time)'), + sa.Column('api_name', sa.String(length=255), nullable=False, comment='API operation name'), + sa.Column('http_method', sa.String(length=10), nullable=False, comment='HTTP method (POST, PUT, DELETE)'), + sa.Column('path', sa.String(length=512), nullable=False, comment='API path'), + sa.Column('status_code', sa.Integer(), nullable=True, comment='HTTP status code'), + sa.Column('request_data', sa.Text(), nullable=True, comment='Request data (JSON)'), + sa.Column('response_data', sa.Text(), nullable=True, comment='Response data (JSON)'), + sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if failed'), + sa.Column('ip_address', sa.String(length=45), nullable=True, comment='Client IP address'), + sa.Column('user_agent', sa.String(length=500), nullable=True, comment='User agent string'), + sa.Column('request_id', sa.String(length=255), nullable=False, comment='Request ID for tracking'), + sa.Column('start_time', sa.BigInteger(), nullable=False, comment='Request start time (milliseconds since epoch)'), + sa.Column('end_time', sa.BigInteger(), nullable=True, comment='Request end time (milliseconds since epoch)'), + sa.Column('gmt_created', sa.DateTime(timezone=True), nullable=False, comment='Created time'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('idx_audit_api_name', 'audit_log', ['api_name'], unique=False) + op.create_index('idx_audit_gmt_created', 'audit_log', ['gmt_created'], unique=False) + op.create_index('idx_audit_http_method', 'audit_log', ['http_method'], unique=False) + op.create_index('idx_audit_request_id', 'audit_log', ['request_id'], unique=False) + op.create_index('idx_audit_resource_id', 'audit_log', ['resource_id'], unique=False) + op.create_index('idx_audit_resource_type', 'audit_log', ['resource_type'], unique=False) + op.create_index('idx_audit_start_time', 'audit_log', ['start_time'], unique=False) + op.create_index('idx_audit_status_code', 'audit_log', ['status_code'], unique=False) + op.create_index('idx_audit_user_id', 'audit_log', ['user_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('idx_audit_user_id', table_name='audit_log') + op.drop_index('idx_audit_status_code', table_name='audit_log') + op.drop_index('idx_audit_start_time', table_name='audit_log') + op.drop_index('idx_audit_resource_type', table_name='audit_log') + op.drop_index('idx_audit_resource_id', table_name='audit_log') + op.drop_index('idx_audit_request_id', table_name='audit_log') + op.drop_index('idx_audit_http_method', table_name='audit_log') + op.drop_index('idx_audit_gmt_created', table_name='audit_log') + op.drop_index('idx_audit_api_name', table_name='audit_log') + op.drop_table('audit_log') + # ### end Alembic commands ### diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index 539d57c0a..9d100da38 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -17,6 +17,7 @@ import re import time import uuid +from datetime import datetime from typing import Any, Dict, List, Optional from sqlalchemy import and_, desc, select @@ -40,14 +41,24 @@ def __init__(self): # Map FastAPI tags to audit resources self.tag_resource_map = { + # Support both singular and plural forms + "collection": AuditResource.COLLECTION, "collections": AuditResource.COLLECTION, + "document": AuditResource.DOCUMENT, "documents": AuditResource.DOCUMENT, + "bot": AuditResource.BOT, "bots": AuditResource.BOT, + "chat": AuditResource.CHAT, "chats": AuditResource.CHAT, + "message": AuditResource.MESSAGE, "messages": AuditResource.MESSAGE, + "apikey": AuditResource.API_KEY, "apikeys": AuditResource.API_KEY, + "llm_provider": AuditResource.LLM_PROVIDER, "llm_providers": AuditResource.LLM_PROVIDER, + "llm_provider_model": AuditResource.LLM_PROVIDER_MODEL, "llm_provider_models": AuditResource.LLM_PROVIDER_MODEL, + "user": AuditResource.USER, "users": AuditResource.USER, "config": AuditResource.CONFIG, } @@ -206,7 +217,7 @@ async def log_audit( ) # Save to database asynchronously - async with get_async_session() as session: + async for session in get_async_session(): session.add(audit_log) await session.commit() @@ -220,12 +231,12 @@ async def list_audit_logs( api_name: Optional[str] = None, http_method: Optional[str] = None, status_code: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, limit: int = 1000 ) -> List[AuditLog]: """List audit logs with filtering""" - async with get_async_session() as session: + async for session in get_async_session(): # Build query stmt = select(AuditLog) diff --git a/aperag/views/audit.py b/aperag/views/audit.py index f5e486591..aaf43e1bb 100644 --- a/aperag/views/audit.py +++ b/aperag/views/audit.py @@ -22,18 +22,11 @@ from aperag.config import get_async_session from aperag.schema import view_models from aperag.service.audit_service import audit_service -from aperag.views.auth import get_current_active_user, get_current_admin +from aperag.views.auth import current_user, get_current_admin router = APIRouter() -async def verify_admin_user(current_user: User = Depends(get_current_active_user)) -> User: - """Verify that the current user is an admin""" - if not current_user.is_superuser: - raise HTTPException(status_code=403, detail="Admin access required") - return current_user - - @router.get("/audit-logs", tags=["audit"], name="ListAuditLogs", response_model=view_models.AuditLogList) async def list_audit_logs( user_id: Optional[str] = Query(None, description="Filter by user ID"), @@ -46,7 +39,7 @@ async def list_audit_logs( start_date: Optional[datetime] = Query(None, description="Filter by start date"), end_date: Optional[datetime] = Query(None, description="Filter by end date"), limit: int = Query(1000, le=5000, description="Maximum number of records"), - current_user: User = Depends(verify_admin_user) + user: User = Depends(current_user) ): """List audit logs with filtering""" @@ -66,8 +59,8 @@ async def list_audit_logs( api_name=api_name, http_method=http_method, status_code=status_code, - start_date=start_date.isoformat() if start_date else None, - end_date=end_date.isoformat() if end_date else None, + start_date=start_date, + end_date=end_date, limit=limit ) @@ -78,7 +71,7 @@ async def list_audit_logs( id=str(log.id), user_id=log.user_id, username=log.username, - resource_type=log.resource_type.value if log.resource_type else None, + resource_type=log.resource_type.value if hasattr(log.resource_type, 'value') else log.resource_type, resource_id=getattr(log, 'resource_id', None), # This is set during query api_name=log.api_name, http_method=log.http_method, @@ -102,7 +95,7 @@ async def list_audit_logs( @router.get("/audit-logs/{audit_id}", tags=["audit"], name="GetAuditLog", response_model=view_models.AuditLog) async def get_audit_log( audit_id: str, - current_user: User = Depends(verify_admin_user) + user: User = Depends(current_user) ): """Get a specific audit log by ID""" @@ -153,7 +146,7 @@ async def list_audit_logs_view( limit: int = Query(20, ge=1, le=100, description="Items per page"), resource_type: Optional[str] = Query(None, description="Filter by resource type"), api_name: Optional[str] = Query(None, description="Filter by API name"), - user: User = Depends(get_current_admin), + user: User = Depends(current_user), ) -> view_models.AuditLogList: """List audit logs with filtering and pagination""" return await audit_service.list_audit_logs( diff --git a/frontend/src/locales/zh-CN.ts b/frontend/src/locales/zh-CN.ts index 84b26a803..ed3694ac4 100644 --- a/frontend/src/locales/zh-CN.ts +++ b/frontend/src/locales/zh-CN.ts @@ -368,7 +368,7 @@ export default { 'settings.title': '设置', 'settings.profile': '个人资料', 'settings.apiKeys': 'API 密钥', - 'settings.auditLogs': '审计日志', + 'settings.auditLogs': '操作日志', 'settings.models': '模型管理', 'settings.invitations': '邀请管理', diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx index 4704c2e66..f4106bdf9 100644 --- a/frontend/src/pages/settings/auditLogs.tsx +++ b/frontend/src/pages/settings/auditLogs.tsx @@ -13,19 +13,20 @@ import { message, Typography, Descriptions, - InputNumber, + Row, + Col, + Divider, Tooltip, - Alert, } from 'antd'; -import { SearchOutlined, ReloadOutlined, EyeOutlined } from '@ant-design/icons'; -import { useIntl } from '@umijs/max'; +import { SearchOutlined, ReloadOutlined, EyeOutlined, ClearOutlined } from '@ant-design/icons'; +import { useIntl } from 'umi'; import type { ColumnsType } from 'antd/es/table'; import dayjs from 'dayjs'; -import { AuditApi } from '@/api'; +import { AuditApi } from '@/api/apis/audit-api'; import type { AuditLog } from '@/api/models'; const { RangePicker } = DatePicker; -const { Text } = Typography; +const { Text, Title } = Typography; const { Option } = Select; const AuditLogsPage: React.FC = () => { @@ -42,12 +43,6 @@ const AuditLogsPage: React.FC = () => { 'model_service_provider', 'user', 'config' ]; - const httpMethodOptions = [ - { value: 'POST', label: 'POST' }, - { value: 'PUT', label: 'PUT' }, - { value: 'DELETE', label: 'DELETE' }, - ]; - // Format duration const formatDuration = (ms?: number): string => { if (!ms) return '-'; @@ -65,23 +60,11 @@ const AuditLogsPage: React.FC = () => { } }; - // Get status color - const getStatusColor = (statusCode?: number): string => { - if (!statusCode) return 'default'; - if (statusCode >= 200 && statusCode < 300) return 'green'; - if (statusCode >= 400 && statusCode < 500) return 'orange'; - if (statusCode >= 500) return 'red'; - return 'default'; - }; - - // Get HTTP method color - const getHttpMethodColor = (method?: string): string => { - switch (method) { - case 'POST': return 'blue'; - case 'PUT': return 'orange'; - case 'DELETE': return 'red'; - default: return 'default'; - } + // Get status display + const getStatusDisplay = (statusCode?: number): { text: string; color: string } => { + if (!statusCode) return { text: '未知', color: 'default' }; + if (statusCode >= 200 && statusCode < 300) return { text: '成功', color: 'success' }; + return { text: '失败', color: 'error' }; }; // Fetch audit logs @@ -89,11 +72,11 @@ const AuditLogsPage: React.FC = () => { setLoading(true); try { const api = new AuditApi(); - const response = await api.listAuditLogs(params); + const response = await api.listAuditLogs(params || {}); setData(response.data.items || []); } catch (error) { console.error('Failed to fetch audit logs:', error); - message.error(intl.formatMessage({ id: 'audit.fetchError' })); + message.error('获取审计日志失败'); } finally { setLoading(false); } @@ -101,13 +84,25 @@ const AuditLogsPage: React.FC = () => { // Initial load useEffect(() => { - fetchData({ limit: 1000 }); + // 默认展示最近一天的数据 + const endTime = dayjs(); + const startTime = endTime.subtract(1, 'day'); + fetchData({ + limit: 100, + startDate: startTime.toISOString(), + endDate: endTime.toISOString() + }); + + // 设置表单默认值 + form.setFieldsValue({ + dateRange: [startTime, endTime] + }); }, []); // Handle search const handleSearch = () => { const values = form.getFieldsValue(); - const params: any = { ...values, limit: 1000 }; + const params: any = { ...values, limit: 100 }; // Handle date range if (values.dateRange) { @@ -122,7 +117,7 @@ const AuditLogsPage: React.FC = () => { // Handle reset const handleReset = () => { form.resetFields(); - fetchData({ limit: 1000 }); + fetchData({ limit: 100 }); }; // Handle view details @@ -133,320 +128,367 @@ const AuditLogsPage: React.FC = () => { setSelectedRecord(log.data); setDetailModalVisible(true); } catch (error) { - message.error(intl.formatMessage({ id: 'audit.fetchDetailError' })); + message.error('获取详细信息失败'); } }; // Table columns const columns: ColumnsType = [ { - title: intl.formatMessage({ id: 'audit.username' }), + title: '用户名', dataIndex: 'username', key: 'username', width: 120, - render: (text?: string) => text || '-', + render: (text?: string) => ( + {text || '-'} + ), }, { - title: intl.formatMessage({ id: 'audit.apiName' }), + title: 'API 名称', dataIndex: 'api_name', key: 'api_name', - width: 150, + width: 200, render: (text?: string) => ( - {text && text.length > 20 ? `${text.substring(0, 20)}...` : text || '-'} + {text && text.length > 30 ? `${text.substring(0, 30)}...` : text || '-'} ), }, { - title: intl.formatMessage({ id: 'audit.httpMethod' }), - dataIndex: 'http_method', - key: 'http_method', - width: 100, - render: (method?: string) => ( - {method || '-'} - ), - }, - { - title: intl.formatMessage({ id: 'audit.resourceType' }), + title: '资源类型', dataIndex: 'resource_type', key: 'resource_type', width: 120, render: (type?: string) => { - return type ? {type} : '-'; + return type ? ( + + {type} + + ) : '-'; }, }, { - title: intl.formatMessage({ id: 'audit.resourceId' }), + title: '资源 ID', dataIndex: 'resource_id', key: 'resource_id', - width: 120, + width: 140, render: (id?: string) => ( id ? ( - - {id.length > 15 ? `${id.substring(0, 15)}...` : id} - + + {id.length > 18 ? `${id.substring(0, 18)}...` : id} + ) : '-' ), }, { - title: intl.formatMessage({ id: 'audit.statusCode' }), + title: '状态', dataIndex: 'status_code', key: 'status_code', width: 100, + align: 'center', render: (code?: number) => { - if (!code) return '-'; - return {code}; + const status = getStatusDisplay(code); + return ( + + {status.text} + + ); }, }, { - title: intl.formatMessage({ id: 'audit.duration' }), - dataIndex: 'duration_ms', - key: 'duration_ms', - width: 100, - render: (ms?: number) => formatDuration(ms), - }, - { - title: intl.formatMessage({ id: 'audit.ipAddress' }), - dataIndex: 'ip_address', - key: 'ip_address', - width: 120, - render: (ip?: string) => ip || '-', + title: '开始时间', + dataIndex: 'start_time', + key: 'start_time', + width: 160, + render: (time?: number) => ( + time ? ( + + {dayjs(time).format('MM-DD HH:mm:ss')} + + ) : '-' + ), }, { - title: intl.formatMessage({ id: 'audit.created' }), - dataIndex: 'created', - key: 'created', - width: 150, - render: (text: string) => dayjs(text).format('YYYY-MM-DD HH:mm:ss'), + title: '结束时间', + dataIndex: 'end_time', + key: 'end_time', + width: 160, + render: (time?: number) => ( + time ? ( + + {dayjs(time).format('MM-DD HH:mm:ss')} + + ) : '-' + ), }, { - title: intl.formatMessage({ id: 'audit.actions' }), + title: '操作', key: 'actions', width: 80, - render: (_, record: AuditLog) => ( + align: 'center', + render: (_, record) => ( ), }, ]; return ( -
+
- - - -
- - - +
+ + 审计日志 + + + 查看系统操作的详细审计记录 + +
- - + + + + - + - - - - - - - - - + - + - - - - + + + + -
- intl.formatMessage( - { id: 'audit.pagination' }, - { start: range[0], end: range[1], total } - ), - }} - scroll={{ x: 1200 }} - size="small" - /> - +
+ `显示 ${range[0] || 0}-${range[1] || 0} 条，共 ${total} 条记录`, + pageSizeOptions: ['20', '50', '100'], + defaultPageSize: 20, + }} + scroll={{ x: 1200 }} + size="small" + bordered + /> setDetailModalVisible(false)} - footer={null} - width={800} + footer={[ + + ]} + width={900} + style={{ top: 20 }} > {selectedRecord && ( - - - {selectedRecord.id} + + + {selectedRecord.id} - - {selectedRecord.username || '-'} + + + {selectedRecord.username || '-'} - - {selectedRecord.user_id || '-'} + + {selectedRecord.user_id || '-'} - + + {selectedRecord.api_name || '-'} - - - {selectedRecord.http_method} - + + + {(() => { + const status = getStatusDisplay(selectedRecord.status_code || undefined); + return ( + + {status.text} + + ); + })()} + + + {selectedRecord.status_code || '-'} - - {selectedRecord.path} + + + + {selectedRecord.path} + - + + {selectedRecord.resource_type ? ( {selectedRecord.resource_type} ) : '-'} - + {selectedRecord.resource_id || '-'} - - - {selectedRecord.status_code} - - - + + {selectedRecord.start_time ? dayjs(selectedRecord.start_time).format('YYYY-MM-DD HH:mm:ss') : '-' } - + {selectedRecord.end_time ? dayjs(selectedRecord.end_time).format('YYYY-MM-DD HH:mm:ss') : '-' } - - {formatDuration(selectedRecord.duration_ms)} + + + + {formatDuration(selectedRecord.duration_ms || undefined)} + - - {selectedRecord.ip_address || '-'} + + {selectedRecord.ip_address || '-'} - - + + + {selectedRecord.user_agent || '-'} - + + {selectedRecord.request_id || '-'} - + + {selectedRecord.created ? dayjs(selectedRecord.created).format('YYYY-MM-DD HH:mm:ss') : '-' } + {selectedRecord.error_message && ( - - {selectedRecord.error_message} + + + {selectedRecord.error_message} + )} + {selectedRecord.request_data && ( - + 
                   {formatJsonData(selectedRecord.request_data)}
)} + {selectedRecord.response_data && ( - + 
                   {formatJsonData(selectedRecord.response_data)}
From 6cbccb7c4eed83a53bcf7621870172bf3fc5beab Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Fri, 20 Jun 2025 18:39:07 +0800 Subject: [PATCH 03/19] chore: tidy up --- aperag/service/audit_service.py | 13 ++++++++++++- aperag/views/audit.py | 13 +++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index 9d100da38..e63b85e24 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -270,7 +270,18 @@ async def list_audit_logs( # Extract resource_id for each log during query time for log in audit_logs: if log.resource_type and log.path: - log.resource_id = self.extract_resource_id_from_path(log.path, log.resource_type) + # Convert string to enum if needed + resource_type_enum = log.resource_type + if isinstance(log.resource_type, str): + try: + resource_type_enum = AuditResource(log.resource_type) + except ValueError: + resource_type_enum = None + + if resource_type_enum: + log.resource_id = self.extract_resource_id_from_path(log.path, resource_type_enum) + else: + log.resource_id = None # Calculate duration if both times are available if log.start_time and log.end_time: diff --git a/aperag/views/audit.py b/aperag/views/audit.py index aaf43e1bb..547d10124 100644 --- a/aperag/views/audit.py +++ b/aperag/views/audit.py @@ -110,7 +110,16 @@ async def get_audit_log( # Extract resource_id for this specific log resource_id = None if audit_log.resource_type and audit_log.path: - resource_id = audit_service.extract_resource_id_from_path(audit_log.path, audit_log.resource_type) + # Convert string to enum if needed + resource_type_enum = audit_log.resource_type + if isinstance(audit_log.resource_type, str): + try: + resource_type_enum = AuditResource(audit_log.resource_type) + except ValueError: + resource_type_enum = None + + if resource_type_enum: + resource_id = audit_service.extract_resource_id_from_path(audit_log.path, resource_type_enum) # Calculate duration if both times are available duration_ms = None @@ -121,7 +130,7 @@ async def get_audit_log( id=str(audit_log.id), user_id=audit_log.user_id, username=audit_log.username, - resource_type=audit_log.resource_type.value if audit_log.resource_type else None, + resource_type=audit_log.resource_type.value if hasattr(audit_log.resource_type, 'value') else audit_log.resource_type, resource_id=resource_id, api_name=audit_log.api_name, http_method=audit_log.http_method, From 3bdbdfa8029f0c0847df440ce3ec7eac6076f04d Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Fri, 20 Jun 2025 18:44:39 +0800 Subject: [PATCH 04/19] chore: tidy up --- aperag/views/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperag/views/audit.py b/aperag/views/audit.py index 547d10124..81b5e5bd0 100644 --- a/aperag/views/audit.py +++ b/aperag/views/audit.py @@ -99,7 +99,7 @@ async def get_audit_log( ): """Get a specific audit log by ID""" - async with get_async_session() as session: + async for session in get_async_session(): stmt = select(AuditLog).where(AuditLog.id == audit_id) result = await session.execute(stmt) audit_log = result.scalar_one_or_none() From a34d1c2438e7bc284e9f75500b9db12c12d0d5c5 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Fri, 20 Jun 2025 22:34:03 +0800 Subject: [PATCH 05/19] chore: tidy up --- aperag/middleware/audit_middleware.py | 21 +- aperag/views/auth.py | 2 + frontend/src/locales/en-US.ts | 38 +++ frontend/src/locales/zh-CN.ts | 38 +++ frontend/src/pages/settings/auditLogs.tsx | 360 ++++++++-------------- 5 files changed, 219 insertions(+), 240 deletions(-) diff --git a/aperag/middleware/audit_middleware.py b/aperag/middleware/audit_middleware.py index f216f156a..947a29363 100644 --- a/aperag/middleware/audit_middleware.py +++ b/aperag/middleware/audit_middleware.py @@ -142,13 +142,20 @@ async def dispatch(self, request: Request, call_next): # Only proceed with audit if we have both api_name and resource_type if api_name and resource_type: # Try to extract response data for non-streaming responses - if hasattr(response, 'body'): - try: - response_body = response.body.decode() if response.body else None - if response_body: - response_data = json.loads(response_body) - except: - pass + try: + # For FastAPI Response objects, we need to read the response body + if hasattr(response, 'body') and response.body: + response_body = response.body.decode('utf-8') + response_data = json.loads(response_body) + elif hasattr(response, '_body') and response._body: + response_body = response._body.decode('utf-8') + response_data = json.loads(response_body) + # Skip streaming responses and large responses + elif response.status_code < 400 and response.headers.get('content-type', '').startswith('application/json'): + # For successful JSON responses, record a simplified response + response_data = {"status": "success", "code": response.status_code} + except Exception as e: + logger.debug(f"Could not extract response data: {e}") # Log audit asynchronously try: diff --git a/aperag/views/auth.py b/aperag/views/auth.py index 49c0f9728..b01288fb3 100644 --- a/aperag/views/auth.py +++ b/aperag/views/auth.py @@ -207,11 +207,13 @@ async def current_user( api_user = await authenticate_api_key(request, session) if api_user: request.state.user_id = api_user.id + request.state.username = api_user.username return api_user # Then try JWT/Cookie authentication if user: request.state.user_id = user.id + request.state.username = user.username return user raise HTTPException(status_code=401, detail="Unauthorized") diff --git a/frontend/src/locales/en-US.ts b/frontend/src/locales/en-US.ts index 92712c0bc..c48a4bd77 100644 --- a/frontend/src/locales/en-US.ts +++ b/frontend/src/locales/en-US.ts @@ -13,6 +13,44 @@ export default { ...collection, ...apiKeys, + // Audit logs + 'audit.logs.title': 'Audit Logs', + 'audit.logs.description': 'View detailed audit records of system operations', + 'audit.logs.username': 'Username', + 'audit.logs.apiName': 'API Name', + 'audit.logs.resourceType': 'Resource Type', + 'audit.logs.resourceId': 'Resource ID', + 'audit.logs.status': 'Status', + 'audit.logs.startTime': 'Start Time', + 'audit.logs.endTime': 'End Time', + 'audit.logs.timeRange': 'Time Range', + 'audit.logs.apiNamePlaceholder': 'Enter API name', + 'audit.logs.detail.title': 'Audit Log Details', + 'audit.logs.detail.apiName': 'API Name', + 'audit.logs.detail.status': 'Status', + 'audit.logs.detail.path': 'Request Path', + 'audit.logs.detail.startTime': 'Start Time', + 'audit.logs.detail.endTime': 'End Time', + 'audit.logs.detail.duration': 'Duration', + 'audit.logs.detail.ipAddress': 'IP Address', + 'audit.logs.detail.userAgent': 'User Agent', + 'audit.logs.detail.errorMessage': 'Error Message', + 'audit.logs.detail.requestData': 'Request Data', + 'audit.logs.detail.responseData': 'Response Data', + 'audit.logs.detail.noData': 'No data', + 'audit.logs.fetchError': 'Failed to fetch audit logs', + 'audit.logs.detailError': 'Failed to get audit details', + + // Common + 'common.search': 'Search', + 'common.actions': 'Actions', + 'common.detail': 'Detail', + 'common.system': 'System', + 'common.status.unknown': 'Unknown', + 'common.status.success': 'Success', + 'common.status.failed': 'Failed', + 'common.pagination.total': 'Showing {start}-{end} of {total} records', + 'text.welcome': 'Welcome to ApeRAG', 'text.authorizing': 'Authorizing', 'text.authorize.error': 'Authorization', diff --git a/frontend/src/locales/zh-CN.ts b/frontend/src/locales/zh-CN.ts index ed3694ac4..20ddc2d3b 100644 --- a/frontend/src/locales/zh-CN.ts +++ b/frontend/src/locales/zh-CN.ts @@ -13,6 +13,44 @@ export default { ...collection, ...apiKeys, + // Audit logs + 'audit.logs.title': '操作日志', + 'audit.logs.description': '查看系统操作的详细审计记录', + 'audit.logs.username': '用户名', + 'audit.logs.apiName': 'API 名称', + 'audit.logs.resourceType': '资源类型', + 'audit.logs.resourceId': '资源 ID', + 'audit.logs.status': '状态', + 'audit.logs.startTime': '开始时间', + 'audit.logs.endTime': '结束时间', + 'audit.logs.timeRange': '时间范围', + 'audit.logs.apiNamePlaceholder': '输入API名称', + 'audit.logs.detail.title': '审计日志详情', + 'audit.logs.detail.apiName': 'API 名称', + 'audit.logs.detail.status': '状态', + 'audit.logs.detail.path': '请求路径', + 'audit.logs.detail.startTime': '开始时间', + 'audit.logs.detail.endTime': '结束时间', + 'audit.logs.detail.duration': '持续时间', + 'audit.logs.detail.ipAddress': 'IP 地址', + 'audit.logs.detail.userAgent': 'User Agent', + 'audit.logs.detail.errorMessage': '错误信息', + 'audit.logs.detail.requestData': '请求内容', + 'audit.logs.detail.responseData': '响应内容', + 'audit.logs.detail.noData': '无数据', + 'audit.logs.fetchError': '获取审计日志失败', + 'audit.logs.detailError': '获取详细信息失败', + + // Common + 'common.search': '搜索', + 'common.actions': '操作', + 'common.detail': '详情', + 'common.system': '系统', + 'common.status.unknown': '未知', + 'common.status.success': '成功', + 'common.status.failed': '失败', + 'common.pagination.total': '显示 {start}-{end} 条，共 {total} 条记录', + 'text.welcome': '欢迎使用ApeRAG', 'text.authorizing': '授权中', 'text.authorize.error': '认证失败', diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx index f4106bdf9..4ad0ad939 100644 --- a/frontend/src/pages/settings/auditLogs.tsx +++ b/frontend/src/pages/settings/auditLogs.tsx @@ -5,20 +5,17 @@ import { Form, Input, Button, - Select, DatePicker, Space, Tag, - Modal, + Drawer, message, Typography, Descriptions, - Row, - Col, Divider, Tooltip, } from 'antd'; -import { SearchOutlined, ReloadOutlined, EyeOutlined, ClearOutlined } from '@ant-design/icons'; +import { SearchOutlined, EyeOutlined } from '@ant-design/icons'; import { useIntl } from 'umi'; import type { ColumnsType } from 'antd/es/table'; import dayjs from 'dayjs'; @@ -27,7 +24,6 @@ import type { AuditLog } from '@/api/models'; const { RangePicker } = DatePicker; const { Text, Title } = Typography; -const { Option } = Select; const AuditLogsPage: React.FC = () => { const intl = useIntl(); @@ -35,13 +31,7 @@ const AuditLogsPage: React.FC = () => { const [loading, setLoading] = useState(false); const [data, setData] = useState([]); const [selectedRecord, setSelectedRecord] = useState(null); - const [detailModalVisible, setDetailModalVisible] = useState(false); - - const resourceTypes = [ - 'collection', 'document', 'bot', 'chat', 'message', - 'api_key', 'llm_provider', 'llm_provider_model', - 'model_service_provider', 'user', 'config' - ]; + const [detailDrawerVisible, setDetailDrawerVisible] = useState(false); // Format duration const formatDuration = (ms?: number): string => { @@ -50,21 +40,22 @@ const AuditLogsPage: React.FC = () => { return `${(ms / 1000).toFixed(2)}s`; }; - // Format JSON data - const formatJsonData = (data?: string): string => { - if (!data) return ''; - try { - return JSON.stringify(JSON.parse(data), null, 2); - } catch { - return data; - } - }; + // Get status display const getStatusDisplay = (statusCode?: number): { text: string; color: string } => { - if (!statusCode) return { text: '未知', color: 'default' }; - if (statusCode >= 200 && statusCode < 300) return { text: '成功', color: 'success' }; - return { text: '失败', color: 'error' }; + if (!statusCode) return { + text: intl.formatMessage({ id: 'common.status.unknown', defaultMessage: 'Unknown' }), + color: 'default' + }; + if (statusCode >= 200 && statusCode < 300) return { + text: intl.formatMessage({ id: 'common.status.success', defaultMessage: 'Success' }), + color: 'success' + }; + return { + text: intl.formatMessage({ id: 'common.status.failed', defaultMessage: 'Failed' }), + color: 'error' + }; }; // Fetch audit logs @@ -76,7 +67,7 @@ const AuditLogsPage: React.FC = () => { setData(response.data.items || []); } catch (error) { console.error('Failed to fetch audit logs:', error); - message.error('获取审计日志失败'); + message.error(intl.formatMessage({ id: 'audit.logs.fetchError', defaultMessage: 'Failed to fetch audit logs' })); } finally { setLoading(false); } @@ -114,37 +105,35 @@ const AuditLogsPage: React.FC = () => { fetchData(params); }; - // Handle reset - const handleReset = () => { - form.resetFields(); - fetchData({ limit: 100 }); - }; - // Handle view details const handleViewDetails = async (record: AuditLog) => { try { const api = new AuditApi(); const log = await api.getAuditLog({ auditId: record.id! }); + console.log('Audit log details:', log.data); + console.log('Request data:', log.data.request_data); + console.log('Response data:', log.data.response_data); setSelectedRecord(log.data); - setDetailModalVisible(true); + setDetailDrawerVisible(true); } catch (error) { - message.error('获取详细信息失败'); + console.error('Failed to get audit details:', error); + message.error(intl.formatMessage({ id: 'audit.logs.detailError', defaultMessage: 'Failed to get audit details' })); } }; // Table columns const columns: ColumnsType = [ { - title: '用户名', + title: intl.formatMessage({ id: 'audit.logs.username', defaultMessage: 'Username' }), dataIndex: 'username', key: 'username', width: 120, render: (text?: string) => ( - {text || '-'} + {text || intl.formatMessage({ id: 'common.system', defaultMessage: 'System' })} ), }, { - title: 'API 名称', + title: intl.formatMessage({ id: 'audit.logs.apiName', defaultMessage: 'API Name' }), dataIndex: 'api_name', key: 'api_name', width: 200, @@ -157,7 +146,7 @@ const AuditLogsPage: React.FC = () => { ), }, { - title: '资源类型', + title: intl.formatMessage({ id: 'audit.logs.resourceType', defaultMessage: 'Resource Type' }), dataIndex: 'resource_type', key: 'resource_type', width: 120, @@ -170,7 +159,7 @@ const AuditLogsPage: React.FC = () => { }, }, { - title: '资源 ID', + title: intl.formatMessage({ id: 'audit.logs.resourceId', defaultMessage: 'Resource ID' }), dataIndex: 'resource_id', key: 'resource_id', width: 140, @@ -185,11 +174,11 @@ const AuditLogsPage: React.FC = () => { ), }, { - title: '状态', + title: intl.formatMessage({ id: 'audit.logs.status', defaultMessage: 'Status' }), dataIndex: 'status_code', key: 'status_code', width: 100, - align: 'center', + align: 'center' as const, render: (code?: number) => { const status = getStatusDisplay(code); return ( @@ -200,7 +189,7 @@ const AuditLogsPage: React.FC = () => { }, }, { - title: '开始时间', + title: intl.formatMessage({ id: 'audit.logs.startTime', defaultMessage: 'Start Time' }), dataIndex: 'start_time', key: 'start_time', width: 160, @@ -213,7 +202,7 @@ const AuditLogsPage: React.FC = () => { ), }, { - title: '结束时间', + title: intl.formatMessage({ id: 'audit.logs.endTime', defaultMessage: 'End Time' }), dataIndex: 'end_time', key: 'end_time', width: 160, @@ -226,10 +215,10 @@ const AuditLogsPage: React.FC = () => { ), }, { - title: '操作', + title: intl.formatMessage({ id: 'common.actions', defaultMessage: 'Actions' }), key: 'actions', width: 80, - align: 'center', + align: 'center' as const, render: (_, record) => ( ), }, @@ -248,10 +237,10 @@ const AuditLogsPage: React.FC = () => {
- 审计日志 + {intl.formatMessage({ id: 'audit.logs.title', defaultMessage: 'Audit Logs' })} - 查看系统操作的详细审计记录 + {intl.formatMessage({ id: 'audit.logs.description', defaultMessage: 'View detailed audit records of system operations' })}
@@ -262,69 +251,35 @@ const AuditLogsPage: React.FC = () => { style={{ marginBottom: '24px' }} > - + - - - - - + - - - - - + @@ -340,7 +295,10 @@ const AuditLogsPage: React.FC = () => { showSizeChanger: true, showQuickJumper: true, showTotal: (total, range) => - `显示 ${range[0] || 0}-${range[1] || 0} 条，共 ${total} 条记录`, + intl.formatMessage( + { id: 'common.pagination.total', defaultMessage: 'Showing {start}-{end} of {total} records' }, + { start: range[0] || 0, end: range[1] || 0, total } + ), pageSizeOptions: ['20', '50', '100'], defaultPageSize: 20, }} @@ -350,153 +308,89 @@ const AuditLogsPage: React.FC = () => { /> - setDetailModalVisible(false)} - footer={[ - - ]} - width={900} - style={{ top: 20 }} + setDetailDrawerVisible(false)} + width={800} + destroyOnClose > {selectedRecord && ( - - - {selectedRecord.id} - - - - {selectedRecord.username || '-'} - - - {selectedRecord.user_id || '-'} - - - - {selectedRecord.api_name || '-'} - - - - {(() => { - const status = getStatusDisplay(selectedRecord.status_code || undefined); - return ( - - {status.text} - - ); - })()} - - - {selectedRecord.status_code || '-'} - - - - - {selectedRecord.path} - - - - - {selectedRecord.resource_type ? ( - {selectedRecord.resource_type} - ) : '-'} - - - - {selectedRecord.resource_id || '-'} - - - - - {selectedRecord.start_time ? - dayjs(selectedRecord.start_time).format('YYYY-MM-DD HH:mm:ss') : '-' - } - - - {selectedRecord.end_time ? - dayjs(selectedRecord.end_time).format('YYYY-MM-DD HH:mm:ss') : '-' - } - - - - - {formatDuration(selectedRecord.duration_ms || undefined)} - - - - {selectedRecord.ip_address || '-'} - - - - - {selectedRecord.user_agent || '-'} - - - - - - {selectedRecord.request_id || '-'} - - - - - {selectedRecord.created ? - dayjs(selectedRecord.created).format('YYYY-MM-DD HH:mm:ss') : '-' - } - - - {selectedRecord.error_message && ( - - - {selectedRecord.error_message} - - - )} - - {selectedRecord.request_data && ( - - 
+            

+              
+                {intl.formatMessage({ id: 'audit.logs.detail.requestData', defaultMessage: 'Request Data' })}
+              
+              

+                
-                  {formatJsonData(selectedRecord.request_data)}
+                  {selectedRecord?.request_data ? 
+                    ((() => {
+                      try {
+                        return JSON.stringify(JSON.parse(selectedRecord.request_data), null, 2);
+                      } catch {
+                        return selectedRecord.request_data;
+                      }
+                    })()) : 
+                    intl.formatMessage({ id: 'audit.logs.detail.noData', defaultMessage: 'No data' })
+                  }
                 

-              
-            )}
+              

+            

             
-            {selectedRecord.response_data && (
-              
-                
+              
+                {intl.formatMessage({ id: 'audit.logs.detail.responseData', defaultMessage: 'Response Data' })}
+              
+              

+                
-                  {formatJsonData(selectedRecord.response_data)}
+                  {selectedRecord?.response_data ? 
+                    ((() => {
+                      try {
+                        return JSON.stringify(JSON.parse(selectedRecord.response_data), null, 2);
+                      } catch {
+                        return selectedRecord.response_data;
+                      }
+                    })()) : 
+                    intl.formatMessage({ id: 'audit.logs.detail.noData', defaultMessage: 'No data' })
+                  }
                 

-              
-            )}
-          
+              

+            
+          
         )}
-      
+      
     
   );
 };

From bbf4c3c06060cd05650c42fd66bd03f6659b05bd Mon Sep 17 00:00:00 2001
From: Guo Ziang 
Date: Sat, 21 Jun 2025 00:39:30 +0800
Subject: [PATCH 06/19] chore: tidy up

---
 aperag/api/components/schemas/audit.yaml      |   2 +-
 aperag/api/paths/audit.yaml                   |  38 ---
 aperag/app.py                                 |   4 -
 aperag/db/models.py                           |   7 +
 aperag/middleware/audit_middleware.py         | 234 ------------------
 ...c810.py => 20250621002836-2768dfee8bbc.py} |   8 +-
 aperag/schema/view_models.py                  |   5 +-
 aperag/service/audit_service.py               |   7 +
 aperag/utils/audit_decorator.py               | 213 ++++++++++++++++
 aperag/views/api_key.py                       |   4 +
 aperag/views/auth.py                          |  12 +-
 aperag/views/flow.py                          |   2 +
 aperag/views/llm.py                           |   9 +-
 aperag/views/main.py                          |  23 ++
 14 files changed, 279 insertions(+), 289 deletions(-)
 delete mode 100644 aperag/middleware/audit_middleware.py
 rename aperag/migration/versions/{20250620172242-3369b50dc810.py => 20250621002836-2768dfee8bbc.py} (92%)
 create mode 100644 aperag/utils/audit_decorator.py

diff --git a/aperag/api/components/schemas/audit.yaml b/aperag/api/components/schemas/audit.yaml
index 51d082fb9..dd8710d49 100644
--- a/aperag/api/components/schemas/audit.yaml
+++ b/aperag/api/components/schemas/audit.yaml
@@ -15,7 +15,7 @@ auditLog:
       description: Username for display
     resource_type:
       type: string
-      enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config]
+      enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, flow, search_test]
       nullable: true
       description: Type of resource
     resource_id:
diff --git a/aperag/api/paths/audit.yaml b/aperag/api/paths/audit.yaml
index 44b808e7d..839d8db3f 100644
--- a/aperag/api/paths/audit.yaml
+++ b/aperag/api/paths/audit.yaml
@@ -6,50 +6,12 @@ audit_logs:
     description: List audit logs with filtering options
     operationId: list_audit_logs
     parameters:
-      - name: user_id
-        in: query
-        required: false
-        schema:
-          type: string
-        description: Filter by user ID
-      - name: username
-        in: query
-        required: false
-        schema:
-          type: string
-        description: Filter by username
-      - name: resource_type
-        in: query
-        required: false
-        schema:
-          type: string
-          enum: [collection, document, bot, chat, message, api_key, llm_provider, llm_provider_model, model_service_provider, user, config]
-        description: Filter by resource type
-      - name: resource_id
-        in: query
-        required: false
-        schema:
-          type: string
-        description: Filter by resource ID
       - name: api_name
         in: query
         required: false
         schema:
           type: string
         description: Filter by API name
-      - name: http_method
-        in: query
-        required: false
-        schema:
-          type: string
-          enum: [POST, PUT, DELETE]
-        description: Filter by HTTP method
-      - name: status_code
-        in: query
-        required: false
-        schema:
-          type: integer
-        description: Filter by status code
       - name: start_date
         in: query
         required: false
diff --git a/aperag/app.py b/aperag/app.py
index 05d5bec45..f8ad72fdf 100644
--- a/aperag/app.py
+++ b/aperag/app.py
@@ -16,7 +16,6 @@
 
 from aperag.exception_handlers import register_exception_handlers
 from aperag.llm.litellm_track import register_opik_llm_track
-from aperag.middleware.audit_middleware import AuditMiddleware
 from aperag.views.api_key import router as api_key_router
 from aperag.views.audit import router as audit_router
 from aperag.views.auth import router as auth_router
@@ -37,9 +36,6 @@
 
 register_opik_llm_track()
 
-# Add audit middleware - should be added before other middlewares/routers
-app.add_middleware(AuditMiddleware, enabled=True)
-
 app.include_router(auth_router, prefix="/api/v1")
 app.include_router(main_router, prefix="/api/v1")
 app.include_router(api_key_router, prefix="/api/v1")
diff --git a/aperag/db/models.py b/aperag/db/models.py
index 2016056b0..dde51852e 100644
--- a/aperag/db/models.py
+++ b/aperag/db/models.py
@@ -764,6 +764,13 @@ class AuditResource(str, Enum):
     MODEL_SERVICE_PROVIDER = "model_service_provider"
     USER = "user"
     CONFIG = "config"
+    INVITATION = "invitation"
+    AUTH = "auth"
+    CHAT_COMPLETION = "chat_completion"
+    SEARCH_TEST = "search_test"
+    LLM = "llm"
+    FLOW = "flow"
+    SYSTEM = "system"
 
 
 class AuditLog(Base):
diff --git a/aperag/middleware/audit_middleware.py b/aperag/middleware/audit_middleware.py
deleted file mode 100644
index 947a29363..000000000
--- a/aperag/middleware/audit_middleware.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2025 ApeCloud, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import logging
-import time
-from typing import Any, Dict, Optional, Tuple
-
-from fastapi import Request, Response
-from starlette.middleware.base import BaseHTTPMiddleware
-
-from aperag.service.audit_service import audit_service
-
-logger = logging.getLogger(__name__)
-
-
-class AuditMiddleware(BaseHTTPMiddleware):
-    """Middleware for automatic audit logging"""
-
-    def __init__(self, app, enabled: bool = True):
-        super().__init__(app)
-        self.enabled = enabled
-        self.exclude_paths = [
-            "/docs", "/openapi.json", "/redoc", "/static",
-            "/health", "/favicon.ico", "/metrics"
-        ]
-
-    def _should_audit(self, path: str, method: str) -> bool:
-        """Check if the request should be audited"""
-        if not self.enabled:
-            return False
-        
-        # Skip GET requests - only audit change operations
-        if method.upper() == "GET":
-            return False
-        
-        # Skip excluded paths
-        for exclude_path in self.exclude_paths:
-            if path.startswith(exclude_path):
-                return False
-        
-        # Only audit API endpoints
-        if not path.startswith("/api/"):
-            return False
-            
-        return True
-
-    def _get_audit_info_from_route(self, request: Request) -> Tuple[Optional[str], Optional[str]]:
-        """Get API name and resource type from route name and tags"""
-        try:
-            if hasattr(request, 'scope') and 'route' in request.scope:
-                route = request.scope['route']
-                
-                # Get API name from route name
-                api_name = None
-                if hasattr(route, 'name') and route.name:
-                    api_name = route.name
-                elif hasattr(route, 'endpoint') and hasattr(route.endpoint, '__name__'):
-                    api_name = route.endpoint.__name__
-                
-                # Get resource type from tags
-                resource_type = None
-                if hasattr(route, 'tags') and route.tags:
-                    resource_type = audit_service.get_resource_type_from_tags(route.tags)
-                
-                # Debug logging
-                logger.debug(f"Route info - Path: {request.url.path}, API name: {api_name}, Tags: {getattr(route, 'tags', None)}, Resource type: {resource_type}")
-                
-                # Both API name and resource type are required
-                if api_name and resource_type:
-                    return api_name, resource_type
-                elif api_name or resource_type:
-                    # Log when we have partial info to help debugging
-                    logger.warning(f"Partial audit info - Path: {request.url.path}, API name: {api_name}, Resource type: {resource_type}, Tags: {getattr(route, 'tags', None)}")
-                        
-        except Exception as e:
-            logger.warning(f"Failed to get audit info from route: {e}")
-            
-        return None, None
-
-    async def _extract_request_data(self, request: Request) -> Optional[Dict[str, Any]]:
-        """Extract request data safely"""
-        try:
-            # Get JSON body if available
-            if request.headers.get("content-type", "").startswith("application/json"):
-                body = await request.body()
-                if body:
-                    return json.loads(body.decode())
-            
-            # Get form data if available
-            elif request.headers.get("content-type", "").startswith("application/x-www-form-urlencoded"):
-                form_data = await request.form()
-                return dict(form_data)
-            
-            # Get query parameters
-            if request.query_params:
-                return dict(request.query_params)
-                
-        except Exception as e:
-            logger.warning(f"Failed to extract request data: {e}")
-            
-        return None
-
-    async def dispatch(self, request: Request, call_next):
-        # Check if audit is needed
-        if not self._should_audit(request.url.path, request.method):
-            return await call_next(request)
-
-        # Record start time in milliseconds
-        start_time_ms = int(time.time() * 1000)
-        request_data = None
-        response_data = None
-        error_message = None
-        status_code = 200
-        end_time_ms = None
-
-        try:
-            # Extract request data before calling next
-            request_data = await self._extract_request_data(request)
-            
-            # Call the actual endpoint
-            response = await call_next(request)
-            status_code = response.status_code
-            
-            # Record end time
-            end_time_ms = int(time.time() * 1000)
-            
-            # Get audit info from route (now available after call_next)
-            api_name, resource_type = self._get_audit_info_from_route(request)
-            
-            # Only proceed with audit if we have both api_name and resource_type
-            if api_name and resource_type:
-                # Try to extract response data for non-streaming responses
-                try:
-                    # For FastAPI Response objects, we need to read the response body
-                    if hasattr(response, 'body') and response.body:
-                        response_body = response.body.decode('utf-8')
-                        response_data = json.loads(response_body)
-                    elif hasattr(response, '_body') and response._body:
-                        response_body = response._body.decode('utf-8')
-                        response_data = json.loads(response_body)
-                    # Skip streaming responses and large responses
-                    elif response.status_code < 400 and response.headers.get('content-type', '').startswith('application/json'):
-                        # For successful JSON responses, record a simplified response
-                        response_data = {"status": "success", "code": response.status_code}
-                except Exception as e:
-                    logger.debug(f"Could not extract response data: {e}")
-                
-                # Log audit asynchronously
-                try:
-                    # Get user info from request state (set by auth middleware)
-                    user_id = getattr(request.state, 'user_id', None)
-                    username = getattr(request.state, 'username', None)
-                    
-                    # Extract client info
-                    ip_address, user_agent = audit_service._extract_client_info(request)
-                    
-                    # Log audit in background (don't await to avoid blocking)
-                    import asyncio
-                    asyncio.create_task(
-                        audit_service.log_audit(
-                            user_id=user_id,
-                            username=username,
-                            resource_type=resource_type,
-                            api_name=api_name,
-                            http_method=request.method,
-                            path=request.url.path,
-                            status_code=status_code,
-                            start_time=start_time_ms,
-                            end_time=end_time_ms,
-                            request_data=request_data,
-                            response_data=response_data,
-                            error_message=error_message,
-                            ip_address=ip_address,
-                            user_agent=user_agent
-                        )
-                    )
-                except Exception as audit_error:
-                    logger.error(f"Failed to log audit: {audit_error}")
-                    
-        except Exception as e:
-            error_message = str(e)
-            status_code = 500
-            end_time_ms = int(time.time() * 1000)
-            
-            # Still try to log the error audit if route info is available
-            try:
-                api_name, resource_type = self._get_audit_info_from_route(request)
-                if api_name and resource_type:
-                    # Get user info from request state (set by auth middleware)
-                    user_id = getattr(request.state, 'user_id', None)
-                    username = getattr(request.state, 'username', None)
-                    
-                    # Extract client info
-                    ip_address, user_agent = audit_service._extract_client_info(request)
-                    
-                    # Log audit in background (don't await to avoid blocking)
-                    import asyncio
-                    asyncio.create_task(
-                        audit_service.log_audit(
-                            user_id=user_id,
-                            username=username,
-                            resource_type=resource_type,
-                            api_name=api_name,
-                            http_method=request.method,
-                            path=request.url.path,
-                            status_code=status_code,
-                            start_time=start_time_ms,
-                            end_time=end_time_ms,
-                            request_data=request_data,
-                            response_data=response_data,
-                            error_message=error_message,
-                            ip_address=ip_address,
-                            user_agent=user_agent
-                        )
-                    )
-            except Exception as audit_error:
-                logger.error(f"Failed to log error audit: {audit_error}")
-            
-            # Re-raise for normal error handling
-            raise
-
-        return response 
\ No newline at end of file
diff --git a/aperag/migration/versions/20250620172242-3369b50dc810.py b/aperag/migration/versions/20250621002836-2768dfee8bbc.py
similarity index 92%
rename from aperag/migration/versions/20250620172242-3369b50dc810.py
rename to aperag/migration/versions/20250621002836-2768dfee8bbc.py
index 72bd701d8..6ca1ff9e7 100644
--- a/aperag/migration/versions/20250620172242-3369b50dc810.py
+++ b/aperag/migration/versions/20250621002836-2768dfee8bbc.py
@@ -1,8 +1,8 @@
 """empty message
 
-Revision ID: 3369b50dc810
+Revision ID: 2768dfee8bbc
 Revises: 12ea6d2bf365
-Create Date: 2025-06-20 17:22:42.655570
+Create Date: 2025-06-21 00:28:36.443046
 
 """
 from typing import Sequence, Union
@@ -12,7 +12,7 @@
 
 
 # revision identifiers, used by Alembic.
-revision: str = '3369b50dc810'
+revision: str = '2768dfee8bbc'
 down_revision: Union[str, None] = '12ea6d2bf365'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
@@ -25,7 +25,7 @@ def upgrade() -> None:
     sa.Column('id', sa.String(length=36), nullable=False),
     sa.Column('user_id', sa.String(length=36), nullable=True, comment='User ID'),
     sa.Column('username', sa.String(length=255), nullable=True, comment='Username'),
-    sa.Column('resource_type', sa.Enum('collection', 'document', 'bot', 'chat', 'message', 'api_key', 'llm_provider', 'llm_provider_model', 'model_service_provider', 'user', 'config', name='auditresource'), nullable=True, comment='Resource type'),
+    sa.Column('resource_type', sa.Enum('collection', 'document', 'bot', 'chat', 'message', 'api_key', 'llm_provider', 'llm_provider_model', 'model_service_provider', 'user', 'config', 'invitation', 'auth', 'chat_completion', 'search_test', 'llm', 'flow', 'system', name='auditresource'), nullable=True, comment='Resource type'),
     sa.Column('resource_id', sa.String(length=255), nullable=True, comment='Resource ID (extracted at query time)'),
     sa.Column('api_name', sa.String(length=255), nullable=False, comment='API operation name'),
     sa.Column('http_method', sa.String(length=10), nullable=False, comment='HTTP method (POST, PUT, DELETE)'),
diff --git a/aperag/schema/view_models.py b/aperag/schema/view_models.py
index 56a6f62e1..6d2b1e10f 100644
--- a/aperag/schema/view_models.py
+++ b/aperag/schema/view_models.py
@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  openapi.merged.yaml
-#   timestamp: 2025-06-20T08:44:29+00:00
+#   timestamp: 2025-06-20T16:37:07+00:00
 
 from __future__ import annotations
 
@@ -1069,7 +1069,8 @@ class AuditLog(BaseModel):
             'llm_provider_model',
             'model_service_provider',
             'user',
-            'config',
+            'flow',
+            'search_test',
         ]
     ] = Field(None, description='Type of resource')
     resource_id: Optional[str] = Field(
diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py
index e63b85e24..b7ac47fe7 100644
--- a/aperag/service/audit_service.py
+++ b/aperag/service/audit_service.py
@@ -61,6 +61,13 @@ def __init__(self):
             "user": AuditResource.USER,
             "users": AuditResource.USER,
             "config": AuditResource.CONFIG,
+            "invitation": AuditResource.INVITATION,
+            "invitations": AuditResource.INVITATION,
+            "auth": AuditResource.AUTH,
+            "chat_completion": AuditResource.CHAT_COMPLETION,
+            "search_test": AuditResource.SEARCH_TEST,
+            "llm": AuditResource.LLM,
+            "flow": AuditResource.FLOW,
         }
 
     def _filter_sensitive_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py
new file mode 100644
index 000000000..b49c46f71
--- /dev/null
+++ b/aperag/utils/audit_decorator.py
@@ -0,0 +1,213 @@
+# Copyright 2025 ApeCloud, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import json
+import logging
+import time
+from typing import Any, Dict, Optional
+
+from fastapi import Request
+
+from aperag.service.audit_service import audit_service
+
+logger = logging.getLogger(__name__)
+
+
+def audit_api(resource_type: str, api_name: str = None):
+    """
+    Decorator for API endpoints to enable automatic audit logging
+    
+    Args:
+        resource_type: The resource type for audit (e.g., 'collection', 'user', etc.)
+        api_name: Optional API name override (defaults to function name)
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        async def wrapper(*args, **kwargs):
+            # Find the request object in the arguments
+            request = None
+            for v in kwargs.values():
+                if isinstance(v, Request):
+                    request = v
+                    break
+            
+            if not request:
+                # If no request found, just call the original function
+                return await func(*args, **kwargs)
+            
+            # Skip GET requests - only audit change operations
+            if request.method.upper() == "GET":
+                return await func(*args, **kwargs)
+            
+            # Record start time
+            start_time_ms = int(time.time() * 1000)
+            actual_api_name = api_name or func.__name__
+            
+            try:
+                # Extract request data
+                request_data = await _extract_request_data(request)
+                
+                # Call the original function
+                response = await func(*args, **kwargs)
+                
+                # Record end time
+                end_time_ms = int(time.time() * 1000)
+                
+                # Extract response data
+                response_data = _extract_response_data(response)
+                
+                # Log audit asynchronously
+                await _log_audit_async(
+                    request=request,
+                    resource_type=resource_type,
+                    api_name=actual_api_name,
+                    start_time_ms=start_time_ms,
+                    end_time_ms=end_time_ms,
+                    status_code=200,  # Success
+                    request_data=request_data,
+                    response_data=response_data,
+                    error_message=None
+                )
+                
+                return response
+                
+            except Exception as e:
+                # Record end time for error case
+                end_time_ms = int(time.time() * 1000)
+                
+                # Extract request data if not already done
+                try:
+                    request_data = await _extract_request_data(request)
+                except:
+                    request_data = None
+                
+                # Log audit for error case
+                await _log_audit_async(
+                    request=request,
+                    resource_type=resource_type,
+                    api_name=actual_api_name,
+                    start_time_ms=start_time_ms,
+                    end_time_ms=end_time_ms,
+                    status_code=500,  # Error
+                    request_data=request_data,
+                    response_data={"error": str(e)},
+                    error_message=str(e)
+                )
+                
+                # Re-raise the exception
+                raise
+                
+        return wrapper
+    return decorator
+
+
+async def _extract_request_data(request: Request) -> Optional[Dict[str, Any]]:
+    """Extract request data safely without consuming the body"""
+    try:
+        # Try to get the body if it hasn't been consumed yet
+        # In FastAPI, we need to be careful not to consume the body
+        # that's needed by the actual endpoint
+        
+        # Get query parameters (safe to read multiple times)
+        if request.query_params:
+            return dict(request.query_params)
+        
+        # For now, let's just extract safe data to avoid body consumption issues
+        # We can enhance this later if needed
+        return {
+            "method": request.method,
+            "path": request.url.path,
+            "query_params": dict(request.query_params) if request.query_params else None,
+            "headers": dict(request.headers) if hasattr(request, 'headers') else None
+        }
+            
+    except Exception as e:
+        logger.warning(f"Failed to extract request data: {e}")
+        
+    return None
+
+
+def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]:
+    """Extract response data from the returned response object"""
+    try:
+        # If response is already a dict (common for JSON APIs)
+        if isinstance(response, dict):
+            return response
+        
+        # If response has a dict() method (Pydantic models)
+        elif hasattr(response, 'dict'):
+            return response.dict()
+        
+        # If response has a model_dump() method (Pydantic v2)
+        elif hasattr(response, 'model_dump'):
+            return response.model_dump()
+        
+        # If response is a list of dicts or models
+        elif isinstance(response, list):
+            result = []
+            for item in response:
+                if isinstance(item, dict):
+                    result.append(item)
+                elif hasattr(item, 'dict'):
+                    result.append(item.dict())
+                elif hasattr(item, 'model_dump'):
+                    result.append(item.model_dump())
+                else:
+                    result.append(str(item))
+            return {"items": result}
+        
+        # For other types, try to convert to string
+        else:
+            return {"response": str(response)}
+            
+    except Exception as e:
+        logger.debug(f"Failed to extract response data: {e}")
+        return {"status": "success", "type": type(response).__name__}
+
+
+async def _log_audit_async(request: Request, resource_type: str, api_name: str,
+                          start_time_ms: int, end_time_ms: int, status_code: int,
+                          request_data: dict, response_data: dict, error_message: str = None):
+    """Log audit information asynchronously"""
+    try:
+        # Get user info from request state
+        user_id = getattr(request.state, 'user_id', None)
+        username = getattr(request.state, 'username', None)
+        
+        # Extract client info
+        ip_address, user_agent = audit_service._extract_client_info(request)
+        
+        # Log audit in background
+        import asyncio
+        asyncio.create_task(
+            audit_service.log_audit(
+                user_id=user_id,
+                username=username,
+                resource_type=resource_type,
+                api_name=api_name,
+                http_method=request.method,
+                path=request.url.path,
+                status_code=status_code,
+                start_time=start_time_ms,
+                end_time=end_time_ms,
+                request_data=request_data,
+                response_data=response_data,
+                error_message=error_message,
+                ip_address=ip_address,
+                user_agent=user_agent
+            )
+        )
+    except Exception as audit_error:
+        logger.error(f"Failed to log audit: {audit_error}") 
\ No newline at end of file
diff --git a/aperag/views/api_key.py b/aperag/views/api_key.py
index b5d4821dc..ef857c272 100644
--- a/aperag/views/api_key.py
+++ b/aperag/views/api_key.py
@@ -18,6 +18,7 @@
 from aperag.db.models import User
 from aperag.schema.view_models import ApiKeyCreate, ApiKeyList, ApiKeyUpdate
 from aperag.service.api_key_service import api_key_service
+from aperag.utils.audit_decorator import audit_api
 from aperag.views.auth import current_user
 
 router = APIRouter()
@@ -30,6 +31,7 @@ async def list_api_keys_view(request: Request, user: User = Depends(current_user
 
 
 @router.post("/apikeys", tags=["apikey"], name="CreateApiKey")
+@audit_api(resource_type="apikey", api_name="CreateApiKey")
 async def create_api_key_view(
     request: Request,
     api_key_create: ApiKeyCreate,
@@ -40,12 +42,14 @@ async def create_api_key_view(
 
 
 @router.delete("/apikeys/{apikey_id}", tags=["apikey"], name="DeleteApiKey")
+@audit_api(resource_type="apikey", api_name="DeleteApiKey")
 async def delete_api_key_view(request: Request, apikey_id: str, user: User = Depends(current_user)):
     """Delete an API key"""
     return await api_key_service.delete_api_key(str(user.id), apikey_id)
 
 
 @router.put("/apikeys/{apikey_id}", tags=["apikey"], name="UpdateApiKey")
+@audit_api(resource_type="apikey", api_name="UpdateApiKey")
 async def update_api_key_view(
     request: Request,
     apikey_id: str,
diff --git a/aperag/views/auth.py b/aperag/views/auth.py
index b01288fb3..a1422c216 100644
--- a/aperag/views/auth.py
+++ b/aperag/views/auth.py
@@ -26,6 +26,7 @@
 from aperag.db.models import ApiKey, ApiKeyStatus, Invitation, Role, User
 from aperag.db.ops import async_db_ops
 from aperag.schema import view_models
+from aperag.utils.audit_decorator import audit_api
 from aperag.utils.utils import utc_now
 
 logger = logging.getLogger(__name__)
@@ -241,8 +242,9 @@ async def get_current_admin(session: AsyncSessionDep, user: User = Depends(get_c
 
 
 @router.post("/invite", tags=["invitation"], name="CreateInvitation")
+@audit_api(resource_type="invitation", api_name="CreateInvitation")
 async def create_invitation_view(
-    data: view_models.InvitationCreate, session: AsyncSessionDep, user: User = Depends(get_current_admin)
+    request: Request, data: view_models.InvitationCreate, session: AsyncSessionDep, user: User = Depends(get_current_admin)
 ) -> view_models.Invitation:
     # Check if user already exists
     from sqlalchemy import select
@@ -299,8 +301,9 @@ async def list_invitations_view(
 
 
 @router.post("/register", tags=["auth"], name="Register")
+@audit_api(resource_type="user", api_name="Register")
 async def register_view(
-    data: view_models.Register, session: AsyncSessionDep, user_manager: UserManager = Depends(get_user_manager)
+    request: Request, data: view_models.Register, session: AsyncSessionDep, user_manager: UserManager = Depends(get_user_manager)
 ) -> view_models.User:
     from sqlalchemy import select
 
@@ -440,7 +443,9 @@ async def list_users_view(session: AsyncSessionDep, user: User = Depends(get_cur
 
 
 @router.post("/change-password", tags=["user"], name="ChangePassword")
+@audit_api(resource_type="user", api_name="ChangePassword")
 async def change_password_view(
+    request: Request,
     data: view_models.ChangePassword,
     session: AsyncSessionDep,
     user_manager: UserManager = Depends(get_user_manager),
@@ -470,7 +475,8 @@ async def change_password_view(
 
 
 @router.delete("/users/{user_id}", tags=["user"], name="DeleteUser")
-async def delete_user_view(user_id: str, session: AsyncSessionDep, user: User = Depends(get_current_admin)):
+@audit_api(resource_type="user", api_name="DeleteUser")
+async def delete_user_view(request: Request, user_id: str, session: AsyncSessionDep, user: User = Depends(get_current_admin)):
     from sqlalchemy import select
 
     result = await session.execute(select(User).where(User.id == user_id))
diff --git a/aperag/views/flow.py b/aperag/views/flow.py
index a0057a75f..6e5996469 100644
--- a/aperag/views/flow.py
+++ b/aperag/views/flow.py
@@ -19,6 +19,7 @@
 from aperag.db.models import User
 from aperag.schema.view_models import WorkflowDefinition
 from aperag.service.flow_service import flow_service_global
+from aperag.utils.audit_decorator import audit_api
 from aperag.views.auth import current_user
 
 router = APIRouter()
@@ -32,6 +33,7 @@ async def get_flow_view(
 
 
 @router.put("/bots/{bot_id}/flow", tags=["flow"], name="UpdateFlow")
+@audit_api(resource_type="flow", api_name="UpdateFlow")
 async def update_flow_view(
     request: Request,
     bot_id: str,
diff --git a/aperag/views/llm.py b/aperag/views/llm.py
index 930523088..4bef6923d 100644
--- a/aperag/views/llm.py
+++ b/aperag/views/llm.py
@@ -14,7 +14,7 @@
 
 import logging
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Request
 
 from aperag.db.models import User
 from aperag.db.ops import async_db_ops
@@ -40,6 +40,7 @@
     RerankResponse,
     RerankUsage,
 )
+from aperag.utils.audit_decorator import audit_api
 from aperag.views.auth import current_user
 
 logger = logging.getLogger(__name__)
@@ -48,7 +49,8 @@
 
 
 @router.post("/embeddings", response_model=EmbeddingResponse, tags=["llm"], name="CreateEmbeddings")
-async def create_embeddings(request: EmbeddingRequest, user: User = Depends(current_user)):
+@audit_api(resource_type="llm", api_name="CreateEmbeddings")
+async def create_embeddings(http_request: Request, request: EmbeddingRequest, user: User = Depends(current_user)):
     """
     Create embeddings for the given input text(s).
     Compatible with OpenAI embeddings API format.
@@ -157,7 +159,8 @@ async def _get_provider_info(provider: str, model: str, user_id: str, api_type:
 
 
 @router.post("/rerank", response_model=RerankResponse, tags=["llm"], name="CreateRerank")
-async def create_rerank(request: RerankRequest, user: User = Depends(current_user)):
+@audit_api(resource_type="llm", api_name="CreateRerank")
+async def create_rerank(http_request: Request, request: RerankRequest, user: User = Depends(current_user)):
     """
     Rerank documents based on relevance to a query.
     Compatible with industry-standard rerank API format used by Cohere, Jina AI, etc.
diff --git a/aperag/views/main.py b/aperag/views/main.py
index 494ea0d7f..2cee4df24 100644
--- a/aperag/views/main.py
+++ b/aperag/views/main.py
@@ -37,6 +37,7 @@
     update_llm_provider_model,
 )
 from aperag.service.prompt_template_service import list_prompt_templates
+from aperag.utils.audit_decorator import audit_api
 
 # Import authentication dependencies
 from aperag.views.auth import UserManager, authenticate_websocket_user, current_user, get_user_manager
@@ -55,6 +56,7 @@ async def list_prompt_templates_view(
 
 
 @router.post("/collections", tags=["collection"], name="CreateCollection")
+@audit_api(resource_type="collection", api_name="CreateCollection")
 async def create_collection_view(
     request: Request,
     collection: view_models.CollectionCreate,
@@ -76,6 +78,7 @@ async def get_collection_view(
 
 
 @router.put("/collections/{collection_id}", tags=["collection"], name="UpdateCollection")
+@audit_api(resource_type="collection", api_name="UpdateCollection")
 async def update_collection_view(
     request: Request,
     collection_id: str,
@@ -86,6 +89,7 @@ async def update_collection_view(
 
 
 @router.delete("/collections/{collection_id}", tags=["collection"], name="DeleteCollection")
+@audit_api(resource_type="collection", api_name="DeleteCollection")
 async def delete_collection_view(
     request: Request, collection_id: str, user: User = Depends(current_user)
 ) -> view_models.Collection:
@@ -93,6 +97,7 @@ async def delete_collection_view(
 
 
 @router.post("/collections/{collection_id}/documents", tags=["document"], name="CreateDocuments")
+@audit_api(resource_type="document", api_name="CreateDocuments")
 async def create_documents_view(
     request: Request,
     collection_id: str,
@@ -120,6 +125,7 @@ async def get_document_view(
 
 
 @router.put("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="UpdateDocument")
+@audit_api(resource_type="document", api_name="UpdateDocument")
 async def update_document_view(
     request: Request,
     collection_id: str,
@@ -131,6 +137,7 @@ async def update_document_view(
 
 
 @router.delete("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="DeleteDocument")
+@audit_api(resource_type="document", api_name="DeleteDocument")
 async def delete_document_view(
     request: Request,
     collection_id: str,
@@ -141,6 +148,7 @@ async def delete_document_view(
 
 
 @router.delete("/collections/{collection_id}/documents", tags=["document"], name="DeleteDocuments")
+@audit_api(resource_type="document", api_name="DeleteDocuments")
 async def delete_documents_view(
     request: Request,
     collection_id: str,
@@ -151,6 +159,7 @@ async def delete_documents_view(
 
 
 @router.post("/bots/{bot_id}/chats", tags=["chat"], name="CreateChat")
+@audit_api(resource_type="chat", api_name="CreateChat")
 async def create_chat_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.Chat:
     return await chat_service_global.create_chat(str(user.id), bot_id)
 
@@ -168,6 +177,7 @@ async def get_chat_view(
 
 
 @router.put("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="UpdateChat")
+@audit_api(resource_type="chat", api_name="UpdateChat")
 async def update_chat_view(
     request: Request,
     bot_id: str,
@@ -179,6 +189,7 @@ async def update_chat_view(
 
 
 @router.post("/bots/{bot_id}/chats/{chat_id}/messages/{message_id}", tags=["message"], name="FeedbackMessage")
+@audit_api(resource_type="message", api_name="FeedbackMessage")
 async def feedback_message_view(
     request: Request,
     bot_id: str,
@@ -193,12 +204,14 @@ async def feedback_message_view(
 
 
 @router.delete("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="DeleteChat")
+@audit_api(resource_type="chat", api_name="DeleteChat")
 async def delete_chat_view(request: Request, bot_id: str, chat_id: str, user: User = Depends(current_user)):
     await chat_service_global.delete_chat(str(user.id), bot_id, chat_id)
     return Response(status_code=204)
 
 
 @router.post("/bots", tags=["bot"], name="CreateBot")
+@audit_api(resource_type="bot", api_name="CreateBot")
 async def create_bot_view(
     request: Request,
     bot_in: view_models.BotCreate,
@@ -218,6 +231,7 @@ async def get_bot_view(request: Request, bot_id: str, user: User = Depends(curre
 
 
 @router.put("/bots/{bot_id}", tags=["bot"], name="UpdateBot")
+@audit_api(resource_type="bot", api_name="UpdateBot")
 async def update_bot_view(
     request: Request,
     bot_id: str,
@@ -228,6 +242,7 @@ async def update_bot_view(
 
 
 @router.delete("/bots/{bot_id}", tags=["bot"], name="DeleteBot")
+@audit_api(resource_type="bot", api_name="DeleteBot")
 async def delete_bot_view(request: Request, bot_id: str, user: User = Depends(current_user)):
     await bot_service.delete_bot(str(user.id), bot_id)
     return Response(status_code=204)
@@ -260,6 +275,7 @@ async def frontend_chat_completions_view(request: Request, user: User = Depends(
 
 
 @router.post("/collections/{collection_id}/searchTests", tags=["search_test"], name="CreateSearchTest")
+@audit_api(resource_type="search_test", api_name="CreateSearchTest")
 async def create_search_test_view(
     request: Request,
     collection_id: str,
@@ -270,6 +286,7 @@ async def create_search_test_view(
 
 
 @router.delete("/collections/{collection_id}/searchTests/{search_test_id}", tags=["search_test"], name="DeleteSearchTest")
+@audit_api(resource_type="search_test", api_name="DeleteSearchTest")
 async def delete_search_test_view(
     request: Request,
     collection_id: str,
@@ -323,6 +340,7 @@ async def get_llm_configuration_view(request: Request, user: User = Depends(curr
 
 
 @router.post("/llm_providers", tags=["llm_provider"], name="CreateLLMProvider")
+@audit_api(resource_type="llm_provider", api_name="CreateLLMProvider")
 async def create_llm_provider_view(
     request: Request,
     provider_data: view_models.LlmProviderCreateWithApiKey,
@@ -345,6 +363,7 @@ async def get_llm_provider_view(request: Request, provider_name: str, user: User
 
 
 @router.put("/llm_providers/{provider_name}", tags=["llm_provider"], name="UpdateLLMProvider")
+@audit_api(resource_type="llm_provider", api_name="UpdateLLMProvider")
 async def update_llm_provider_view(
     request: Request,
     provider_name: str,
@@ -359,6 +378,7 @@ async def update_llm_provider_view(
 
 
 @router.delete("/llm_providers/{provider_name}", tags=["llm_provider"], name="DeleteLLMProvider")
+@audit_api(resource_type="llm_provider", api_name="DeleteLLMProvider")
 async def delete_llm_provider_view(request: Request, provider_name: str, user: User = Depends(current_user)):
     """Delete an LLM provider"""
     from aperag.db.models import Role
@@ -388,6 +408,7 @@ async def get_provider_models_view(request: Request, provider_name: str, user: U
 
 
 @router.post("/llm_providers/{provider_name}/models", tags=["llm_provider_model"], name="CreateProviderModel")
+@audit_api(resource_type="llm_provider_model", api_name="CreateProviderModel")
 async def create_provider_model_view(request: Request, provider_name: str, user: User = Depends(current_user)):
     """Create a new model for a specific provider"""
     import json
@@ -401,6 +422,7 @@ async def create_provider_model_view(request: Request, provider_name: str, user:
 
 
 @router.put("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="UpdateProviderModel")
+@audit_api(resource_type="llm_provider_model", api_name="UpdateProviderModel")
 async def update_provider_model_view(
     request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user)
 ):
@@ -416,6 +438,7 @@ async def update_provider_model_view(
 
 
 @router.delete("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="DeleteProviderModel")
+@audit_api(resource_type="llm_provider_model", api_name="DeleteProviderModel")
 async def delete_provider_model_view(
     request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user)
 ):

From 115ada46119c80ca06fb4a2a08ee9851b036432b Mon Sep 17 00:00:00 2001
From: Guo Ziang 
Date: Sat, 21 Jun 2025 00:49:10 +0800
Subject: [PATCH 07/19] chore: tidy up

---
 frontend/src/pages/settings/auditLogs.tsx | 24 ++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx
index 4ad0ad939..0c8619226 100644
--- a/frontend/src/pages/settings/auditLogs.tsx
+++ b/frontend/src/pages/settings/auditLogs.tsx
@@ -14,6 +14,7 @@ import {
   Descriptions,
   Divider,
   Tooltip,
+  theme,
 } from 'antd';
 import { SearchOutlined, EyeOutlined } from '@ant-design/icons';
 import { useIntl } from 'umi';
@@ -27,6 +28,7 @@ const { Text, Title } = Typography;
 
 const AuditLogsPage: React.FC = () => {
   const intl = useIntl();
+  const { token } = theme.useToken();
   const [form] = Form.useForm();
   const [loading, setLoading] = useState(false);
   const [data, setData] = useState([]);
@@ -192,11 +194,11 @@ const AuditLogsPage: React.FC = () => {
       title: intl.formatMessage({ id: 'audit.logs.startTime', defaultMessage: 'Start Time' }),
       dataIndex: 'start_time',
       key: 'start_time',
-      width: 160,
+      width: 180,
       render: (time?: number) => (
         time ? (
           
-            {dayjs(time).format('MM-DD HH:mm:ss')}
+            {dayjs(time).format('YYYY-MM-DD HH:mm:ss.SSS')}
           
         ) : '-'
       ),
@@ -205,11 +207,11 @@ const AuditLogsPage: React.FC = () => {
       title: intl.formatMessage({ id: 'audit.logs.endTime', defaultMessage: 'End Time' }),
       dataIndex: 'end_time',
       key: 'end_time',
-      width: 160,
+      width: 180,
       render: (time?: number) => (
         time ? (
           
-            {dayjs(time).format('MM-DD HH:mm:ss')}
+            {dayjs(time).format('YYYY-MM-DD HH:mm:ss.SSS')}
           
         ) : '-'
       ),
@@ -302,7 +304,7 @@ const AuditLogsPage: React.FC = () => {
             pageSizeOptions: ['20', '50', '100'],
             defaultPageSize: 20,
           }}
-          scroll={{ x: 1200 }}
+          scroll={{ x: 1240 }}
           size="small"
           bordered
         />
@@ -324,9 +326,9 @@ const AuditLogsPage: React.FC = () => {
               

@@ -337,7 +339,7 @@ const AuditLogsPage: React.FC = () => {
                   whiteSpace: 'pre-wrap',
                   wordBreak: 'break-word',
                   fontFamily: 'Monaco, Menlo, "Ubuntu Mono", Consolas, monospace',
-                  color: '#d4d4d4',
+                  color: token.colorText,
                 }}>
                   {selectedRecord?.request_data ? 
                     ((() => {
@@ -360,9 +362,9 @@ const AuditLogsPage: React.FC = () => {
               

@@ -373,7 +375,7 @@ const AuditLogsPage: React.FC = () => {
                   whiteSpace: 'pre-wrap',
                   wordBreak: 'break-word',
                   fontFamily: 'Monaco, Menlo, "Ubuntu Mono", Consolas, monospace',
-                  color: '#d4d4d4',
+                  color: token.colorText,
                 }}>
                   {selectedRecord?.response_data ? 
                     ((() => {

From d0d68ee92f07d013dc6677948f6d298c7484a1c8 Mon Sep 17 00:00:00 2001
From: Guo Ziang 
Date: Sat, 21 Jun 2025 11:59:25 +0800
Subject: [PATCH 08/19] chore: tidy up

---
 aperag/utils/audit_decorator.py           | 175 +++++++++++++++++++---
 aperag/views/auth.py                      |   2 +-
 frontend/src/pages/settings/auditLogs.tsx |  88 +++++++----
 3 files changed, 215 insertions(+), 50 deletions(-)

diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py
index b49c46f71..97398b575 100644
--- a/aperag/utils/audit_decorator.py
+++ b/aperag/utils/audit_decorator.py
@@ -56,15 +56,15 @@ async def wrapper(*args, **kwargs):
             actual_api_name = api_name or func.__name__
             
             try:
-                # Extract request data
-                request_data = await _extract_request_data(request)
-                
-                # Call the original function
+                # Call the original function first to get the parsed data
                 response = await func(*args, **kwargs)
                 
                 # Record end time
                 end_time_ms = int(time.time() * 1000)
                 
+                # Extract request data from function arguments (after parsing)
+                request_data = _extract_request_data_from_args(request, kwargs)
+                
                 # Extract response data
                 response_data = _extract_response_data(response)
                 
@@ -87,11 +87,11 @@ async def wrapper(*args, **kwargs):
                 # Record end time for error case
                 end_time_ms = int(time.time() * 1000)
                 
-                # Extract request data if not already done
+                # Extract request data if possible
                 try:
-                    request_data = await _extract_request_data(request)
+                    request_data = _extract_request_data_from_args(request, kwargs)
                 except:
-                    request_data = None
+                    request_data = {"method": request.method, "path": request.url.path}
                 
                 # Log audit for error case
                 await _log_audit_async(
@@ -113,30 +113,163 @@ async def wrapper(*args, **kwargs):
     return decorator
 
 
+def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[Dict[str, Any]]:
+    """Extract request data from function arguments (after FastAPI parsing)"""
+    try:
+        # Extract parsed data from function arguments
+        # FastAPI injects parsed JSON data as function parameters
+        parsed_data = {}
+        for key, value in kwargs.items():
+            # Skip the request object itself
+            if isinstance(value, Request):
+                continue
+            
+            # Skip User objects and other database model objects
+            if hasattr(value, '__tablename__'):  # SQLAlchemy model
+                continue
+                
+            # Try to serialize the value
+            try:
+                if hasattr(value, 'dict'):  # Pydantic model
+                    serialized = value.dict()
+                    # Clean up the serialized data - remove null values and filter sensitive data
+                    cleaned_data = _clean_data_for_audit(serialized)
+                    if cleaned_data:  # Only add if there's actual data
+                        parsed_data[key] = cleaned_data
+                elif hasattr(value, 'model_dump'):  # Pydantic v2
+                    serialized = value.model_dump()
+                    # Clean up the serialized data
+                    cleaned_data = _clean_data_for_audit(serialized)
+                    if cleaned_data:  # Only add if there's actual data
+                        parsed_data[key] = cleaned_data
+                elif isinstance(value, (dict, list, str, int, float, bool)):
+                    # For basic types, also clean the data
+                    cleaned_data = _clean_data_for_audit(value)
+                    if cleaned_data is not None:  # Allow False, 0, empty string but not None
+                        parsed_data[key] = cleaned_data
+                else:
+                    # For other types, convert to string but skip if it looks like an object
+                    str_value = str(value)
+                    if not (' object at 0x' in str_value):  # Skip object representations
+                        parsed_data[key] = str_value
+            except Exception:
+                # Skip problematic values
+                continue
+        
+        # Return the actual data directly, not wrapped in any structure
+        # If there's only one main data object, return it directly
+        if len(parsed_data) == 1:
+            return list(parsed_data.values())[0]
+        elif len(parsed_data) > 1:
+            return parsed_data
+        else:
+            return None
+            
+    except Exception as e:
+        logger.warning(f"Failed to extract request data from args: {e}")
+        return None
+
+
+def _clean_data_for_audit(data):
+    """Clean data for audit logging - remove null values and sensitive information"""
+    if data is None:
+        return None
+    
+    if isinstance(data, dict):
+        cleaned = {}
+        for key, value in data.items():
+            # Skip null/None values
+            if value is None:
+                continue
+            
+            # Filter out sensitive fields
+            key_lower = key.lower()
+            if any(sensitive in key_lower for sensitive in ['password', 'secret', 'token', 'key']):
+                cleaned[key] = "***FILTERED***"
+            else:
+                # Recursively clean nested data
+                cleaned_value = _clean_data_for_audit(value)
+                if cleaned_value is not None or isinstance(cleaned_value, (bool, int, float, str)):
+                    # Keep non-null values and primitive types (including False, 0, empty string)
+                    if not (isinstance(cleaned_value, dict) and len(cleaned_value) == 0):
+                        # Don't add empty dicts
+                        cleaned[key] = cleaned_value
+        
+        return cleaned if cleaned else None
+    
+    elif isinstance(data, list):
+        cleaned = []
+        for item in data:
+            cleaned_item = _clean_data_for_audit(item)
+            if cleaned_item is not None:
+                cleaned.append(cleaned_item)
+        return cleaned if cleaned else None
+    
+    else:
+        # For primitive types, return as-is
+        return data
+
+
 async def _extract_request_data(request: Request) -> Optional[Dict[str, Any]]:
     """Extract request data safely without consuming the body"""
     try:
-        # Try to get the body if it hasn't been consumed yet
-        # In FastAPI, we need to be careful not to consume the body
-        # that's needed by the actual endpoint
+        request_data = {
+            "method": request.method,
+            "path": request.url.path,
+        }
         
-        # Get query parameters (safe to read multiple times)
+        # Add query parameters if present
         if request.query_params:
-            return dict(request.query_params)
+            request_data["query_params"] = dict(request.query_params)
+        
+        # For POST/PUT/PATCH requests, try to extract JSON body
+        if request.method.upper() in ["POST", "PUT", "PATCH"]:
+            try:
+                # Check if content type is JSON
+                content_type = request.headers.get("content-type", "")
+                if "application/json" in content_type:
+                    # Read the body - this is safe in FastAPI decorators
+                    # because the body will be re-read by FastAPI's dependency injection
+                    body = await request.body()
+                    if body:
+                        import json
+                        body_data = json.loads(body.decode('utf-8'))
+                        request_data["body"] = body_data
+                elif "application/x-www-form-urlencoded" in content_type:
+                    # Handle form data
+                    form_data = await request.form()
+                    request_data["form_data"] = dict(form_data)
+                elif "multipart/form-data" in content_type:
+                    # Handle multipart form data (files)
+                    form_data = await request.form()
+                    form_dict = {}
+                    for key, value in form_data.items():
+                        if hasattr(value, 'filename'):  # File upload
+                            form_dict[key] = f""
+                        else:
+                            form_dict[key] = str(value)
+                    request_data["form_data"] = form_dict
+            except Exception as body_error:
+                logger.debug(f"Failed to extract request body: {body_error}")
+                request_data["body_error"] = str(body_error)
         
-        # For now, let's just extract safe data to avoid body consumption issues
-        # We can enhance this later if needed
+        # Add relevant headers (filter out sensitive ones)
+        filtered_headers = {}
+        for key, value in request.headers.items():
+            key_lower = key.lower()
+            if key_lower not in ['authorization', 'cookie', 'x-api-key']:
+                filtered_headers[key] = value
+        request_data["headers"] = filtered_headers
+        
+        return request_data
+            
+    except Exception as e:
+        logger.warning(f"Failed to extract request data: {e}")
         return {
             "method": request.method,
             "path": request.url.path,
-            "query_params": dict(request.query_params) if request.query_params else None,
-            "headers": dict(request.headers) if hasattr(request, 'headers') else None
+            "error": str(e)
         }
-            
-    except Exception as e:
-        logger.warning(f"Failed to extract request data: {e}")
-        
-    return None
 
 
 def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]:
diff --git a/aperag/views/auth.py b/aperag/views/auth.py
index a1422c216..1f9dafa57 100644
--- a/aperag/views/auth.py
+++ b/aperag/views/auth.py
@@ -301,7 +301,7 @@ async def list_invitations_view(
 
 
 @router.post("/register", tags=["auth"], name="Register")
-@audit_api(resource_type="user", api_name="Register")
+@audit_api(resource_type="user", api_name="RegisterUser")
 async def register_view(
     request: Request, data: view_models.Register, session: AsyncSessionDep, user_manager: UserManager = Depends(get_user_manager)
 ) -> view_models.User:
diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx
index 0c8619226..25912836b 100644
--- a/frontend/src/pages/settings/auditLogs.tsx
+++ b/frontend/src/pages/settings/auditLogs.tsx
@@ -16,7 +16,7 @@ import {
   Tooltip,
   theme,
 } from 'antd';
-import { SearchOutlined, EyeOutlined } from '@ant-design/icons';
+import { SearchOutlined, EyeOutlined, CopyOutlined } from '@ant-design/icons';
 import { useIntl } from 'umi';
 import type { ColumnsType } from 'antd/es/table';
 import dayjs from 'dayjs';
@@ -123,6 +123,34 @@ const AuditLogsPage: React.FC = () => {
     }
   };
 
+  // Handle copy to clipboard
+  const handleCopy = async (text: string, type: 'request' | 'response') => {
+    try {
+      await navigator.clipboard.writeText(text);
+      message.success(
+        intl.formatMessage(
+          { id: 'audit.logs.copySuccess', defaultMessage: '{type} data copied to clipboard' },
+          { type: type === 'request' ? 'Request' : 'Response' }
+        )
+      );
+    } catch (error) {
+      message.error(intl.formatMessage({ id: 'audit.logs.copyError', defaultMessage: 'Failed to copy to clipboard' }));
+    }
+  };
+
+  // Format JSON data for display
+  const formatJsonData = (data: any): string => {
+    if (!data) return '';
+    try {
+      if (typeof data === 'string') {
+        return JSON.stringify(JSON.parse(data), null, 2);
+      }
+      return JSON.stringify(data, null, 2);
+    } catch {
+      return typeof data === 'string' ? data : JSON.stringify(data);
+    }
+  };
+
   // Table columns
   const columns: ColumnsType = [
     {
@@ -269,7 +297,7 @@ const AuditLogsPage: React.FC = () => {
                   intl.formatMessage({ id: 'audit.logs.startTime', defaultMessage: 'Start Time' }),
                   intl.formatMessage({ id: 'audit.logs.endTime', defaultMessage: 'End Time' })
                 ]}
-                style={{ width: 350 }}
+                style={{ width: 420 }}
               />
             
 
@@ -320,9 +348,20 @@ const AuditLogsPage: React.FC = () => {
         {selectedRecord && (
           
             

-              
-                {intl.formatMessage({ id: 'audit.logs.detail.requestData', defaultMessage: 'Request Data' })}
-              
+              

+                
+                  {intl.formatMessage({ id: 'audit.logs.detail.requestData', defaultMessage: 'Request Data' })}
+                
+                
+              

               
 {
                   fontFamily: 'Monaco, Menlo, "Ubuntu Mono", Consolas, monospace',
                   color: token.colorText,
                 }}>
-                  {selectedRecord?.request_data ? 
-                    ((() => {
-                      try {
-                        return JSON.stringify(JSON.parse(selectedRecord.request_data), null, 2);
-                      } catch {
-                        return selectedRecord.request_data;
-                      }
-                    })()) : 
-                    intl.formatMessage({ id: 'audit.logs.detail.noData', defaultMessage: 'No data' })
-                  }
+                  {formatJsonData(selectedRecord.request_data)}
                 

               
             
             
             

-              
-                {intl.formatMessage({ id: 'audit.logs.detail.responseData', defaultMessage: 'Response Data' })}
-              
+              

+                
+                  {intl.formatMessage({ id: 'audit.logs.detail.responseData', defaultMessage: 'Response Data' })}
+                
+                
+              

               
 {
                   fontFamily: 'Monaco, Menlo, "Ubuntu Mono", Consolas, monospace',
                   color: token.colorText,
                 }}>
-                  {selectedRecord?.response_data ? 
-                    ((() => {
-                      try {
-                        return JSON.stringify(JSON.parse(selectedRecord.response_data), null, 2);
-                      } catch {
-                        return selectedRecord.response_data;
-                      }
-                    })()) : 
-                    intl.formatMessage({ id: 'audit.logs.detail.noData', defaultMessage: 'No data' })
-                  }
+                  {formatJsonData(selectedRecord.response_data)}
From b8ec444049eee1f5be94a3057be88118000c3e80 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sat, 21 Jun 2025 12:01:47 +0800 Subject: [PATCH 09/19] chore: tidy up --- aperag/utils/audit_decorator.py | 62 --------------------------------- 1 file changed, 62 deletions(-) diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py index 97398b575..0147e0e8d 100644 --- a/aperag/utils/audit_decorator.py +++ b/aperag/utils/audit_decorator.py @@ -210,68 +210,6 @@ def _clean_data_for_audit(data): return data -async def _extract_request_data(request: Request) -> Optional[Dict[str, Any]]: - """Extract request data safely without consuming the body""" - try: - request_data = { - "method": request.method, - "path": request.url.path, - } - - # Add query parameters if present - if request.query_params: - request_data["query_params"] = dict(request.query_params) - - # For POST/PUT/PATCH requests, try to extract JSON body - if request.method.upper() in ["POST", "PUT", "PATCH"]: - try: - # Check if content type is JSON - content_type = request.headers.get("content-type", "") - if "application/json" in content_type: - # Read the body - this is safe in FastAPI decorators - # because the body will be re-read by FastAPI's dependency injection - body = await request.body() - if body: - import json - body_data = json.loads(body.decode('utf-8')) - request_data["body"] = body_data - elif "application/x-www-form-urlencoded" in content_type: - # Handle form data - form_data = await request.form() - request_data["form_data"] = dict(form_data) - elif "multipart/form-data" in content_type: - # Handle multipart form data (files) - form_data = await request.form() - form_dict = {} - for key, value in form_data.items(): - if hasattr(value, 'filename'): # File upload - form_dict[key] = f"" - else: - form_dict[key] = str(value) - request_data["form_data"] = form_dict - except Exception as body_error: - logger.debug(f"Failed to extract request body: {body_error}") - request_data["body_error"] = str(body_error) - - # Add relevant headers (filter out sensitive ones) - filtered_headers = {} - for key, value in request.headers.items(): - key_lower = key.lower() - if key_lower not in ['authorization', 'cookie', 'x-api-key']: - filtered_headers[key] = value - request_data["headers"] = filtered_headers - - return request_data - - except Exception as e: - logger.warning(f"Failed to extract request data: {e}") - return { - "method": request.method, - "path": request.url.path, - "error": str(e) - } - - def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]: """Extract response data from the returned response object""" try: From 0247a5d26f8f19d8f8970ca8f6875eafaf3803f8 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sat, 21 Jun 2025 12:15:55 +0800 Subject: [PATCH 10/19] chore: tidy up --- aperag/db/models.py | 2 +- aperag/views/audit.py | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/aperag/db/models.py b/aperag/db/models.py index dde51852e..0e3b3c6cc 100644 --- a/aperag/db/models.py +++ b/aperag/db/models.py @@ -795,7 +795,7 @@ class AuditLog(Base): request_id = Column(String(255), nullable=False, comment="Request ID for tracking") start_time = Column(BigInteger, nullable=False, comment="Request start time (milliseconds since epoch)") end_time = Column(BigInteger, nullable=True, comment="Request end time (milliseconds since epoch)") - gmt_created = Column(DateTime(timezone=True), nullable=False, default=func.now(), comment="Created time") + gmt_created = Column(DateTime(timezone=True), nullable=False, default=utc_now, comment="Created time") # Index for better query performance __table_args__ = ( diff --git a/aperag/views/audit.py b/aperag/views/audit.py index 81b5e5bd0..3459ba736 100644 --- a/aperag/views/audit.py +++ b/aperag/views/audit.py @@ -149,21 +149,3 @@ async def get_audit_log( ) -@router.get("/audit/logs", tags=["audit"], name="ListAuditLogs") -async def list_audit_logs_view( - page: int = Query(1, ge=1, description="Page number"), - limit: int = Query(20, ge=1, le=100, description="Items per page"), - resource_type: Optional[str] = Query(None, description="Filter by resource type"), - api_name: Optional[str] = Query(None, description="Filter by API name"), - user: User = Depends(current_user), -) -> view_models.AuditLogList: - """List audit logs with filtering and pagination""" - return await audit_service.list_audit_logs( - page=page, - limit=limit, - resource_type=resource_type, - api_name=api_name, - ) - - - \ No newline at end of file From 04e8bdeb1271696a92a8e52c711e42c979369aae Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 00:11:39 +0800 Subject: [PATCH 11/19] chore: tidy up --- aperag/service/audit_service.py | 38 ---- aperag/utils/audit_decorator.py | 341 +++++++++++++++++--------------- 2 files changed, 184 insertions(+), 195 deletions(-) diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index b7ac47fe7..460f0cd8b 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -115,44 +115,6 @@ def json_serializer(obj): logger.warning(f"Failed to serialize data: {e}") return str(data) - def _extract_client_info(self, request) -> tuple[Optional[str], Optional[str]]: - """Extract client IP and User-Agent from request""" - try: - # Get IP address - ip_address = None - if hasattr(request, 'client') and request.client: - ip_address = request.client.host - - # Check for forwarded headers - if hasattr(request, 'headers'): - forwarded_for = request.headers.get('X-Forwarded-For') - if forwarded_for: - ip_address = forwarded_for.split(',')[0].strip() - elif request.headers.get('X-Real-IP'): - ip_address = request.headers.get('X-Real-IP') - - # Get User-Agent - user_agent = None - if hasattr(request, 'headers'): - user_agent = request.headers.get('User-Agent') - - return ip_address, user_agent - except Exception as e: - logger.warning(f"Failed to extract client info: {e}") - return None, None - - def get_resource_type_from_tags(self, tags: List[str]) -> Optional[AuditResource]: - """Get resource type from FastAPI tags""" - if not tags: - return None - - # Find the first tag that matches our resource mapping - for tag in tags: - if tag in self.tag_resource_map: - return self.tag_resource_map[tag] - - return None - def extract_resource_id_from_path(self, path: str, resource_type: AuditResource) -> Optional[str]: """Extract resource ID from path - called during query time""" try: diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py index 0147e0e8d..10534c773 100644 --- a/aperag/utils/audit_decorator.py +++ b/aperag/utils/audit_decorator.py @@ -25,92 +25,82 @@ logger = logging.getLogger(__name__) -def audit_api(resource_type: str, api_name: str = None): - """ - Decorator for API endpoints to enable automatic audit logging - - Args: - resource_type: The resource type for audit (e.g., 'collection', 'user', etc.) - api_name: Optional API name override (defaults to function name) - """ - def decorator(func): - @functools.wraps(func) - async def wrapper(*args, **kwargs): - # Find the request object in the arguments - request = None - for v in kwargs.values(): - if isinstance(v, Request): - request = v - break - - if not request: - # If no request found, just call the original function - return await func(*args, **kwargs) - - # Skip GET requests - only audit change operations - if request.method.upper() == "GET": - return await func(*args, **kwargs) +def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]: + """Extract response data from the returned response object""" + try: + # If response is already a dict (common for JSON APIs) + if isinstance(response, dict): + return response + + # If response has a dict() method (Pydantic models) + elif hasattr(response, 'dict'): + return response.dict() + + # If response has a model_dump() method (Pydantic v2) + elif hasattr(response, 'model_dump'): + return response.model_dump() + + # If response is a list of dicts or models + elif isinstance(response, list): + result = [] + for item in response: + if isinstance(item, dict): + result.append(item) + elif hasattr(item, 'dict'): + result.append(item.dict()) + elif hasattr(item, 'model_dump'): + result.append(item.model_dump()) + else: + result.append(str(item)) + return {"items": result} + + # For other types, try to convert to string + else: + return {"response": str(response)} - # Record start time - start_time_ms = int(time.time() * 1000) - actual_api_name = api_name or func.__name__ + except Exception as e: + logger.debug(f"Failed to extract response data: {e}") + return {"status": "success", "type": type(response).__name__} + + +def _clean_data_for_audit(data): + """Clean data for audit logging - remove null values and sensitive information""" + if data is None: + return None + + if isinstance(data, dict): + cleaned = {} + for key, value in data.items(): + # Skip null/None values + if value is None: + continue - try: - # Call the original function first to get the parsed data - response = await func(*args, **kwargs) - - # Record end time - end_time_ms = int(time.time() * 1000) - - # Extract request data from function arguments (after parsing) - request_data = _extract_request_data_from_args(request, kwargs) - - # Extract response data - response_data = _extract_response_data(response) - - # Log audit asynchronously - await _log_audit_async( - request=request, - resource_type=resource_type, - api_name=actual_api_name, - start_time_ms=start_time_ms, - end_time_ms=end_time_ms, - status_code=200, # Success - request_data=request_data, - response_data=response_data, - error_message=None - ) - - return response - - except Exception as e: - # Record end time for error case - end_time_ms = int(time.time() * 1000) - - # Extract request data if possible - try: - request_data = _extract_request_data_from_args(request, kwargs) - except: - request_data = {"method": request.method, "path": request.url.path} - - # Log audit for error case - await _log_audit_async( - request=request, - resource_type=resource_type, - api_name=actual_api_name, - start_time_ms=start_time_ms, - end_time_ms=end_time_ms, - status_code=500, # Error - request_data=request_data, - response_data={"error": str(e)}, - error_message=str(e) - ) - - # Re-raise the exception - raise - - return wrapper - return decorator + # Filter out sensitive fields + key_lower = key.lower() + if any(sensitive in key_lower for sensitive in ['password', 'secret', 'token', 'key']): + cleaned[key] = "***FILTERED***" + else: + # Recursively clean nested data + cleaned_value = _clean_data_for_audit(value) + if cleaned_value is not None or isinstance(cleaned_value, (bool, int, float, str)): + # Keep non-null values and primitive types (including False, 0, empty string) + if not (isinstance(cleaned_value, dict) and len(cleaned_value) == 0): + # Don't add empty dicts + cleaned[key] = cleaned_value + + return cleaned if cleaned else None + + elif isinstance(data, list): + cleaned = [] + for item in data: + cleaned_item = _clean_data_for_audit(item) + if cleaned_item is not None: + cleaned.append(cleaned_item) + return cleaned if cleaned else None + + else: + # For primitive types, return as-is + return data def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[Dict[str, Any]]: @@ -170,82 +160,31 @@ def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[ return None -def _clean_data_for_audit(data): - """Clean data for audit logging - remove null values and sensitive information""" - if data is None: - return None - - if isinstance(data, dict): - cleaned = {} - for key, value in data.items(): - # Skip null/None values - if value is None: - continue - - # Filter out sensitive fields - key_lower = key.lower() - if any(sensitive in key_lower for sensitive in ['password', 'secret', 'token', 'key']): - cleaned[key] = "***FILTERED***" - else: - # Recursively clean nested data - cleaned_value = _clean_data_for_audit(value) - if cleaned_value is not None or isinstance(cleaned_value, (bool, int, float, str)): - # Keep non-null values and primitive types (including False, 0, empty string) - if not (isinstance(cleaned_value, dict) and len(cleaned_value) == 0): - # Don't add empty dicts - cleaned[key] = cleaned_value - - return cleaned if cleaned else None - - elif isinstance(data, list): - cleaned = [] - for item in data: - cleaned_item = _clean_data_for_audit(item) - if cleaned_item is not None: - cleaned.append(cleaned_item) - return cleaned if cleaned else None - - else: - # For primitive types, return as-is - return data - - -def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]: - """Extract response data from the returned response object""" +def _extract_client_info(request) -> tuple[Optional[str], Optional[str]]: + """Extract client IP and User-Agent from request""" try: - # If response is already a dict (common for JSON APIs) - if isinstance(response, dict): - return response + # Get IP address + ip_address = None + if hasattr(request, 'client') and request.client: + ip_address = request.client.host - # If response has a dict() method (Pydantic models) - elif hasattr(response, 'dict'): - return response.dict() + # Check for forwarded headers + if hasattr(request, 'headers'): + forwarded_for = request.headers.get('X-Forwarded-For') + if forwarded_for: + ip_address = forwarded_for.split(',')[0].strip() + elif request.headers.get('X-Real-IP'): + ip_address = request.headers.get('X-Real-IP') - # If response has a model_dump() method (Pydantic v2) - elif hasattr(response, 'model_dump'): - return response.model_dump() + # Get User-Agent + user_agent = None + if hasattr(request, 'headers'): + user_agent = request.headers.get('User-Agent') - # If response is a list of dicts or models - elif isinstance(response, list): - result = [] - for item in response: - if isinstance(item, dict): - result.append(item) - elif hasattr(item, 'dict'): - result.append(item.dict()) - elif hasattr(item, 'model_dump'): - result.append(item.model_dump()) - else: - result.append(str(item)) - return {"items": result} - - # For other types, try to convert to string - else: - return {"response": str(response)} - + return ip_address, user_agent except Exception as e: - logger.debug(f"Failed to extract response data: {e}") - return {"status": "success", "type": type(response).__name__} + logger.warning(f"Failed to extract client info: {e}") + return None, None async def _log_audit_async(request: Request, resource_type: str, api_name: str, @@ -258,7 +197,7 @@ async def _log_audit_async(request: Request, resource_type: str, api_name: str, username = getattr(request.state, 'username', None) # Extract client info - ip_address, user_agent = audit_service._extract_client_info(request) + ip_address, user_agent = _extract_client_info(request) # Log audit in background import asyncio @@ -281,4 +220,92 @@ async def _log_audit_async(request: Request, resource_type: str, api_name: str, ) ) except Exception as audit_error: - logger.error(f"Failed to log audit: {audit_error}") \ No newline at end of file + logger.error(f"Failed to log audit: {audit_error}") + +def audit_api(resource_type: str, api_name: str = None): + """ + Decorator for API endpoints to enable automatic audit logging + + Args: + resource_type: The resource type for audit (e.g., 'collection', 'user', etc.) + api_name: Optional API name override (defaults to function name) + """ + def decorator(func): + @functools.wraps(func) + async def wrapper(*args, **kwargs): + # Find the request object in the arguments + request = None + for v in kwargs.values(): + if isinstance(v, Request): + request = v + break + + if not request: + # If no request found, just call the original function + return await func(*args, **kwargs) + + # Skip GET requests - only audit change operations + if request.method.upper() == "GET": + return await func(*args, **kwargs) + + # Record start time + start_time_ms = int(time.time() * 1000) + actual_api_name = api_name or func.__name__ + + try: + # Call the original function first to get the parsed data + response = await func(*args, **kwargs) + + # Record end time + end_time_ms = int(time.time() * 1000) + + # Extract request data from function arguments (after parsing) + request_data = _extract_request_data_from_args(request, kwargs) + + # Extract response data + response_data = _extract_response_data(response) + + # Log audit asynchronously + await _log_audit_async( + request=request, + resource_type=resource_type, + api_name=actual_api_name, + start_time_ms=start_time_ms, + end_time_ms=end_time_ms, + status_code=200, # Success + request_data=request_data, + response_data=response_data, + error_message=None + ) + + return response + + except Exception as e: + # Record end time for error case + end_time_ms = int(time.time() * 1000) + + # Extract request data if possible + try: + request_data = _extract_request_data_from_args(request, kwargs) + except: + request_data = {"method": request.method, "path": request.url.path} + + # Log audit for error case + await _log_audit_async( + request=request, + resource_type=resource_type, + api_name=actual_api_name, + start_time_ms=start_time_ms, + end_time_ms=end_time_ms, + status_code=500, # Error + request_data=request_data, + response_data={"error": str(e)}, + error_message=str(e) + ) + + # Re-raise the exception + raise + + return wrapper + return decorator + From 64f4eef97358a978891f13f0e538f37d94cb2333 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 18:41:32 +0800 Subject: [PATCH 12/19] chore: tidy up --- tests/e2e_test/README.md | 17 + tests/e2e_test/test_audit.py | 1169 ++++++++++++++++++++++++++++++++++ 2 files changed, 1186 insertions(+) create mode 100644 tests/e2e_test/test_audit.py diff --git a/tests/e2e_test/README.md b/tests/e2e_test/README.md index dcb07244c..912f04ec4 100644 --- a/tests/e2e_test/README.md +++ b/tests/e2e_test/README.md @@ -137,6 +137,23 @@ RERANK_MODEL_PROVIDER=siliconflow RERANK_MODEL_NAME=BAAI/bge-large-zh-1.5 ``` +## 🧪 Available Test Suites + +### Audit Logging Tests +Comprehensive tests for the audit logging functionality: +- **File**: `test_audit.py` +- **Documentation**: See `README_AUDIT.md` for detailed information +- **Coverage**: Collection, Document, Bot, Chat, LLM Provider, and LLM Provider Model operations +- **Features**: Sensitive data filtering, timestamp validation, user tracking + +```bash +# Run all audit tests +pytest tests/e2e_test/test_audit.py -v + +# Run specific audit test class +pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v +``` + ## 🧪 Available Fixtures The E2E tests provide the following pytest fixtures that can be used directly in tests: diff --git a/tests/e2e_test/test_audit.py b/tests/e2e_test/test_audit.py new file mode 100644 index 000000000..b0740fb46 --- /dev/null +++ b/tests/e2e_test/test_audit.py @@ -0,0 +1,1169 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import time +from http import HTTPStatus +from typing import Dict, Any, Optional + +import pytest + +from tests.e2e_test.config import ( + COMPLETION_MODEL_CUSTOM_PROVIDER, + COMPLETION_MODEL_NAME, + COMPLETION_MODEL_PROVIDER, + EMBEDDING_MODEL_CUSTOM_PROVIDER, + EMBEDDING_MODEL_NAME, + EMBEDDING_MODEL_PROVIDER, +) + + +class AuditLogTestHelper: + """Helper class for audit log testing""" + + def __init__(self, cookie_client): + self.cookie_client = cookie_client + + def get_audit_logs(self, **filters) -> list: + """Get audit logs with optional filters""" + params = {k: v for k, v in filters.items() if v is not None} + resp = self.cookie_client.get("/api/v1/audit-logs", params=params) + assert resp.status_code == HTTPStatus.OK, f"Failed to get audit logs: {resp.text}" + return resp.json()["items"] + + def find_audit_log(self, resource_type: str, api_name: str, resource_id: Optional[str] = None, + http_method: str = None, max_wait_seconds: int = 10, + match_response_data: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: + """Find a specific audit log by criteria with retry mechanism""" + start_time = time.time() + while time.time() - start_time < max_wait_seconds: + logs = self.get_audit_logs( + resource_type=resource_type, + api_name=api_name, + http_method=http_method + ) + + for log in logs: + # Check resource_id if provided and available + if resource_id and log.get("resource_id") and log.get("resource_id") != resource_id: + continue + if http_method and log.get("http_method") != http_method: + continue + + # For create operations, match against response data when resource_id is not available in path + if (resource_id and not log.get("resource_id") and + match_response_data and log.get("response_data")): + try: + response_data = json.loads(log.get("response_data", "{}")) + match_found = True + for key, expected_value in match_response_data.items(): + if response_data.get(key) != expected_value: + match_found = False + break + if not match_found: + continue + except (json.JSONDecodeError, AttributeError): + continue + + return log + + time.sleep(0.5) # Wait before retry + + return None + + def assert_audit_log_content(self, log: Dict[str, Any], expected_fields: Dict[str, Any]): + """Assert audit log contains expected fields and values""" + assert log is not None, "Audit log not found" + + for field, expected_value in expected_fields.items(): + actual_value = log.get(field) + if expected_value is not None: + assert actual_value == expected_value, f"Field {field}: expected {expected_value}, got {actual_value}" + else: + assert field in log, f"Field {field} not found in audit log" + + # Basic assertions for all audit logs + assert log.get("start_time") is not None, "start_time should not be None" + assert log.get("end_time") is not None, "end_time should not be None" + assert log.get("duration_ms") is not None, "duration_ms should not be None" + assert log.get("status_code") in [200, 201, 204], f"Unexpected status code: {log.get('status_code')}" + assert log.get("user_id") is not None, "user_id should not be None" + assert log.get("username") is not None, "username should not be None" + + def parse_json_field(self, log: Dict[str, Any], field_name: str) -> Dict[str, Any]: + """Parse JSON string field from audit log""" + # Handle both dict and object access + if hasattr(log, field_name): + json_str = getattr(log, field_name, "{}") + else: + json_str = log.get(field_name, "{}") + + if json_str: + try: + return json.loads(json_str) + except (json.JSONDecodeError, TypeError): + return {} + return {} + + +@pytest.fixture +def audit_helper(cookie_client): + """Create audit log test helper""" + return AuditLogTestHelper(cookie_client) + + +class TestCollectionAudit: + """Test audit logs for collection operations""" + + def test_create_collection_audit(self, client, audit_helper): + """Test that creating a collection generates audit log""" + # Create collection + collection_data = { + "title": "Audit Test Collection", + "type": "document", + "config": { + "source": "system", + "enable_knowledge_graph": False, + "embedding": { + "model": EMBEDDING_MODEL_NAME, + "model_service_provider": EMBEDDING_MODEL_PROVIDER, + "custom_llm_provider": EMBEDDING_MODEL_CUSTOM_PROVIDER, + }, + }, + } + + resp = client.post("/api/v1/collections", json=collection_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to create collection: {resp.text}" + collection = resp.json() + collection_id = collection["id"] + + try: + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="collection", + api_name="CreateCollection", + resource_id=collection_id, + http_method="POST", + match_response_data={"id": collection_id} + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "collection", + "api_name": "CreateCollection", + "http_method": "POST", + "path": "/api/v1/collections" + }) + + # Verify request data contains collection info + request_data = audit_helper.parse_json_field(audit_log, "request_data") + assert "title" in request_data, "Request data should contain title" + assert request_data["title"] == collection_data["title"] + + # Verify response data contains collection ID + response_data = audit_helper.parse_json_field(audit_log, "response_data") + assert "id" in response_data, "Response data should contain collection ID" + assert response_data["id"] == collection_id + + finally: + # Cleanup + client.delete(f"/api/v1/collections/{collection_id}") + + def test_update_collection_audit(self, client, audit_helper, collection): + """Test that updating a collection generates audit log""" + collection_id = collection["id"] + + # Update collection + update_data = { + "title": "Updated Audit Test Collection", + "description": "Updated description for audit test", + "config": collection["config"] + } + + resp = client.put(f"/api/v1/collections/{collection_id}", json=update_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to update collection: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="collection", + api_name="UpdateCollection", + resource_id=collection_id, + http_method="PUT" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "collection", + "api_name": "UpdateCollection", + "resource_id": collection_id, + "http_method": "PUT", + "path": f"/api/v1/collections/{collection_id}" + }) + + # Verify request data contains update info + request_data = audit_helper.parse_json_field(audit_log, "request_data") + # The request data structure might be nested, check both formats + if "title" in request_data: + assert request_data["title"] == update_data["title"] + elif "collection" in request_data and "title" in request_data["collection"]: + assert request_data["collection"]["title"] == update_data["title"] + else: + assert False, f"title field not found in request_data: {request_data}" + + def test_delete_collection_audit(self, client, audit_helper): + """Test that deleting a collection generates audit log""" + # Create collection first + collection_data = { + "title": "Collection to Delete", + "type": "document", + "config": { + "source": "system", + "enable_knowledge_graph": False, + "embedding": { + "model": EMBEDDING_MODEL_NAME, + "model_service_provider": EMBEDDING_MODEL_PROVIDER, + "custom_llm_provider": EMBEDDING_MODEL_CUSTOM_PROVIDER, + }, + }, + } + + resp = client.post("/api/v1/collections", json=collection_data) + assert resp.status_code == HTTPStatus.OK + collection_id = resp.json()["id"] + + # Delete collection + resp = client.delete(f"/api/v1/collections/{collection_id}") + assert resp.status_code == HTTPStatus.OK, f"Failed to delete collection: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="collection", + api_name="DeleteCollection", + resource_id=collection_id, + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "collection", + "api_name": "DeleteCollection", + "resource_id": collection_id, + "http_method": "DELETE", + "path": f"/api/v1/collections/{collection_id}" + }) + + +class TestDocumentAudit: + """Test audit logs for document operations""" + + def test_create_document_audit(self, client, audit_helper, collection): + """Test that creating documents generates audit log""" + collection_id = collection["id"] + + # Upload document + files = {"files": ("audit_test.txt", "This is a test document for audit.", "text/plain")} + resp = client.post(f"/api/v1/collections/{collection_id}/documents", files=files) + assert resp.status_code == HTTPStatus.OK, f"Failed to create document: {resp.text}" + + documents = resp.json()["items"] + assert len(documents) > 0 + document_id = documents[0]["id"] + + try: + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="document", + api_name="CreateDocuments", + http_method="POST" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "document", + "api_name": "CreateDocuments", + "http_method": "POST", + "path": f"/api/v1/collections/{collection_id}/documents" + }) + + # Verify response data contains document info + response_data = audit_helper.parse_json_field(audit_log, "response_data") + assert "items" in response_data + assert len(response_data["items"]) > 0 + + finally: + # Cleanup + client.delete(f"/api/v1/collections/{collection_id}/documents/{document_id}") + + def test_delete_document_audit(self, client, audit_helper, collection, document): + """Test that deleting a document generates audit log""" + collection_id = collection["id"] + document_id = document["id"] + + # Delete document + resp = client.delete(f"/api/v1/collections/{collection_id}/documents/{document_id}") + assert resp.status_code == HTTPStatus.OK, f"Failed to delete document: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="document", + api_name="DeleteDocument", + resource_id=document_id, + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "document", + "api_name": "DeleteDocument", + "resource_id": document_id, + "http_method": "DELETE", + "path": f"/api/v1/collections/{collection_id}/documents/{document_id}" + }) + + +class TestBotAudit: + """Test audit logs for bot operations""" + + def test_create_bot_audit(self, client, audit_helper, collection): + """Test that creating a bot generates audit log""" + # Create bot + config = { + "model_name": COMPLETION_MODEL_NAME, + "model_service_provider": COMPLETION_MODEL_PROVIDER, + "llm": {"context_window": 3500, "similarity_score_threshold": 0.5, "similarity_topk": 3, "temperature": 0.1}, + } + bot_data = { + "title": "Audit Test Bot", + "description": "Bot for audit testing", + "type": "knowledge", + "config": json.dumps(config), + "collection_ids": [collection["id"]], + } + + resp = client.post("/api/v1/bots", json=bot_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to create bot: {resp.text}" + bot = resp.json() + bot_id = bot["id"] + + try: + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="bot", + api_name="CreateBot", + resource_id=bot_id, + http_method="POST", + match_response_data={"id": bot_id} + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "bot", + "api_name": "CreateBot", + "http_method": "POST", + "path": "/api/v1/bots" + }) + + # Verify request data contains bot info + request_data = audit_helper.parse_json_field(audit_log, "request_data") + assert "title" in request_data + assert request_data["title"] == bot_data["title"] + + finally: + # Cleanup + client.delete(f"/api/v1/bots/{bot_id}") + + @pytest.mark.skip(reason="Bot update requires valid LLM Provider configuration with API keys which are not available in test environment") + def test_update_bot_audit(self, client, audit_helper, bot): + """Test that updating a bot generates audit log""" + bot_id = bot["id"] + + # Update bot - include required fields + update_data = { + "title": "Updated Audit Test Bot", + "description": "Updated bot description", + "config": bot["config"], # Include original config to avoid JSON parsing error + "collection_ids": bot["collection_ids"] # Include original collection_ids + } + + resp = client.put(f"/api/v1/bots/{bot_id}", json=update_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to update bot: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="bot", + api_name="UpdateBot", + resource_id=bot_id, + http_method="PUT" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "bot", + "api_name": "UpdateBot", + "resource_id": bot_id, + "http_method": "PUT", + "path": f"/api/v1/bots/{bot_id}" + }) + + # Verify request data contains update info + request_data = audit_helper.parse_json_field(audit_log, "request_data") + assert "title" in request_data + assert request_data["title"] == update_data["title"] + + def test_delete_bot_audit(self, client, audit_helper, collection): + """Test that deleting a bot generates audit log""" + # Create bot first + config = { + "model_name": COMPLETION_MODEL_NAME, + "model_service_provider": COMPLETION_MODEL_PROVIDER, + "llm": {"context_window": 3500, "similarity_score_threshold": 0.5, "similarity_topk": 3, "temperature": 0.1}, + } + bot_data = { + "title": "Bot to Delete", + "description": "Bot for delete audit test", + "type": "knowledge", + "config": json.dumps(config), + "collection_ids": [collection["id"]], + } + + resp = client.post("/api/v1/bots", json=bot_data) + assert resp.status_code == HTTPStatus.OK + bot_id = resp.json()["id"] + + # Delete bot + resp = client.delete(f"/api/v1/bots/{bot_id}") + assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete bot: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="bot", + api_name="DeleteBot", + resource_id=bot_id, + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "bot", + "api_name": "DeleteBot", + "resource_id": bot_id, + "http_method": "DELETE", + "path": f"/api/v1/bots/{bot_id}" + }) + + +class TestChatAudit: + """Test audit logs for chat operations""" + + def test_create_chat_audit(self, client, audit_helper, bot): + """Test that creating a chat generates audit log""" + bot_id = bot["id"] + + # Create chat + resp = client.post(f"/api/v1/bots/{bot_id}/chats") + assert resp.status_code == HTTPStatus.OK, f"Failed to create chat: {resp.text}" + chat = resp.json() + chat_id = chat["id"] + + try: + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="chat", + api_name="CreateChat", + resource_id=chat_id, + http_method="POST", + match_response_data={"id": chat_id} + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "chat", + "api_name": "CreateChat", + "http_method": "POST", + "path": f"/api/v1/bots/{bot_id}/chats" + }) + + # Verify response data contains chat info + response_data = audit_helper.parse_json_field(audit_log, "response_data") + assert "id" in response_data + assert response_data["id"] == chat_id + + finally: + # Cleanup + client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") + + def test_update_chat_audit(self, client, audit_helper, bot): + """Test that updating a chat generates audit log""" + bot_id = bot["id"] + + # Create chat first + resp = client.post(f"/api/v1/bots/{bot_id}/chats") + assert resp.status_code == HTTPStatus.OK + chat = resp.json() + chat_id = chat["id"] + + try: + # Update chat + update_data = {"title": "Updated Chat Title"} + resp = client.put(f"/api/v1/bots/{bot_id}/chats/{chat_id}", json=update_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to update chat: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="chat", + api_name="UpdateChat", + resource_id=chat_id, + http_method="PUT" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "chat", + "api_name": "UpdateChat", + "resource_id": chat_id, + "http_method": "PUT", + "path": f"/api/v1/bots/{bot_id}/chats/{chat_id}" + }) + + # Verify request data contains update info + request_data = audit_helper.parse_json_field(audit_log, "request_data") + # The request data structure might be nested, check both formats + if "title" in request_data: + assert request_data["title"] == update_data["title"] + elif "chat_in" in request_data and "title" in request_data["chat_in"]: + assert request_data["chat_in"]["title"] == update_data["title"] + else: + assert False, f"title field not found in request_data: {request_data}" + + finally: + # Cleanup + client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") + + def test_delete_chat_audit(self, client, audit_helper, bot): + """Test that deleting a chat generates audit log""" + bot_id = bot["id"] + + # Create chat first + resp = client.post(f"/api/v1/bots/{bot_id}/chats") + assert resp.status_code == HTTPStatus.OK + chat_id = resp.json()["id"] + + # Delete chat + resp = client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") + assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete chat: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="chat", + api_name="DeleteChat", + resource_id=chat_id, + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "chat", + "api_name": "DeleteChat", + "resource_id": chat_id, + "http_method": "DELETE", + "path": f"/api/v1/bots/{bot_id}/chats/{chat_id}" + }) + + +class TestLLMProviderAudit: + """Test audit logs for LLM provider operations""" + + def test_create_llm_provider_audit(self, cookie_client, audit_helper): + """Test that creating an LLM provider generates audit log""" + # Create LLM provider + provider_data = { + "name": "audit-test-provider", + "label": "Audit Test Provider", + "base_url": "https://api.example.com/v1", + "completion_dialect": "openai", + "embedding_dialect": "openai", + "api_key": "test-api-key" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to create LLM provider: {resp.text}" + provider = resp.json() + provider_name = provider["name"] + + try: + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="CreateLLMProvider", + resource_id=provider_name, + http_method="POST", + match_response_data={"name": provider_name} + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider", + "api_name": "CreateLLMProvider", + "http_method": "POST", + "path": "/api/v1/llm_providers" + }) + + # Verify request data (API key should be filtered) + request_data = audit_helper.parse_json_field(audit_log, "request_data") + assert "label" in request_data + assert request_data["label"] == provider_data["label"] + # API key should be filtered out or redacted + if "api_key" in request_data: + assert request_data["api_key"] == "***FILTERED***" + + finally: + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + + def test_update_llm_provider_audit(self, cookie_client, audit_helper): + """Test that updating an LLM provider generates audit log""" + # Create provider first + provider_data = { + "name": "update-test-provider", + "label": "Provider for Update Test", + "base_url": "https://api.example.com/v1", + "api_key": "initial-key" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK + provider_name = resp.json()["name"] + + try: + # Update provider + update_data = { + "label": "Updated Provider Label", + "base_url": "https://api.updated.com/v1", + "api_key": "updated-key" + } + + resp = cookie_client.put(f"/api/v1/llm_providers/{provider_name}", json=update_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to update LLM provider: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="UpdateLLMProvider", + resource_id=provider_name, + http_method="PUT" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider", + "api_name": "UpdateLLMProvider", + "resource_id": provider_name, + "http_method": "PUT", + "path": f"/api/v1/llm_providers/{provider_name}" + }) + + # Verify request data contains update info (API key should be filtered) + request_data = audit_helper.parse_json_field(audit_log, "request_data") + # The request data structure might be nested, check both formats + if "label" in request_data: + assert request_data["label"] == update_data["label"] + if "api_key" in request_data: + assert request_data["api_key"] == "***FILTERED***" + elif "provider_data" in request_data and "label" in request_data["provider_data"]: + assert request_data["provider_data"]["label"] == update_data["label"] + if "api_key" in request_data["provider_data"]: + assert request_data["provider_data"]["api_key"] == "***FILTERED***" + else: + assert False, f"label field not found in request_data: {request_data}" + + finally: + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + + def test_delete_llm_provider_audit(self, cookie_client, audit_helper): + """Test that deleting an LLM provider generates audit log""" + # Create provider first + provider_data = { + "name": "delete-test-provider", + "label": "Provider for Delete Test", + "base_url": "https://api.example.com/v1" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK + provider_name = resp.json()["name"] + + # Delete provider + resp = cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete LLM provider: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="DeleteLLMProvider", + resource_id=provider_name, + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider", + "api_name": "DeleteLLMProvider", + "resource_id": provider_name, + "http_method": "DELETE", + "path": f"/api/v1/llm_providers/{provider_name}" + }) + + +class TestLLMProviderModelAudit: + """Test audit logs for LLM provider model operations""" + + @pytest.fixture + def test_provider(self, cookie_client): + """Create a test LLM provider for model tests""" + provider_data = { + "name": "model-test-provider", + "label": "Provider for Model Test", + "base_url": "https://api.example.com/v1" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK + provider = resp.json() + + yield provider + + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider['name']}") + + def test_create_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): + """Test that creating an LLM provider model generates audit log""" + provider_name = test_provider["name"] + + # Create provider model + import uuid + unique_id = str(uuid.uuid4())[:8] + model_data = { + "api": "completion", + "model": f"test-model-{unique_id}", + "custom_llm_provider": "openai", + "max_tokens": 4096, + "tags": ["test"] + } + + resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to create provider model: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider_model", + api_name="CreateProviderModel", + http_method="POST" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider_model", + "api_name": "CreateProviderModel", + "http_method": "POST", + "path": f"/api/v1/llm_providers/{provider_name}/models" + }) + + # Verify response data contains model info (since request data extraction has limitations) + response_data = audit_helper.parse_json_field(audit_log, "response_data") + assert "model" in response_data + assert response_data["model"] == model_data["model"] + assert "custom_llm_provider" in response_data + assert response_data["custom_llm_provider"] == model_data["custom_llm_provider"] + + def test_update_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): + """Test that updating an LLM provider model generates audit log""" + provider_name = test_provider["name"] + + # Create model first + import uuid + unique_id = str(uuid.uuid4())[:8] + model_data = { + "api": "completion", + "model": f"test-model-{unique_id}", + "custom_llm_provider": "openai", + "max_tokens": 4096 + } + + resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) + assert resp.status_code == HTTPStatus.OK + + # Update model + update_data = { + "custom_llm_provider": "anthropic", + "max_tokens": 8192, + "tags": ["updated", "test"] + } + + api = model_data["api"] + model = model_data["model"] + resp = cookie_client.put(f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}", json=update_data) + assert resp.status_code == HTTPStatus.OK, f"Failed to update provider model: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider_model", + api_name="UpdateProviderModel", + http_method="PUT" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider_model", + "api_name": "UpdateProviderModel", + "http_method": "PUT", + "path": f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}" + }) + + # Verify response data contains update info (since request data extraction has limitations) + response_data = audit_helper.parse_json_field(audit_log, "response_data") + assert "custom_llm_provider" in response_data + assert response_data["custom_llm_provider"] == update_data["custom_llm_provider"] + + def test_delete_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): + """Test that deleting an LLM provider model generates audit log""" + provider_name = test_provider["name"] + + # Create model first + import uuid + unique_id = str(uuid.uuid4())[:8] + model_data = { + "api": "completion", + "model": f"test-model-{unique_id}", + "custom_llm_provider": "openai" + } + + resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) + assert resp.status_code == HTTPStatus.OK + + # Delete model + api = model_data["api"] + model = model_data["model"] + resp = cookie_client.delete(f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}") + assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete provider model: {resp.text}" + + # Check audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider_model", + api_name="DeleteProviderModel", + http_method="DELETE" + ) + + audit_helper.assert_audit_log_content(audit_log, { + "resource_type": "llm_provider_model", + "api_name": "DeleteProviderModel", + "http_method": "DELETE", + "path": f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}" + }) + + +class TestAuditLogRetrieval: + """Test audit log retrieval and filtering functionality""" + + def test_list_audit_logs(self, cookie_client, audit_helper): + """Test basic audit log listing functionality""" + # Get audit logs + logs = audit_helper.get_audit_logs(limit=10) + assert isinstance(logs, list), "Audit logs should be a list" + + # Verify log structure + if logs: # Only check if there are logs + log = logs[0] + required_fields = [ + "id", "resource_type", "api_name", "http_method", "path", + "status_code", "start_time", "end_time", "created" + ] + for field in required_fields: + assert field in log, f"Field {field} should be present in audit log" + + def test_audit_log_filtering(self, cookie_client, audit_helper): + """Test audit log filtering by resource type and other criteria""" + # Test filtering by resource type + collection_logs = audit_helper.get_audit_logs(resource_type="collection") + if collection_logs: + for log in collection_logs: + assert log["resource_type"] == "collection", "All logs should be collection type" + + # Test filtering by HTTP method + post_logs = audit_helper.get_audit_logs(http_method="POST") + if post_logs: + for log in post_logs: + assert log["http_method"] == "POST", "All logs should be POST method" + + # Test filtering by status code + success_logs = audit_helper.get_audit_logs(status_code=200) + if success_logs: + for log in success_logs: + assert log["status_code"] == 200, "All logs should have status code 200" + + def test_audit_log_detail(self, cookie_client, audit_helper): + """Test retrieving individual audit log details""" + # Get a log first + logs = audit_helper.get_audit_logs(limit=1) + if not logs: + pytest.skip("No audit logs available for testing") + + log_id = logs[0]["id"] + + # Get detailed log + resp = cookie_client.get(f"/api/v1/audit-logs/{log_id}") + assert resp.status_code == HTTPStatus.OK, f"Failed to get audit log detail: {resp.text}" + + detailed_log = resp.json() + assert detailed_log["id"] == log_id, "Log ID should match" + + # Verify all expected fields are present + required_fields = [ + "id", "resource_type", "api_name", "http_method", "path", + "status_code", "start_time", "end_time", "created" + ] + for field in required_fields: + assert field in detailed_log, f"Field {field} should be present in detailed audit log" + + +class TestAuditSensitiveDataFiltering: + """Test that sensitive data is properly filtered in audit logs""" + + def test_api_key_filtering_in_audit(self, cookie_client, audit_helper): + """Test that API keys are filtered in audit logs""" + # Create LLM provider with API key + provider_data = { + "name": "sensitive-test-provider", + "label": "Sensitive Data Test Provider", + "base_url": "https://api.example.com/v1", + "api_key": "sk-very-secret-api-key-12345" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK + provider_name = resp.json()["name"] + + try: + # Find the audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="CreateLLMProvider", + resource_id=provider_name, + http_method="POST" + ) + + # Verify API key is filtered + request_data = audit_helper.parse_json_field(audit_log, "request_data") + if "api_key" in request_data: + # API key should be filtered out or replaced with a placeholder + assert request_data["api_key"] != provider_data["api_key"], "API key should be filtered" + assert request_data["api_key"] == "***FILTERED***", "API key should be replaced with placeholder" + + finally: + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + + +class TestAuditLogIntegrity: + """Test audit log data integrity and consistency""" + + def test_audit_log_timestamps(self, cookie_client, audit_helper): + """Test that audit log timestamps are consistent and valid""" + # Create a simple resource to generate audit log + provider_data = { + "name": "timestamp-test-provider", + "label": "Timestamp Test Provider", + "base_url": "https://api.example.com/v1" + } + + before_request = int(time.time() * 1000) # milliseconds + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + after_request = int(time.time() * 1000) # milliseconds + + assert resp.status_code == HTTPStatus.OK + provider_name = resp.json()["name"] + + try: + # Find the audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="CreateLLMProvider", + resource_id=provider_name, + http_method="POST", + match_response_data={"name": provider_name} + ) + + # Verify timestamps + start_time = audit_log.get("start_time") + end_time = audit_log.get("end_time") + duration_ms = audit_log.get("duration_ms") + + assert start_time is not None, "start_time should not be None" + assert end_time is not None, "end_time should not be None" + assert duration_ms is not None, "duration_ms should not be None" + + # Verify timestamp ranges + assert before_request <= start_time <= after_request, "start_time should be within request timeframe" + assert before_request <= end_time <= after_request + 5000, "end_time should be reasonable" # Allow 5s buffer + + # Verify duration calculation + expected_duration = end_time - start_time + assert duration_ms == expected_duration, f"duration_ms should match calculated duration: {duration_ms} != {expected_duration}" + + # Verify start_time <= end_time + assert start_time <= end_time, "start_time should be <= end_time" + + finally: + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + + def test_audit_log_user_info(self, cookie_client, audit_helper): + """Test that audit logs contain correct user information""" + # Create a simple resource to generate audit log + provider_data = { + "name": "user-info-test-provider", + "label": "User Info Test Provider", + "base_url": "https://api.example.com/v1" + } + + resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) + assert resp.status_code == HTTPStatus.OK + provider_name = resp.json()["name"] + + try: + # Find the audit log + audit_log = audit_helper.find_audit_log( + resource_type="llm_provider", + api_name="CreateLLMProvider", + resource_id=provider_name, + http_method="POST", + match_response_data={"name": provider_name} + ) + + # Verify user information is present + assert audit_log.get("user_id") is not None, "user_id should be present" + assert audit_log.get("username") is not None, "username should be present" + + # Verify user_id is a valid UUID-like string + user_id = audit_log.get("user_id") + assert len(user_id) > 0, "user_id should not be empty" + + # Verify username is a valid string + username = audit_log.get("username") + assert len(username) > 0, "username should not be empty" + assert isinstance(username, str), "username should be a string" + + finally: + # Cleanup + cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") + + +# Additional helper functions for testing audit logs in specific scenarios + +def verify_audit_log_for_resource_operation(audit_helper, resource_type: str, operation: str, + resource_id: str = None, expected_path: str = None): + """Helper function to verify audit log for a specific resource operation""" + http_method_map = { + "create": "POST", + "update": "PUT", + "delete": "DELETE" + } + + api_name_map = { + "collection": { + "create": "CreateCollection", + "update": "UpdateCollection", + "delete": "DeleteCollection" + }, + "document": { + "create": "CreateDocuments", + "update": "UpdateDocument", + "delete": "DeleteDocument" + }, + "bot": { + "create": "CreateBot", + "update": "UpdateBot", + "delete": "DeleteBot" + }, + "chat": { + "create": "CreateChat", + "update": "UpdateChat", + "delete": "DeleteChat" + }, + "llm_provider": { + "create": "CreateLLMProvider", + "update": "UpdateLLMProvider", + "delete": "DeleteLLMProvider" + }, + "llm_provider_model": { + "create": "CreateProviderModel", + "update": "UpdateProviderModel", + "delete": "DeleteProviderModel" + } + } + + http_method = http_method_map.get(operation) + api_name = api_name_map.get(resource_type, {}).get(operation) + + if not http_method or not api_name: + raise ValueError(f"Invalid resource_type '{resource_type}' or operation '{operation}'") + + audit_log = audit_helper.find_audit_log( + resource_type=resource_type, + api_name=api_name, + resource_id=resource_id, + http_method=http_method + ) + + expected_fields = { + "resource_type": resource_type, + "api_name": api_name, + "http_method": http_method + } + + if resource_id: + expected_fields["resource_id"] = resource_id + + if expected_path: + expected_fields["path"] = expected_path + + audit_helper.assert_audit_log_content(audit_log, expected_fields) + + return audit_log + + +""" +Test Usage Examples: + +1. Run all audit tests: + pytest tests/e2e_test/test_audit.py -v + +2. Run specific test class: + pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v + +3. Run specific test method: + pytest tests/e2e_test/test_audit.py::TestCollectionAudit::test_create_collection_audit -v + +4. Run tests with specific markers (if added): + pytest tests/e2e_test/test_audit.py -m "audit" -v + +5. Run audit tests for specific resource types: + pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v # Collection tests + pytest tests/e2e_test/test_audit.py::TestBotAudit -v # Bot tests + pytest tests/e2e_test/test_audit.py::TestChatAudit -v # Chat tests + +The test cases cover: +- Creation/Update/Deletion operations for all specified resource types +- Audit log content validation (resource_type, api_name, timestamps, etc.) +- Sensitive data filtering (API keys, passwords, etc.) +- Audit log retrieval and filtering functionality +- Data integrity and consistency checks +- User information tracking in audit logs + +Each test follows the pattern: +1. Perform operation that should generate audit log +2. Search for the audit log using the helper +3. Validate audit log content and structure +4. Clean up resources created during test + +The AuditLogTestHelper class provides convenient methods for: +- Retrieving audit logs with filters +- Finding specific audit logs by criteria +- Asserting audit log content matches expectations +- Handling retry logic for eventual consistency +""" From 33500db3ab7c81b9a744f31eda135b5145230c99 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 20:37:44 +0800 Subject: [PATCH 13/19] chore: tidy up --- frontend/src/locales/zh-CN.ts | 116 ---------------------------------- 1 file changed, 116 deletions(-) diff --git a/frontend/src/locales/zh-CN.ts b/frontend/src/locales/zh-CN.ts index 20ddc2d3b..a880f27c5 100644 --- a/frontend/src/locales/zh-CN.ts +++ b/frontend/src/locales/zh-CN.ts @@ -350,120 +350,4 @@ export default { 'audit.pagination': '第 {start}-{end} 条，共 {total} 条', 'audit.fetchError': '获取审计日志失败', 'audit.fetchDetailError': '获取审计日志详情失败', - - 'menu.welcome': '欢迎', - 'menu.more-blocks': '更多区块', - 'menu.home': '首页', - 'menu.admin': '管理页', - 'menu.admin.sub-page': '二级管理页', - 'menu.login': '登录', - 'menu.register': '注册', - 'menu.register-result': '注册结果', - 'menu.dashboard': '仪表板', - 'menu.dashboard.analysis': '分析页', - 'menu.dashboard.monitor': '监控页', - 'menu.dashboard.workplace': '工作台', - 'menu.exception.403': '403', - 'menu.exception.404': '404', - 'menu.exception.500': '500', - 'menu.form': '表单页', - 'menu.form.basic-form': '基础表单', - 'menu.form.step-form': '分步表单', - 'menu.form.step-form.info': '分步表单（填写转账信息）', - 'menu.form.step-form.confirm': '分步表单（确认转账信息）', - 'menu.form.step-form.result': '分步表单（完成）', - 'menu.form.advanced-form': '高级表单', - 'menu.list': '列表页', - 'menu.list.table-list': '查询表格', - 'menu.list.basic-list': '标准列表', - 'menu.list.card-list': '卡片列表', - 'menu.list.search-list': '搜索列表', - 'menu.list.search-list.articles': '搜索列表（文章）', - 'menu.list.search-list.projects': '搜索列表（项目）', - 'menu.list.search-list.applications': '搜索列表（应用）', - 'menu.profile': '详情页', - 'menu.profile.basic': '基础详情页', - 'menu.profile.advanced': '高级详情页', - 'menu.result': '结果页', - 'menu.result.success': '成功页', - 'menu.result.fail': '失败页', - 'menu.exception': '异常页', - 'menu.exception.not-permission': '403', - 'menu.exception.not-find': '404', - 'menu.exception.server-error': '500', - 'menu.exception.trigger': '触发错误', - 'menu.account': '个人页', - 'menu.account.center': '个人中心', - 'menu.account.settings': '个人设置', - 'menu.account.trigger': '触发报错', - 'menu.account.logout': '退出登录', - 'app.copyright.produced': '蚂蚁集团体验技术部出品', - 'app.preview.down.block': '下载此页面到本地项目', - 'app.welcome.link.fetch-blocks': '获取全部区块', - 'app.welcome.link.block-list': '基于 block 开发，快速构建标准页面', - - // 设置相关 - 'settings.title': '设置', - 'settings.profile': '个人资料', - 'settings.apiKeys': 'API 密钥', - 'settings.auditLogs': '操作日志', - 'settings.models': '模型管理', - 'settings.invitations': '邀请管理', - - // API Keys 相关 - 'apiKeys.title': 'API 密钥', - 'apiKeys.description': 'API 密钥用于访问 ApeRAG API。请妥善保管您的密钥，不要与他人分享。', - 'apiKeys.create': '创建 API 密钥', - 'apiKeys.name': '名称', - 'apiKeys.key': '密钥', - 'apiKeys.created': '创建时间', - 'apiKeys.lastUsed': '最后使用', - 'apiKeys.status': '状态', - 'apiKeys.actions': '操作', - 'apiKeys.edit': '编辑', - 'apiKeys.delete': '删除', - 'apiKeys.copy': '复制', - 'apiKeys.copied': '已复制到剪贴板', - 'apiKeys.active': '活跃', - 'apiKeys.inactive': '停用', - 'apiKeys.createTitle': '创建 API 密钥', - 'apiKeys.editTitle': '编辑 API 密钥', - 'apiKeys.nameLabel': '密钥名称', - 'apiKeys.namePlaceholder': '请输入密钥名称', - 'apiKeys.descriptionLabel': '描述', - 'apiKeys.descriptionPlaceholder': '请输入描述（可选）', - 'apiKeys.expiresAtLabel': '过期时间', - 'apiKeys.expiresAtPlaceholder': '选择过期时间（可选）', - 'apiKeys.isActiveLabel': '状态', - 'apiKeys.confirm': '确定', - 'apiKeys.cancel': '取消', - 'apiKeys.deleteConfirm': '确定要删除这个 API 密钥吗？', - 'apiKeys.deleteSuccess': 'API 密钥删除成功', - 'apiKeys.createSuccess': 'API 密钥创建成功', - 'apiKeys.updateSuccess': 'API 密钥更新成功', - 'apiKeys.fetchError': '获取 API 密钥列表失败', - 'apiKeys.neverUsed': '从未使用', - 'apiKeys.expired': '已过期', - - // 通用 - 'common.success': '成功', - 'common.error': '错误', - 'common.warning': '警告', - 'common.info': '信息', - 'common.confirm': '确认', - 'common.cancel': '取消', - 'common.save': '保存', - 'common.edit': '编辑', - 'common.delete': '删除', - 'common.create': '创建', - 'common.update': '更新', - 'common.view': '查看', - 'common.loading': '加载中...', - 'common.noData': '暂无数据', - 'common.operation': '操作', - 'common.status': '状态', - 'common.name': '名称', - 'common.description': '描述', - 'common.createdAt': '创建时间', - 'common.updatedAt': '更新时间', }; From 44af984017dc127519ed68ea7aff257d5398e7f5 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 20:40:21 +0800 Subject: [PATCH 14/19] chore: tidy up --- frontend/src/locales/zh-CN.ts | 38 ----------------------------------- 1 file changed, 38 deletions(-) diff --git a/frontend/src/locales/zh-CN.ts b/frontend/src/locales/zh-CN.ts index a880f27c5..8404e43b9 100644 --- a/frontend/src/locales/zh-CN.ts +++ b/frontend/src/locales/zh-CN.ts @@ -312,42 +312,4 @@ export default { 'searchTest.history_question': '历史问题', 'searchTest.confirmDeleteHistory': '确定要删除这条历史记录吗？', 'searchTest.similarity': '相似度', - - audit: '---------------', - 'audit.title': '审计日志', - 'audit.description': '系统会自动记录所有重要操作的审计日志，帮助管理员监控和审计系统使用情况。', - 'audit.username': '用户名', - 'audit.userId': '用户ID', - 'audit.apiName': 'API名称', - 'audit.httpMethod': 'HTTP方法', - 'audit.path': 'API路径', - 'audit.resourceType': '资源类型', - 'audit.resourceId': '资源ID', - 'audit.statusCode': '状态码', - 'audit.duration': '耗时', - 'audit.ipAddress': 'IP地址', - 'audit.userAgent': 'User Agent', - 'audit.requestId': '请求ID', - 'audit.startTime': '开始时间', - 'audit.endTime': '结束时间', - 'audit.created': '创建时间', - 'audit.errorMessage': '错误信息', - 'audit.requestData': '请求数据', - 'audit.responseData': '响应数据', - 'audit.actions': '操作', - 'audit.search': '搜索', - 'audit.reset': '重置', - 'audit.refresh': '刷新', - 'audit.dateRange': '时间范围', - 'audit.startDate': '开始时间', - 'audit.endDate': '结束时间', - 'audit.enterUsername': '请输入用户名', - 'audit.enterApiName': '请输入API名称', - 'audit.selectResourceType': '请选择资源类型', - 'audit.selectHttpMethod': '请选择HTTP方法', - 'audit.detailTitle': '审计日志详情', - 'audit.id': 'ID', - 'audit.pagination': '第 {start}-{end} 条，共 {total} 条', - 'audit.fetchError': '获取审计日志失败', - 'audit.fetchDetailError': '获取审计日志详情失败', }; From 0e4b6380c92031c280543f7424d4e6a3290fd698 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 20:45:38 +0800 Subject: [PATCH 15/19] chore: tidy up --- tests/e2e_test/README.md | 17 - tests/e2e_test/test_audit.py | 1169 ---------------------------------- 2 files changed, 1186 deletions(-) delete mode 100644 tests/e2e_test/test_audit.py diff --git a/tests/e2e_test/README.md b/tests/e2e_test/README.md index 912f04ec4..dcb07244c 100644 --- a/tests/e2e_test/README.md +++ b/tests/e2e_test/README.md @@ -137,23 +137,6 @@ RERANK_MODEL_PROVIDER=siliconflow RERANK_MODEL_NAME=BAAI/bge-large-zh-1.5 ``` -## 🧪 Available Test Suites - -### Audit Logging Tests -Comprehensive tests for the audit logging functionality: -- **File**: `test_audit.py` -- **Documentation**: See `README_AUDIT.md` for detailed information -- **Coverage**: Collection, Document, Bot, Chat, LLM Provider, and LLM Provider Model operations -- **Features**: Sensitive data filtering, timestamp validation, user tracking - -```bash -# Run all audit tests -pytest tests/e2e_test/test_audit.py -v - -# Run specific audit test class -pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v -``` - ## 🧪 Available Fixtures The E2E tests provide the following pytest fixtures that can be used directly in tests: diff --git a/tests/e2e_test/test_audit.py b/tests/e2e_test/test_audit.py deleted file mode 100644 index b0740fb46..000000000 --- a/tests/e2e_test/test_audit.py +++ /dev/null @@ -1,1169 +0,0 @@ -# Copyright 2025 ApeCloud, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import time -from http import HTTPStatus -from typing import Dict, Any, Optional - -import pytest - -from tests.e2e_test.config import ( - COMPLETION_MODEL_CUSTOM_PROVIDER, - COMPLETION_MODEL_NAME, - COMPLETION_MODEL_PROVIDER, - EMBEDDING_MODEL_CUSTOM_PROVIDER, - EMBEDDING_MODEL_NAME, - EMBEDDING_MODEL_PROVIDER, -) - - -class AuditLogTestHelper: - """Helper class for audit log testing""" - - def __init__(self, cookie_client): - self.cookie_client = cookie_client - - def get_audit_logs(self, **filters) -> list: - """Get audit logs with optional filters""" - params = {k: v for k, v in filters.items() if v is not None} - resp = self.cookie_client.get("/api/v1/audit-logs", params=params) - assert resp.status_code == HTTPStatus.OK, f"Failed to get audit logs: {resp.text}" - return resp.json()["items"] - - def find_audit_log(self, resource_type: str, api_name: str, resource_id: Optional[str] = None, - http_method: str = None, max_wait_seconds: int = 10, - match_response_data: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: - """Find a specific audit log by criteria with retry mechanism""" - start_time = time.time() - while time.time() - start_time < max_wait_seconds: - logs = self.get_audit_logs( - resource_type=resource_type, - api_name=api_name, - http_method=http_method - ) - - for log in logs: - # Check resource_id if provided and available - if resource_id and log.get("resource_id") and log.get("resource_id") != resource_id: - continue - if http_method and log.get("http_method") != http_method: - continue - - # For create operations, match against response data when resource_id is not available in path - if (resource_id and not log.get("resource_id") and - match_response_data and log.get("response_data")): - try: - response_data = json.loads(log.get("response_data", "{}")) - match_found = True - for key, expected_value in match_response_data.items(): - if response_data.get(key) != expected_value: - match_found = False - break - if not match_found: - continue - except (json.JSONDecodeError, AttributeError): - continue - - return log - - time.sleep(0.5) # Wait before retry - - return None - - def assert_audit_log_content(self, log: Dict[str, Any], expected_fields: Dict[str, Any]): - """Assert audit log contains expected fields and values""" - assert log is not None, "Audit log not found" - - for field, expected_value in expected_fields.items(): - actual_value = log.get(field) - if expected_value is not None: - assert actual_value == expected_value, f"Field {field}: expected {expected_value}, got {actual_value}" - else: - assert field in log, f"Field {field} not found in audit log" - - # Basic assertions for all audit logs - assert log.get("start_time") is not None, "start_time should not be None" - assert log.get("end_time") is not None, "end_time should not be None" - assert log.get("duration_ms") is not None, "duration_ms should not be None" - assert log.get("status_code") in [200, 201, 204], f"Unexpected status code: {log.get('status_code')}" - assert log.get("user_id") is not None, "user_id should not be None" - assert log.get("username") is not None, "username should not be None" - - def parse_json_field(self, log: Dict[str, Any], field_name: str) -> Dict[str, Any]: - """Parse JSON string field from audit log""" - # Handle both dict and object access - if hasattr(log, field_name): - json_str = getattr(log, field_name, "{}") - else: - json_str = log.get(field_name, "{}") - - if json_str: - try: - return json.loads(json_str) - except (json.JSONDecodeError, TypeError): - return {} - return {} - - -@pytest.fixture -def audit_helper(cookie_client): - """Create audit log test helper""" - return AuditLogTestHelper(cookie_client) - - -class TestCollectionAudit: - """Test audit logs for collection operations""" - - def test_create_collection_audit(self, client, audit_helper): - """Test that creating a collection generates audit log""" - # Create collection - collection_data = { - "title": "Audit Test Collection", - "type": "document", - "config": { - "source": "system", - "enable_knowledge_graph": False, - "embedding": { - "model": EMBEDDING_MODEL_NAME, - "model_service_provider": EMBEDDING_MODEL_PROVIDER, - "custom_llm_provider": EMBEDDING_MODEL_CUSTOM_PROVIDER, - }, - }, - } - - resp = client.post("/api/v1/collections", json=collection_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to create collection: {resp.text}" - collection = resp.json() - collection_id = collection["id"] - - try: - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="collection", - api_name="CreateCollection", - resource_id=collection_id, - http_method="POST", - match_response_data={"id": collection_id} - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "collection", - "api_name": "CreateCollection", - "http_method": "POST", - "path": "/api/v1/collections" - }) - - # Verify request data contains collection info - request_data = audit_helper.parse_json_field(audit_log, "request_data") - assert "title" in request_data, "Request data should contain title" - assert request_data["title"] == collection_data["title"] - - # Verify response data contains collection ID - response_data = audit_helper.parse_json_field(audit_log, "response_data") - assert "id" in response_data, "Response data should contain collection ID" - assert response_data["id"] == collection_id - - finally: - # Cleanup - client.delete(f"/api/v1/collections/{collection_id}") - - def test_update_collection_audit(self, client, audit_helper, collection): - """Test that updating a collection generates audit log""" - collection_id = collection["id"] - - # Update collection - update_data = { - "title": "Updated Audit Test Collection", - "description": "Updated description for audit test", - "config": collection["config"] - } - - resp = client.put(f"/api/v1/collections/{collection_id}", json=update_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to update collection: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="collection", - api_name="UpdateCollection", - resource_id=collection_id, - http_method="PUT" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "collection", - "api_name": "UpdateCollection", - "resource_id": collection_id, - "http_method": "PUT", - "path": f"/api/v1/collections/{collection_id}" - }) - - # Verify request data contains update info - request_data = audit_helper.parse_json_field(audit_log, "request_data") - # The request data structure might be nested, check both formats - if "title" in request_data: - assert request_data["title"] == update_data["title"] - elif "collection" in request_data and "title" in request_data["collection"]: - assert request_data["collection"]["title"] == update_data["title"] - else: - assert False, f"title field not found in request_data: {request_data}" - - def test_delete_collection_audit(self, client, audit_helper): - """Test that deleting a collection generates audit log""" - # Create collection first - collection_data = { - "title": "Collection to Delete", - "type": "document", - "config": { - "source": "system", - "enable_knowledge_graph": False, - "embedding": { - "model": EMBEDDING_MODEL_NAME, - "model_service_provider": EMBEDDING_MODEL_PROVIDER, - "custom_llm_provider": EMBEDDING_MODEL_CUSTOM_PROVIDER, - }, - }, - } - - resp = client.post("/api/v1/collections", json=collection_data) - assert resp.status_code == HTTPStatus.OK - collection_id = resp.json()["id"] - - # Delete collection - resp = client.delete(f"/api/v1/collections/{collection_id}") - assert resp.status_code == HTTPStatus.OK, f"Failed to delete collection: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="collection", - api_name="DeleteCollection", - resource_id=collection_id, - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "collection", - "api_name": "DeleteCollection", - "resource_id": collection_id, - "http_method": "DELETE", - "path": f"/api/v1/collections/{collection_id}" - }) - - -class TestDocumentAudit: - """Test audit logs for document operations""" - - def test_create_document_audit(self, client, audit_helper, collection): - """Test that creating documents generates audit log""" - collection_id = collection["id"] - - # Upload document - files = {"files": ("audit_test.txt", "This is a test document for audit.", "text/plain")} - resp = client.post(f"/api/v1/collections/{collection_id}/documents", files=files) - assert resp.status_code == HTTPStatus.OK, f"Failed to create document: {resp.text}" - - documents = resp.json()["items"] - assert len(documents) > 0 - document_id = documents[0]["id"] - - try: - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="document", - api_name="CreateDocuments", - http_method="POST" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "document", - "api_name": "CreateDocuments", - "http_method": "POST", - "path": f"/api/v1/collections/{collection_id}/documents" - }) - - # Verify response data contains document info - response_data = audit_helper.parse_json_field(audit_log, "response_data") - assert "items" in response_data - assert len(response_data["items"]) > 0 - - finally: - # Cleanup - client.delete(f"/api/v1/collections/{collection_id}/documents/{document_id}") - - def test_delete_document_audit(self, client, audit_helper, collection, document): - """Test that deleting a document generates audit log""" - collection_id = collection["id"] - document_id = document["id"] - - # Delete document - resp = client.delete(f"/api/v1/collections/{collection_id}/documents/{document_id}") - assert resp.status_code == HTTPStatus.OK, f"Failed to delete document: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="document", - api_name="DeleteDocument", - resource_id=document_id, - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "document", - "api_name": "DeleteDocument", - "resource_id": document_id, - "http_method": "DELETE", - "path": f"/api/v1/collections/{collection_id}/documents/{document_id}" - }) - - -class TestBotAudit: - """Test audit logs for bot operations""" - - def test_create_bot_audit(self, client, audit_helper, collection): - """Test that creating a bot generates audit log""" - # Create bot - config = { - "model_name": COMPLETION_MODEL_NAME, - "model_service_provider": COMPLETION_MODEL_PROVIDER, - "llm": {"context_window": 3500, "similarity_score_threshold": 0.5, "similarity_topk": 3, "temperature": 0.1}, - } - bot_data = { - "title": "Audit Test Bot", - "description": "Bot for audit testing", - "type": "knowledge", - "config": json.dumps(config), - "collection_ids": [collection["id"]], - } - - resp = client.post("/api/v1/bots", json=bot_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to create bot: {resp.text}" - bot = resp.json() - bot_id = bot["id"] - - try: - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="bot", - api_name="CreateBot", - resource_id=bot_id, - http_method="POST", - match_response_data={"id": bot_id} - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "bot", - "api_name": "CreateBot", - "http_method": "POST", - "path": "/api/v1/bots" - }) - - # Verify request data contains bot info - request_data = audit_helper.parse_json_field(audit_log, "request_data") - assert "title" in request_data - assert request_data["title"] == bot_data["title"] - - finally: - # Cleanup - client.delete(f"/api/v1/bots/{bot_id}") - - @pytest.mark.skip(reason="Bot update requires valid LLM Provider configuration with API keys which are not available in test environment") - def test_update_bot_audit(self, client, audit_helper, bot): - """Test that updating a bot generates audit log""" - bot_id = bot["id"] - - # Update bot - include required fields - update_data = { - "title": "Updated Audit Test Bot", - "description": "Updated bot description", - "config": bot["config"], # Include original config to avoid JSON parsing error - "collection_ids": bot["collection_ids"] # Include original collection_ids - } - - resp = client.put(f"/api/v1/bots/{bot_id}", json=update_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to update bot: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="bot", - api_name="UpdateBot", - resource_id=bot_id, - http_method="PUT" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "bot", - "api_name": "UpdateBot", - "resource_id": bot_id, - "http_method": "PUT", - "path": f"/api/v1/bots/{bot_id}" - }) - - # Verify request data contains update info - request_data = audit_helper.parse_json_field(audit_log, "request_data") - assert "title" in request_data - assert request_data["title"] == update_data["title"] - - def test_delete_bot_audit(self, client, audit_helper, collection): - """Test that deleting a bot generates audit log""" - # Create bot first - config = { - "model_name": COMPLETION_MODEL_NAME, - "model_service_provider": COMPLETION_MODEL_PROVIDER, - "llm": {"context_window": 3500, "similarity_score_threshold": 0.5, "similarity_topk": 3, "temperature": 0.1}, - } - bot_data = { - "title": "Bot to Delete", - "description": "Bot for delete audit test", - "type": "knowledge", - "config": json.dumps(config), - "collection_ids": [collection["id"]], - } - - resp = client.post("/api/v1/bots", json=bot_data) - assert resp.status_code == HTTPStatus.OK - bot_id = resp.json()["id"] - - # Delete bot - resp = client.delete(f"/api/v1/bots/{bot_id}") - assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete bot: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="bot", - api_name="DeleteBot", - resource_id=bot_id, - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "bot", - "api_name": "DeleteBot", - "resource_id": bot_id, - "http_method": "DELETE", - "path": f"/api/v1/bots/{bot_id}" - }) - - -class TestChatAudit: - """Test audit logs for chat operations""" - - def test_create_chat_audit(self, client, audit_helper, bot): - """Test that creating a chat generates audit log""" - bot_id = bot["id"] - - # Create chat - resp = client.post(f"/api/v1/bots/{bot_id}/chats") - assert resp.status_code == HTTPStatus.OK, f"Failed to create chat: {resp.text}" - chat = resp.json() - chat_id = chat["id"] - - try: - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="chat", - api_name="CreateChat", - resource_id=chat_id, - http_method="POST", - match_response_data={"id": chat_id} - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "chat", - "api_name": "CreateChat", - "http_method": "POST", - "path": f"/api/v1/bots/{bot_id}/chats" - }) - - # Verify response data contains chat info - response_data = audit_helper.parse_json_field(audit_log, "response_data") - assert "id" in response_data - assert response_data["id"] == chat_id - - finally: - # Cleanup - client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") - - def test_update_chat_audit(self, client, audit_helper, bot): - """Test that updating a chat generates audit log""" - bot_id = bot["id"] - - # Create chat first - resp = client.post(f"/api/v1/bots/{bot_id}/chats") - assert resp.status_code == HTTPStatus.OK - chat = resp.json() - chat_id = chat["id"] - - try: - # Update chat - update_data = {"title": "Updated Chat Title"} - resp = client.put(f"/api/v1/bots/{bot_id}/chats/{chat_id}", json=update_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to update chat: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="chat", - api_name="UpdateChat", - resource_id=chat_id, - http_method="PUT" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "chat", - "api_name": "UpdateChat", - "resource_id": chat_id, - "http_method": "PUT", - "path": f"/api/v1/bots/{bot_id}/chats/{chat_id}" - }) - - # Verify request data contains update info - request_data = audit_helper.parse_json_field(audit_log, "request_data") - # The request data structure might be nested, check both formats - if "title" in request_data: - assert request_data["title"] == update_data["title"] - elif "chat_in" in request_data and "title" in request_data["chat_in"]: - assert request_data["chat_in"]["title"] == update_data["title"] - else: - assert False, f"title field not found in request_data: {request_data}" - - finally: - # Cleanup - client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") - - def test_delete_chat_audit(self, client, audit_helper, bot): - """Test that deleting a chat generates audit log""" - bot_id = bot["id"] - - # Create chat first - resp = client.post(f"/api/v1/bots/{bot_id}/chats") - assert resp.status_code == HTTPStatus.OK - chat_id = resp.json()["id"] - - # Delete chat - resp = client.delete(f"/api/v1/bots/{bot_id}/chats/{chat_id}") - assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete chat: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="chat", - api_name="DeleteChat", - resource_id=chat_id, - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "chat", - "api_name": "DeleteChat", - "resource_id": chat_id, - "http_method": "DELETE", - "path": f"/api/v1/bots/{bot_id}/chats/{chat_id}" - }) - - -class TestLLMProviderAudit: - """Test audit logs for LLM provider operations""" - - def test_create_llm_provider_audit(self, cookie_client, audit_helper): - """Test that creating an LLM provider generates audit log""" - # Create LLM provider - provider_data = { - "name": "audit-test-provider", - "label": "Audit Test Provider", - "base_url": "https://api.example.com/v1", - "completion_dialect": "openai", - "embedding_dialect": "openai", - "api_key": "test-api-key" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to create LLM provider: {resp.text}" - provider = resp.json() - provider_name = provider["name"] - - try: - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="CreateLLMProvider", - resource_id=provider_name, - http_method="POST", - match_response_data={"name": provider_name} - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider", - "api_name": "CreateLLMProvider", - "http_method": "POST", - "path": "/api/v1/llm_providers" - }) - - # Verify request data (API key should be filtered) - request_data = audit_helper.parse_json_field(audit_log, "request_data") - assert "label" in request_data - assert request_data["label"] == provider_data["label"] - # API key should be filtered out or redacted - if "api_key" in request_data: - assert request_data["api_key"] == "***FILTERED***" - - finally: - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - - def test_update_llm_provider_audit(self, cookie_client, audit_helper): - """Test that updating an LLM provider generates audit log""" - # Create provider first - provider_data = { - "name": "update-test-provider", - "label": "Provider for Update Test", - "base_url": "https://api.example.com/v1", - "api_key": "initial-key" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK - provider_name = resp.json()["name"] - - try: - # Update provider - update_data = { - "label": "Updated Provider Label", - "base_url": "https://api.updated.com/v1", - "api_key": "updated-key" - } - - resp = cookie_client.put(f"/api/v1/llm_providers/{provider_name}", json=update_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to update LLM provider: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="UpdateLLMProvider", - resource_id=provider_name, - http_method="PUT" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider", - "api_name": "UpdateLLMProvider", - "resource_id": provider_name, - "http_method": "PUT", - "path": f"/api/v1/llm_providers/{provider_name}" - }) - - # Verify request data contains update info (API key should be filtered) - request_data = audit_helper.parse_json_field(audit_log, "request_data") - # The request data structure might be nested, check both formats - if "label" in request_data: - assert request_data["label"] == update_data["label"] - if "api_key" in request_data: - assert request_data["api_key"] == "***FILTERED***" - elif "provider_data" in request_data and "label" in request_data["provider_data"]: - assert request_data["provider_data"]["label"] == update_data["label"] - if "api_key" in request_data["provider_data"]: - assert request_data["provider_data"]["api_key"] == "***FILTERED***" - else: - assert False, f"label field not found in request_data: {request_data}" - - finally: - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - - def test_delete_llm_provider_audit(self, cookie_client, audit_helper): - """Test that deleting an LLM provider generates audit log""" - # Create provider first - provider_data = { - "name": "delete-test-provider", - "label": "Provider for Delete Test", - "base_url": "https://api.example.com/v1" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK - provider_name = resp.json()["name"] - - # Delete provider - resp = cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete LLM provider: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="DeleteLLMProvider", - resource_id=provider_name, - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider", - "api_name": "DeleteLLMProvider", - "resource_id": provider_name, - "http_method": "DELETE", - "path": f"/api/v1/llm_providers/{provider_name}" - }) - - -class TestLLMProviderModelAudit: - """Test audit logs for LLM provider model operations""" - - @pytest.fixture - def test_provider(self, cookie_client): - """Create a test LLM provider for model tests""" - provider_data = { - "name": "model-test-provider", - "label": "Provider for Model Test", - "base_url": "https://api.example.com/v1" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK - provider = resp.json() - - yield provider - - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider['name']}") - - def test_create_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): - """Test that creating an LLM provider model generates audit log""" - provider_name = test_provider["name"] - - # Create provider model - import uuid - unique_id = str(uuid.uuid4())[:8] - model_data = { - "api": "completion", - "model": f"test-model-{unique_id}", - "custom_llm_provider": "openai", - "max_tokens": 4096, - "tags": ["test"] - } - - resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to create provider model: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider_model", - api_name="CreateProviderModel", - http_method="POST" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider_model", - "api_name": "CreateProviderModel", - "http_method": "POST", - "path": f"/api/v1/llm_providers/{provider_name}/models" - }) - - # Verify response data contains model info (since request data extraction has limitations) - response_data = audit_helper.parse_json_field(audit_log, "response_data") - assert "model" in response_data - assert response_data["model"] == model_data["model"] - assert "custom_llm_provider" in response_data - assert response_data["custom_llm_provider"] == model_data["custom_llm_provider"] - - def test_update_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): - """Test that updating an LLM provider model generates audit log""" - provider_name = test_provider["name"] - - # Create model first - import uuid - unique_id = str(uuid.uuid4())[:8] - model_data = { - "api": "completion", - "model": f"test-model-{unique_id}", - "custom_llm_provider": "openai", - "max_tokens": 4096 - } - - resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) - assert resp.status_code == HTTPStatus.OK - - # Update model - update_data = { - "custom_llm_provider": "anthropic", - "max_tokens": 8192, - "tags": ["updated", "test"] - } - - api = model_data["api"] - model = model_data["model"] - resp = cookie_client.put(f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}", json=update_data) - assert resp.status_code == HTTPStatus.OK, f"Failed to update provider model: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider_model", - api_name="UpdateProviderModel", - http_method="PUT" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider_model", - "api_name": "UpdateProviderModel", - "http_method": "PUT", - "path": f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}" - }) - - # Verify response data contains update info (since request data extraction has limitations) - response_data = audit_helper.parse_json_field(audit_log, "response_data") - assert "custom_llm_provider" in response_data - assert response_data["custom_llm_provider"] == update_data["custom_llm_provider"] - - def test_delete_llm_provider_model_audit(self, cookie_client, audit_helper, test_provider): - """Test that deleting an LLM provider model generates audit log""" - provider_name = test_provider["name"] - - # Create model first - import uuid - unique_id = str(uuid.uuid4())[:8] - model_data = { - "api": "completion", - "model": f"test-model-{unique_id}", - "custom_llm_provider": "openai" - } - - resp = cookie_client.post(f"/api/v1/llm_providers/{provider_name}/models", json=model_data) - assert resp.status_code == HTTPStatus.OK - - # Delete model - api = model_data["api"] - model = model_data["model"] - resp = cookie_client.delete(f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}") - assert resp.status_code in [HTTPStatus.OK, HTTPStatus.NO_CONTENT], f"Failed to delete provider model: {resp.text}" - - # Check audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider_model", - api_name="DeleteProviderModel", - http_method="DELETE" - ) - - audit_helper.assert_audit_log_content(audit_log, { - "resource_type": "llm_provider_model", - "api_name": "DeleteProviderModel", - "http_method": "DELETE", - "path": f"/api/v1/llm_providers/{provider_name}/models/{api}/{model}" - }) - - -class TestAuditLogRetrieval: - """Test audit log retrieval and filtering functionality""" - - def test_list_audit_logs(self, cookie_client, audit_helper): - """Test basic audit log listing functionality""" - # Get audit logs - logs = audit_helper.get_audit_logs(limit=10) - assert isinstance(logs, list), "Audit logs should be a list" - - # Verify log structure - if logs: # Only check if there are logs - log = logs[0] - required_fields = [ - "id", "resource_type", "api_name", "http_method", "path", - "status_code", "start_time", "end_time", "created" - ] - for field in required_fields: - assert field in log, f"Field {field} should be present in audit log" - - def test_audit_log_filtering(self, cookie_client, audit_helper): - """Test audit log filtering by resource type and other criteria""" - # Test filtering by resource type - collection_logs = audit_helper.get_audit_logs(resource_type="collection") - if collection_logs: - for log in collection_logs: - assert log["resource_type"] == "collection", "All logs should be collection type" - - # Test filtering by HTTP method - post_logs = audit_helper.get_audit_logs(http_method="POST") - if post_logs: - for log in post_logs: - assert log["http_method"] == "POST", "All logs should be POST method" - - # Test filtering by status code - success_logs = audit_helper.get_audit_logs(status_code=200) - if success_logs: - for log in success_logs: - assert log["status_code"] == 200, "All logs should have status code 200" - - def test_audit_log_detail(self, cookie_client, audit_helper): - """Test retrieving individual audit log details""" - # Get a log first - logs = audit_helper.get_audit_logs(limit=1) - if not logs: - pytest.skip("No audit logs available for testing") - - log_id = logs[0]["id"] - - # Get detailed log - resp = cookie_client.get(f"/api/v1/audit-logs/{log_id}") - assert resp.status_code == HTTPStatus.OK, f"Failed to get audit log detail: {resp.text}" - - detailed_log = resp.json() - assert detailed_log["id"] == log_id, "Log ID should match" - - # Verify all expected fields are present - required_fields = [ - "id", "resource_type", "api_name", "http_method", "path", - "status_code", "start_time", "end_time", "created" - ] - for field in required_fields: - assert field in detailed_log, f"Field {field} should be present in detailed audit log" - - -class TestAuditSensitiveDataFiltering: - """Test that sensitive data is properly filtered in audit logs""" - - def test_api_key_filtering_in_audit(self, cookie_client, audit_helper): - """Test that API keys are filtered in audit logs""" - # Create LLM provider with API key - provider_data = { - "name": "sensitive-test-provider", - "label": "Sensitive Data Test Provider", - "base_url": "https://api.example.com/v1", - "api_key": "sk-very-secret-api-key-12345" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK - provider_name = resp.json()["name"] - - try: - # Find the audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="CreateLLMProvider", - resource_id=provider_name, - http_method="POST" - ) - - # Verify API key is filtered - request_data = audit_helper.parse_json_field(audit_log, "request_data") - if "api_key" in request_data: - # API key should be filtered out or replaced with a placeholder - assert request_data["api_key"] != provider_data["api_key"], "API key should be filtered" - assert request_data["api_key"] == "***FILTERED***", "API key should be replaced with placeholder" - - finally: - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - - -class TestAuditLogIntegrity: - """Test audit log data integrity and consistency""" - - def test_audit_log_timestamps(self, cookie_client, audit_helper): - """Test that audit log timestamps are consistent and valid""" - # Create a simple resource to generate audit log - provider_data = { - "name": "timestamp-test-provider", - "label": "Timestamp Test Provider", - "base_url": "https://api.example.com/v1" - } - - before_request = int(time.time() * 1000) # milliseconds - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - after_request = int(time.time() * 1000) # milliseconds - - assert resp.status_code == HTTPStatus.OK - provider_name = resp.json()["name"] - - try: - # Find the audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="CreateLLMProvider", - resource_id=provider_name, - http_method="POST", - match_response_data={"name": provider_name} - ) - - # Verify timestamps - start_time = audit_log.get("start_time") - end_time = audit_log.get("end_time") - duration_ms = audit_log.get("duration_ms") - - assert start_time is not None, "start_time should not be None" - assert end_time is not None, "end_time should not be None" - assert duration_ms is not None, "duration_ms should not be None" - - # Verify timestamp ranges - assert before_request <= start_time <= after_request, "start_time should be within request timeframe" - assert before_request <= end_time <= after_request + 5000, "end_time should be reasonable" # Allow 5s buffer - - # Verify duration calculation - expected_duration = end_time - start_time - assert duration_ms == expected_duration, f"duration_ms should match calculated duration: {duration_ms} != {expected_duration}" - - # Verify start_time <= end_time - assert start_time <= end_time, "start_time should be <= end_time" - - finally: - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - - def test_audit_log_user_info(self, cookie_client, audit_helper): - """Test that audit logs contain correct user information""" - # Create a simple resource to generate audit log - provider_data = { - "name": "user-info-test-provider", - "label": "User Info Test Provider", - "base_url": "https://api.example.com/v1" - } - - resp = cookie_client.post("/api/v1/llm_providers", json=provider_data) - assert resp.status_code == HTTPStatus.OK - provider_name = resp.json()["name"] - - try: - # Find the audit log - audit_log = audit_helper.find_audit_log( - resource_type="llm_provider", - api_name="CreateLLMProvider", - resource_id=provider_name, - http_method="POST", - match_response_data={"name": provider_name} - ) - - # Verify user information is present - assert audit_log.get("user_id") is not None, "user_id should be present" - assert audit_log.get("username") is not None, "username should be present" - - # Verify user_id is a valid UUID-like string - user_id = audit_log.get("user_id") - assert len(user_id) > 0, "user_id should not be empty" - - # Verify username is a valid string - username = audit_log.get("username") - assert len(username) > 0, "username should not be empty" - assert isinstance(username, str), "username should be a string" - - finally: - # Cleanup - cookie_client.delete(f"/api/v1/llm_providers/{provider_name}") - - -# Additional helper functions for testing audit logs in specific scenarios - -def verify_audit_log_for_resource_operation(audit_helper, resource_type: str, operation: str, - resource_id: str = None, expected_path: str = None): - """Helper function to verify audit log for a specific resource operation""" - http_method_map = { - "create": "POST", - "update": "PUT", - "delete": "DELETE" - } - - api_name_map = { - "collection": { - "create": "CreateCollection", - "update": "UpdateCollection", - "delete": "DeleteCollection" - }, - "document": { - "create": "CreateDocuments", - "update": "UpdateDocument", - "delete": "DeleteDocument" - }, - "bot": { - "create": "CreateBot", - "update": "UpdateBot", - "delete": "DeleteBot" - }, - "chat": { - "create": "CreateChat", - "update": "UpdateChat", - "delete": "DeleteChat" - }, - "llm_provider": { - "create": "CreateLLMProvider", - "update": "UpdateLLMProvider", - "delete": "DeleteLLMProvider" - }, - "llm_provider_model": { - "create": "CreateProviderModel", - "update": "UpdateProviderModel", - "delete": "DeleteProviderModel" - } - } - - http_method = http_method_map.get(operation) - api_name = api_name_map.get(resource_type, {}).get(operation) - - if not http_method or not api_name: - raise ValueError(f"Invalid resource_type '{resource_type}' or operation '{operation}'") - - audit_log = audit_helper.find_audit_log( - resource_type=resource_type, - api_name=api_name, - resource_id=resource_id, - http_method=http_method - ) - - expected_fields = { - "resource_type": resource_type, - "api_name": api_name, - "http_method": http_method - } - - if resource_id: - expected_fields["resource_id"] = resource_id - - if expected_path: - expected_fields["path"] = expected_path - - audit_helper.assert_audit_log_content(audit_log, expected_fields) - - return audit_log - - -""" -Test Usage Examples: - -1. Run all audit tests: - pytest tests/e2e_test/test_audit.py -v - -2. Run specific test class: - pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v - -3. Run specific test method: - pytest tests/e2e_test/test_audit.py::TestCollectionAudit::test_create_collection_audit -v - -4. Run tests with specific markers (if added): - pytest tests/e2e_test/test_audit.py -m "audit" -v - -5. Run audit tests for specific resource types: - pytest tests/e2e_test/test_audit.py::TestCollectionAudit -v # Collection tests - pytest tests/e2e_test/test_audit.py::TestBotAudit -v # Bot tests - pytest tests/e2e_test/test_audit.py::TestChatAudit -v # Chat tests - -The test cases cover: -- Creation/Update/Deletion operations for all specified resource types -- Audit log content validation (resource_type, api_name, timestamps, etc.) -- Sensitive data filtering (API keys, passwords, etc.) -- Audit log retrieval and filtering functionality -- Data integrity and consistency checks -- User information tracking in audit logs - -Each test follows the pattern: -1. Perform operation that should generate audit log -2. Search for the audit log using the helper -3. Validate audit log content and structure -4. Clean up resources created during test - -The AuditLogTestHelper class provides convenient methods for: -- Retrieving audit logs with filters -- Finding specific audit logs by criteria -- Asserting audit log content matches expectations -- Handling retry logic for eventual consistency -""" From 23d397d77e44820f4a51044c752ee41d6d655bab Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 20:51:27 +0800 Subject: [PATCH 16/19] chore: tidy up --- frontend/src/pages/settings/auditLogs.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx index 25912836b..295cada2f 100644 --- a/frontend/src/pages/settings/auditLogs.tsx +++ b/frontend/src/pages/settings/auditLogs.tsx @@ -159,7 +159,7 @@ const AuditLogsPage: React.FC = () => { key: 'username', width: 120, render: (text?: string) => ( - {text || intl.formatMessage({ id: 'common.system', defaultMessage: 'System' })} + {text || '-'} ), }, { From 96304b8f192a886396558632caa67e8b38f0690f Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 21:04:19 +0800 Subject: [PATCH 17/19] chore: tidy up --- aperag/db/models.py | 5 +- aperag/exceptions.py | 2 - aperag/service/audit_service.py | 73 +++++++++-------- aperag/utils/audit_decorator.py | 138 +++++++++++++++++--------------- aperag/views/api_key.py | 16 ++-- aperag/views/audit.py | 89 ++++++++++---------- aperag/views/auth.py | 42 ++++++---- aperag/views/chat_completion.py | 2 +- aperag/views/config.py | 2 +- aperag/views/flow.py | 8 +- aperag/views/llm.py | 10 +-- aperag/views/main.py | 126 +++++++++++++++-------------- 12 files changed, 267 insertions(+), 246 deletions(-) diff --git a/aperag/db/models.py b/aperag/db/models.py index 0e3b3c6cc..e77d06d54 100644 --- a/aperag/db/models.py +++ b/aperag/db/models.py @@ -24,14 +24,12 @@ Boolean, Column, DateTime, + Index, Integer, String, Text, UniqueConstraint, select, - Index, - Float, - func, ) from sqlalchemy import Enum as SQLEnum from sqlalchemy.ext.declarative import declarative_base @@ -753,6 +751,7 @@ def update_spec(self, desired_state: IndexDesiredState = None, created_by: str = class AuditResource(str, Enum): """Audit resource types""" + COLLECTION = "collection" DOCUMENT = "document" BOT = "bot" diff --git a/aperag/exceptions.py b/aperag/exceptions.py index e460e9dc8..bf9fa09a4 100644 --- a/aperag/exceptions.py +++ b/aperag/exceptions.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import traceback from enum import Enum from http import HTTPStatus from typing import Any, Dict, Optional @@ -90,7 +89,6 @@ def __init__(self, error_code: ErrorCode, message: Optional[str] = None, details self.error_code = error_code self.message = message or error_code.error_name.replace("_", " ").title() self.details = details or {} - traceback.print_stack() super().__init__(self.message) @property diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index 460f0cd8b..a0aaceddd 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -15,7 +15,6 @@ import json import logging import re -import time import uuid from datetime import datetime from typing import Any, Dict, List, Optional @@ -35,10 +34,17 @@ def __init__(self): self.enabled = True # Sensitive fields that should be filtered from logs self.sensitive_fields = { - "password", "token", "api_key", "secret", "authorization", - "access_token", "refresh_token", "private_key", "credential" + "password", + "token", + "api_key", + "secret", + "authorization", + "access_token", + "refresh_token", + "private_key", + "credential", } - + # Map FastAPI tags to audit resources self.tag_resource_map = { # Support both singular and plural forms @@ -84,8 +90,7 @@ def _filter_sensitive_data(self, data: Dict[str, Any]) -> Dict[str, Any]: filtered[key] = self._filter_sensitive_data(value) elif isinstance(value, list): filtered[key] = [ - self._filter_sensitive_data(item) if isinstance(item, dict) else item - for item in value + self._filter_sensitive_data(item) if isinstance(item, dict) else item for item in value ] else: filtered[key] = value @@ -95,21 +100,21 @@ def _safe_json_serialize(self, data: Any) -> str: """Safely serialize data to JSON string""" if data is None: return None - + try: # Filter sensitive data first if isinstance(data, dict): data = self._filter_sensitive_data(data) - + # Handle special types that aren't JSON serializable def json_serializer(obj): - if hasattr(obj, 'dict'): # Pydantic models + if hasattr(obj, "dict"): # Pydantic models return obj.dict() - elif hasattr(obj, '__dict__'): # Regular objects + elif hasattr(obj, "__dict__"): # Regular objects return obj.__dict__ else: return str(obj) - + return json.dumps(data, default=json_serializer, ensure_ascii=False) except Exception as e: logger.warning(f"Failed to serialize data: {e}") @@ -120,15 +125,15 @@ def extract_resource_id_from_path(self, path: str, resource_type: AuditResource) try: # Define ID extraction patterns for different resource types id_patterns = { - AuditResource.MESSAGE: r'/messages/([^/]+)', - AuditResource.CHAT: r'/chats/([^/]+)', - AuditResource.DOCUMENT: r'/documents/([^/]+)', - AuditResource.BOT: r'/bots/([^/]+)', - AuditResource.COLLECTION: r'/collections/([^/]+)', - AuditResource.API_KEY: r'/apikeys/([^/]+)', - AuditResource.LLM_PROVIDER: r'/llm_providers/([^/]+)', - AuditResource.LLM_PROVIDER_MODEL: r'/models/([^/]+/[^/]+)', - AuditResource.USER: r'/users/([^/]+)', + AuditResource.MESSAGE: r"/messages/([^/]+)", + AuditResource.CHAT: r"/chats/([^/]+)", + AuditResource.DOCUMENT: r"/documents/([^/]+)", + AuditResource.BOT: r"/bots/([^/]+)", + AuditResource.COLLECTION: r"/collections/([^/]+)", + AuditResource.API_KEY: r"/apikeys/([^/]+)", + AuditResource.LLM_PROVIDER: r"/llm_providers/([^/]+)", + AuditResource.LLM_PROVIDER_MODEL: r"/models/([^/]+/[^/]+)", + AuditResource.USER: r"/users/([^/]+)", } pattern = id_patterns.get(resource_type) @@ -136,10 +141,10 @@ def extract_resource_id_from_path(self, path: str, resource_type: AuditResource) match = re.search(pattern, path) if match: return match.group(1) - + except Exception as e: logger.warning(f"Failed to extract resource ID: {e}") - + return None async def log_audit( @@ -158,7 +163,7 @@ async def log_audit( error_message: Optional[str] = None, ip_address: Optional[str] = None, user_agent: Optional[str] = None, - request_id: Optional[str] = None + request_id: Optional[str] = None, ): """Log an audit entry""" if not self.enabled: @@ -182,7 +187,7 @@ async def log_audit( error_message=error_message, ip_address=ip_address, user_agent=user_agent, - request_id=request_id or str(uuid.uuid4()) + request_id=request_id or str(uuid.uuid4()), ) # Save to database asynchronously @@ -202,13 +207,13 @@ async def list_audit_logs( status_code: Optional[int] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, - limit: int = 1000 + limit: int = 1000, ) -> List[AuditLog]: """List audit logs with filtering""" async for session in get_async_session(): # Build query stmt = select(AuditLog) - + # Add filters conditions = [] if user_id: @@ -225,17 +230,17 @@ async def list_audit_logs( conditions.append(AuditLog.gmt_created >= start_date) if end_date: conditions.append(AuditLog.gmt_created <= end_date) - + if conditions: stmt = stmt.where(and_(*conditions)) - + # Order by creation time (newest first) and limit stmt = stmt.order_by(desc(AuditLog.gmt_created)).limit(limit) - + # Execute query result = await session.execute(stmt) audit_logs = result.scalars().all() - + # Extract resource_id for each log during query time for log in audit_logs: if log.resource_type and log.path: @@ -246,20 +251,20 @@ async def list_audit_logs( resource_type_enum = AuditResource(log.resource_type) except ValueError: resource_type_enum = None - + if resource_type_enum: log.resource_id = self.extract_resource_id_from_path(log.path, resource_type_enum) else: log.resource_id = None - + # Calculate duration if both times are available if log.start_time and log.end_time: log.duration_ms = log.end_time - log.start_time else: log.duration_ms = None - + return audit_logs # Global audit service instance -audit_service = AuditService() \ No newline at end of file +audit_service = AuditService() diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py index 10534c773..a38c5e71d 100644 --- a/aperag/utils/audit_decorator.py +++ b/aperag/utils/audit_decorator.py @@ -13,7 +13,6 @@ # limitations under the License. import functools -import json import logging import time from typing import Any, Dict, Optional @@ -31,33 +30,33 @@ def _extract_response_data(response: Any) -> Optional[Dict[str, Any]]: # If response is already a dict (common for JSON APIs) if isinstance(response, dict): return response - + # If response has a dict() method (Pydantic models) - elif hasattr(response, 'dict'): + elif hasattr(response, "dict"): return response.dict() - + # If response has a model_dump() method (Pydantic v2) - elif hasattr(response, 'model_dump'): + elif hasattr(response, "model_dump"): return response.model_dump() - + # If response is a list of dicts or models elif isinstance(response, list): result = [] for item in response: if isinstance(item, dict): result.append(item) - elif hasattr(item, 'dict'): + elif hasattr(item, "dict"): result.append(item.dict()) - elif hasattr(item, 'model_dump'): + elif hasattr(item, "model_dump"): result.append(item.model_dump()) else: result.append(str(item)) return {"items": result} - + # For other types, try to convert to string else: return {"response": str(response)} - + except Exception as e: logger.debug(f"Failed to extract response data: {e}") return {"status": "success", "type": type(response).__name__} @@ -67,17 +66,17 @@ def _clean_data_for_audit(data): """Clean data for audit logging - remove null values and sensitive information""" if data is None: return None - + if isinstance(data, dict): cleaned = {} for key, value in data.items(): # Skip null/None values if value is None: continue - + # Filter out sensitive fields key_lower = key.lower() - if any(sensitive in key_lower for sensitive in ['password', 'secret', 'token', 'key']): + if any(sensitive in key_lower for sensitive in ["password", "secret", "token", "key"]): cleaned[key] = "***FILTERED***" else: # Recursively clean nested data @@ -87,9 +86,9 @@ def _clean_data_for_audit(data): if not (isinstance(cleaned_value, dict) and len(cleaned_value) == 0): # Don't add empty dicts cleaned[key] = cleaned_value - + return cleaned if cleaned else None - + elif isinstance(data, list): cleaned = [] for item in data: @@ -97,7 +96,7 @@ def _clean_data_for_audit(data): if cleaned_item is not None: cleaned.append(cleaned_item) return cleaned if cleaned else None - + else: # For primitive types, return as-is return data @@ -113,20 +112,20 @@ def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[ # Skip the request object itself if isinstance(value, Request): continue - + # Skip User objects and other database model objects - if hasattr(value, '__tablename__'): # SQLAlchemy model + if hasattr(value, "__tablename__"): # SQLAlchemy model continue - + # Try to serialize the value try: - if hasattr(value, 'dict'): # Pydantic model + if hasattr(value, "dict"): # Pydantic model serialized = value.dict() # Clean up the serialized data - remove null values and filter sensitive data cleaned_data = _clean_data_for_audit(serialized) if cleaned_data: # Only add if there's actual data parsed_data[key] = cleaned_data - elif hasattr(value, 'model_dump'): # Pydantic v2 + elif hasattr(value, "model_dump"): # Pydantic v2 serialized = value.model_dump() # Clean up the serialized data cleaned_data = _clean_data_for_audit(serialized) @@ -140,12 +139,12 @@ def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[ else: # For other types, convert to string but skip if it looks like an object str_value = str(value) - if not (' object at 0x' in str_value): # Skip object representations + if " object at 0x" not in str_value: # Skip object representations parsed_data[key] = str_value except Exception: # Skip problematic values continue - + # Return the actual data directly, not wrapped in any structure # If there's only one main data object, return it directly if len(parsed_data) == 1: @@ -154,7 +153,7 @@ def _extract_request_data_from_args(request: Request, kwargs: dict) -> Optional[ return parsed_data else: return None - + except Exception as e: logger.warning(f"Failed to extract request data from args: {e}") return None @@ -165,42 +164,51 @@ def _extract_client_info(request) -> tuple[Optional[str], Optional[str]]: try: # Get IP address ip_address = None - if hasattr(request, 'client') and request.client: + if hasattr(request, "client") and request.client: ip_address = request.client.host - + # Check for forwarded headers - if hasattr(request, 'headers'): - forwarded_for = request.headers.get('X-Forwarded-For') + if hasattr(request, "headers"): + forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: - ip_address = forwarded_for.split(',')[0].strip() - elif request.headers.get('X-Real-IP'): - ip_address = request.headers.get('X-Real-IP') - + ip_address = forwarded_for.split(",")[0].strip() + elif request.headers.get("X-Real-IP"): + ip_address = request.headers.get("X-Real-IP") + # Get User-Agent user_agent = None - if hasattr(request, 'headers'): - user_agent = request.headers.get('User-Agent') - + if hasattr(request, "headers"): + user_agent = request.headers.get("User-Agent") + return ip_address, user_agent except Exception as e: logger.warning(f"Failed to extract client info: {e}") return None, None -async def _log_audit_async(request: Request, resource_type: str, api_name: str, - start_time_ms: int, end_time_ms: int, status_code: int, - request_data: dict, response_data: dict, error_message: str = None): +async def _log_audit_async( + request: Request, + resource_type: str, + api_name: str, + start_time_ms: int, + end_time_ms: int, + status_code: int, + request_data: dict, + response_data: dict, + error_message: str = None, +): """Log audit information asynchronously""" try: # Get user info from request state - user_id = getattr(request.state, 'user_id', None) - username = getattr(request.state, 'username', None) - + user_id = getattr(request.state, "user_id", None) + username = getattr(request.state, "username", None) + # Extract client info ip_address, user_agent = _extract_client_info(request) - + # Log audit in background import asyncio + asyncio.create_task( audit_service.log_audit( user_id=user_id, @@ -216,20 +224,22 @@ async def _log_audit_async(request: Request, resource_type: str, api_name: str, response_data=response_data, error_message=error_message, ip_address=ip_address, - user_agent=user_agent + user_agent=user_agent, ) ) except Exception as audit_error: - logger.error(f"Failed to log audit: {audit_error}") + logger.error(f"Failed to log audit: {audit_error}") -def audit_api(resource_type: str, api_name: str = None): + +def audit(resource_type: str, api_name: str = None): """ Decorator for API endpoints to enable automatic audit logging - + Args: resource_type: The resource type for audit (e.g., 'collection', 'user', etc.) api_name: Optional API name override (defaults to function name) """ + def decorator(func): @functools.wraps(func) async def wrapper(*args, **kwargs): @@ -239,32 +249,32 @@ async def wrapper(*args, **kwargs): if isinstance(v, Request): request = v break - + if not request: # If no request found, just call the original function return await func(*args, **kwargs) - + # Skip GET requests - only audit change operations if request.method.upper() == "GET": return await func(*args, **kwargs) - + # Record start time start_time_ms = int(time.time() * 1000) actual_api_name = api_name or func.__name__ - + try: # Call the original function first to get the parsed data response = await func(*args, **kwargs) - + # Record end time end_time_ms = int(time.time() * 1000) - + # Extract request data from function arguments (after parsing) request_data = _extract_request_data_from_args(request, kwargs) - + # Extract response data response_data = _extract_response_data(response) - + # Log audit asynchronously await _log_audit_async( request=request, @@ -275,21 +285,21 @@ async def wrapper(*args, **kwargs): status_code=200, # Success request_data=request_data, response_data=response_data, - error_message=None + error_message=None, ) - + return response - + except Exception as e: # Record end time for error case end_time_ms = int(time.time() * 1000) - + # Extract request data if possible try: request_data = _extract_request_data_from_args(request, kwargs) - except: + except Exception: request_data = {"method": request.method, "path": request.url.path} - + # Log audit for error case await _log_audit_async( request=request, @@ -300,12 +310,12 @@ async def wrapper(*args, **kwargs): status_code=500, # Error request_data=request_data, response_data={"error": str(e)}, - error_message=str(e) + error_message=str(e), ) - + # Re-raise the exception raise - + return wrapper - return decorator + return decorator diff --git a/aperag/views/api_key.py b/aperag/views/api_key.py index ef857c272..72dcb6ddd 100644 --- a/aperag/views/api_key.py +++ b/aperag/views/api_key.py @@ -18,20 +18,20 @@ from aperag.db.models import User from aperag.schema.view_models import ApiKeyCreate, ApiKeyList, ApiKeyUpdate from aperag.service.api_key_service import api_key_service -from aperag.utils.audit_decorator import audit_api +from aperag.utils.audit_decorator import audit from aperag.views.auth import current_user router = APIRouter() -@router.get("/apikeys", tags=["apikey"], name="ListApiKeys") +@router.get("/apikeys") async def list_api_keys_view(request: Request, user: User = Depends(current_user)) -> ApiKeyList: """List all API keys for the current user""" return await api_key_service.list_api_keys(str(user.id)) -@router.post("/apikeys", tags=["apikey"], name="CreateApiKey") -@audit_api(resource_type="apikey", api_name="CreateApiKey") +@router.post("/apikeys") +@audit(resource_type="apikey", api_name="CreateApiKey") async def create_api_key_view( request: Request, api_key_create: ApiKeyCreate, @@ -41,15 +41,15 @@ async def create_api_key_view( return await api_key_service.create_api_key(str(user.id), api_key_create) -@router.delete("/apikeys/{apikey_id}", tags=["apikey"], name="DeleteApiKey") -@audit_api(resource_type="apikey", api_name="DeleteApiKey") +@router.delete("/apikeys/{apikey_id}") +@audit(resource_type="apikey", api_name="DeleteApiKey") async def delete_api_key_view(request: Request, apikey_id: str, user: User = Depends(current_user)): """Delete an API key""" return await api_key_service.delete_api_key(str(user.id), apikey_id) -@router.put("/apikeys/{apikey_id}", tags=["apikey"], name="UpdateApiKey") -@audit_api(resource_type="apikey", api_name="UpdateApiKey") +@router.put("/apikeys/{apikey_id}") +@audit(resource_type="apikey", api_name="UpdateApiKey") async def update_api_key_view( request: Request, apikey_id: str, diff --git a/aperag/views/audit.py b/aperag/views/audit.py index 3459ba736..e4b0ce473 100644 --- a/aperag/views/audit.py +++ b/aperag/views/audit.py @@ -13,21 +13,21 @@ # limitations under the License. from datetime import datetime -from typing import List, Optional +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import desc, select +from sqlalchemy import select -from aperag.db.models import AuditLog, AuditResource, User from aperag.config import get_async_session +from aperag.db.models import AuditLog, AuditResource, User from aperag.schema import view_models from aperag.service.audit_service import audit_service -from aperag.views.auth import current_user, get_current_admin +from aperag.views.auth import current_user router = APIRouter() -@router.get("/audit-logs", tags=["audit"], name="ListAuditLogs", response_model=view_models.AuditLogList) +@router.get("/audit-logs") async def list_audit_logs( user_id: Optional[str] = Query(None, description="Filter by user ID"), username: Optional[str] = Query(None, description="Filter by username"), @@ -39,13 +39,13 @@ async def list_audit_logs( start_date: Optional[datetime] = Query(None, description="Filter by start date"), end_date: Optional[datetime] = Query(None, description="Filter by end date"), limit: int = Query(1000, le=5000, description="Maximum number of records"), - user: User = Depends(current_user) + user: User = Depends(current_user), ): """List audit logs with filtering""" - + # Convert string enums to actual enum values audit_resource = None - + if resource_type: try: audit_resource = AuditResource(resource_type) @@ -61,52 +61,51 @@ async def list_audit_logs( status_code=status_code, start_date=start_date, end_date=end_date, - limit=limit + limit=limit, ) # Convert to view models items = [] for log in audit_logs: - items.append(view_models.AuditLog( - id=str(log.id), - user_id=log.user_id, - username=log.username, - resource_type=log.resource_type.value if hasattr(log.resource_type, 'value') else log.resource_type, - resource_id=getattr(log, 'resource_id', None), # This is set during query - api_name=log.api_name, - http_method=log.http_method, - path=log.path, - status_code=log.status_code, - start_time=log.start_time, - end_time=log.end_time, - duration_ms=getattr(log, 'duration_ms', None), # Calculated during query - request_data=log.request_data, - response_data=log.response_data, - error_message=log.error_message, - ip_address=log.ip_address, - user_agent=log.user_agent, - request_id=log.request_id, - created=log.gmt_created - )) + items.append( + view_models.AuditLog( + id=str(log.id), + user_id=log.user_id, + username=log.username, + resource_type=log.resource_type.value if hasattr(log.resource_type, "value") else log.resource_type, + resource_id=getattr(log, "resource_id", None), # This is set during query + api_name=log.api_name, + http_method=log.http_method, + path=log.path, + status_code=log.status_code, + start_time=log.start_time, + end_time=log.end_time, + duration_ms=getattr(log, "duration_ms", None), # Calculated during query + request_data=log.request_data, + response_data=log.response_data, + error_message=log.error_message, + ip_address=log.ip_address, + user_agent=log.user_agent, + request_id=log.request_id, + created=log.gmt_created, + ) + ) return view_models.AuditLogList(items=items) -@router.get("/audit-logs/{audit_id}", tags=["audit"], name="GetAuditLog", response_model=view_models.AuditLog) -async def get_audit_log( - audit_id: str, - user: User = Depends(current_user) -): +@router.get("/audit-logs/{audit_id}") +async def get_audit_log(audit_id: str, user: User = Depends(current_user)): """Get a specific audit log by ID""" - + async for session in get_async_session(): stmt = select(AuditLog).where(AuditLog.id == audit_id) result = await session.execute(stmt) audit_log = result.scalar_one_or_none() - + if not audit_log: raise HTTPException(status_code=404, detail="Audit log not found") - + # Extract resource_id for this specific log resource_id = None if audit_log.resource_type and audit_log.path: @@ -117,20 +116,22 @@ async def get_audit_log( resource_type_enum = AuditResource(audit_log.resource_type) except ValueError: resource_type_enum = None - + if resource_type_enum: resource_id = audit_service.extract_resource_id_from_path(audit_log.path, resource_type_enum) - + # Calculate duration if both times are available duration_ms = None if audit_log.start_time and audit_log.end_time: duration_ms = audit_log.end_time - audit_log.start_time - + return view_models.AuditLog( id=str(audit_log.id), user_id=audit_log.user_id, username=audit_log.username, - resource_type=audit_log.resource_type.value if hasattr(audit_log.resource_type, 'value') else audit_log.resource_type, + resource_type=audit_log.resource_type.value + if hasattr(audit_log.resource_type, "value") + else audit_log.resource_type, resource_id=resource_id, api_name=audit_log.api_name, http_method=audit_log.http_method, @@ -145,7 +146,5 @@ async def get_audit_log( ip_address=audit_log.ip_address, user_agent=audit_log.user_agent, request_id=audit_log.request_id, - created=audit_log.gmt_created + created=audit_log.gmt_created, ) - - diff --git a/aperag/views/auth.py b/aperag/views/auth.py index 1f9dafa57..6338aff54 100644 --- a/aperag/views/auth.py +++ b/aperag/views/auth.py @@ -26,7 +26,7 @@ from aperag.db.models import ApiKey, ApiKeyStatus, Invitation, Role, User from aperag.db.ops import async_db_ops from aperag.schema import view_models -from aperag.utils.audit_decorator import audit_api +from aperag.utils.audit_decorator import audit from aperag.utils.utils import utc_now logger = logging.getLogger(__name__) @@ -241,10 +241,13 @@ async def get_current_admin(session: AsyncSessionDep, user: User = Depends(get_c # --- API Implementation --- -@router.post("/invite", tags=["invitation"], name="CreateInvitation") -@audit_api(resource_type="invitation", api_name="CreateInvitation") +@router.post("/invite") +@audit(resource_type="invitation", api_name="CreateInvitation") async def create_invitation_view( - request: Request, data: view_models.InvitationCreate, session: AsyncSessionDep, user: User = Depends(get_current_admin) + request: Request, + data: view_models.InvitationCreate, + session: AsyncSessionDep, + user: User = Depends(get_current_admin), ) -> view_models.Invitation: # Check if user already exists from sqlalchemy import select @@ -276,7 +279,7 @@ async def create_invitation_view( ) -@router.get("/invitations", tags=["invitation"], name="ListInvitations") +@router.get("/invitations") async def list_invitations_view( session: AsyncSessionDep, user: User = Depends(get_current_admin) ) -> view_models.InvitationList: @@ -300,10 +303,13 @@ async def list_invitations_view( return view_models.InvitationList(items=invitations) -@router.post("/register", tags=["auth"], name="Register") -@audit_api(resource_type="user", api_name="RegisterUser") +@router.post("/register") +@audit(resource_type="user", api_name="RegisterUser") async def register_view( - request: Request, data: view_models.Register, session: AsyncSessionDep, user_manager: UserManager = Depends(get_user_manager) + request: Request, + data: view_models.Register, + session: AsyncSessionDep, + user_manager: UserManager = Depends(get_user_manager), ) -> view_models.User: from sqlalchemy import select @@ -357,7 +363,7 @@ async def register_view( ) -@router.post("/login", tags=["auth"], name="Login") +@router.post("/login") async def login_view( request: Request, response: Response, @@ -400,14 +406,14 @@ async def login_view( ) -@router.post("/logout", tags=["auth"], name="Logout") +@router.post("/logout") async def logout_view(response: Response): # Clear authentication cookie response.delete_cookie(key="session") return {"success": True} -@router.get("/user", tags=["user"], name="GetCurrentUser") +@router.get("/user") async def get_user_view(request: Request, session: AsyncSessionDep, user: Optional[User] = Depends(current_user)): """Get user info, return 401 if not authenticated""" if not user: @@ -423,7 +429,7 @@ async def get_user_view(request: Request, session: AsyncSessionDep, user: Option ) -@router.get("/users", tags=["user"], name="ListUsers") +@router.get("/users") async def list_users_view(session: AsyncSessionDep, user: User = Depends(get_current_admin)) -> view_models.UserList: from sqlalchemy import select @@ -442,8 +448,8 @@ async def list_users_view(session: AsyncSessionDep, user: User = Depends(get_cur return view_models.UserList(items=users) -@router.post("/change-password", tags=["user"], name="ChangePassword") -@audit_api(resource_type="user", api_name="ChangePassword") +@router.post("/change-password") +@audit(resource_type="user", api_name="ChangePassword") async def change_password_view( request: Request, data: view_models.ChangePassword, @@ -474,9 +480,11 @@ async def change_password_view( ) -@router.delete("/users/{user_id}", tags=["user"], name="DeleteUser") -@audit_api(resource_type="user", api_name="DeleteUser") -async def delete_user_view(request: Request, user_id: str, session: AsyncSessionDep, user: User = Depends(get_current_admin)): +@router.delete("/users/{user_id}") +@audit(resource_type="user", api_name="DeleteUser") +async def delete_user_view( + request: Request, user_id: str, session: AsyncSessionDep, user: User = Depends(get_current_admin) +): from sqlalchemy import select result = await session.execute(select(User).where(User.id == user_id)) diff --git a/aperag/views/chat_completion.py b/aperag/views/chat_completion.py index 57469694a..f023353b8 100644 --- a/aperag/views/chat_completion.py +++ b/aperag/views/chat_completion.py @@ -26,7 +26,7 @@ router = APIRouter() -@router.post("/chat/completions", tags=["chat_completion"], name="OpenAIChatCompletions") +@router.post("/chat/completions") async def openai_chat_completions_view(request: Request, user: User = Depends(current_user)): try: body_data = await request.json() diff --git a/aperag/views/config.py b/aperag/views/config.py index b1f4dc65b..8f3f529c1 100644 --- a/aperag/views/config.py +++ b/aperag/views/config.py @@ -21,7 +21,7 @@ router = APIRouter() -@router.get("", tags=["config"], name="GetConfig") +@router.get("") async def config_view() -> Config: auth = Auth( type=settings.auth_type, diff --git a/aperag/views/flow.py b/aperag/views/flow.py index 6e5996469..74dad4751 100644 --- a/aperag/views/flow.py +++ b/aperag/views/flow.py @@ -19,21 +19,21 @@ from aperag.db.models import User from aperag.schema.view_models import WorkflowDefinition from aperag.service.flow_service import flow_service_global -from aperag.utils.audit_decorator import audit_api +from aperag.utils.audit_decorator import audit from aperag.views.auth import current_user router = APIRouter() -@router.get("/bots/{bot_id}/flow", tags=["flow"], name="GetFlow") +@router.get("/bots/{bot_id}/flow") async def get_flow_view( request: Request, bot_id: str, user: User = Depends(current_user) ) -> Union[WorkflowDefinition, dict]: return await flow_service_global.get_flow(str(user.id), bot_id) -@router.put("/bots/{bot_id}/flow", tags=["flow"], name="UpdateFlow") -@audit_api(resource_type="flow", api_name="UpdateFlow") +@router.put("/bots/{bot_id}/flow") +@audit(resource_type="flow", api_name="UpdateFlow") async def update_flow_view( request: Request, bot_id: str, diff --git a/aperag/views/llm.py b/aperag/views/llm.py index 4bef6923d..0e05ebb31 100644 --- a/aperag/views/llm.py +++ b/aperag/views/llm.py @@ -40,7 +40,7 @@ RerankResponse, RerankUsage, ) -from aperag.utils.audit_decorator import audit_api +from aperag.utils.audit_decorator import audit from aperag.views.auth import current_user logger = logging.getLogger(__name__) @@ -48,8 +48,8 @@ router = APIRouter() -@router.post("/embeddings", response_model=EmbeddingResponse, tags=["llm"], name="CreateEmbeddings") -@audit_api(resource_type="llm", api_name="CreateEmbeddings") +@router.post("/embeddings", response_model=EmbeddingResponse) +@audit(resource_type="llm", api_name="CreateEmbeddings") async def create_embeddings(http_request: Request, request: EmbeddingRequest, user: User = Depends(current_user)): """ Create embeddings for the given input text(s). @@ -158,8 +158,8 @@ async def _get_provider_info(provider: str, model: str, user_id: str, api_type: ) from e -@router.post("/rerank", response_model=RerankResponse, tags=["llm"], name="CreateRerank") -@audit_api(resource_type="llm", api_name="CreateRerank") +@router.post("/rerank", response_model=RerankResponse) +@audit(resource_type="llm", api_name="CreateRerank") async def create_rerank(http_request: Request, request: RerankRequest, user: User = Depends(current_user)): """ Rerank documents based on relevance to a query. diff --git a/aperag/views/main.py b/aperag/views/main.py index 2cee4df24..716f776d7 100644 --- a/aperag/views/main.py +++ b/aperag/views/main.py @@ -37,7 +37,7 @@ update_llm_provider_model, ) from aperag.service.prompt_template_service import list_prompt_templates -from aperag.utils.audit_decorator import audit_api +from aperag.utils.audit_decorator import audit # Import authentication dependencies from aperag.views.auth import UserManager, authenticate_websocket_user, current_user, get_user_manager @@ -47,7 +47,7 @@ router = APIRouter() -@router.get("/prompt-templates", tags=["prompt_template"], name="ListPromptTemplates") +@router.get("/prompt-templates") async def list_prompt_templates_view( request: Request, user: User = Depends(current_user) ) -> view_models.PromptTemplateList: @@ -55,8 +55,8 @@ async def list_prompt_templates_view( return list_prompt_templates(language) -@router.post("/collections", tags=["collection"], name="CreateCollection") -@audit_api(resource_type="collection", api_name="CreateCollection") +@router.post("/collections") +@audit(resource_type="collection", api_name="CreateCollection") async def create_collection_view( request: Request, collection: view_models.CollectionCreate, @@ -65,20 +65,20 @@ async def create_collection_view( return await collection_service.create_collection(str(user.id), collection) -@router.get("/collections", tags=["collection"], name="ListCollections") +@router.get("/collections") async def list_collections_view(request: Request, user: User = Depends(current_user)) -> view_models.CollectionList: return await collection_service.list_collections(str(user.id)) -@router.get("/collections/{collection_id}", tags=["collection"], name="GetCollection") +@router.get("/collections/{collection_id}") async def get_collection_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.Collection: return await collection_service.get_collection(str(user.id), collection_id) -@router.put("/collections/{collection_id}", tags=["collection"], name="UpdateCollection") -@audit_api(resource_type="collection", api_name="UpdateCollection") +@router.put("/collections/{collection_id}") +@audit(resource_type="collection", api_name="UpdateCollection") async def update_collection_view( request: Request, collection_id: str, @@ -88,16 +88,16 @@ async def update_collection_view( return await collection_service.update_collection(str(user.id), collection_id, collection) -@router.delete("/collections/{collection_id}", tags=["collection"], name="DeleteCollection") -@audit_api(resource_type="collection", api_name="DeleteCollection") +@router.delete("/collections/{collection_id}") +@audit(resource_type="collection", api_name="DeleteCollection") async def delete_collection_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.Collection: return await collection_service.delete_collection(str(user.id), collection_id) -@router.post("/collections/{collection_id}/documents", tags=["document"], name="CreateDocuments") -@audit_api(resource_type="document", api_name="CreateDocuments") +@router.post("/collections/{collection_id}/documents") +@audit(resource_type="document", api_name="CreateDocuments") async def create_documents_view( request: Request, collection_id: str, @@ -107,14 +107,14 @@ async def create_documents_view( return await document_service.create_documents(str(user.id), collection_id, files) -@router.get("/collections/{collection_id}/documents", tags=["document"], name="ListDocuments") +@router.get("/collections/{collection_id}/documents") async def list_documents_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.DocumentList: return await document_service.list_documents(str(user.id), collection_id) -@router.get("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="GetDocument") +@router.get("/collections/{collection_id}/documents/{document_id}") async def get_document_view( request: Request, collection_id: str, @@ -124,8 +124,8 @@ async def get_document_view( return await document_service.get_document(str(user.id), collection_id, document_id) -@router.put("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="UpdateDocument") -@audit_api(resource_type="document", api_name="UpdateDocument") +@router.put("/collections/{collection_id}/documents/{document_id}") +@audit(resource_type="document", api_name="UpdateDocument") async def update_document_view( request: Request, collection_id: str, @@ -136,8 +136,8 @@ async def update_document_view( return await document_service.update_document(str(user.id), collection_id, document_id, document) -@router.delete("/collections/{collection_id}/documents/{document_id}", tags=["document"], name="DeleteDocument") -@audit_api(resource_type="document", api_name="DeleteDocument") +@router.delete("/collections/{collection_id}/documents/{document_id}") +@audit(resource_type="document", api_name="DeleteDocument") async def delete_document_view( request: Request, collection_id: str, @@ -147,8 +147,8 @@ async def delete_document_view( return await document_service.delete_document(str(user.id), collection_id, document_id) -@router.delete("/collections/{collection_id}/documents", tags=["document"], name="DeleteDocuments") -@audit_api(resource_type="document", api_name="DeleteDocuments") +@router.delete("/collections/{collection_id}/documents") +@audit(resource_type="document", api_name="DeleteDocuments") async def delete_documents_view( request: Request, collection_id: str, @@ -158,26 +158,26 @@ async def delete_documents_view( return await document_service.delete_documents(str(user.id), collection_id, document_ids) -@router.post("/bots/{bot_id}/chats", tags=["chat"], name="CreateChat") -@audit_api(resource_type="chat", api_name="CreateChat") +@router.post("/bots/{bot_id}/chats") +@audit(resource_type="chat", api_name="CreateChat") async def create_chat_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.Chat: return await chat_service_global.create_chat(str(user.id), bot_id) -@router.get("/bots/{bot_id}/chats", tags=["chat"], name="ListChats") +@router.get("/bots/{bot_id}/chats") async def list_chats_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.ChatList: return await chat_service_global.list_chats(str(user.id), bot_id) -@router.get("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="GetChat") +@router.get("/bots/{bot_id}/chats/{chat_id}") async def get_chat_view( request: Request, bot_id: str, chat_id: str, user: User = Depends(current_user) ) -> view_models.ChatDetails: return await chat_service_global.get_chat(str(user.id), bot_id, chat_id) -@router.put("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="UpdateChat") -@audit_api(resource_type="chat", api_name="UpdateChat") +@router.put("/bots/{bot_id}/chats/{chat_id}") +@audit(resource_type="chat", api_name="UpdateChat") async def update_chat_view( request: Request, bot_id: str, @@ -188,8 +188,8 @@ async def update_chat_view( return await chat_service_global.update_chat(str(user.id), bot_id, chat_id, chat_in) -@router.post("/bots/{bot_id}/chats/{chat_id}/messages/{message_id}", tags=["message"], name="FeedbackMessage") -@audit_api(resource_type="message", api_name="FeedbackMessage") +@router.post("/bots/{bot_id}/chats/{chat_id}/messages/{message_id}") +@audit(resource_type="message", api_name="FeedbackMessage") async def feedback_message_view( request: Request, bot_id: str, @@ -203,15 +203,15 @@ async def feedback_message_view( ) -@router.delete("/bots/{bot_id}/chats/{chat_id}", tags=["chat"], name="DeleteChat") -@audit_api(resource_type="chat", api_name="DeleteChat") +@router.delete("/bots/{bot_id}/chats/{chat_id}") +@audit(resource_type="chat", api_name="DeleteChat") async def delete_chat_view(request: Request, bot_id: str, chat_id: str, user: User = Depends(current_user)): await chat_service_global.delete_chat(str(user.id), bot_id, chat_id) return Response(status_code=204) -@router.post("/bots", tags=["bot"], name="CreateBot") -@audit_api(resource_type="bot", api_name="CreateBot") +@router.post("/bots") +@audit(resource_type="bot", api_name="CreateBot") async def create_bot_view( request: Request, bot_in: view_models.BotCreate, @@ -220,18 +220,18 @@ async def create_bot_view( return await bot_service.create_bot(str(user.id), bot_in) -@router.get("/bots", tags=["bot"], name="ListBots") +@router.get("/bots") async def list_bots_view(request: Request, user: User = Depends(current_user)) -> view_models.BotList: return await bot_service.list_bots(str(user.id)) -@router.get("/bots/{bot_id}", tags=["bot"], name="GetBot") +@router.get("/bots/{bot_id}") async def get_bot_view(request: Request, bot_id: str, user: User = Depends(current_user)) -> view_models.Bot: return await bot_service.get_bot(str(user.id), bot_id) -@router.put("/bots/{bot_id}", tags=["bot"], name="UpdateBot") -@audit_api(resource_type="bot", api_name="UpdateBot") +@router.put("/bots/{bot_id}") +@audit(resource_type="bot", api_name="UpdateBot") async def update_bot_view( request: Request, bot_id: str, @@ -241,14 +241,14 @@ async def update_bot_view( return await bot_service.update_bot(str(user.id), bot_id, bot_in) -@router.delete("/bots/{bot_id}", tags=["bot"], name="DeleteBot") -@audit_api(resource_type="bot", api_name="DeleteBot") +@router.delete("/bots/{bot_id}") +@audit(resource_type="bot", api_name="DeleteBot") async def delete_bot_view(request: Request, bot_id: str, user: User = Depends(current_user)): await bot_service.delete_bot(str(user.id), bot_id) return Response(status_code=204) -@router.post("/available_models", tags=["model"], name="ListAvailableModels") +@router.post("/available_models") async def get_available_models_view( request: Request, tag_filter_request: Optional[view_models.TagFilterRequest] = Body(None), @@ -262,7 +262,7 @@ async def get_available_models_view( return await llm_available_model_service.get_available_models(str(user.id), tag_filter_request) -@router.post("/chat/completions/frontend", tags=["chat_completion"], name="FrontendChatCompletions") +@router.post("/chat/completions/frontend") async def frontend_chat_completions_view(request: Request, user: User = Depends(current_user)): body = await request.body() message = body.decode("utf-8") @@ -274,8 +274,8 @@ async def frontend_chat_completions_view(request: Request, user: User = Depends( return await chat_service_global.frontend_chat_completions(str(user.id), message, stream, bot_id, chat_id, msg_id) -@router.post("/collections/{collection_id}/searchTests", tags=["search_test"], name="CreateSearchTest") -@audit_api(resource_type="search_test", api_name="CreateSearchTest") +@router.post("/collections/{collection_id}/searchTests") +@audit(resource_type="search_test", api_name="CreateSearchTest") async def create_search_test_view( request: Request, collection_id: str, @@ -285,8 +285,10 @@ async def create_search_test_view( return await collection_service.create_search_test(str(user.id), collection_id, data) -@router.delete("/collections/{collection_id}/searchTests/{search_test_id}", tags=["search_test"], name="DeleteSearchTest") -@audit_api(resource_type="search_test", api_name="DeleteSearchTest") +@router.delete( + "/collections/{collection_id}/searchTests/{search_test_id}", tags=["search_test"], name="DeleteSearchTest" +) +@audit(resource_type="search_test", api_name="DeleteSearchTest") async def delete_search_test_view( request: Request, collection_id: str, @@ -296,14 +298,14 @@ async def delete_search_test_view( return await collection_service.delete_search_test(str(user.id), collection_id, search_test_id) -@router.get("/collections/{collection_id}/searchTests", tags=["search_test"], name="ListSearchTests") +@router.get("/collections/{collection_id}/searchTests") async def list_search_tests_view( request: Request, collection_id: str, user: User = Depends(current_user) ) -> view_models.SearchTestResultList: return await collection_service.list_search_tests(str(user.id), collection_id) -@router.post("/bots/{bot_id}/flow/debug", tags=["bot"], name="DebugBotFlow") +@router.post("/bots/{bot_id}/flow/debug") async def debug_flow_stream_view( request: Request, bot_id: str, @@ -330,7 +332,7 @@ async def websocket_chat_endpoint( # LLM Configuration API endpoints -@router.get("/llm_configuration", tags=["llm_provider"], name="GetLLMConfiguration") +@router.get("/llm_configuration") async def get_llm_configuration_view(request: Request, user: User = Depends(current_user)): """Get complete LLM configuration including providers and models""" from aperag.db.models import Role @@ -339,8 +341,8 @@ async def get_llm_configuration_view(request: Request, user: User = Depends(curr return await get_llm_configuration(str(user.id), is_admin) -@router.post("/llm_providers", tags=["llm_provider"], name="CreateLLMProvider") -@audit_api(resource_type="llm_provider", api_name="CreateLLMProvider") +@router.post("/llm_providers") +@audit(resource_type="llm_provider", api_name="CreateLLMProvider") async def create_llm_provider_view( request: Request, provider_data: view_models.LlmProviderCreateWithApiKey, @@ -353,7 +355,7 @@ async def create_llm_provider_view( return await create_llm_provider(provider_data.model_dump(), str(user.id), is_admin) -@router.get("/llm_providers/{provider_name}", tags=["llm_provider"], name="GetLLMProvider") +@router.get("/llm_providers/{provider_name}") async def get_llm_provider_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Get a specific LLM provider""" from aperag.db.models import Role @@ -362,8 +364,8 @@ async def get_llm_provider_view(request: Request, provider_name: str, user: User return await get_llm_provider(provider_name, str(user.id), is_admin) -@router.put("/llm_providers/{provider_name}", tags=["llm_provider"], name="UpdateLLMProvider") -@audit_api(resource_type="llm_provider", api_name="UpdateLLMProvider") +@router.put("/llm_providers/{provider_name}") +@audit(resource_type="llm_provider", api_name="UpdateLLMProvider") async def update_llm_provider_view( request: Request, provider_name: str, @@ -377,8 +379,8 @@ async def update_llm_provider_view( return await update_llm_provider(provider_name, provider_data.model_dump(), str(user.id), is_admin) -@router.delete("/llm_providers/{provider_name}", tags=["llm_provider"], name="DeleteLLMProvider") -@audit_api(resource_type="llm_provider", api_name="DeleteLLMProvider") +@router.delete("/llm_providers/{provider_name}") +@audit(resource_type="llm_provider", api_name="DeleteLLMProvider") async def delete_llm_provider_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Delete an LLM provider""" from aperag.db.models import Role @@ -387,7 +389,7 @@ async def delete_llm_provider_view(request: Request, provider_name: str, user: U return await delete_llm_provider(provider_name, str(user.id), is_admin) -@router.get("/llm_provider_models", tags=["llm_provider_model"], name="ListAllLLMProviderModels") +@router.get("/llm_provider_models") async def list_llm_provider_models_view( request: Request, provider_name: str = None, user: User = Depends(current_user) ): @@ -398,7 +400,7 @@ async def list_llm_provider_models_view( return await list_llm_provider_models(provider_name, str(user.id), is_admin) -@router.get("/llm_providers/{provider_name}/models", tags=["llm_provider_model"], name="ListProviderModels") +@router.get("/llm_providers/{provider_name}/models") async def get_provider_models_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Get all models for a specific provider""" from aperag.db.models import Role @@ -407,8 +409,8 @@ async def get_provider_models_view(request: Request, provider_name: str, user: U return await list_llm_provider_models(provider_name=provider_name, user_id=str(user.id), is_admin=is_admin) -@router.post("/llm_providers/{provider_name}/models", tags=["llm_provider_model"], name="CreateProviderModel") -@audit_api(resource_type="llm_provider_model", api_name="CreateProviderModel") +@router.post("/llm_providers/{provider_name}/models") +@audit(resource_type="llm_provider_model", api_name="CreateProviderModel") async def create_provider_model_view(request: Request, provider_name: str, user: User = Depends(current_user)): """Create a new model for a specific provider""" import json @@ -421,8 +423,8 @@ async def create_provider_model_view(request: Request, provider_name: str, user: return await create_llm_provider_model(provider_name, data, str(user.id), is_admin) -@router.put("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="UpdateProviderModel") -@audit_api(resource_type="llm_provider_model", api_name="UpdateProviderModel") +@router.put("/llm_providers/{provider_name}/models/{api}/{model}") +@audit(resource_type="llm_provider_model", api_name="UpdateProviderModel") async def update_provider_model_view( request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user) ): @@ -437,8 +439,8 @@ async def update_provider_model_view( return await update_llm_provider_model(provider_name, api, model, data, str(user.id), is_admin) -@router.delete("/llm_providers/{provider_name}/models/{api}/{model}", tags=["llm_provider_model"], name="DeleteProviderModel") -@audit_api(resource_type="llm_provider_model", api_name="DeleteProviderModel") +@router.delete("/llm_providers/{provider_name}/models/{api}/{model}") +@audit(resource_type="llm_provider_model", api_name="DeleteProviderModel") async def delete_provider_model_view( request: Request, provider_name: str, api: str, model: str, user: User = Depends(current_user) ): From f8ca523ed0904469e1bc4d799b557f19669dbc76 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 21:19:24 +0800 Subject: [PATCH 18/19] chore: tidy up --- aperag/views/api_key.py | 6 +++--- frontend/src/pages/settings/auditLogs.tsx | 12 +++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/aperag/views/api_key.py b/aperag/views/api_key.py index 72dcb6ddd..462dc796c 100644 --- a/aperag/views/api_key.py +++ b/aperag/views/api_key.py @@ -31,7 +31,7 @@ async def list_api_keys_view(request: Request, user: User = Depends(current_user @router.post("/apikeys") -@audit(resource_type="apikey", api_name="CreateApiKey") +@audit(resource_type="api_key", api_name="CreateApiKey") async def create_api_key_view( request: Request, api_key_create: ApiKeyCreate, @@ -42,14 +42,14 @@ async def create_api_key_view( @router.delete("/apikeys/{apikey_id}") -@audit(resource_type="apikey", api_name="DeleteApiKey") +@audit(resource_type="api_key", api_name="DeleteApiKey") async def delete_api_key_view(request: Request, apikey_id: str, user: User = Depends(current_user)): """Delete an API key""" return await api_key_service.delete_api_key(str(user.id), apikey_id) @router.put("/apikeys/{apikey_id}") -@audit(resource_type="apikey", api_name="UpdateApiKey") +@audit(resource_type="api_key", api_name="UpdateApiKey") async def update_api_key_view( request: Request, apikey_id: str, diff --git a/frontend/src/pages/settings/auditLogs.tsx b/frontend/src/pages/settings/auditLogs.tsx index 295cada2f..cb1bfaa39 100644 --- a/frontend/src/pages/settings/auditLogs.tsx +++ b/frontend/src/pages/settings/auditLogs.tsx @@ -157,7 +157,7 @@ const AuditLogsPage: React.FC = () => { title: intl.formatMessage({ id: 'audit.logs.username', defaultMessage: 'Username' }), dataIndex: 'username', key: 'username', - width: 120, + width: 180, render: (text?: string) => ( {text || '-'} ), @@ -192,14 +192,12 @@ const AuditLogsPage: React.FC = () => { title: intl.formatMessage({ id: 'audit.logs.resourceId', defaultMessage: 'Resource ID' }), dataIndex: 'resource_id', key: 'resource_id', - width: 140, + width: 240, render: (id?: string) => ( id ? ( - - - {id.length > 18 ? `${id.substring(0, 18)}...` : id} - - + + {id} + ) : '-' ), }, From ae7c70baf99ae414700c34a2646279700e761a80 Mon Sep 17 00:00:00 2001 From: Guo Ziang Date: Sun, 22 Jun 2025 21:21:04 +0800 Subject: [PATCH 19/19] chore: tidy up --- aperag/service/audit_service.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index a0aaceddd..6fa1a1e8b 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -45,37 +45,6 @@ def __init__(self): "credential", } - # Map FastAPI tags to audit resources - self.tag_resource_map = { - # Support both singular and plural forms - "collection": AuditResource.COLLECTION, - "collections": AuditResource.COLLECTION, - "document": AuditResource.DOCUMENT, - "documents": AuditResource.DOCUMENT, - "bot": AuditResource.BOT, - "bots": AuditResource.BOT, - "chat": AuditResource.CHAT, - "chats": AuditResource.CHAT, - "message": AuditResource.MESSAGE, - "messages": AuditResource.MESSAGE, - "apikey": AuditResource.API_KEY, - "apikeys": AuditResource.API_KEY, - "llm_provider": AuditResource.LLM_PROVIDER, - "llm_providers": AuditResource.LLM_PROVIDER, - "llm_provider_model": AuditResource.LLM_PROVIDER_MODEL, - "llm_provider_models": AuditResource.LLM_PROVIDER_MODEL, - "user": AuditResource.USER, - "users": AuditResource.USER, - "config": AuditResource.CONFIG, - "invitation": AuditResource.INVITATION, - "invitations": AuditResource.INVITATION, - "auth": AuditResource.AUTH, - "chat_completion": AuditResource.CHAT_COMPLETION, - "search_test": AuditResource.SEARCH_TEST, - "llm": AuditResource.LLM, - "flow": AuditResource.FLOW, - } - def _filter_sensitive_data(self, data: Dict[str, Any]) -> Dict[str, Any]: """Filter sensitive information from data""" if not isinstance(data, dict):