Add log PVC cleanup monitor and configurable log rotation (#520)

* Add log PVC cleanup monitor

* Add configurable log rotation across services

* Use POSIX shell source syntax in Makefile

Author: hhhhsc701

Makefile

@@ -358,7 +358,7 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
 		exit 1; \
 	fi
 	@if [ "$*" = "label-studio" ]; then \
-		if [ -f /tmp/datamate-helm-args.sh ]; then source /tmp/datamate-helm-args.sh; fi; \
+		if [ -f /tmp/datamate-helm-args.sh ]; then . /tmp/datamate-helm-args.sh; fi; \
 		helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
     elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
 		kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
@@ -369,7 +369,7 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
 		chmod +x scripts/k8s/node-setup.sh; \
 		./scripts/k8s/node-setup.sh --namespace $(NAMESPACE); \
 		if [ -f /tmp/datamate-helm-args.sh ]; then \
-			source /tmp/datamate-helm-args.sh; \
+			. /tmp/datamate-helm-args.sh; \
 		fi; \
 		chmod +x scripts/k8s/collect-secrets.sh; \
 		eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
@@ -399,7 +399,7 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
 		MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
 		MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
 		if [ -f /tmp/datamate-helm-args.sh ]; then \
-			source /tmp/datamate-helm-args.sh; \
+			. /tmp/datamate-helm-args.sh; \
 		fi; \
 		helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
 			--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \

backend/api-gateway/src/main/resources/application.yml

@@ -41,4 +41,7 @@ oms:
     url: ${OMS_SERVICE_URL:https://omsservice:18082}
   # 服务器端口配置
 server:
-  port: 8080  # 必须有这个配置
+  port: 8080  # 必须有这个配置
+
+logging:
+  config: classpath:log4j2.xml

backend/api-gateway/src/main/resources/log4j2.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" monitorInterval="30">
+    <Properties>
+        <Property name="LOG_PATH">/var/log/datamate/gateway</Property>
+        <Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
+        <Property name="MAX_FILE_SIZE">${env:LOG_ROTATION_MAX_SIZE:-100MB}</Property>
+        <Property name="MAX_HISTORY">${env:LOG_ROTATION_BACKUP_COUNT:-30}</Property>
+        <Property name="LOG_LEVEL">INFO</Property>
+        <Property name="SQL_LOG_LEVEL">WARN</Property>
+    </Properties>
+
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+        </Console>
+
+        <RollingFile name="File" fileName="${LOG_PATH}/app.log" filePattern="${LOG_PATH}/app-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="${LOG_PATTERN}" />
+            <Policies>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+                <SizeBasedTriggeringPolicy size="${MAX_FILE_SIZE}" />
+            </Policies>
+            <DefaultRolloverStrategy max="${MAX_HISTORY}" />
+        </RollingFile>
+    </Appenders>
+
+    <Loggers>
+        <Logger name="com.datamate" level="${LOG_LEVEL}" />
+        <Logger name="org.springframework.security" level="${LOG_LEVEL}" />
+        <Logger name="org.hibernate.SQL" level="${SQL_LOG_LEVEL}" />
+        <Logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="${SQL_LOG_LEVEL}" />
+
+        <Root level="INFO">
+            <AppenderRef ref="Console" />
+            <AppenderRef ref="File" />
+        </Root>
+    </Loggers>
+</Configuration>

backend/services/main-application/src/main/resources/log4j2.xml

@@ -3,8 +3,8 @@
     <Properties>
         <Property name="LOG_PATH">/var/log/datamate/backend</Property>
         <Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</Property>
-        <Property name="MAX_FILE_SIZE">100MB</Property>
-        <Property name="MAX_HISTORY">30</Property>
+        <Property name="MAX_FILE_SIZE">${env:LOG_ROTATION_MAX_SIZE:-100MB}</Property>
+        <Property name="MAX_HISTORY">${env:LOG_ROTATION_BACKUP_COUNT:-30}</Property>
         <Property name="LOG_LEVEL">INFO</Property>
         <Property name="SQL_LOG_LEVEL">WARN</Property>
     </Properties>

deployment/docker/datamate/docker-compose.yml

@@ -12,6 +12,8 @@ services:
       - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
       - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file}
       - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false}
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
     volumes:
       - dataset_volume:/dataset
       - flow_volume:/flow
@@ -31,6 +33,14 @@ services:
       - "18000:18000"
     environment:
       - log_level=DEBUG
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+      - LOG_PVC_MONITOR_ENABLED=${LOG_PVC_MONITOR_ENABLED:-true}
+      - LOG_PVC_MONITOR_PATH=${LOG_PVC_MONITOR_PATH:-/var/log/datamate}
+      - LOG_PVC_MONITOR_THRESHOLD=${LOG_PVC_MONITOR_THRESHOLD:-0.9}
+      - LOG_PVC_MONITOR_INTERVAL_SECONDS=${LOG_PVC_MONITOR_INTERVAL_SECONDS:-300}
+      - LOG_PVC_MONITOR_DELETE_BATCH_SIZE=${LOG_PVC_MONITOR_DELETE_BATCH_SIZE:-10}
+      - LOG_PVC_MONITOR_FILE_SUFFIXES=${LOG_PVC_MONITOR_FILE_SUFFIXES:-log,out,err}
       - PGSQL_HOST=datamate-database
       - PGSQL_PORT=5432
       - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
@@ -60,6 +70,10 @@ services:
     environment:
       - JWT_SECRET=${JWT_SECRET:-}
       - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false}
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
+    volumes:
+      - log_volume:/var/log/datamate
     networks: [ datamate ]
 
   datamate-frontend:
@@ -68,6 +82,9 @@ services:
     restart: on-failure
     ports:
       - "30000:3000"
+    environment:
+      - LOG_ROTATION_MAX_SIZE=${LOG_ROTATION_MAX_SIZE:-100MB}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
     volumes:
       - frontend_log_volume:/var/log/datamate/frontend
     networks: [ datamate ]
@@ -82,6 +99,7 @@ services:
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file}
+      - LOG_ROTATION_BACKUP_COUNT=${LOG_ROTATION_BACKUP_COUNT:-30}
     volumes:
       - postgresql_volume:/var/lib/postgresql/data
       - database_log_volume:/var/log/datamate/database
@@ -96,6 +114,8 @@ services:
     environment:
       RAY_DEDUP_LOGS: "0"
       RAY_TQDM_PATCH_PRINT: "0"
+      LOG_ROTATION_MAX_SIZE: ${LOG_ROTATION_MAX_SIZE:-100MB}
+      LOG_ROTATION_BACKUP_COUNT: ${LOG_ROTATION_BACKUP_COUNT:-30}
       PG_HOST: "datamate-database"
       PG_PORT: "5432"
       PG_USER: "postgres"

deployment/helm/datamate/values.yaml

@@ -115,6 +115,8 @@ database:
         secretKeyRef:
           name: datamate-conf
           key: HOME_PAGE_URL
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: "30"
   volumes:
     - *dataVolume
     - *logVolume
@@ -148,6 +150,10 @@ backend:
       value: "http://datamate-milvus:19530"
     - name: datamate.jwt.enable
       value: &DATAMATE_JWT_ENABLE 'false'
+    - name: LOG_ROTATION_MAX_SIZE
+      value: &LOG_ROTATION_MAX_SIZE "100MB"
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: &LOG_ROTATION_BACKUP_COUNT "30"
   volumes:
     - *datasetVolume
     - *flowVolume
@@ -178,6 +184,22 @@ backend-python:
           key: DB_PASSWORD
     - name: datamate_jwt_enable
       value: *DATAMATE_JWT_ENABLE
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
+    - name: LOG_PVC_MONITOR_ENABLED
+      value: "true"
+    - name: LOG_PVC_MONITOR_PATH
+      value: "/var/log/datamate"
+    - name: LOG_PVC_MONITOR_THRESHOLD
+      value: "0.9"
+    - name: LOG_PVC_MONITOR_INTERVAL_SECONDS
+      value: "300"
+    - name: LOG_PVC_MONITOR_DELETE_BATCH_SIZE
+      value: "10"
+    - name: LOG_PVC_MONITOR_FILE_SUFFIXES
+      value: "log,out,err"
     - name: milvus_uri
       value: "http://datamate-milvus:19530"
     - name: LABEL_STUDIO_USER_TOKEN
@@ -220,6 +242,10 @@ gateway:
       value: *DATAMATE_JWT_ENABLE
     - name: OMS_AUTH_ENABLED
       value: "false"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
   volumes:
     - *logVolume
   volumeMounts:
@@ -246,6 +272,10 @@ frontend:
           key: DOMAIN
     - name: REAL_IP_MODE
       value: "off"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
   volumes:
     - *logVolume
     - name: cert-volume
@@ -282,6 +312,10 @@ runtime:
           key: DB_PASSWORD
     - name: PG_DATABASE
       value: "datamate"
+    - name: LOG_ROTATION_MAX_SIZE
+      value: *LOG_ROTATION_MAX_SIZE
+    - name: LOG_ROTATION_BACKUP_COUNT
+      value: *LOG_ROTATION_BACKUP_COUNT
   volumes:
     - *datasetVolume
     - *flowVolume
@@ -332,6 +366,10 @@ ray-cluster:
         value: "0"
       - name: RAY_TQDM_PATCH_PRINT
         value: "0"
+      - name: LOG_ROTATION_MAX_SIZE
+        value: *LOG_ROTATION_MAX_SIZE
+      - name: LOG_ROTATION_BACKUP_COUNT
+        value: *LOG_ROTATION_BACKUP_COUNT
       - name: PG_HOST
         value: "datamate-database"
       - name: PG_PORT
@@ -389,6 +427,10 @@ ray-cluster:
         value: "0"
       - name: RAY_TQDM_PATCH_PRINT
         value: "0"
+      - name: LOG_ROTATION_MAX_SIZE
+        value: *LOG_ROTATION_MAX_SIZE
+      - name: LOG_ROTATION_BACKUP_COUNT
+        value: *LOG_ROTATION_BACKUP_COUNT
       - name: PG_HOST
         value: "datamate-database"
       - name: PG_PORT
@@ -413,6 +455,13 @@ ray-cluster:
       requests:
         cpu: "1"
         memory: "2G"
+    lifecycle: &rayLogRotationLifecycle
+      postStart:
+        exec:
+          command:
+            - /bin/bash
+            - -c
+            - "nohup /usr/local/bin/log-rotate-copytruncate.sh /tmp/ray >/tmp/log-rotation.out 2>/tmp/log-rotation.err &"
     volumes:
       - *datasetVolume
       - *flowVolume
@@ -447,6 +496,10 @@ ray-cluster:
           value: "0"
         - name: RAY_TQDM_PATCH_PRINT
           value: "0"
+        - name: LOG_ROTATION_MAX_SIZE
+          value: *LOG_ROTATION_MAX_SIZE
+        - name: LOG_ROTATION_BACKUP_COUNT
+          value: *LOG_ROTATION_BACKUP_COUNT
         - name: PG_HOST
           value: "datamate-database"
         - name: PG_PORT
@@ -473,6 +526,7 @@ ray-cluster:
           cpu: "1"
           memory: "2G"
           huawei.com/Ascend910: 1
+      lifecycle: *rayLogRotationLifecycle
       volumes:
         - *datasetVolume
         - *flowVolume
@@ -511,6 +565,10 @@ ray-cluster:
           value: "0"
         - name: RAY_TQDM_PATCH_PRINT
           value: "0"
+        - name: LOG_ROTATION_MAX_SIZE
+          value: *LOG_ROTATION_MAX_SIZE
+        - name: LOG_ROTATION_BACKUP_COUNT
+          value: *LOG_ROTATION_BACKUP_COUNT
         - name: PG_HOST
           value: "datamate-database"
         - name: PG_PORT
@@ -537,6 +595,7 @@ ray-cluster:
           cpu: "1"
           memory: "2G"
           nvidia.com/gpu: 1
+      lifecycle: *rayLogRotationLifecycle
       volumes:
         - *datasetVolume
         - *flowVolume

runtime/datamate-python/.env.example

@@ -5,6 +5,14 @@ PORT=18000
 DEBUG=true
 LOG_LEVEL=DEBUG
 LOG_FILE_DIR=./logs
+LOG_ROTATION_MAX_SIZE=100MB
+LOG_ROTATION_BACKUP_COUNT=30
+LOG_PVC_MONITOR_ENABLED=true
+LOG_PVC_MONITOR_PATH=./logs
+LOG_PVC_MONITOR_THRESHOLD=0.9
+LOG_PVC_MONITOR_INTERVAL_SECONDS=300
+LOG_PVC_MONITOR_DELETE_BATCH_SIZE=10
+LOG_PVC_MONITOR_FILE_SUFFIXES=log,out,err
 RAG_STORAGE_DIR=./rag_storage
 
 # DataBase

runtime/datamate-python/app/core/config.py

@@ -27,6 +27,14 @@ class Config:
     log_level: str = "INFO"
     debug: bool = True
     log_file_dir: str = "/var/log/datamate/backend-python"
+    log_pvc_monitor_enabled: bool = True
+    log_pvc_monitor_path: str = "/var/log/datamate"
+    log_pvc_monitor_threshold: float = 0.9
+    log_pvc_monitor_interval_seconds: int = 300
+    log_pvc_monitor_delete_batch_size: int = 10
+    log_pvc_monitor_file_suffixes: str = "log,out,err"
+    log_rotation_max_size: str = "100MB"
+    log_rotation_backup_count: int = 30
     rag_storage_dir: str = "/data/rag_storage"
 
     # Database