Skip to content

Commit 9ab60c2

Browse files
yaojin3616yaojin3616
committed
Merge remote-tracking branch 'origin/main' into pr-448-tool-call-pair-integrity
# Conflicts: # backend/app/api/feishu.py # backend/app/api/websocket.py
2 parents f763d2c + 8a53778 commit 9ab60c2

421 files changed

Lines changed: 74160 additions & 17179 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.example‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,39 @@ FEISHU_REDIRECT_URI=http://localhost:3000/auth/feishu/callback
2121
# Default: local host -> ~/.clawith/data/agents ; container runtime -> /data/agents
2222
# AGENT_DATA_DIR=
2323

24+
# File storage backend. Use "s3" for S3-compatible object storage.
25+
# When STORAGE_BACKEND=s3, local fallback lets old files under STORAGE_LOCAL_ROOT
26+
# be read and copied into S3 on first access during migration.
27+
# STORAGE_BACKEND=local
28+
# STORAGE_LOCAL_ROOT=
29+
# STORAGE_LOCAL_FALLBACK_ENABLED=true
30+
# S3_BUCKET=
31+
# S3_REGION=
32+
# S3_ENDPOINT_URL=
33+
# S3_ACCESS_KEY_ID=
34+
# S3_SECRET_ACCESS_KEY=
35+
# S3_PREFIX=agents
36+
# S3_MAX_POOL_CONNECTIONS=50
37+
# S3_WRITE_WORKERS=32
38+
39+
# Google Cloud Storage (S3-compatible API) — set these instead of MinIO values:
40+
# STORAGE_BACKEND=s3
41+
# S3_BUCKET=your-gcs-bucket-name
42+
# S3_REGION=auto
43+
# S3_ENDPOINT_URL=https://storage.googleapis.com
44+
# S3_ACCESS_KEY_ID=your-hmac-access-key
45+
# S3_SECRET_ACCESS_KEY=your-hmac-secret
46+
# S3_PREFIX=agents
47+
48+
# Local MinIO settings used by docker-compose.multi-instance.yml.
49+
# Change the password before exposing MinIO outside local development.
50+
MINIO_ROOT_USER=clawith
51+
MINIO_ROOT_PASSWORD=clawith-minio-secret
52+
MINIO_BUCKET=clawith
53+
MINIO_API_PORT=9000
54+
MINIO_CONSOLE_PORT=9001
55+
API_PORT=8000
56+
2457
# Jina AI API key (for jina_search and jina_read tools — get one at https://jina.ai)
2558
# Without a key, the tools still work but with lower rate limits
2659
JINA_API_KEY=
@@ -37,3 +70,13 @@ PUBLIC_BASE_URL=
3770

3871
# Password reset token lifetime in minutes
3972
PASSWORD_RESET_TOKEN_EXPIRE_MINUTES=30
73+
74+
# Frontend port (default: 3008)
75+
# FRONTEND_PORT=3008
76+
77+
# API upstream for nginx proxy (default: backend:8000)
78+
# API_UPSTREAM=backend:8000
79+
80+
# Python pip index URL (for China mirrors)
81+
# CLAWITH_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
82+
# CLAWITH_PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn

‎.github/drone.yml‎

Lines changed: 302 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
# ============================================================
2+
# Clawith CI/CD Pipeline
3+
# 基于 Drone CI 的自动化构建、本地部署测试和升级测试
4+
#
5+
# 流程:
6+
# 1. 代码克隆 (获取完整历史和 tags)
7+
# 2. 构建 Docker 镜像
8+
# 3. 本地部署测试 (直接利用本地 docker socket 和 docker-compose.ci.yml)
9+
# 4. 本地升级测试 (测试旧版到新版的迁移)
10+
# 5. 打包传输到服务器 (可选)
11+
# ============================================================
12+
kind: pipeline
13+
type: docker
14+
name: build-and-test
15+
16+
workspace:
17+
path: /drone/src
18+
19+
clone:
20+
disable: true
21+
22+
steps:
23+
# --------------------------------------------------------
24+
# Step 1: 代码克隆 (获取完整历史和 tags)
25+
# --------------------------------------------------------
26+
- name: clone
27+
image: alpine/git
28+
pull: if-not-exists
29+
environment:
30+
http_proxy:
31+
from_secret: PROXY
32+
https_proxy:
33+
from_secret: PROXY
34+
commands:
35+
- git config --global core.compression 0
36+
- git config --global http.postBuffer 524288000
37+
- git clone --no-single-branch https://github.com/dataelement/Clawith.git .
38+
# 将 shallow clone 转换为完整克隆，获取完整 commit 历史
39+
- git fetch --unshallow --tags || git fetch --tags
40+
- git checkout $DRONE_COMMIT
41+
- echo "当前 commit $(git log --oneline -1)"
42+
- echo "上一个 tag $(git describe --tags --abbrev=0 2>/dev/null || echo '无 tag')"
43+
44+
# --------------------------------------------------------
45+
# Step 2: 构建新旧版本 Docker 镜像
46+
# --------------------------------------------------------
47+
- name: build-images
48+
image: docker:24.0.6
49+
pull: if-not-exists
50+
privileged: true
51+
volumes:
52+
- name: docker-socket
53+
path: /var/run/docker.sock
54+
environment:
55+
http_proxy:
56+
from_secret: PROXY
57+
https_proxy:
58+
from_secret: PROXY
59+
commands:
60+
- apk add --no-cache git
61+
- echo "========================================"
62+
- echo "开始构建 Docker 镜像"
63+
- echo "事件 $DRONE_BUILD_EVENT"
64+
- echo "目标分支 $DRONE_BRANCH"
65+
- echo "提交 $DRONE_COMMIT_SHORT"
66+
- echo "========================================"
67+
68+
# 获取上一个 tag
69+
- PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
70+
- |
71+
if [ -z "$PREV_TAG" ]; then
72+
echo "警告: 未找到历史 tag，升级测试将被跳过"
73+
echo "SKIP_UPGRADE_TEST=true" > .env.drone
74+
# 仅构建当前版本
75+
docker build -t clawith-backend:new --build-arg CLAWITH_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -f backend/Dockerfile ./backend
76+
docker build -t clawith-frontend:new -f frontend/Dockerfile ./frontend
77+
echo "当前版本镜像构建完成"
78+
exit 0
79+
fi
80+
- echo "上一个 tag $PREV_TAG"
81+
- |
82+
echo "SKIP_UPGRADE_TEST=false" > .env.drone
83+
84+
# --- 构建旧版本镜像 ---
85+
- echo "--- 构建旧版本镜像 ($PREV_TAG) ---"
86+
- git checkout $PREV_TAG
87+
- docker build -t clawith-backend:old --build-arg CLAWITH_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -f backend/Dockerfile ./backend
88+
- docker build -t clawith-frontend:old -f frontend/Dockerfile ./frontend
89+
- echo "旧版本镜像构建完成"
90+
91+
# --- 构建当前版本镜像 ---
92+
- echo "--- 构建当前版本镜像 ($DRONE_COMMIT_SHORT) ---"
93+
- git checkout $DRONE_COMMIT
94+
- docker build -t clawith-backend:new --build-arg CLAWITH_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -f backend/Dockerfile ./backend
95+
- docker build -t clawith-frontend:new -f frontend/Dockerfile ./frontend
96+
- echo "当前版本镜像构建完成"
97+
98+
# --------------------------------------------------------
99+
# Step 3: 本地部署测试 (Deploy Test)
100+
# --------------------------------------------------------
101+
- name: local-deploy-test
102+
image: docker:24.0.6
103+
privileged: true
104+
volumes:
105+
- name: docker-socket
106+
path: /var/run/docker.sock
107+
commands:
108+
- apk add --no-cache docker-cli-compose curl bash
109+
- echo "========================================"
110+
- echo "本地部署测试 全新部署验证"
111+
- echo "========================================"
112+
- docker compose -f docker-compose.ci.yml down -v --remove-orphans 2>/dev/null || true
113+
- echo "启动全部服务 (postgres/redis/backend/frontend)..."
114+
- IMAGE_TAG=new docker compose -f docker-compose.ci.yml up -d --no-build
115+
- sleep 5
116+
117+
# 等待基础设施就绪
118+
- echo "等待 postgres 和 redis 健康检查通过..."
119+
- timeout 30s bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" $(docker compose -f docker-compose.ci.yml ps -q postgres) 2>/dev/null)" = "healthy" ]; do sleep 2; done' || (echo "postgres 启动超时" && docker compose -f docker-compose.ci.yml logs postgres && exit 1)
120+
- timeout 30s bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" $(docker compose -f docker-compose.ci.yml ps -q redis) 2>/dev/null)" = "healthy" ]; do sleep 2; done' || (echo "redis 启动超时" && docker compose -f docker-compose.ci.yml logs redis && exit 1)
121+
122+
# 核心验证: backend 启动
123+
- echo "等待 backend 启动 (最多 90 秒)..."
124+
- timeout 90s bash -c 'until docker compose -f docker-compose.ci.yml logs --tail=200 backend 2>&1 | grep -q "Application startup complete"; do sleep 3; done'
125+
- RESULT=$?
126+
- |
127+
if [ $RESULT -ne 0 ]; then
128+
echo "❌ 部署测试失败: backend 启动超时"
129+
docker compose -f docker-compose.ci.yml logs --tail=100 backend
130+
exit 1
131+
fi
132+
133+
# 检查迁移是否失败
134+
- echo "检查数据库迁移状态..."
135+
- |
136+
MIGRATION_LOG=$(docker compose -f docker-compose.ci.yml logs --tail=200 backend 2>&1)
137+
if echo "$MIGRATION_LOG" | grep -Eqi "migration.*FAIL|alembic.*error"; then
138+
echo "❌ 部署测试失败: 检测到数据库迁移失败"
139+
echo "$MIGRATION_LOG" | grep -Ei "migration|alembic|error|FAIL"
140+
exit 1
141+
fi
142+
143+
# 验证 health endpoint (通过 docker compose exec 直接在容器内 curl)
144+
- echo "验证 health endpoint..."
145+
- timeout 10s bash -c 'until docker compose -f docker-compose.ci.yml exec -T backend curl -sf http://localhost:8000/api/health > /dev/null 2>&1; do sleep 2; done' || (echo "❌ health endpoint 不可达" && exit 1)
146+
147+
# 清理环境
148+
- echo "清理测试环境..."
149+
- docker compose -f docker-compose.ci.yml down -v --remove-orphans
150+
- echo "✅ 本地部署测试通过"
151+
152+
# --------------------------------------------------------
153+
# Step 4: 本地升级测试 (Upgrade Test)
154+
# --------------------------------------------------------
155+
- name: local-upgrade-test
156+
image: docker:24.0.6
157+
privileged: true
158+
volumes:
159+
- name: docker-socket
160+
path: /var/run/docker.sock
161+
commands:
162+
- apk add --no-cache docker-cli-compose curl bash
163+
- echo "========================================"
164+
- echo "本地升级测试 旧版 → 新版数据库迁移验证"
165+
- echo "========================================"
166+
- |
167+
if [ -f ".env.drone" ] && grep -q "SKIP_UPGRADE_TEST=true" ".env.drone"; then
168+
echo "⚠️ 未找到历史 tag，跳过升级测试"
169+
exit 0
170+
fi
171+
172+
- docker compose -f docker-compose.ci.yml down -v --remove-orphans 2>/dev/null || true
173+
- docker rm -f backend-upgrade-test 2>/dev/null || true
174+
175+
# 阶段 1: 部署旧版本
176+
- echo "--- 阶段 1 - 部署旧版本 ---"
177+
- IMAGE_TAG=old docker compose -f docker-compose.ci.yml up -d postgres redis
178+
- sleep 5
179+
- timeout 30s bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" $(docker compose -f docker-compose.ci.yml ps -q postgres) 2>/dev/null)" = "healthy" ]; do sleep 2; done'
180+
- timeout 30s bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" $(docker compose -f docker-compose.ci.yml ps -q redis) 2>/dev/null)" = "healthy" ]; do sleep 2; done'
181+
182+
# 启动旧版 backend
183+
- |
184+
docker run -d --name backend-upgrade-test \
185+
--network clawith_network \
186+
-e DATABASE_URL=postgresql+asyncpg://clawith:clawith@postgres:5432/clawith \
187+
-e REDIS_URL=redis://redis:6379/0 \
188+
-e AGENT_DATA_DIR=/data/agents \
189+
-e AGENT_TEMPLATE_DIR=/app/agent_template \
190+
-e SECRET_KEY=ci-test-secret \
191+
-e JWT_SECRET_KEY=ci-test-jwt-secret \
192+
-e CORS_ORIGINS='["*"]' \
193+
clawith-backend:old
194+
195+
- timeout 90s bash -c 'until docker logs backend-upgrade-test 2>&1 | grep -q "Application startup complete"; do sleep 3; done'
196+
- RESULT=$?
197+
- |
198+
if [ $RESULT -ne 0 ]; then
199+
echo "❌ 升级测试失败: 旧版 backend 启动超时"
200+
docker logs --tail=100 backend-upgrade-test
201+
exit 1
202+
fi
203+
204+
- docker exec backend-upgrade-test alembic history 2>&1 | head -5 || true
205+
- docker stop backend-upgrade-test
206+
- docker rm backend-upgrade-test
207+
208+
# 阶段 2: 升级到新版本
209+
- echo "--- 阶段 2 - 升级到新版本 ---"
210+
- |
211+
docker run -d --name backend-upgrade-test \
212+
--network clawith_network \
213+
-e DATABASE_URL=postgresql+asyncpg://clawith:clawith@postgres:5432/clawith \
214+
-e REDIS_URL=redis://redis:6379/0 \
215+
-e AGENT_DATA_DIR=/data/agents \
216+
-e AGENT_TEMPLATE_DIR=/app/agent_template \
217+
-e SECRET_KEY=ci-test-secret \
218+
-e JWT_SECRET_KEY=ci-test-jwt-secret \
219+
-e CORS_ORIGINS='["*"]' \
220+
clawith-backend:new
221+
222+
- timeout 120s bash -c 'until docker logs backend-upgrade-test 2>&1 | grep -q "Application startup complete"; do sleep 3; done'
223+
- RESULT=$?
224+
- |
225+
if [ $RESULT -ne 0 ]; then
226+
echo "❌ 升级测试失败: 新版 backend 启动超时"
227+
docker logs --tail=200 backend-upgrade-test
228+
exit 1
229+
fi
230+
231+
- |
232+
UPGRADE_LOG=$(docker logs --tail=300 backend-upgrade-test 2>&1)
233+
if echo "$UPGRADE_LOG" | grep -Eqi "migration.*FAIL|alembic.*error|WARNING.*migration"; then
234+
echo "❌ 升级测试失败: 检测到数据库迁移异常"
235+
echo "$UPGRADE_LOG" | grep -Ei "migration|alembic|error|FAIL|WARNING"
236+
exit 1
237+
fi
238+
239+
- timeout 10s bash -c 'until docker exec backend-upgrade-test curl -sf http://localhost:8000/api/health > /dev/null 2>&1; do sleep 2; done' || (echo "❌ health endpoint 不可达" && exit 1)
240+
241+
- docker exec backend-upgrade-test alembic history 2>&1 | head -5 || true
242+
- docker exec backend-upgrade-test alembic current 2>&1 || true
243+
244+
- docker rm -f backend-upgrade-test
245+
- docker compose -f docker-compose.ci.yml down -v --remove-orphans
246+
- echo "✅ 本地升级测试通过"
247+
248+
# --------------------------------------------------------
249+
# Step 5: (可选) 导出镜像并传输到私有服务器
250+
# 只有在部署测试通过后才会执行到这一步
251+
# --------------------------------------------------------
252+
- name: save-images
253+
image: docker:24.0.6
254+
privileged: true
255+
volumes:
256+
- name: docker-socket
257+
path: /var/run/docker.sock
258+
commands:
259+
- echo "测试全部通过，开始导出镜像文件..."
260+
- docker save -o clawith-backend-new.tar clawith-backend:new
261+
- docker save -o clawith-frontend-new.tar clawith-frontend:new
262+
- |
263+
if [ -f ".env.drone" ] && grep -q "SKIP_UPGRADE_TEST=false" ".env.drone"; then
264+
docker save -o clawith-backend-old.tar clawith-backend:old
265+
docker save -o clawith-frontend-old.tar clawith-frontend:old
266+
fi
267+
- chmod 644 *.tar
268+
- ls -lh *.tar
269+
270+
- name: scp-images
271+
image: appleboy/drone-scp
272+
pull: if-not-exists
273+
settings:
274+
host:
275+
from_secret: PRIVATE_SERVER_IP
276+
username: root
277+
password:
278+
from_secret: sshpwd
279+
port: 22
280+
command_timeout: 15m
281+
target: /opt/server/clawith-ci/
282+
source:
283+
- clawith-backend-new.tar
284+
- clawith-frontend-new.tar
285+
- clawith-backend-old.tar
286+
- clawith-frontend-old.tar
287+
- docker-compose.ci.yml
288+
- .env.drone
289+
rm: false
290+
291+
trigger:
292+
branch:
293+
- main
294+
- release
295+
event:
296+
- push
297+
- pull_request
298+
299+
volumes:
300+
- name: docker-socket
301+
host:
302+
path: /var/run/docker.sock

0 commit comments

Comments
 (0)