Skip to content

Commit 0716bad

Browse files
authored
优化数据处理/安装部署 (#419)
* 支持界面展示报错 * 支持文件元数据透传 * 支持gpu * 支持下载日志文件 * 系统参数模块 * 支持自定义首页 * 切换数据池镜像
1 parent df162a5 commit 0716bad

26 files changed

Lines changed: 760 additions & 98 deletions

File tree

backend/api-gateway/src/main/java/com/datamate/gateway/ApiGatewayApplication.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ public RouteLocator customRouteLocator(RouteLocatorBuilder builder) {
5555
.route("data-cleaning", r -> r.path("/api/cleaning/**")
5656
.uri("http://datamate-backend-python:18000"))
5757

58+
.route("data-setting", r -> r.path("/api/sys-param/**")
59+
.uri("http://datamate-backend-python:18000"))
60+
5861
.route("deer-flow-frontend", r -> r.path("/chat/**")
5962
.uri("http://deer-flow-frontend:3000"))
6063

deployment/docker/datamate/docker-compose.yml

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -204,16 +204,15 @@ services:
204204
# ==============================
205205
label-studio-pgbouncer:
206206
container_name: label-studio-pgbouncer
207-
image: pgbouncer/pgbouncer:latest
207+
image: edoburu/pgbouncer:latest
208208
restart: on-failure
209-
ports:
210-
- "6432:6432"
211209
environment:
212-
- DATABASES_HOST=datamate-database
213-
- DATABASES_PORT=5432
214-
- DATABASES_NAME=labelstudio
215-
- DATABASES_USER=postgres
216-
- DATABASES_PASSWORD=${DB_PASSWORD:-password}
210+
- DB_HOST=datamate-database
211+
- DB_PORT=5432
212+
- DB_NAME=labelstudio
213+
- DB_USER=postgres
214+
- DB_PASSWORD=${DB_PASSWORD:-password}
215+
- AUTH_TYPE=scram-sha-256
217216
- POOL_MODE=transaction
218217
- MAX_CLIENT_CONN=100
219218
- DEFAULT_POOL_SIZE=20
@@ -241,7 +240,7 @@ services:
241240
- POSTGRE_NAME=labelstudio
242241
- POSTGRE_USER=postgres
243242
- POSTGRE_PASSWORD=${DB_PASSWORD:-password}
244-
- POSTGRE_PORT=6432
243+
- POSTGRE_PORT=5432
245244
- POSTGRE_HOST=label-studio-pgbouncer
246245
- LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
247246
- LOCAL_FILES_SERVING_ENABLED=true

deployment/helm/datamate/values.yaml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public:
4242
DB_PASSWORD: "password"
4343
CERT_PASS: ""
4444
DOMAIN: ""
45+
HOME_PAGE_URL: ""
4546

4647
datasetVolume: &datasetVolume
4748
name: dataset-volume
@@ -90,6 +91,11 @@ database:
9091
secretKeyRef:
9192
name: datamate-conf
9293
key: DB_PASSWORD
94+
- name: HOME_PAGE_URL
95+
valueFrom:
96+
secretKeyRef:
97+
name: datamate-conf
98+
key: HOME_PAGE_URL
9399
volumes:
94100
- *dataVolume
95101
- *logVolume
@@ -420,3 +426,60 @@ ray-cluster:
420426
subPath: site-packages
421427
- mountPath: /usr/local/Ascend
422428
name: ascend
429+
gpuGroup:
430+
disabled: false
431+
replicas: 0
432+
minReplicas: 0
433+
maxReplicas: 8
434+
rayStartParams:
435+
resources: '"{\"gpu\": 1}"'
436+
containerEnv:
437+
- name: RAY_DEDUP_LOGS
438+
value: "0"
439+
- name: RAY_TQDM_PATCH_PRINT
440+
value: "0"
441+
- name: PG_HOST
442+
value: "datamate-database"
443+
- name: PG_PORT
444+
value: "5432"
445+
- name: PG_USER
446+
value: "postgres"
447+
- name: PG_PASSWORD
448+
valueFrom:
449+
secretKeyRef:
450+
name: datamate-conf
451+
key: DB_PASSWORD
452+
- name: PG_DATABASE
453+
value: "datamate"
454+
- name: POD_NAME
455+
valueFrom:
456+
fieldRef:
457+
fieldPath: metadata.name
458+
resources:
459+
limits:
460+
cpu: "8"
461+
memory: "64G"
462+
nvidia.com/gpu: 1
463+
requests:
464+
cpu: "1"
465+
memory: "2G"
466+
nvidia.com/gpu: 1
467+
volumes:
468+
- *datasetVolume
469+
- *flowVolume
470+
- *logVolume
471+
- *operatorVolume
472+
volumeMounts:
473+
- mountPath: /tmp/ray
474+
name: log-volume
475+
subPathExpr: ray/$(POD_NAME)
476+
- mountPath: /dataset
477+
name: dataset-volume
478+
- mountPath: /flow
479+
name: flow-volume
480+
- mountPath: /opt/runtime/datamate/ops/user
481+
name: operator-volume
482+
subPath: extract
483+
- mountPath: /usr/local/lib/ops/site-packages
484+
name: operator-volume
485+
subPath: site-packages

deployment/helm/label-studio/templates/deployment.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ spec:
3838
- name: POSTGRE_PASSWORD
3939
value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
4040
- name: POSTGRE_PORT
41-
value: {{ if .Values.pgbouncer.enabled }}{{ "6432" | quote }}{{ else }}{{ .Values.env.POSTGRE_PORT | quote }}{{ end }}
41+
value: 5432
4242
- name: POSTGRE_HOST
4343
value: {{ if .Values.pgbouncer.enabled }}{{ printf "%s-pgbouncer" (include "label-studio.fullname" .) | quote }}{{ else }}{{ .Values.env.POSTGRE_HOST | quote }}{{ end }}
4444
- name: LABEL_STUDIO_HOST

deployment/helm/label-studio/templates/pgbouncer-deployment.yaml

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,18 @@ spec:
3030
imagePullPolicy: {{ .Values.pgbouncer.image.pullPolicy }}
3131
ports:
3232
- name: pgbouncer
33-
containerPort: 6432
33+
containerPort: 5432
3434
protocol: TCP
3535
env:
36-
- name: DATABASES_HOST
36+
- name: DB_HOST
3737
value: {{ .Values.env.POSTGRE_HOST | quote }}
38-
- name: DATABASES_PORT
38+
- name: DB_PORT
3939
value: {{ .Values.env.POSTGRE_PORT | quote }}
40-
- name: DATABASES_NAME
40+
- name: DB_NAME
4141
value: {{ .Values.env.POSTGRE_NAME | quote }}
42-
- name: DATABASES_USER
42+
- name: DB_USER
4343
value: {{ .Values.env.POSTGRE_USER | quote }}
44-
- name: DATABASES_PASSWORD
44+
- name: DB_PASSWORD
4545
value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
4646
- name: POOL_MODE
4747
value: {{ .Values.pgbouncer.poolMode | quote }}
@@ -51,14 +51,16 @@ spec:
5151
value: {{ .Values.pgbouncer.defaultPoolSize | quote }}
5252
- name: MAX_DB_CONNECTIONS
5353
value: {{ .Values.pgbouncer.maxDbConnections | quote }}
54+
- name: AUTH_TYPE
55+
value: {{ .Values.pgbouncer.authType | quote }}
5456
livenessProbe:
5557
tcpSocket:
56-
port: 6432
58+
port: 5432
5759
initialDelaySeconds: 15
5860
periodSeconds: 20
5961
readinessProbe:
6062
tcpSocket:
61-
port: 6432
63+
port: 5432
6264
initialDelaySeconds: 5
6365
periodSeconds: 10
6466
resources:

deployment/helm/label-studio/templates/pgbouncer-service.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ metadata:
1111
spec:
1212
type: ClusterIP
1313
ports:
14-
- port: 6432
14+
- port: 5432
1515
targetPort: pgbouncer
1616
protocol: TCP
1717
name: pgbouncer

deployment/helm/label-studio/values.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ pgbouncer:
7272
replicaCount: 1
7373

7474
image:
75-
repository: pgbouncer/pgbouncer
75+
repository: edoburu/pgbouncer
7676
tag: "latest"
7777
pullPolicy: IfNotPresent
7878

@@ -81,6 +81,7 @@ pgbouncer:
8181
maxClientConn: 100 # Maximum number of client connections
8282
defaultPoolSize: 20 # Default pool size per database (max connections to PostgreSQL)
8383
maxDbConnections: 20 # Maximum database connections (hard limit)
84+
authType: scram-sha-256
8485

8586
resources: {}
8687

frontend/src/i18n/locales/en/common.json

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1452,12 +1452,14 @@
14521452
"sizeOptimization": "File Size Optimization",
14531453
"reduced": "Reduced by {{percent}}%"
14541454
},
1455-
"logTable": {
1455+
"logTable": {
14561456
"selectRun": "Select Run",
14571457
"currentDisplay": "Current Display: {{num}}th Run",
14581458
"nthRun": "{{num}}th Run",
14591459
"noLogs": "No logs available for this task",
1460-
"streaming": "Streaming..."
1460+
"streaming": "Streaming...",
1461+
"download": "Download Log",
1462+
"downloadFailed": "Failed to download log file"
14611463
},
14621464
"operatorTable": {
14631465
"serialNumber": "Serial Number",

frontend/src/i18n/locales/zh/common.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1457,7 +1457,9 @@
14571457
"currentDisplay": "当前展示: 第 {{num}} 次",
14581458
"nthRun": "第 {{num}} 次",
14591459
"noLogs": "当前任务无可用日志",
1460-
"streaming": "实时流式输出中..."
1460+
"streaming": "实时流式输出中...",
1461+
"download": "下载日志",
1462+
"downloadFailed": "下载日志文件失败"
14611463
},
14621464
"operatorTable": {
14631465
"serialNumber": "序号",

frontend/src/main.tsx

Lines changed: 81 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,39 +11,96 @@ import theme from "./theme";
1111
import {errorConfigStore} from "@/utils/errorConfigStore.ts";
1212
import "@/i18n";
1313

14+
async function checkHomePageRedirect(): Promise<string | null> {
15+
try {
16+
const response = await fetch('/api/sys-param/sys.home.page.url', {
17+
cache: 'no-store'
18+
});
19+
20+
if (response.ok) {
21+
const result = await response.json();
22+
return result.data?.paramValue?.trim() || null;
23+
}
24+
} catch (error) {
25+
console.error('Failed to fetch home page URL:', error);
26+
}
27+
28+
return null;
29+
}
30+
31+
function showLoadingUI() {
32+
const container = document.getElementById("root");
33+
if (!container) return;
34+
35+
container.innerHTML = `
36+
<div style="
37+
min-height: 100vh;
38+
background: linear-gradient(to bottom right, #eff6ff, #e0e7ff);
39+
display: flex;
40+
align-items: center;
41+
justify-content: center;
42+
">
43+
<div style="text-align: center;">
44+
<div style="
45+
width: 40px;
46+
height: 40px;
47+
border: 3px solid #e5e7eb;
48+
border-top-color: #3b82f6;
49+
border-radius: 50%;
50+
animation: spin 1s linear infinite;
51+
"></div>
52+
<style>
53+
@keyframes spin {
54+
to { transform: rotate(360deg); }
55+
}
56+
</style>
57+
</div>
58+
</div>
59+
`;
60+
}
61+
1462
async function bootstrap() {
1563
const container = document.getElementById("root");
1664
if (!container) return;
1765

18-
const root = createRoot(container);
66+
showLoadingUI();
1967

2068
try {
21-
// 2. 【关键步骤】在渲染前,等待配置文件加载完成
22-
// 这一步会发起 fetch 请求去拿 /config/error-code.json
23-
await errorConfigStore.loadConfig();
69+
const [, homePageUrl] = await Promise.all([
70+
errorConfigStore.loadConfig(),
71+
checkHomePageRedirect()
72+
]);
73+
74+
if (homePageUrl) {
75+
const currentPath = window.location.pathname;
76+
const targetPath = new URL(homePageUrl, window.location.origin).pathname;
77+
78+
if (currentPath === '/' && currentPath !== targetPath) {
79+
window.location.href = homePageUrl;
80+
return;
81+
}
82+
}
2483

2584
} catch (e) {
26-
// 容错处理:即使配置文件加载失败(比如404),也不应该导致整个 App 白屏崩溃
27-
// 此时 App 会使用代码里的默认兜底文案
28-
console.error('Error config load failed, using default messages.', e);
29-
} finally {
30-
// 3. 无论配置加载成功与否,最后都执行渲染
31-
root.render(
32-
<StrictMode>
33-
<Provider store={store}>
34-
<ConfigProvider theme={ theme }>
35-
<AntdApp>
36-
<Suspense fallback={<Spin />}>
37-
<TopLoadingBar />
38-
<RouterProvider router={router} />
39-
</Suspense>
40-
</AntdApp>
41-
</ConfigProvider>
42-
</Provider>
43-
</StrictMode>
44-
);
85+
console.error('Config load failed:', e);
4586
}
87+
88+
const root = createRoot(container);
89+
90+
root.render(
91+
<StrictMode>
92+
<Provider store={store}>
93+
<ConfigProvider theme={ theme }>
94+
<AntdApp>
95+
<Suspense fallback={<Spin />}>
96+
<TopLoadingBar />
97+
<RouterProvider router={router} />
98+
</Suspense>
99+
</AntdApp>
100+
</ConfigProvider>
101+
</Provider>
102+
</StrictMode>
103+
);
46104
}
47105

48-
// 4. 执行启动
49106
bootstrap();

0 commit comments

Comments
 (0)