feat: nginx支持https/密码通过secret存放 (#242)

* feat: nginx支持https

* feat: 密码通过secret存放

Author: hhhhsc701

deployment/docker/datamate/docker-compose.yml

@@ -8,6 +8,8 @@ services:
     image: ${REGISTRY:-}datamate-backend
     restart: on-failure
     privileged: true
+    environment:
+      DB_PASSWORD: ${DB_PASSWORD:-password}
     volumes:
       - dataset_volume:/dataset
       - flow_volume:/flow
@@ -25,6 +27,7 @@ services:
     privileged: true
     environment:
       - log_level=DEBUG
+      - pgsql_password=${DB_PASSWORD:-password}
     volumes:
       - dataset_volume:/dataset
       - flow_volume:/flow
@@ -45,7 +48,7 @@ services:
     image: ${REGISTRY:-}datamate-frontend
     restart: on-failure
     ports:
-      - "30000:80"
+      - "30000:3000"
     volumes:
       - frontend_log_volume:/var/log/datamate/frontend
     networks: [ datamate ]
@@ -59,7 +62,7 @@ services:
     restart: on-failure
     environment:
       - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=password
+      - POSTGRES_PASSWORD=${DB_PASSWORD:-password}
     volumes:
       - postgresql_volume:/var/lib/postgresql/data
       - database_log_volume:/var/log/datamate/database
@@ -77,7 +80,7 @@ services:
       PG_HOST: "datamate-database"
       PG_PORT: "5432"
       PG_USER: "postgres"
-      PG_PASSWORD: "password"
+      PG_PASSWORD: ${DB_PASSWORD:-password}
       PG_DATABASE: "datamate"
     command:
       - python
@@ -167,7 +170,7 @@ services:
     volumes:
       - ./conf.yaml:/app/conf.yaml:ro
       - deer-flow-log-volume:/var/log/deer-flow
-    restart: unless-stopped
+    restart: on-failure
     networks:
       - datamate
     profiles:
@@ -180,7 +183,7 @@ services:
       - .env
     depends_on:
       - deer-flow-backend
-    restart: unless-stopped
+    restart: on-failure
     networks:
       - datamate
     profiles:
@@ -195,7 +198,7 @@ services:
     tty: true
     image: heartexlabs/label-studio:latest
     privileged: true
-    restart: unless-stopped
+    restart: on-failure
     user: root
     expose:
       - "8000"
@@ -207,7 +210,7 @@ services:
       - DJANGO_DB=default
       - POSTGRE_NAME=labelstudio
       - POSTGRE_USER=postgres
-      - POSTGRE_PASSWORD=password
+      - POSTGRE_PASSWORD=${DB_PASSWORD:-password}
       - POSTGRE_PORT=5432
       - POSTGRE_HOST=datamate-database
       - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
@@ -276,6 +279,7 @@ services:
 
   milvus:
     container_name: milvus-standalone
+    restart: on-failure
     image: milvusdb/milvus:v2.6.5
     command: ["milvus", "run", "standalone"]
     security_opt:

deployment/helm/datamate/charts/frontend/values.yaml

@@ -56,7 +56,7 @@ service:
   # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
   type: NodePort
   # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
-  port: 80
+  port: 3000
   nodePort: 30000
 
 resources: {}

deployment/helm/datamate/charts/public/templates/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: datamate-conf
+data:
+  {{- range $key, $val := .Values.secrets.data }}
+  {{ $key }}: {{ $val | toString | b64enc | quote }}
+  {{- end }}

deployment/helm/datamate/values.yaml

@@ -37,6 +37,9 @@ public:
       log: 1Gi
       database: 1Gi
       operator: 1Gi
+  secrets:
+    data:
+      DB_PASSWORD: "password"
 
 datasetVolume: &datasetVolume
   name: dataset-volume
@@ -68,13 +71,17 @@ database:
     - name: POSTGRES_USER
       value: postgres
     - name: POSTGRES_PASSWORD
-      value: &dbPass "password"
+      valueFrom:
+        secretKeyRef:
+          name: datamate-conf
+          key: DB_PASSWORD
   volumes:
     - *dataVolume
     - *logVolume
   volumeMounts:
     - name: data-volume
       mountPath: /var/lib/postgresql/data
+      subPath: postgres
     - name: log-volume
       mountPath: /var/log/datamate/database
       subPath: database
@@ -86,7 +93,10 @@ backend:
         - SYS_ADMIN
   env:
     - name: DB_PASSWORD
-      value: *dbPass
+      valueFrom:
+        secretKeyRef:
+          name: datamate-conf
+          key: DB_PASSWORD
     - name: datamate.rag.milvus-uri
       value: "http://milvus:19530"
   volumes:
@@ -106,8 +116,11 @@ backend:
 
 backend-python:
   env:
-    - name: DB_PASSWORD
-      value: *dbPass
+    - name: pgsql_password
+      valueFrom:
+        secretKeyRef:
+          name: datamate-conf
+          key: DB_PASSWORD
   volumes:
     - *datasetVolume
     - *flowVolume
@@ -134,16 +147,10 @@ frontend:
     nodePort: 30000
   volumes:
     - *logVolume
-    - name: datamate-nginx-conf
-      configMap:
-        name: datamate-nginx-conf
   volumeMounts:
     - name: log-volume
       mountPath: /var/log/datamate/frontend
       subPath: frontend
-    - mountPath: /etc/nginx/conf.d/backend.conf
-      name: datamate-nginx-conf
-      subPath: backend.conf
 
 runtime:
   enabled: false
@@ -160,7 +167,10 @@ runtime:
     - name: MYSQL_USER
       value: "root"
     - name: MYSQL_PASSWORD
-      value: *dbPass
+      valueFrom:
+        secretKeyRef:
+          name: datamate-conf
+          key: DB_PASSWORD
     - name: MYSQL_DATABASE
       value: "datamate"
   volumes:
@@ -207,15 +217,18 @@ ray-cluster:
         value: "0"
       - name: RAY_TQDM_PATCH_PRINT
         value: "0"
-      - name: MYSQL_HOST
+      - name: PG_HOST
         value: "datamate-database"
-      - name: MYSQL_PORT
-        value: "3306"
-      - name: MYSQL_USER
-        value: "root"
-      - name: MYSQL_PASSWORD
-        value: *dbPass
-      - name: MYSQL_DATABASE
+      - name: PG_PORT
+        value: "5432"
+      - name: PG_USER
+        value: "postgres"
+      - name: PG_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: datamate-conf
+            key: DB_PASSWORD
+      - name: PG_DATABASE
         value: "datamate"
       - name: RAY_enable_autoscaler_v2
         value: "1"
@@ -259,15 +272,18 @@ ray-cluster:
         value: "0"
       - name: RAY_TQDM_PATCH_PRINT
         value: "0"
-      - name: MYSQL_HOST
+      - name: PG_HOST
         value: "datamate-database"
-      - name: MYSQL_PORT
-        value: "3306"
-      - name: MYSQL_USER
-        value: "root"
-      - name: MYSQL_PASSWORD
-        value: *dbPass
-      - name: MYSQL_DATABASE
+      - name: PG_PORT
+        value: "5432"
+      - name: PG_USER
+        value: "postgres"
+      - name: PG_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: datamate-conf
+            key: DB_PASSWORD
+      - name: PG_DATABASE
         value: "datamate"
     resources:
       limits:
@@ -308,15 +324,18 @@ ray-cluster:
           value: "0"
         - name: RAY_TQDM_PATCH_PRINT
           value: "0"
-        - name: MYSQL_HOST
+        - name: PG_HOST
           value: "datamate-database"
-        - name: MYSQL_PORT
-          value: "3306"
-        - name: MYSQL_USER
-          value: "root"
-        - name: MYSQL_PASSWORD
-          value: *dbPass
-        - name: MYSQL_DATABASE
+        - name: PG_PORT
+          value: "5432"
+        - name: PG_USER
+          value: "postgres"
+        - name: PG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: datamate-conf
+              key: DB_PASSWORD
+        - name: PG_DATABASE
           value: "datamate"
         - name: POD_NAME
           valueFrom:

scripts/images/database/Dockerfile

@@ -14,5 +14,4 @@ COPY scripts/images/database/postgresql.cnf /etc/postgresql/postgresql.conf
 
 COPY scripts/db/ /docker-entrypoint-initdb.d/
 
-USER postgres
 CMD ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]

scripts/images/frontend/Dockerfile

@@ -9,12 +9,23 @@ RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi && \
 
 FROM nginx:1.29 AS runner
 
-COPY --from=builder /app/dist /opt/frontend
-COPY scripts/images/frontend/backend.conf /etc/nginx/conf.d/backend.conf
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt update \
+    && apt install -y dos2unix
 
-RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+COPY --from=builder /app/dist /opt/frontend/statics
+COPY scripts/images/frontend/routes.inc /opt/frontend/routes.inc
+COPY scripts/images/frontend/http_backend.conf /opt/frontend/http_backend.conf
+COPY scripts/images/frontend/https_backend.conf /opt/frontend/https_backend.conf
+
+COPY scripts/images/frontend/start.sh /opt/frontend/start.sh
+
+RUN dos2unix /opt/frontend/start.sh \
+    && chmod +x /opt/frontend/start.sh \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
     && rm -f /etc/nginx/conf.d/default.conf
 
-EXPOSE 80
+EXPOSE 3000
 
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["/opt/frontend/start.sh"]

scripts/images/frontend/backend.conf

@@ -0,0 +1,13 @@
+server {
+    listen       3000;
+    server_name  0.0.0.0;
+
+    access_log /var/log/datamate/frontend/access.log main;
+    error_log /var/log/datamate/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}

scripts/images/frontend/http_backend.conf

@@ -0,0 +1,18 @@
+server {
+    listen       3000 ssl;
+    server_name  0.0.0.0;
+
+    access_log /var/log/datamate/frontend/access.log main;
+    error_log /var/log/datamate/frontend/error.log notice;
+
+    client_max_body_size 1024M;
+
+    ssl_certificate     /etc/nginx/cert/server.crt;
+    ssl_certificate_key /etc/nginx/cert/server.key;
+
+    error_page 497 https://$host$request_uri;
+
+    add_header Set-Cookie "NEXT_LOCALE=zh";
+
+    include /etc/nginx/conf.d/routes.inc;
+}