优化docker和k8s部署脚本

Author: hhhhsc701

Makefile

@@ -23,7 +23,7 @@ install-%:
 	$(MAKE) $*-$$INSTALLER-install
 
 .PHONY: install
-install: install-edatamate
+install: install-data-platform
 
 .PHONY: uninstall-%
 uninstall-%:
@@ -41,6 +41,8 @@ uninstall-%:
 	esac; \
     $(MAKE) $*-$$INSTALLER-uninstall
 
+.PHONY: uninstall
+uninstall: uninstall-data-platform
 
 # build
 .PHONY: mineru-docker-build
@@ -71,26 +73,6 @@ frontend-docker-build:
 runtime-docker-build:
 	sh scripts/images/runtime/build.sh
 
-.PHONY: mineru-k8s-install
-mineru-k8s-install:
-	kubectl apply -f deployment/kubernetes/mineru/deploy.yaml
-
-.PHONY: datax-k8s-install
-datax-k8s-install:
-	kubectl apply -f deployment/kubernetes/datax/deploy.yaml
-
-.PHONY: datax-k8s-uninstall
-datax-k8s-uninstall:
-	kubectl delete -f deployment/kubernetes/datax/deploy.yaml
-
-.PHONY: datax-docker-install
-datax-docker-install:
-	cd deployment/docker/data-platform && docker-compose up -d datax
-
-.PHONY: datax-docker-uninstall
-datax-docker-uninstall:
-	cd deployment/docker/data-platform && docker-compose down datax
-
 .PHONY: backend-docker-install
 backend-docker-install:
 	cd deployment/docker/data-platform && docker-compose up -d backend
@@ -107,27 +89,28 @@ frontend-docker-install:
 frontend-docker-uninstall:
 	cd deployment/docker/data-platform && docker-compose down frontend
 
-.PHONY: data-juicer-helm-install
-data-juicer-helm-install:
-	sh deployment/helm/data-juicer/install.sh
-
-.PHONY: ray-helm-install
-ray-helm-install:
-	sh deployment/helm/ray/install.sh
-
-.PHONY: es-helm-install
-es-helm-install:
-	sh deployment/helm/es/install.sh
+.PHONY: runtime-helm-install
+runtime-helm-install:
+	helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+	helm repo update
+	helm install kuberay-operator kuberay/kuberay-operator --version 1.4.0
+	helm install raycluster deployment/helm/ray/ray-cluster/
 
 .PHONY: unstructured-k8s-install
 unstructured-k8s-install:
 	kubectl apply -f deployment/kubernetes/unstructured/deploy.yaml
 
 .PHONY: mysql-k8s-install
 mysql-k8s-install:
-	kubectl create configmap init-sql --from-file=scripts/db/
+	kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml > deployment/kubernetes/mysql/init-sql.yaml
+	kubectl apply -f deployment/kubernetes/mysql/init-sql.yaml
 	kubectl apply -f deployment/kubernetes/mysql/deploy.yaml
 
+.PHONY: mysql-k8s-uninstall
+mysql-k8s-uninstall:
+	kubectl delete configmap init-sql
+	kubectl delete -f deployment/kubernetes/mysql/deploy.yaml
+
 .PHONY: backend-k8s-install
 backend-k8s-install:
 	kubectl apply -f deployment/kubernetes/backend/deploy.yaml
@@ -144,24 +127,16 @@ frontend-k8s-install:
 frontend-k8s-uninstall:
 	kubectl delete -f deployment/kubernetes/frontend/deploy.yaml
 
-.PHONY: pgsql-k8s-install
-pgsql-k8s-install:
-	kubectl apply -f deployment/kubernetes/postgresql/deploy.yaml
-
-.PHONY: pgsql-k8s-uninstall
-pgsql-k8s-uninstall:
-	kubectl delete -f deployment/kubernetes/postgresql/deploy.yaml
-
-.PHONY: edatamate-docker-install
-edatamate-docker-install:
+.PHONY: data-platform-docker-install
+data-platform-docker-install:
 	cd deployment/docker/data-platform && docker-compose up -d
 
-.PHONY: edatamate-docker-uninstall
-edatamate-docker-uninstall:
+.PHONY: data-platform-docker-uninstall
+data-platform-docker-uninstall:
 	cd deployment/docker/data-platform && docker-compose down
 
-.PHONY: edatamate-k8s-install
-edatamate-k8s-install: pgsql-k8s-install backend-k8s-install frontend-k8s-install datax-k8s-install
+.PHONY: data-platform-k8s-install
+data-platform-k8s-install: mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-helm-install
 
-.PHONY: edatamate-k8s-uninstall
-edatamate-k8s-uninstall: pgsql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall datax-k8s-uninstall
+.PHONY: data-platform-k8s-uninstall
+data-platform-k8s-uninstall: mysql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall

deployment/docker/data-platform/.env

@@ -0,0 +1,4 @@
+DATASET_VOLUME=/opt/data-platform/data/dataset
+FLOW_VOLUME=/opt/data-platform/data/flow
+MYSQL_VOLUME=/opt/data-platform/data/mysql
+LOG_VOLUME=/opt/data-platform/data/log

deployment/docker/data-platform/docker-compose.yml

@@ -2,54 +2,37 @@ services:
   # 1) backend
   backend:
     container_name: backend
-    image: backend         # 等价于 imagePullPolicy: IfNotPresent
+    image: backend
     ports:
       - "8080:8080"
     volumes:
-      - type: bind
-        source: /tmp/data-platform/dataset
-        target: /dataset
+      - ${DATASET_VOLUME}:/dataset
+      - ${FLOW_VOLUME}:/flow
+      - ${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
-  # 2) datax
-  datax:
-    container_name: datax
-    image: datax
-    command: >
-      bash -c "rpcbind && python3 /opt/datax/bin/app.py"
-    cap_add:
-      - SYS_ADMIN               # securityContext.capabilities.add
-    ports:
-      - "8000:8000"
-    volumes:
-      - type: bind
-        source: /tmp/data-platform/dataset
-        target: /dataset
-    networks: [ edatamate ]
-
-  # 3) frontend（NodePort 30000）
+  # 2) frontend（NodePort 30000）
   frontend:
     container_name: frontend
     image: frontend
     ports:
-      - "3000:80"            # nodePort → hostPort
+      - "30000:80"            # nodePort → hostPort
+    volumes:
+      - ${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
-  # 4) mysql
+  # 3) mysql
   mysql:
     container_name: mysql
-    image: mysql:8.0
+    image: mysql:8
     environment:
       MYSQL_ROOT_PASSWORD: Huawei@123
-      MYSQL_DATABASE: dataengine
-      MYSQL_USER: root
-      MYSQL_PASSWORD: Huawei@123
     ports:
       - "3306:3306"
     volumes:
-      - type: bind
-        source: /tmp/data-platform/mysql
-        target: /var/lib/mysql
+      - ${MYSQL_VOLUME}:/var/lib/mysql
+      - ../../../scripts/db:/docker-entrypoint-initdb.d
+      - ${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
 networks: