Merge remote-tracking branch 'origin/develop_930' into develop_930

# Conflicts:
#	Makefile
#	deployment/kubernetes/api-gateway.yaml

Author: hefanli

Makefile

@@ -1,12 +1,19 @@
 MAKEFLAGS += --no-print-directory
 
+VERSION ?= latest
+NAMESPACE ?= datamate
+
 .PHONY: build-%
 build-%:
 	$(MAKE) $*-docker-build
 
 .PHONY: build
 build: backend-docker-build frontend-docker-build runtime-docker-build
 
+.PHONY: create-namespace
+create-namespace:
+	@kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
+
 .PHONY: install-%
 install-%:
 ifeq ($(origin INSTALLER), undefined)
@@ -52,31 +59,27 @@ uninstall: uninstall-data-mate
 # build
 .PHONY: mineru-docker-build
 mineru-docker-build:
-	sh scripts/images/mineru/build.sh
+	docker build -t mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile
 
 .PHONY: datax-docker-build
 datax-docker-build:
-	sh scripts/images/datax/build.sh
-
-.PHONY: data-juicer-docker-build
-data-juicer-docker-build:
-	sh scripts/images/data-juicer/build.sh
+	docker build -t datax:$(VERSION) . -f scripts/images/datax/Dockerfile
 
 .PHONY: unstructured-docker-build
 unstructured-docker-build:
-	sh scripts/images/unstructured/build.sh
+	docker build -t unstructured:$(VERSION) . -f scripts/images/unstructured/Dockerfile
 
 .PHONY: backend-docker-build
 backend-docker-build:
-	sh scripts/images/backend/build.sh
+	docker build -t backend:$(VERSION) . -f scripts/images/backend/Dockerfile
 
 .PHONY: frontend-docker-build
 frontend-docker-build:
-	sh scripts/images/frontend/build.sh
+	docker build -t frontend:$(VERSION) . -f scripts/images/frontend/Dockerfile
 
 .PHONY: runtime-docker-build
 runtime-docker-build:
-	sh scripts/images/runtime/build.sh
+	docker build -t runtime:$(VERSION) . -f scripts/images/runtime/Dockerfile
 
 .PHONY: backend-docker-install
 backend-docker-install:
@@ -103,59 +106,59 @@ runtime-docker-uninstall:
 	cd deployment/docker/data-mate && docker-compose down runtime
 
 .PHONY: runtime-k8s-install
-runtime-k8s-install:
-	helm upgrade kuberay-operator deployment/helm/ray/kuberay-operator --install
-	helm upgrade raycluster deployment/helm/ray/ray-cluster/ --install
-	kubectl apply -f deployment/helm/ray/service.yaml
+runtime-k8s-install: create-namespace
+	helm upgrade kuberay-operator deployment/helm/ray/kuberay-operator --install -n $(NAMESPACE)
+	helm upgrade raycluster deployment/helm/ray/ray-cluster/ --install -n $(NAMESPACE)
+	kubectl apply -f deployment/helm/ray/service.yaml -n $(NAMESPACE)
 
 .PHONY: runtime-k8s-uninstall
 runtime-k8s-uninstall:
-	helm uninstall raycluster
-	helm uninstall kuberay-operator
-	kubectl delete -f deployment/helm/ray/service.yaml
+	helm uninstall raycluster -n $(NAMESPACE)
+	helm uninstall kuberay-operator -n $(NAMESPACE)
+	kubectl delete -f deployment/helm/ray/service.yaml -n $(NAMESPACE)
 
 .PHONY: unstructured-k8s-install
-unstructured-k8s-install:
-	kubectl apply -f deployment/kubernetes/unstructured/deploy.yaml
+unstructured-k8s-install: create-namespace
+	kubectl apply -f deployment/kubernetes/unstructured/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: mysql-k8s-install
-mysql-k8s-install:
-	kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f -
-	kubectl apply -f deployment/kubernetes/mysql/configmap.yaml
-	kubectl apply -f deployment/kubernetes/mysql/deploy.yaml
+mysql-k8s-install: create-namespace
+	kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
+	kubectl apply -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE)
+	kubectl apply -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: mysql-k8s-uninstall
 mysql-k8s-uninstall:
-	kubectl delete configmap init-sql
-	kubectl delete -f deployment/kubernetes/mysql/configmap.yaml
-	kubectl delete -f deployment/kubernetes/mysql/deploy.yaml
+	kubectl delete configmap init-sql -n $(NAMESPACE)
+	kubectl delete -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE)
+	kubectl delete -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: backend-k8s-install
-backend-k8s-install:
-	kubectl apply -f deployment/kubernetes/backend/deploy.yaml
+backend-k8s-install: create-namespace
+	kubectl apply -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: backend-k8s-uninstall
 backend-k8s-uninstall:
-	kubectl delete -f deployment/kubernetes/backend/deploy.yaml
+	kubectl delete -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: frontend-k8s-install
-frontend-k8s-install:
-	kubectl apply -f deployment/kubernetes/frontend/deploy.yaml
+frontend-k8s-install: create-namespace
+	kubectl apply -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: frontend-k8s-uninstall
 frontend-k8s-uninstall:
-	kubectl delete -f deployment/kubernetes/frontend/deploy.yaml
+	kubectl delete -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE)
 
 .PHONY: data-mate-docker-install
 data-mate-docker-install:
-	cd deployment/docker/data-mate && docker-compose up -d
+	cd deployment/docker/datamate && docker-compose up -d
 
 .PHONY: data-mate-docker-uninstall
 data-mate-docker-uninstall:
-	cd deployment/docker/data-mate && docker-compose down
+	cd deployment/docker/datamate && docker-compose down
 
 .PHONY: data-mate-k8s-install
-data-mate-k8s-install: mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install
+data-mate-k8s-install: create-namespace mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install
 
 .PHONY: data-mate-k8s-uninstall
 data-mate-k8s-uninstall: mysql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall runtime-k8s-uninstall

…/docker/data-platform/docker-compose.yml …yment/docker/datamate/docker-compose.ymldeployment/docker/data-platform/docker-compose.yml renamed to deployment/docker/datamate/docker-compose.yml deployment/docker/data-platform/docker-compose.yml renamed to deployment/docker/datamate/docker-compose.yml

@@ -1,93 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: data-mate
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: api-gateway
-  namespace: data-mate
-  labels:
-    app: api-gateway
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      app: api-gateway
-  template:
-    metadata:
-      labels:
-        app: api-gateway
-    spec:
-      containers:
-      - name: api-gateway
-        image: data-mate/api-gateway:latest
-        ports:
-        - containerPort: 8080
-        env:
-        - name: SPRING_PROFILES_ACTIVE
-          value: "kubernetes"
-        - name: SPRING_DATASOURCE_URL
-          value: "jdbc:mysql://mysql-service:3306/datamate"
-        - name: SPRING_REDIS_HOST
-          value: "redis-service"
-        resources:
-          requests:
-            memory: "512Mi"
-            cpu: "250m"
-          limits:
-            memory: "1Gi"
-            cpu: "500m"
-        readinessProbe:
-          httpGet:
-            path: /actuator/health
-            port: 8080
-          initialDelaySeconds: 30
-          periodSeconds: 10
-        livenessProbe:
-          httpGet:
-            path: /actuator/health
-            port: 8080
-          initialDelaySeconds: 60
-          periodSeconds: 30
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: api-gateway-service
-  namespace: data-mate
-spec:
-  selector:
-    app: api-gateway
-  ports:
-  - port: 8080
-    targetPort: 8080
-  type: LoadBalancer
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: data-mate-ingress
-  namespace: data-mate
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-spec:
-  rules:
-  - host: data-mate.local
-    http:
-      paths:
-      - path: /api
-        pathType: Prefix
-        backend:
-          service:
-            name: api-gateway-service
-            port:
-              number: 8080
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: console-service
-            port:
-              number: 3000

deployment/docker/data-platform/utf8.cnf deployment/docker/datamate/utf8.cnfdeployment/docker/data-platform/utf8.cnf renamed to deployment/docker/datamate/utf8.cnf

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
       - name: unstructured
-        image: downloads.unstructured.io/unstructured-io/unstructured:app
+        image: unstructured
         imagePullPolicy: IfNotPresent
         command:
           - python