Health check added for HA deployment

Author: vaibhavsawhney

deployment/haystack-agent/templates/deployment.yaml

@@ -45,16 +45,14 @@ spec:
           value: "${jvm_memory_limit}m"
         - name: "JAVA_XMX"
           value: "${jvm_memory_limit}m"
-#        livenessProbe:
-#          exec:
-#            command:
-#            - grep
-#            - "true"
-#            - /app/isHealthy
-#          initialDelaySeconds: 30
-#          periodSeconds: 5
-#          timeoutSeconds: 5
-#          failureThreshold: 6
+        livenessProbe:
+          exec:
+            command:
+              - /bin/grpc_health_probe
+              - "-addr=:${blobs_service_port}"
+          initialDelaySeconds: 30
+          periodSeconds: 15
+          failureThreshold: 3
       nodeSelector:
         ${node_selecter_label}
       volumes:

docker/Dockerfile

@@ -6,6 +6,10 @@ ENV APP_HOME /app/bin
 
 RUN mkdir -p ${APP_HOME}
 
+RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
+
 COPY bundlers/haystack-agent/target/${APP_NAME}.jar ${APP_HOME}/
 
 COPY docker/default.conf ${APP_HOME}/