Enable TLS in Elasticsearch

Co-authored-by: wych42 <wych101010@gmail.com>

Author: antoineco

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [ tls ]
   pull_request:
-    branches: [ main ]
+    branches: [ tls ]
 
 jobs:
 
@@ -77,7 +77,7 @@ jobs:
 
           docker compose up -d elasticsearch
           source .github/workflows/scripts/lib/testing.sh
-          poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
+          poll_ready "$(container_id elasticsearch)" "https://elasticsearch:9200/" --resolve "elasticsearch:9200:$(service_ip elasticsearch)" --cacert "$(realpath tls/kibana/elasticsearch-ca.pem)" -u 'elastic:changeme'
 
           # Set passwords
 
@@ -295,7 +295,7 @@ jobs:
           docker stack deploy -c ./docker-stack.yml elk
           docker service scale elk_logstash=0 elk_kibana=0
           source .github/workflows/scripts/lib/testing.sh
-          poll_ready "$(container_id elasticsearch)" "http://$(service_ip elasticsearch):9200/" -u 'elastic:changeme'
+          poll_ready "$(container_id elasticsearch)" "https://elasticsearch:9200/" --resolve "elasticsearch:9200:$(service_ip elasticsearch)" --cacert "$(realpath tls/kibana/elasticsearch-ca.pem)" -u 'elastic:changeme'
 
           # Set passwords
 

.github/workflows/docs.yml

@@ -1,12 +1,10 @@
 name: Documentation
 
 on:
-  schedule:
-    - cron: '0 0 * * 0'  # At 00:00 every Sunday
   push:
-    branches: [ main ]
+    branches: [ tls ]
   pull_request:
-    branches: [ main ]
+    branches: [ tls ]
 
 jobs:
 

.github/workflows/scripts/elasticsearch-setup-passwords.exp

@@ -13,7 +13,7 @@ if { [string match "swarm" $MODE] } {
 }
 
 foreach user $users {
-	set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i"
+	set cmd "docker exec -it $cid bin/elasticsearch-reset-password --batch --user $user -i --url https://localhost:9200"
 
 	spawn {*}$cmd
 

.github/workflows/scripts/run-tests-core.sh

@@ -15,8 +15,10 @@ ip_es="$(service_ip elasticsearch)"
 ip_ls="$(service_ip logstash)"
 ip_kb="$(service_ip kibana)"
 
+es_ca_cert="$(realpath $(dirname ${BASH_SOURCE[0]})/../../../tls/kibana/elasticsearch-ca.pem)"
+
 log 'Waiting for readiness of Elasticsearch'
-poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+poll_ready "$cid_es" 'https://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u 'elastic:testpasswd'
 
 log 'Waiting for readiness of Logstash'
 poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
@@ -44,11 +46,11 @@ if ((was_retried)); then
 fi
 
 sleep 5
-curl -X POST "http://${ip_es}:9200/logs-generic-default/_refresh" -u elastic:testpasswd \
-	-s -w '\n'
+curl -X POST "https://elasticsearch:9200/logs-generic-default/_refresh" -u elastic:testpasswd \
+	-s -w '\n' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert"
 
 log 'Searching message in Elasticsearch'
-response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s -u elastic:testpasswd)"
+response="$(curl "https://elasticsearch:9200/logs-generic-default/_search?q=message:dockerelk&pretty" -s --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u elastic:testpasswd)"
 echo "$response"
 declare -i count
 count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

.github/workflows/scripts/run-tests-enterprise-search.sh

@@ -13,14 +13,16 @@ cid_en="$(container_id enterprise-search)"
 ip_es="$(service_ip elasticsearch)"
 ip_en="$(service_ip enterprise-search)"
 
+es_ca_cert="$(realpath $(dirname ${BASH_SOURCE[0]})/../../../tls/kibana/elasticsearch-ca.pem)"
+
 log 'Waiting for readiness of Elasticsearch'
-poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+poll_ready "$cid_es" 'https://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u 'elastic:testpasswd'
 
 log 'Waiting for readiness of Enterprise Search'
 poll_ready "$cid_en" "http://${ip_en}:3002/api/ent/v1/internal/health" -u 'elastic:testpasswd'
 
 log 'Ensuring that App Search API keys were created in Elasticsearch'
-response="$(curl "http://${ip_es}:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty" -s -u elastic:testpasswd)"
+response="$(curl 'https://elasticsearch:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty' -s --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u elastic:testpasswd)"
 echo "$response"
 declare -i count
 count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

.github/workflows/scripts/run-tests-filebeat.sh

@@ -13,8 +13,10 @@ cid_mb="$(container_id filebeat)"
 ip_es="$(service_ip elasticsearch)"
 ip_mb="$(service_ip filebeat)"
 
+es_ca_cert="$(realpath $(dirname ${BASH_SOURCE[0]})/../../../tls/kibana/elasticsearch-ca.pem)"
+
 log 'Waiting for readiness of Elasticsearch'
-poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+poll_ready "$cid_es" 'https://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u 'elastic:testpasswd'
 
 log 'Waiting for readiness of Filebeat'
 poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
@@ -35,7 +37,7 @@ declare -i was_retried=0
 
 # retry for max 60s (30*2s)
 for _ in $(seq 1 30); do
-	response="$(curl "http://${ip_es}:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)"
+	response="$(curl "https://elasticsearch:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u elastic:testpasswd)"
 
 	set +u  # prevent "unbound variable" if assigned value is not an integer
 	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

.github/workflows/scripts/run-tests-logspout.sh

@@ -15,8 +15,10 @@ ip_es="$(service_ip elasticsearch)"
 ip_ls="$(service_ip logstash)"
 ip_lsp="$(service_ip logspout)"
 
+es_ca_cert="$(realpath $(dirname ${BASH_SOURCE[0]})/../../../tls/kibana/elasticsearch-ca.pem)"
+
 log 'Waiting for readiness of Elasticsearch'
-poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+poll_ready "$cid_es" 'https://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u 'elastic:testpasswd'
 
 log 'Waiting for readiness of Logstash'
 poll_ready "$cid_ls" "http://${ip_ls}:9600/_node/pipelines/main?pretty"
@@ -39,7 +41,7 @@ declare -i was_retried=0
 
 # retry for max 60s (30*2s)
 for _ in $(seq 1 30); do
-	response="$(curl "http://${ip_es}:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
+	response="$(curl "https://elasticsearch:9200/logs-generic-default/_search?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u elastic:testpasswd)"
 
 	set +u  # prevent "unbound variable" if assigned value is not an integer
 	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

.github/workflows/scripts/run-tests-metricbeat.sh

@@ -13,8 +13,10 @@ cid_mb="$(container_id metricbeat)"
 ip_es="$(service_ip elasticsearch)"
 ip_mb="$(service_ip metricbeat)"
 
+es_ca_cert="$(realpath $(dirname ${BASH_SOURCE[0]})/../../../tls/kibana/elasticsearch-ca.pem)"
+
 log 'Waiting for readiness of Elasticsearch'
-poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+poll_ready "$cid_es" 'https://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u 'elastic:testpasswd'
 
 log 'Waiting for readiness of Metricbeat'
 poll_ready "$cid_mb" "http://${ip_mb}:5066/?pretty"
@@ -36,7 +38,7 @@ declare -i was_retried=0
 
 # retry for max 60s (30*2s)
 for _ in $(seq 1 30); do
-	response="$(curl "http://${ip_es}:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)"
+	response="$(curl 'https://elasticsearch:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty' -s --resolve "elasticsearch:9200:${ip_es}" --cacert "$es_ca_cert" -u elastic:testpasswd)"
 
 	set +u  # prevent "unbound variable" if assigned value is not an integer
 	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"