Molecule verification enhancement

docs/role-elasticsearch.md

@@ -29,7 +29,7 @@ Role Variables
 * *elasticsearch_disable_systemcallfilterchecks*: Disable system call filter checks. This has a security impact but is necessary on some systems. Please refer to the [docs](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html) for details. (default: `false`)
 * *elasticsearch_http_publish_host*: (String) The network address for HTTP clients to contact the node using sniffing. Accepts an IP address, a hostname, or a special value. (default: `not set`)
 * *elasticsearch_http_publish_port*: (integer) The port of the HTTP publish address. Configure this setting only if you need the publish port to be different from http.port. (default: `not set`)
-* *elasticsearch_check_calculation*: End play in checks (Default: `false`)
+* *elasticsearch_check_calculation*: When set to `true`, the elasticsearch Ansible role runs **only** the node role validation checks and then immediately stops execution on each host via `meta: end_host`. Elasticsearch is **not installed, configured, or started**. The validation checks include: deriving the node role (`master`, `data`, or `other`) from `elasticsearch_node_types`, grouping nodes accordingly, and asserting that the number of master-eligible nodes is odd (required for a valid quorum). This is useful both in production (e.g. as a pre-flight validation before scaling or reconfiguring a cluster) and in CI to quickly verify your inventory topology without the cost of a full deployment. (Default: `false`)
 * *elasticsearch_network_host*: You can configure multipe network addresses where the networking is bind to. You can assign IP addresses or interfaces by their names. You can also use elasticsearch internal variabels as it set as default. Example: `"_ens190_,_local_"` (Default: `"_local_,"_site_"`) (Optional; if not defined `default` is used)
 * *elasticsearch_api_host*: Hostname or IP elasticsearch is listening on. Only used for connection checks by ansible role. (Default: `localhost`)
 * *elasticsearch_extra_config*: You can set additional configuration in YAML-notation as you would write in the `elasaticsearch.yml`. Example:

molecule/beats_default/verify.yml

@@ -1,9 +1,29 @@
 ---
-# This is an example playbook to execute Ansible tests.
-
-- name: Verify
+- name: Verify Beats deployment
   hosts: all
   tasks:
-  - name: Example assertion
-    ansible.builtin.assert:
-      that: true
+    - name: Check filebeat binary exists
+      ansible.builtin.stat:
+        path: /usr/share/filebeat/bin/filebeat
+      register: filebeat_binary
+
+    - name: Assert filebeat binary exists
+      ansible.builtin.assert:
+        that: filebeat_binary.stat.exists
+        fail_msg: "Filebeat binary not found"
+
+    - name: Check filebeat configuration file exists
+      ansible.builtin.stat:
+        path: /etc/filebeat/filebeat.yml
+      register: filebeat_config
+
+    - name: Assert filebeat config exists
+      ansible.builtin.assert:
+        that: filebeat_config.stat.exists
+        fail_msg: "Filebeat configuration file not found"
+
+    - name: Run filebeat config syntax check
+      ansible.builtin.command: filebeat test config -c /etc/filebeat/filebeat.yml
+      register: filebeat_config_test
+      changed_when: false
+      failed_when: filebeat_config_test.rc != 0

molecule/beats_peculiar/verify.yml

@@ -13,13 +13,6 @@
       ansible.builtin.debug:
         var: beats_filebeat_version.stdout
 
-    - name: Fail if Filebeat has the wrong version
-      ansible.builtin.fail:
-        msg: "Filebeat has the wrong version"
-      when:
-        - beats_filebeat_version.stdout.find('7.17.1') == -1
-        - elasticstack_release == 7
-
     - name: Fail if Filebeat has the wrong version
       ansible.builtin.fail:
         msg: "Filebeat has the wrong version"

molecule/elasticsearch_default/verify.yml

@@ -0,0 +1,53 @@
+---
+- name: Verify Elasticsearch deployment
+  hosts: elasticsearch
+  vars:
+    elasticstack_elasticsearch_http_port: 9200
+    elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords
+  tasks:
+    - name: Fetch Elastic password # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        grep "PASSWORD elastic" {{ elasticstack_initial_passwords }} |
+        awk {' print $4 '}
+      register: elastic_pass
+      changed_when: false
+      run_once: true
+
+    - name: Check Elasticsearch cluster health
+      ansible.builtin.uri:
+        url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health"
+        method: GET
+        force_basic_auth: true
+        user: elastic
+        password: "{{ elastic_pass.stdout }}"
+        validate_certs: false
+        status_code: 200
+        return_content: true
+      register: cluster_health
+      until: cluster_health.json is defined and cluster_health.json.status == "green"
+      retries: 20
+      delay: 15
+
+    - name: Check all nodes are visible
+      ansible.builtin.uri:
+        url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes"
+        method: GET
+        force_basic_auth: true
+        user: elastic
+        password: "{{ elastic_pass.stdout }}"
+        validate_certs: false
+        status_code: 200
+        return_content: true
+      register: nodes_output
+      until: nodes_output.status == 200
+      retries: 6
+      delay: 15
+
+    - name: Assert all expected nodes are present by name
+      ansible.builtin.assert:
+        that:
+          - item | replace('_', '') in nodes_output.content
+        fail_msg: "{{ item }} was not found in cluster nodes"
+        success_msg: "{{ item }} found in cluster nodes"
+      loop: "{{ groups['elasticsearch'] }}"

molecule/kibana_default/verify.yml

@@ -0,0 +1,16 @@
+---
+- name: Verify Kibana deployment
+  hosts: all
+  tasks:
+    - name: Gather service facts
+      ansible.builtin.service_facts:
+
+    - name: Assert Kibana service is running
+      ansible.builtin.assert:
+        that: ansible_facts.services['kibana.service'].state == 'running'
+        fail_msg: "Kibana service is not running"
+
+    - name: Wait for Kibana port to be available
+      ansible.builtin.wait_for:
+        port: 5601
+        timeout: 120

molecule/logstash_pipelines/converge.yml

@@ -13,8 +13,12 @@
     logstash_pipelines:
       - name: syslog
         exclusive: false
-        source: https://github.com/widhalmt/syslog-logstash-pipeline.git
-        version: bca66fd6f086a89a4b737103f5c339f66782c78e
+        source: https://github.com/NETWAYS/syslog-logstash-pipeline.git
+        version: 1183c7544d07487ab24c70213b7172133e9bbb6d
+      - name: filebeat
+        exclusive: false
+        source: https://github.com/NETWAYS/filebeat-logstash-pipeline.git
+        version: 97a85f140e0be8d941085b210f7ca9a604088455
       - name: icinga
         exclusive: false
         source: https://github.com/Icinga/icinga-logstash-pipeline.git

molecule/logstash_pipelines/verify.yml

@@ -1,51 +1,43 @@
 ---
-# This is an example playbook to execute Ansible tests.
-
 - name: Check if Logstash configuration does what it should
   hosts: all
   vars:
     elasticstack_beats_port: 5044
   tasks:
-  - name: Run syntax check
-    ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t"
-    changed_when: false
+    - name: Run syntax check
+      ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t"
+      changed_when: false
+
+    - name: Check for open port tcp {{ elasticstack_beats_port }}
+      ansible.builtin.wait_for:
+        port: "{{ elasticstack_beats_port }}"
+
+    - name: Check syslog pipeline directory exists
+      ansible.builtin.stat:
+        path: /etc/logstash/conf.d/syslog
+      register: syslog_pipeline_dir
+
+    - name: Assert syslog pipeline directory exists
+      ansible.builtin.assert:
+        that: syslog_pipeline_dir.stat.isdir
+        fail_msg: "Syslog pipeline directory not found"
 
-  - name: Check for open port tcp {{ elasticstack_beats_port }}
-    ansible.builtin.wait_for:
-      port: "{{ elasticstack_beats_port }}"
+    - name: Check filebeat pipeline directory exists
+      ansible.builtin.stat:
+        path: /etc/logstash/conf.d/filebeat
+      register: filebeat_pipeline_dir
 
-  - name: Create Syslog input
-    ansible.builtin.copy:
-      dest: /etc/logstash/conf.d/syslog/input.conf
-      content: |
-        input {
-          redis {
-            key => "input"
-            host => "localhost"
-            data_type => "list"
-          }
-        }
-      mode: preserve
-      owner: root
-      group: root
+    - name: Assert filebeat pipeline directory exists
+      ansible.builtin.assert:
+        that: filebeat_pipeline_dir.stat.isdir
+        fail_msg: "Filebeat pipeline directory not found"
 
-  - name: Create Syslog output
-    ansible.builtin.copy:
-      dest: /etc/logstash/conf.d/syslog/output.conf
-      content: |
-        output {
-          redis {
-            key => "input"
-            host => "localhost"
-            data_type => "list"
-          }
-        }
-      mode: preserve
-      owner: root
-      group: root
+    - name: Check icinga pipeline directory exists
+      ansible.builtin.stat:
+        path: /etc/logstash/conf.d/icinga
+      register: icinga_pipeline_dir
 
-  - name: Update Pipeline code
-    ansible.builtin.git:
-      repo: https://github.com/widhalmt/syslog-logstash-pipeline.git
-      dest: /etc/logstash/conf.d/syslog/
-      version: master
+    - name: Assert icinga pipeline directory exists
+      ansible.builtin.assert:
+        that: icinga_pipeline_dir.stat.isdir
+        fail_msg: "Icinga pipeline directory not found"

molecule/logstash_specific_version/verify.yml

@@ -9,13 +9,6 @@
       ansible.builtin.command: "/usr/share/logstash/bin/logstash --version | grep ^logstash"
       register: logstash_version
 
-    - name: Fail if Logstash has the wrong version
-      ansible.builtin.fail:
-        msg: "Logstash has the wrong version"
-      when:
-        - logstash_version.stdout.find('7.17.1') == -1
-        - elasticstack_release == 7
-
     - name: Fail if Logstash has the wrong version
       ansible.builtin.fail:
         msg: "Logstash has the wrong version"

molecule/repos_default/verify.yml

@@ -1,9 +1,43 @@
 ---
-# This is an example playbook to execute Ansible tests.
-
-- name: Verify
+- name: Verify Elastic repositories are configured
   hosts: all
   tasks:
-  - name: Install Kibana
-    ansible.builtin.package:
-      name: kibana
+    - name: Check Elastic apt repo file exists (Debian)
+      ansible.builtin.stat:
+        path: /etc/apt/sources.list.d/elasticstack.list
+      register: apt_repo_file
+      when: ansible_os_family == "Debian"
+
+    - name: Assert Elastic apt repo file exists (Debian)
+      ansible.builtin.assert:
+        that: apt_repo_file.stat.exists
+        fail_msg: "Elastic apt repository file not found"
+      when: ansible_os_family == "Debian"
+
+    - name: Check Elastic GPG key exists (Debian)
+      ansible.builtin.stat:
+        path: /usr/share/keyrings/elasticsearch.asc
+      register: apt_gpg_key
+      when: ansible_os_family == "Debian"
+
+    - name: Assert Elastic GPG key exists (Debian)
+      ansible.builtin.assert:
+        that: apt_gpg_key.stat.exists
+        fail_msg: "Elastic GPG key not found"
+      when: ansible_os_family == "Debian"
+
+    - name: Check Elastic yum repo file exists (RedHat)
+      ansible.builtin.stat:
+        path: /etc/yum.repos.d/elastic-release.repo
+      register: yum_repo_file
+      when: ansible_os_family == "RedHat"
+
+    - name: Assert Elastic yum repo file exists (RedHat)
+      ansible.builtin.assert:
+        that: yum_repo_file.stat.exists
+        fail_msg: "Elastic yum repository file not found"
+      when: ansible_os_family == "RedHat"
+
+    - name: Install Kibana
+      ansible.builtin.package:
+        name: kibana

molecule/repos_oss/verify.yml

@@ -1,9 +1,43 @@
 ---
-# This is an example playbook to execute Ansible tests.
-
-- name: Verify
+- name: Verify Elastic OSS repositories are configured
   hosts: all
   tasks:
-  - name: Install logstash-oss
-    ansible.builtin.package:
-      name: logstash-oss
+    - name: Check Elastic OSS apt repo file exists (Debian)
+      ansible.builtin.stat:
+        path: /etc/apt/sources.list.d/elasticstack.list
+      register: apt_repo_file
+      when: ansible_os_family == "Debian"
+
+    - name: Assert Elastic OSS apt repo file exists (Debian)
+      ansible.builtin.assert:
+        that: apt_repo_file.stat.exists
+        fail_msg: "Elastic OSS apt repository file not found"
+      when: ansible_os_family == "Debian"
+
+    - name: Check Elastic GPG key exists (Debian)
+      ansible.builtin.stat:
+        path: /usr/share/keyrings/elasticsearch.asc
+      register: apt_gpg_key
+      when: ansible_os_family == "Debian"
+
+    - name: Assert Elastic GPG key exists (Debian)
+      ansible.builtin.assert:
+        that: apt_gpg_key.stat.exists
+        fail_msg: "Elastic GPG key not found"
+      when: ansible_os_family == "Debian"
+
+    - name: Check Elastic OSS yum repo file exists (RedHat)
+      ansible.builtin.stat:
+        path: /etc/yum.repos.d/elastic-oss-release.repo
+      register: yum_repo_file
+      when: ansible_os_family == "RedHat"
+
+    - name: Assert Elastic OSS yum repo file exists (RedHat)
+      ansible.builtin.assert:
+        that: yum_repo_file.stat.exists
+        fail_msg: "Elastic OSS yum repository file not found"
+      when: ansible_os_family == "RedHat"
+
+    - name: Install logstash-oss
+      ansible.builtin.package:
+        name: logstash-oss