Set loop label to prevent the passwords in rsyslog_environments from being logged

Move auth log processing to a separate file instead of running the same loop with condition many times

Author: pmeulen

roles/rsyslog/tasks/process_auth_log_for_environment.yml

@@ -0,0 +1,78 @@
+---
+
+- name: Create log_logins table for each log_login environment
+  community.mysql.mysql_db:
+    name: "{{ rsyslog_environment.db_loglogins_name }}"
+    login_user: "{{ rsyslog_environment.db_loglogins_user }}"
+    login_password: "{{ rsyslog_environment.db_loglogins_password }}"
+    login_host: "{{ rsyslog_environment.db_loglogins_host }}"
+    state: import
+    target: /var/tmp/log_logins.sql
+  changed_when: false
+
+- name: Create lastseen table for each log_login environment
+  community.mysql.mysql_db:
+    name: "{{ rsyslog_environment.db_lastseen_name }}"
+    login_user: "{{ rsyslog_environment.db_lastseen_user }}"
+    login_password: "{{ rsyslog_environment.db_lastseen_password }}"
+    login_host: "{{ rsyslog_environment.db_lastseen_host }}"
+    state: import
+    target: /var/tmp/lastseen.sql
+  changed_when: false
+
+- name: Create a python script that parses eb log_logins per environment
+  ansible.builtin.template:
+    src: parse_ebauth_to_mysql.py.j2
+    dest: /usr/local/sbin/parse_ebauth_to_mysql_{{ rsyslog_environment.name }}.py
+    mode: 0740
+    owner: root
+    group: root
+
+- name: Create a python script that parses stepup log_logins per environment
+  ansible.builtin.template:
+    src: parse_stepupauth_to_mysql.py.j2
+    dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ rsyslog_environment.name }}.py
+    mode: 0740
+    owner: root
+    group: root
+
+- name: Put log_logins logrotate scripts for eb
+  ansible.builtin.template:
+    src: logrotate_ebauth.j2
+    dest: /etc/logrotate.d/logrotate_ebauth_{{ rsyslog_environment.name }}
+    mode: 0644
+    owner: root
+    group: root
+
+- name: Put log_logins logrotate scripts for stepup
+  ansible.builtin.template:
+    src: logrotate_stepupauth.j2
+    dest: /etc/logrotate.d/logrotate_stepupauth_{{ rsyslog_environment.name }}
+    mode: 0644
+    owner: root
+    group: root
+
+- name: Create logdirectory for log_logins cleanup script
+  ansible.builtin.file:
+    path: "{{ rsyslog_dir }}/apps/{{ rsyslog_environment.name }}/loglogins_cleanup/"
+    state: directory
+    owner: root
+    group: "{{ rsyslog_read_group }}"
+    mode: 0750
+
+- name: Put log_logins cleanup script
+  ansible.builtin.template:
+    src: clean_loglogins.j2
+    dest: /usr/local/sbin/clean_loglogins_{{ rsyslog_environment.name }}
+    owner: root
+    group: root
+    mode: 0700
+
+- name: Create cronjobs to run the log_logins script
+  ansible.builtin.cron:
+    name: Delete old {{ rsyslog_environment.name }} log_login data
+    user: root
+    minute: "20"
+    hour: "02"
+    job: "/usr/local/sbin/clean_loglogins_{{ rsyslog_environment.name }}"
+    cron_file: loglogins_cleanup_{{ rsyslog_environment.name }}

roles/rsyslog/tasks/process_auth_logs.yml

@@ -9,103 +9,16 @@
     - log_logins.sql
     - lastseen.sql
 
-- name: Create log_logins table for each log_login environment
-  community.mysql.mysql_db:
-    name: "{{ item.db_loglogins_name }}"
-    login_user: "{{ item.db_loglogins_user }}"
-    login_password: "{{ item.db_loglogins_password }}"
-    login_host: "{{ item.db_loglogins_host }}"
-    state: import
-    target: /var/tmp/log_logins.sql
-  changed_when: false
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Create lastseen table for each log_login environment
-  community.mysql.mysql_db:
-    name: "{{ item.db_lastseen_name }}"
-    login_user: "{{ item.db_lastseen_user }}"
-    login_password: "{{ item.db_lastseen_password }}"
-    login_host: "{{ item.db_lastseen_host }}"
-    state: import
-    target: /var/tmp/lastseen.sql
-  changed_when: false
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
 - name: add python mysql module for parse_ebauth_to_mysql script
   apt:
     name: python3-mysqldb
     state: present
   when: ansible_os_family == "Debian"
 
-- name: Create a python script that parses eb log_logins per environment
-  ansible.builtin.template:
-    src: parse_ebauth_to_mysql.py.j2
-    dest: /usr/local/sbin/parse_ebauth_to_mysql_{{ item.name }}.py
-    mode: 0740
-    owner: root
-    group: root
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Create a python script that parses stepup log_logins per environment
-  ansible.builtin.template:
-    src: parse_stepupauth_to_mysql.py.j2
-    dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py
-    mode: 0740
-    owner: root
-    group: root
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Put log_logins logrotate scripts for eb
-  ansible.builtin.template:
-    src: logrotate_ebauth.j2
-    dest: /etc/logrotate.d/logrotate_ebauth_{{ item.name }}
-    mode: 0644
-    owner: root
-    group: root
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Put log_logins logrotate scripts for stepup
-  ansible.builtin.template:
-    src: logrotate_stepupauth.j2
-    dest: /etc/logrotate.d/logrotate_stepupauth_{{ item.name }}
-    mode: 0644
-    owner: root
-    group: root
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Create logdirectory for log_logins cleanup script
-  ansible.builtin.file:
-    path: "{{ rsyslog_dir }}/apps/{{ item.name }}/loglogins_cleanup/"
-    state: directory
-    owner: root
-    group: "{{ rsyslog_read_group }}"
-    mode: 0750
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Put log_logins cleanup script
-  ansible.builtin.template:
-    src: clean_loglogins.j2
-    dest: /usr/local/sbin/clean_loglogins_{{ item.name }}
-    owner: root
-    group: root
-    mode: 0700
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
-
-- name: Create cronjobs to run the log_logins script
-  ansible.builtin.cron:
-    name: Delete old {{ item.name }} log_login data
-    user: root
-    minute: "20"
-    hour: "02"
-    job: "/usr/local/sbin/clean_loglogins_{{ item.name }}"
-    cron_file: loglogins_cleanup_{{ item.name }}
-  with_items: "{{ rsyslog_environments }}"
-  when: item.db_loglogins_name is defined
+- name: Process auth logs for each rsyslog environment
+  ansible.builtin.include_tasks: process_auth_log_for_environment.yml
+  loop: "{{ rsyslog_environments }}"
+  loop_control:
+    loop_var: rsyslog_environment
+    label: "{{ rsyslog_environment.name }}"
+  when: rsyslog_environment.db_loglogins_name is defined

roles/rsyslog/tasks/rsyslog_central.yml

@@ -51,6 +51,9 @@
     dest: /etc/rsyslog.d/templates/{{ item.name }}.conf
     backup: true
   with_items: "{{ rsyslog_environments }}"
+  loop_control:
+    label: "{{ item.name }}"
+
   notify:
     - "restart rsyslog"
 
@@ -60,6 +63,8 @@
     dest: /etc/rsyslog.d/rulesets/{{ item.name }}.conf
     backup: true
   with_items: "{{ rsyslog_environments }}"
+  loop_control:
+    label: "{{ item.name }}"
   notify:
     - "restart rsyslog"
 
@@ -69,6 +74,8 @@
     dest: /etc/rsyslog.d/listeners/{{ item.name }}.conf
     backup: true
   with_items: "{{ rsyslog_environments }}"
+  loop_control:
+    label: "{{ item.name }}"
   notify:
     - "restart rsyslog"
 

roles/rsyslog/templates/clean_loglogins.j2

@@ -1,9 +1,9 @@
 #!/bin/bash
 # Script to clean up the log_logins from mySQL
-LOGFILE="{{ rsyslog_dir }}/apps/{{ item.name }}/loglogins_cleanup/loglogins_cleanup.log"
+LOGFILE="{{ rsyslog_dir }}/apps/{{ rsyslog_environment.name }}/loglogins_cleanup/loglogins_cleanup.log"
 echo `date '+%h %d %H:%M:%S'` Starting cleanup of log_logins | tee -a $LOGFILE
 LOGINSTAMP=$(date -d "-{{ loglogins_max_age }} months" +%Y-%m-%d)
-OLDESTTIMESTAMP=$(mysql -u {{ item.db_loglogins_user }} -p{{ item.db_loglogins_password }} -h {{ item.db_loglogins_host }} {{ item.db_loglogins_name }} -se "select (DATE_FORMAT(loginstamp,'%Y-%m-%d')) from log_logins order by loginstamp asc limit 1")
+OLDESTTIMESTAMP=$(mysql -u {{ rsyslog_environment.db_loglogins_user }} -p{{ rsyslog_environment.db_loglogins_password }} -h {{ rsyslog_environment.db_loglogins_host }} {{ rsyslog_environment.db_loglogins_name }} -se "select (DATE_FORMAT(loginstamp,'%Y-%m-%d')) from log_logins order by loginstamp asc limit 1")
 if [ -z "$OLDESTTIMESTAMP" ]
         then echo "No logins found in log_logins" | tee -a $LOGFILE
         exit
@@ -21,6 +21,6 @@ if [ "$TIMESTAMPDIFF" -gt 5 ]
         echo "The log_login cleanup script wants to delete more than 5 days of logins on the {{ ansible_hostname }}. Please investigate" | mail -r "{{ noreply_email }}" -s "log_login script on {{ ansible_hostname }} needs attention" "{{ error_mail_to }}"	
 	exit
 else
-	DELETEDROWS=$(mysql -u {{ item.db_loglogins_user }} -p{{ item.db_loglogins_password }} -h {{ item.db_loglogins_host }} -sNe "delete from log_logins where loginstamp < '$LOGINSTAMP'; select row_count();" {{ item.db_loglogins_name }})
+	DELETEDROWS=$(mysql -u {{ rsyslog_environment.db_loglogins_user }} -p{{ rsyslog_environment.db_loglogins_password }} -h {{ rsyslog_environment.db_loglogins_host }} -sNe "delete from log_logins where loginstamp < '$LOGINSTAMP'; select row_count();" {{ rsyslog_environment.db_loglogins_name }})
 	echo `date '+%h %d %H:%M:%S'` We have deleted $DELETEDROWS rows.  | tee -a $LOGFILE
 fi

roles/rsyslog/templates/logrotate_ebauth.j2

@@ -1,4 +1,4 @@
-{{ rsyslog_dir }}/log_logins/{{ item.name }}/eb-authentication.log
+{{ rsyslog_dir }}/log_logins/{{ rsyslog_environment.name }}/eb-authentication.log
 {
     missingok
     daily
@@ -10,7 +10,7 @@
     delaycompress
     create 0640 root {{ rsyslog_read_group }} 
     postrotate
-      /usr/local/sbin/parse_ebauth_to_mysql_{{ item.name }}.py > /dev/null
+      /usr/local/sbin/parse_ebauth_to_mysql_{{ rsyslog_environment.name }}.py > /dev/null
       systemctl kill -s HUP rsyslog.service
     endscript
 }

roles/rsyslog/templates/logrotate_stepupauth.j2

@@ -10,7 +10,9 @@
     delaycompress
     create 0640 root {{ rsyslog_read_group }}
     postrotate
-      /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null
+      # PiMe: temporarily disable parsing of the rotated stepup authentication log because to MySQL lastseen
+      # because the first rotated log will be huge
+      # /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null
       systemctl kill -s HUP rsyslog.service
     endscript
 }

roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2

@@ -10,11 +10,11 @@ import json
 import MySQLdb
 from dateutil.parser import parse
 
-mysql_host="{{ item.db_loglogins_host }}"
-mysql_user="{{ item.db_loglogins_user }}"
-mysql_password="{{ item.db_loglogins_password }}"
-mysql_db="{{ item.db_loglogins_name }}"
-workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/"
+mysql_host="{{ rsyslog_environment.db_loglogins_host }}"
+mysql_user="{{ rsyslog_environment.db_loglogins_user }}"
+mysql_password="{{ rsyslog_environment.db_loglogins_password }}"
+mysql_db="{{ rsyslog_environment.db_loglogins_name }}"
+workdir="{{ rsyslog_dir }}/log_logins/{{ rsyslog_environment.name}}/"
 
 db = MySQLdb.connect(mysql_host,mysql_user,mysql_password,mysql_db )
 cursor = db.cursor()

roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2

@@ -11,11 +11,11 @@ import MySQLdb
 from dateutil.parser import parse
 
 # Configuration variables (to be injected by Ansible/Jinja2)
-mysql_host="{{ item.db_loglogins_host }}"
-mysql_user="{{ item.db_loglogins_user }}"
-mysql_password="{{ item.db_loglogins_password }}"
-mysql_db="{{ item.db_loglogins_name }}"
-workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/"
+mysql_host="{{ rsyslog_environment.db_loglogins_host }}"
+mysql_user="{{ rsyslog_environment.db_loglogins_user }}"
+mysql_password="{{ rsyslog_environment.db_loglogins_password }}"
+mysql_db="{{ rsyslog_environment.db_loglogins_name }}"
+workdir="{{ rsyslog_dir }}/log_logins/{{ rsyslog_environment.name}}/"
 
 # Establish database connection
 try: