Skip to content

Commit bb72165

Browse files
crosmullercrosmuller
authored
Add check empty logfiles scripts and some cleaning (#606)
Add check empty logfiles scripts Clean up obsolete rsyslog rules
1 parent ede0360 commit bb72165

5 files changed

Lines changed: 111 additions & 11 deletions

File tree

‎roles/rsyslog/defaults/main.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,12 @@ rsyslog_queue_dir: "/var/spool/rsyslog"
3131

3232
# rsyslog_imjournal_statefile # default is imjournal.state which means imjournal.state relative to the rsyslog workdir
3333
# rsyslog_workdirectory # default /var/spool/rsyslog
34+
35+
# Empty log check script, optional
36+
rsyslog_enable_warn_empty_script: false
37+
rsyslog_warn_empty_log_recipient: admin@example.com
38+
rsyslog_monitor_for_emptylogs_path: "{{ rsyslog_dir }}/apps/prod_sc"
39+
rsyslog_checkemptylogs_cron_minute: "0"
40+
rsyslog_checkemptylogs_cron_hour: "9"
41+
rsyslog_checkemptylogs_cron_weekdays: "1-5"
42+
rsyslog_checkemptylogs_dir: "/usr/local/bin"

‎roles/rsyslog/tasks/rsyslog_central.yml‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,25 @@
7777
src: centralsyslog.j2
7878
dest: /etc/logrotate.d/centralsyslog
7979

80-
- name: put ryslog config file
80+
- name: Put ryslog config file
8181
template:
8282
src: "rsyslog.conf.j2"
8383
dest: "/etc/rsyslog.conf"
8484
notify:
8585
- "restart rsyslog"
86+
87+
- name: Put log empty warn script and cronjob
88+
when: rsyslog_enable_warn_empty_script
89+
block:
90+
- name: Put log empty script
91+
ansible.builtin.template:
92+
src: warn-empty-log.sh.j2
93+
dest: "{{ rsyslog_checkemptylogs_dir }}/warn-empty-log.sh"
94+
backup: True
95+
- name: Create cronjob
96+
ansible.builtin.cron:
97+
name: "check empty logs"
98+
minute: "{{ rsyslog_checkemptylogs_cron_minute }}"
99+
hour: "{{ rsyslog_checkemptylogs_cron_hour }}"
100+
weekday: "{{ rsyslog_checkemptylogs_cron_weekdays }}"
101+
job: "{{ rsyslog_checkemptylogs_dir }}/warn-empty-log.sh -m"

‎roles/rsyslog/templates/sc_ruleset.conf.j2‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,7 @@ if $programname == "engineblock" and $msg contains '{"channel":"authentication"'
1616
:programname, isequal, "Apache-EBAPI" { action(type="omfile" DynaFile="apache-eb-api-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
1717
:programname, isequal, "manageserver" { action(type="omfile" DynaFile="manage-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
1818
:programname, isequal, "managegui" { action(type="omfile" DynaFile="apache-manage-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
19-
:programname, isequal, "PDPANALYTICS" { action(type="omfile" DynaFile="pdpanalytics-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
2019
:programname, isequal, "pdp" { action(type="omfile" DynaFile="pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
21-
:programname, isequal, "pdpserver" { action(type="omfile" DynaFile="pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
22-
:programname, isequal, "pdpgui" { action(type="omfile" DynaFile="apache-pdp-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
2320
if $programname == "profile" and $msg startswith "{" then { action(type="omfile" DynaFile="profile-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
2421
:programname, isequal, "profile" { action(type="omfile" DynaFile="apache-profile-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
2522
:programname, isequal, "teamsserver" { action(type="omfile" DynaFile="teams-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
@@ -55,8 +52,6 @@ if $programname == "gateway" and $msg contains '{"message":"Intrinsic Loa Reques
5552
if $programname == "{{ stepupapp }}" and $msg startswith "{{ stepupapp }}" then { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
5653
:programname, isequal, "{{ stepupapp }}" { action(type="omfile" DynaFile="stepup-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
5754
:programname, isequal, "Apache-{{ stepupapp }}" { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
58-
if $programname == "{{ stepupapp }}" and $msg startswith "{{ stepupapp }}" then { action(type="omfile" DynaFile="apache-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
59-
:programname, isequal, "{{ stepupapp }}" { action(type="omfile" DynaFile="stepup-{{ stepupapp }}-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
6055
{% endfor %}
6156
:programname, isequal, "Apache-azuremfa" { action(type="omfile" DynaFile="apache-azure-mfa-{{ item.name }}" {{ rsyslog_dir_file_modes }} ) stop }
6257
if $programname == "spdashboard" and $msg startswith "spdashboard" then { action(type="omfile" DynaFile="apache-spdashboard-{{item.name }}" {{ rsyslog_dir_file_modes }} ) stop }

‎roles/rsyslog/templates/sc_template.conf.j2‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,12 @@ $template apache-servicedesk-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.na
4040
$template apache-eduid-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/eduid/eduid-apache.log"
4141
$template spdashboard-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/spdashboard/spdashboard.log"
4242
$template apache-spdashboard-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/spdashboard/apache.log"
43-
$template inviteclient-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteclient.log"
44-
$template invitewelcome-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//invitewelcome.log"
45-
$template inviteserver-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteserver.log"
43+
$template inviteclient-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteclient.log"
44+
$template invitewelcome-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//invitewelcome.log"
45+
$template inviteserver-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteserver.log"
4646
$template invitejson-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite/invitejson.log"
47-
$template inviteprovisioningmock-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/invite//inviteprovisioningmock.log"
48-
$template loadbalancer-{{ item.name }}, "/opt/surfconext/logs/apps/{{ item.name }}/traefik/traefik.log"
47+
$template inviteprovisioningmock-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/invite//inviteprovisioningmock.log"
48+
$template loadbalancer-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/traefik/traefik.log"
4949
{% for stepupapp in stepupapps %}
5050
$template stepup-{{ stepupapp }}-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/{{ stepupapp }}/{{ stepupapp }}.log
5151
$template apache-{{ stepupapp }}-{{ item.name }}, "{{ rsyslog_dir }}/apps/{{ item.name }}/{{ stepupapp }}/{{ stepupapp }}-apache.log

‎roles/rsyslog/templates/warn-empty-log.sh.j2‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
3+
# Script to check for empty log files in {{ rsyslog_monitor_for_emptylogs_path }}
4+
# Usage: ./check_empty_logs.sh [-m]
5+
# -m: send mail if empty files are found
6+
7+
set -e
8+
9+
SEND_MAIL=false
10+
RECIPIENT="{{ rsyslog_warn_empty_log_recipient }}"
11+
HOSTNAME=$(hostname --fqdn)
12+
13+
# Parse command line options
14+
while getopts "m" opt; do
15+
case $opt in
16+
m)
17+
SEND_MAIL=true
18+
;;
19+
\?)
20+
echo "Invalid option: -$OPTARG" >&2
21+
exit 1
22+
;;
23+
esac
24+
done
25+
26+
# Find empty log files and read into array
27+
mapfile -t empty_files < <(find {{ rsyslog_monitor_for_emptylogs_path }} -maxdepth 2 -type f -size 0)
28+
29+
# If no empty files found, exit successfully
30+
{% raw %}
31+
if [ ${#empty_files[@]} -eq 0 ]; then
32+
exit 0
33+
fi
34+
{% endraw %}
35+
36+
# Empty files were found
37+
{% raw %}
38+
file_count=${#empty_files[@]}
39+
{% endraw %}
40+
41+
# Determine singular or plural
42+
if [ $file_count -eq 1 ]; then
43+
file_word="file"
44+
verb="was"
45+
else
46+
file_word="files"
47+
verb="were"
48+
fi
49+
50+
# Create mail body
51+
mail_body=$(cat <<EOF
52+
Subject: Alert: Empty log files detected on ${HOSTNAME}
53+
54+
Dear SURFconext team,
55+
56+
This is an automated alert to inform you that ${file_count} empty log ${file_word} ${verb} found on ${HOSTNAME} in /opt/surfconext/logs/apps.
57+
58+
Empty log files should not occur under normal operation and may indicate a configuration issue or a problem with the logging system.
59+
60+
List of empty log files:
61+
62+
$(printf '%s\n' "${empty_files[@]}")
63+
64+
Please investigate this issue.
65+
66+
--
67+
Your friendly automated monitoring system
68+
EOF
69+
)
70+
71+
# Send mail if -m option was provided
72+
if [ "$SEND_MAIL" = true ]; then
73+
echo "$mail_body" | mail -s "Alert: Empty log ${file_word} detected on ${HOSTNAME}" "$RECIPIENT"
74+
echo "Mail sent to $RECIPIENT about $file_count empty log ${file_word}" | logger
75+
else
76+
echo "Found $file_count empty log ${file_word}, but mail not sent (use -m to send):"
77+
echo "$mail_body"
78+
fi
79+
80+
exit 1

0 commit comments

Comments
 (0)