Skip to content

Commit 5f60ae5

Browse files
pmeulenpmeulen
authored
Merge pull request #671 from OpenConext/feature/refactor_log_logins_cleanup_script
Refactor clean_loglogins script
2 parents 561999f + acd2039 commit 5f60ae5

1 file changed

Lines changed: 122 additions & 22 deletions

File tree

‎roles/rsyslog/templates/clean_loglogins.j2‎

Lines changed: 122 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,126 @@
11
#!/bin/bash
2-
# Script to clean up the log_logins from mySQL
3-
LOGFILE="{{ rsyslog_dir }}/apps/{{ rsyslog_environment.name }}/loglogins_cleanup/loglogins_cleanup.log"
4-
echo `date '+%h %d %H:%M:%S'` Starting cleanup of log_logins | tee -a $LOGFILE
5-
LOGINSTAMP=$(date -d "-{{ loglogins_max_age }} months" +%Y-%m-%d)
6-
OLDESTTIMESTAMP=$(mysql -u {{ rsyslog_environment.db_loglogins_user }} -p{{ rsyslog_environment.db_loglogins_password }} -h {{ rsyslog_environment.db_loglogins_host }} {{ rsyslog_environment.db_loglogins_name }} -se "select (DATE_FORMAT(loginstamp,'%Y-%m-%d')) from log_logins order by loginstamp asc limit 1")
7-
if [ -z "$OLDESTTIMESTAMP" ]
8-
then echo "No logins found in log_logins" | tee -a $LOGFILE
9-
exit
2+
# Script to remove logins that are older than LOG_LOGINS_MAX_AGE months from the log_logins table
3+
# in the ${DB_LOG_LOGINS_NAME} database on the ${DB_LOG_LOGINS_HOST} host
4+
# As a failsafe, the script will refuse to run if more than ${MAX_DELETE_DAYS} days worth of logins would be removed,
5+
# unless the --force option is provided.
6+
# The script is intended to be run daily from a cron job.
7+
8+
MAX_DELETE_DAYS=5
9+
LOG_LOGINS_MAX_AGE="{{ loglogins_max_age }}" # Number of MONTHS of logins to keep
10+
RSYSLOG_DIR="{{ rsyslog_dir }}"
11+
RSYSLOG_ENVIRONMENT_NAME="{{ rsyslog_environment.name }}"
12+
DB_LOG_LOGINS_NAME="{{ rsyslog_environment.db_loglogins_name }}"
13+
DB_LOG_LOGINS_USER="{{ rsyslog_environment.db_loglogins_user }}"
14+
DB_LOG_LOGINS_PASSWORD="{{ rsyslog_environment.db_loglogins_password }}"
15+
DB_LOG_LOGINS_HOST="{{ rsyslog_environment.db_loglogins_host }}"
16+
NOREPLY_EMAIL="{{ noreply_email }}"
17+
ERROR_MAIL_TO="{{ error_mail_to }}"
18+
ANSIBLE_HOSTNAME="{{ ansible_hostname }}"
19+
20+
LOG_FILE="${RSYSLOG_DIR}/apps/${RSYSLOG_ENVIRONMENT_NAME}/loglogins_cleanup/loglogins_cleanup.log"
21+
FORCE=0 # Whether to run if more than ${MAX_DELETE_DAYS} days of log data would be removed.
22+
23+
SCRIPT_NAME=$0
24+
25+
for ARG in "$@"; do
26+
case "$ARG" in
27+
--force)
28+
FORCE=1
29+
;;
30+
*)
31+
echo "Usage: $0 [--force]"
32+
exit 1
33+
;;
34+
esac
35+
done
36+
37+
log_message() {
38+
echo "$(date '+%h %d %H:%M:%S')" "$@" | tee -a "$LOG_FILE"
39+
}
40+
41+
send_mail() {
42+
local message="$*"
43+
44+
log_message "Mailing error to ${ERROR_MAIL_TO}"
45+
46+
echo "$message" | mail \
47+
-r "${NOREPLY_EMAIL}" \
48+
-s "log_login script on ${ANSIBLE_HOSTNAME} needs attention" \
49+
"${ERROR_MAIL_TO}"
50+
}
51+
52+
53+
log_message "Starting cleanup of old log_logins data"
54+
55+
LOGIN_STAMP=$(date -d "-${LOG_LOGINS_MAX_AGE} months" +%Y-%m-%d)
56+
log_message "Removing all logins older than ${LOG_LOGINS_MAX_AGE} months (before ${LOGIN_STAMP})"
57+
58+
log_message "Using database: ${DB_LOG_LOGINS_USER}@${DB_LOG_LOGINS_HOST}:${DB_LOG_LOGINS_NAME}"
59+
60+
MYSQL_DB=(
61+
"mysql"
62+
"-u" "${DB_LOG_LOGINS_USER}"
63+
"-p${DB_LOG_LOGINS_PASSWORD}"
64+
"-h" "${DB_LOG_LOGINS_HOST}"
65+
)
66+
67+
# Find the oldest timestamp in the log_logins table:
68+
# SELECT (DATE_FORMAT(loginstamp,'%Y-%m-%d')) FROM log_logins ORDER BY loginstamp ASC LIMIT 1
69+
QUERY=(
70+
"${MYSQL_DB[@]}"
71+
# -s: silent; -e: query
72+
"-se" "SELECT (DATE_FORMAT(loginstamp,'%Y-%m-%d')) FROM log_logins ORDER BY loginstamp ASC LIMIT 1"
73+
"${DB_LOG_LOGINS_NAME}"
74+
)
75+
# DEBUG: Print query
76+
# echo "${QUERY[@]}"
77+
if ! OLDEST_TIMESTAMP=$( "${QUERY[@]}" ); then
78+
log_message "Error running mysql query (OLDEST_TIMESTAMP)"
79+
send_mail "Error running mysql query (OLDEST_TIMESTAMP)"
80+
exit 1
81+
fi
82+
if [ -z "$OLDEST_TIMESTAMP" ]; then
83+
log_message "No logins found in log_logins. Nothing to cleanup."
84+
send_mail "No logins found in log_logins. Nothing to cleanup."
85+
# The log_logins table is empty. Not treating this as an error. Exit normally,
86+
exit 0
87+
fi
88+
89+
# If we are going to delete more than ${MAX_DELETE_DAYS} days, something might be wrong
90+
# Warn and exit if this is the case, unless FORCE==1
91+
LOGIN_STAMP_UNIX=$(date -d "$LOGIN_STAMP" +%s)
92+
OLDEST_TIMESTAMP_UNIX=$(date -d "$OLDEST_TIMESTAMP" +%s)
93+
TIMESTAMP_DIFF=$(( (LOGIN_STAMP_UNIX-OLDEST_TIMESTAMP_UNIX)/3600/24 ))
94+
log_message "The oldest timestamp in the database is $OLDEST_TIMESTAMP"
95+
if [ "$TIMESTAMP_DIFF" -lt "0" ]; then
96+
log_message "There is no data to delete. We're done."
97+
exit 0
1098
fi
1199

12-
# If we are going to delete more than 5 days, something is wrong
13-
LOGINSTAMPUNIX=$(date -d $LOGINSTAMP +%s)
14-
OLDESTTIMESTAMPUNIX=$(date -d $OLDESTTIMESTAMP +%s)
15-
TIMESTAMPDIFF=$(( ($LOGINSTAMPUNIX-$OLDESTTIMESTAMPUNIX)/3600/24 ))
16-
echo `date '+%h %d %H:%M:%S'` We will delete $TIMESTAMPDIFF days of log_logins data | tee -a $LOGFILE
17-
if [ "$TIMESTAMPDIFF" -gt 5 ]
18-
then
19-
echo `date '+%h %d %H:%M:%S'` Something is up, you need to delete more than 5 days worth of logindata | tee -a $LOGFILE
20-
21-
echo "The log_login cleanup script wants to delete more than 5 days of logins on the {{ ansible_hostname }}. Please investigate" | mail -r "{{ noreply_email }}" -s "log_login script on {{ ansible_hostname }} needs attention" "{{ error_mail_to }}"
22-
exit
23-
else
24-
DELETEDROWS=$(mysql -u {{ rsyslog_environment.db_loglogins_user }} -p{{ rsyslog_environment.db_loglogins_password }} -h {{ rsyslog_environment.db_loglogins_host }} -sNe "delete from log_logins where loginstamp < '$LOGINSTAMP'; select row_count();" {{ rsyslog_environment.db_loglogins_name }})
25-
echo `date '+%h %d %H:%M:%S'` We have deleted $DELETEDROWS rows. | tee -a $LOGFILE
100+
log_message "We will delete $TIMESTAMP_DIFF days of log_logins data"
101+
if [ "$TIMESTAMP_DIFF" -gt "$MAX_DELETE_DAYS" ] && [ "$FORCE" -ne 1 ]; then
102+
log_message "Something is up, you need to delete more than ${MAX_DELETE_DAYS} days worth of login data"
103+
send_mail "The ${SCRIPT_NAME} script wants to delete more than ${MAX_DELETE_DAYS} days of logins on ${ANSIBLE_HOSTNAME}. Please investigate"
104+
exit 0
105+
fi
106+
if [ "$TIMESTAMP_DIFF" -gt "$MAX_DELETE_DAYS" ] && [ "$FORCE" -eq 1 ]; then
107+
log_message "Force option provided; deleting more than ${MAX_DELETE_DAYS} days worth of login data"
26108
fi
109+
110+
# Delete rows
111+
# DELETE FROM log_logins WHERE loginstamp < '$LOGIN_STAMP'; SELECT row_count();"
112+
QUERY=(
113+
"${MYSQL_DB[@]}"
114+
# -s: silent; -N: Don't write column names in results; -e: query;
115+
"-sNe" "DELETE FROM log_logins WHERE loginstamp < '${LOGIN_STAMP}'; SELECT row_count();"
116+
"${DB_LOG_LOGINS_NAME}"
117+
)
118+
# DEBUG: Print query
119+
# echo "${QUERY[@]}"
120+
if ! DELETED_ROWS=$( "${QUERY[@]}" ); then
121+
log_message "Error running mysql query (DELETED_ROWS)"
122+
send_mail "Error running mysql query (DELETED_ROWS)"
123+
exit 1
124+
fi
125+
126+
log_message "We have deleted $DELETED_ROWS rows. Done."

0 commit comments

Comments
 (0)