From fa500ab4df385aa64e2ed2b611c41cf44320cd1b Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 11:37:29 +0100 Subject: [PATCH 1/3] rsyslog: Only opdate the lastseen tabel for newer dates --- roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 index b37f4720c..7e0bc7bcb 100644 --- a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 @@ -21,11 +21,17 @@ cursor = db.cursor() def update_lastseen(user_id, date): query = """ - REPLACE INTO last_login (userid, lastseen) + INSERT INTO last_login (userid, lastseen) VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) """ - cursor.execute(query, (user_id, date)) - db.commit() + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") def load_in_mysql(a,b,c,d,e,f,g,h): sql = """insert into log_logins(idpentityid,spentityid,loginstamp,userid,keyid,sessionid,requestid,trustedproxyentityid) values(%s,%s,%s,%s,%s,%s,%s,%s)""" @@ -73,4 +79,3 @@ for filename in os.listdir(workdir): cursor.close() db.close() - From c4e408e7c04e40b072061a1bcd2600dc8b40ac71 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 12:50:09 +0100 Subject: [PATCH 2/3] rsyslog: Also rotate and parse stepup-logs --- .../rsyslog/templates/logrotate_stepupauth.j2 | 16 +++ .../parse_stepupauthauth_to_mysql.py.j2 | 133 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 roles/rsyslog/templates/logrotate_stepupauth.j2 create mode 100644 roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 diff --git a/roles/rsyslog/templates/logrotate_stepupauth.j2 b/roles/rsyslog/templates/logrotate_stepupauth.j2 new file mode 100644 index 000000000..be1a50652 --- /dev/null +++ b/roles/rsyslog/templates/logrotate_stepupauth.j2 @@ -0,0 +1,16 @@ +{{ rsyslog_dir }}/log_logins/{{ item.name }}/stepup-authentication.log +{ + missingok + daily + rotate 180 + sharedscripts + dateext + dateyesterday + compress + delaycompress + create 0640 root {{ rsyslog_read_group }} + postrotate + /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null + systemctl kill -s HUP rsyslog.service + endscript +} diff --git a/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 new file mode 100644 index 000000000..2fe55d6ab --- /dev/null +++ b/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +# This script parses rotated stepup-authentication.log files produced by engineblock. +# It filters for successful logins (authentication_result:OK) and inserts the data +# into the log_logins and last_login MySQL tables. +# This script is intended to be run separately during logrotate. + +import os +import sys +import json +import MySQLdb +from dateutil.parser import parse + +# Configuration variables (to be injected by Ansible/Jinja2) +mysql_host="{{ item.db_loglogins_host }}" +mysql_user="{{ item.db_loglogins_user }}" +mysql_password="{{ item.db_loglogins_password }}" +mysql_db="{{ item.db_loglogins_name }}" +workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/" + +# Establish database connection +try: + db = MySQLdb.connect(mysql_host,mysql_user,mysql_password,mysql_db ) + cursor = db.cursor() +except Exception as e: + print(f"Error connecting to MySQL: {e}") + sys.exit(1) + +# --- Database Functions --- + +def update_lastseen(user_id, date): + """ + Updates the last_login table. + Uses GREATEST() to ensure only newer dates overwrite the existing 'lastseen' value. + """ + query = """ + INSERT INTO last_login (userid, lastseen) + VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) + """ + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") + +def load_stepup_in_mysql(idp, sp, loginstamp, userid, requestid): + """ + Inserts Step-up login data into the log_logins table. + Fills keyid, sessionid, and trustedproxyentityid with NULL. + """ + # Columns in log_logins: idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid + + keyid = None + sessionid = None + trustedproxyentityid = None + + sql = """ + INSERT INTO log_logins(idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid) + VALUES(%s, %s, %s, %s, %s, %s, %s, %s) + """ + try: + cursor.execute(sql, (idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error inserting stepup data: {e}") + # Print the data that failed insertion + print((idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + +# --- Parsing Function --- + +def parse_stepup_lines(a): + """ + Opens the stepup log file, parses each line, filters for successful logins, + and loads the data into MySQL. + """ + input_file = open((a), 'r') + for line in input_file: + try: + # Assumes JSON data starts after the first ']:' + jsonline = line.split(']:',2)[1] + data = json.loads(jsonline) + except: + continue + + # 1. Filtering condition: Only parse logs having authentication_result:OK + if data.get("authentication_result") != "OK": + continue + + # 2. Extract required fields + user_id = data.get("identity_id") + timestamp = data.get("datetime") + request_id = data.get("request_id") + sp_entity_id = data.get("requesting_sp") + idp_entity_id = data.get("authenticating_idp") + + # Basic data validation + if not user_id or not timestamp: + continue + + try: + # 3. Format date and time for MySQL + loginstamp = parse(timestamp).strftime("%Y-%m-%d %H:%M:%S") + last_login_date = parse(timestamp).strftime("%Y-%m-%d") + except: + continue + + # 4. Insert into MySQL + load_stepup_in_mysql(idp_entity_id, sp_entity_id, loginstamp, user_id, request_id) + + # 5. Update last login date + update_lastseen(user_id, last_login_date) + + +# --- Main Execution --- + +## Loop over the files and parse them one by one +for filename in os.listdir(workdir): + filetoparse=(os.path.join(workdir, filename)) + + # Check for Stepup files, ignore compressed files + if os.path.isfile(filetoparse) and filename.startswith("stepup-authentication.log-") and not filename.endswith(".gz"): + print(f"Parsing stepup log file: {filename}") + parse_stepup_lines(filetoparse) + else: + continue + +# Close database connection +cursor.close() +db.close() +print("Stepup log parsing complete.") From a6a52262fd2c89f2faaf710269e6ff352914ff57 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 12:53:04 +0100 Subject: [PATCH 3/3] rsyslog: Add ansible tasks for stepup log parsing --- roles/rsyslog/tasks/process_auth_logs.yml | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/roles/rsyslog/tasks/process_auth_logs.yml b/roles/rsyslog/tasks/process_auth_logs.yml index e62027530..e123fb33f 100644 --- a/roles/rsyslog/tasks/process_auth_logs.yml +++ b/roles/rsyslog/tasks/process_auth_logs.yml @@ -39,7 +39,7 @@ state: present when: ansible_os_family == "Debian" -- name: Create a python script that parses log_logins per environment +- name: Create a python script that parses eb log_logins per environment ansible.builtin.template: src: parse_ebauth_to_mysql.py.j2 dest: /usr/local/sbin/parse_ebauth_to_mysql_{{ item.name }}.py @@ -49,7 +49,17 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined -- name: Put log_logins logrotate scripts +- name: Create a python script that parses stepup log_logins per environment + ansible.builtin.template: + src: parse_ebauth_to_mysql.py.j2 + dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py + mode: 0740 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + +- name: Put log_logins logrotate scripts for eb ansible.builtin.template: src: logrotate_ebauth.j2 dest: /etc/logrotate.d/logrotate_ebauth_{{ item.name }} @@ -59,6 +69,16 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined +- name: Put log_logins logrotate scripts for stepup + ansible.builtin.template: + src: logrotate_ebauth.j2 + dest: /etc/logrotate.d/logrotate_stepupauth_{{ item.name }} + mode: 0644 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + - name: Create logdirectory for log_logins cleanup script ansible.builtin.file: path: "{{ rsyslog_dir }}/apps/{{ item.name }}/loglogins_cleanup/"