feat: nfs timeout notification

tasks/rosetta/configure_nfs_monitoring.yml

+---
+- name: create directory for systemd units
+  ansible.builtin.file:
+    path: "/usr/local/lib/systemd/system/"
+    mode: "0755"
+    state: directory
+    owner: "root"
+    group: "root"
+
+- name: install NFS monitoring services
+  ansible.builtin.template:
+    src: "usr/local/lib/systemd/system/{{ item }}.j2"
+    dest: "/usr/local/lib/systemd/system/{{ item }}"
+    mode: "0600"
+    owner: "root"
+    group: "root"
+  loop:
+    - "nfs_timeout_notification.service"
+    - "nfs_timeout_notification.service.sh"
+  notify: daemon-reload
+
+- name: enable NFS monitoring services
+  ansible.builtin.systemd_service:
+    name: "{{ item.name }}"
+    enabled: "{{ item.enabled | default(true) }}"
+    state: "{{ item.state | default('started') }}"
+    daemon_reload: true
+  loop:
+    - name: "nfs_timeout_notification.service"

tasks/rosetta/main_rosetta.yml

@@ -52,3 +52,6 @@
 - name: configure shell environment
   ansible.builtin.import_tasks: "rosetta/configure_shell.yml"
   tags: [shell, csh, alias, aliases]
+- name: configure NFS monitoring
+  ansible.builtin.import_tasks: "rosetta/configure_nfs_monitoring.yml"
+  tags: [rosetta, systemd, nfs_monitoring]

templates/usr/local/lib/systemd/system/nfs_timeout_notification.service.j2

+[Unit]
+Description=monitor journal for NFS timeouts and notify staff
+After=remote-fs.target
+
+[Service]
+Type=simple
+RemainAfterExit=no
+Restart=no
+ExecStart=/bin/bash /usr/local/lib/systemd/system/nfs_timeout_notification.service.sh
+User={{ vault_nfs_timeout_notification_service.owner }}
+Group={{ vault_nfs_timeout_notification_service.group }}
+
+[Install]
+WantedBy=multi-user.target

templates/usr/local/lib/systemd/system/nfs_timeout_notification.service.sh.j2

+#!/bin/bash
+
+WAIT_BETWEEN_MAILS_IN_SECONDS={{ vault_nfs_timeout_notification_service.wait_between_mails_in_seconds }};
+NO_MAIL_UNTIL_EPOCH=0;
+journalctl -f | while read -r LINE; do
+  TIMEOUT=$(echo "${LINE}" | grep "nfs.*timed out");
+  if [[ ! -z "${TIMEOUT}" ]]; then
+    TIME=$(date +%s);
+    if [[ "${TIME}" -ge "${NO_MAIL_UNTIL}" ]]; then
+      NO_MAIL_UNTIL_EPOCH=$((TIME + WAIT_BETWEEN_MAILS_IN_SECONDS));
+      echo "NFS timeout detected, sending mail to staff";
+      echo "${TIMEOUT}" | /usr/bin/mail -s "NFS timeout detected on $(hostname -f)" {{ vault_nfs_timeout_notification_service.staff_mail }};
+    fi;
+  fi;
+  TIMEOUT="";
+done;
\ No newline at end of file