system.autoUpgrade with E-mail Notificaitons?

Help
1

When using something like:

  system.autoUpgrade = {
    enable = true;
    allowReboot = true;
  };

Is there away to receive e-mail notification of things like errors, reboots, logs, etc.? This is a feature I really like about unattended-upgrades on Debian/Ubuntu.

It would be nice if it could use something like msmtp the way ZFS can be setup to e-mail notification.

https://nixos.wiki/wiki/ZFS#Alternative_1:_Enable_Mail_Notification_without_Re-compliation

Thank you

2

I have

        systemd.services.status = {
            description                             = "Status";
            startAt                                 = "6:30";
            serviceConfig = {
                ExecStart                           = "/etc/nixos/run-status.sh";
                Restart                             = "on-failure";
                RestartSec                          = "300";
            };
        };

with /etc/nixos/run-status.sh:

#!/run/current-system/sw/bin/bash

set -auE

PATH="$PATH:/run/current-system/sw/bin"

echo -e "Subject: Status - $(hostname)\n\n$(/etc/nixos/status.sh)" | tee | msmtp status@<example.com>

and /etc/nixos/status.sh:

#!/run/current-system/sw/bin/bash

set -auE

PATH="$PATH:/run/current-system/sw/bin"

mv /var/tmp/status.log /var/tmp/old-status.log
shasum $0                                                     | tee    /var/tmp/status.log
top -bn1 -Eg -w170 -1 | head -20                              | tee -a /var/tmp/status.log
echo
df -h /dev/disk/by-label/*                                    | tee -a /var/tmp/status.log
echo
journalctl -u firewall -S '70s ago' | grep -m3 blacklisted    | tee -a /var/tmp/status.log
echo
for networkdev in eno1 eno2
do
    mv /var/tmp/${networkdev} /var/tmp/${networkdev}-orig
    ethtool -S ${networkdev} > /var/tmp/${networkdev}

    rx_bytes_old=0$(grep rx_bytes /var/tmp/${networkdev}-orig | sed -e 's,^.*: ,,')
    tx_bytes_old=0$(grep tx_bytes /var/tmp/${networkdev}-orig | sed -e 's,^.*: ,,')
    rx_bytes=0$(grep rx_bytes /var/tmp/${networkdev}          | sed -e 's,^.*: ,,')
    tx_bytes=0$(grep tx_bytes /var/tmp/${networkdev}          | sed -e 's,^.*: ,,')

    rx_Gbytes_sum=$(printf "%.3f" $(bc <<< "scale=3; (${rx_bytes} - ${rx_bytes_old})/1024/1024/1024"))
    tx_Gbytes_sum=$(printf "%.3f" $(bc <<< "scale=3; (${tx_bytes} - ${tx_bytes_old})/1024/1024/1024"))

    echo "${networkdev} receive:  ${rx_Gbytes_sum}GB"         | tee -a /var/tmp/status.log
    echo "${networkdev} transmit: ${tx_Gbytes_sum}GB"         | tee -a /var/tmp/status.log
done
echo
for backupservice in $(systemctl -a | grep backup.*service | sed 's,.service.*$,,')
  do systemctl status ${backupservice} -n1000 | egrep -w 'Loaded|Main'
done
echo
for file in $(find /sys/devices/ -type f -name 'temp')
do
    echo -n "${file}:  "
    cat ${file}
done                                                          | tee -a /var/tmp/status.log
echo
diff -yW170 --left-column /var/tmp/old-status.log /var/tmp/status.log || true
echo
journalctl -S yesterday | grep -w error
echo
journalctl -u nixos-upgrade -S today -l --no-pager

running on all my systems which gives me a nice overview in my mailbox every morning.

3 Likes
3

I use the systemd onFailure attribute of the nixos-upgrade service inspired by this thread and this blog post.

  # Create an email notification service for failed jobs
  systemd.services."notify-email@" =
    let address = "system@my.email.server";
    in {
      environment.SERVICE_ID = "%i";
      script = ''
        TEMPFILE=$(mktemp)
        echo "From: ${address}" > $TEMPFILE
        echo "To: ${address}" >> $TEMPFILE
        echo "Subject: Failure in $SERVICE_ID" >> $TEMPFILE
        echo -e "\nGot an error with $SERVICE_ID\n\n" >> $TEMPFILE
        set +e
        systemctl status $SERVICE_ID >> $TEMPFILE
        set -e
        ${pkgs.msmtp}/bin/msmtp \
            --file=/home/me/.config/msmtp/config \
            --account=system \
            ${address} < $TEMPFILE
      '';
    };

  # Send an email whenever auto upgrade fails
  systemd.services.nixos-upgrade.onFailure =
    lib.mkIf config.systemd.services."notify-email@".enable
    [ "notify-email@%i.service" ];

Here’s how it actually looks in my full config.

The NixOS systemd script attribute automatically sets -e, so if you expect an error code you have to set +e or suppress it for the script to complete.

1 Like
4

Since I found this thread helpful, I’d like to also post my own spin of the config posted by @noah. The basic idea is the same, but it avoids creating a tempfile, uses the system default MTA from $PATH (instead of hard-coding MSMTP) and uses a messages format very similar to how systemd-cron also implemented this functionality.

{ config, pkgs, lib, ... }:

let
   emailAddress = "postmaster@ninetailed.ninja";
   services = ["nixos-upgrade" "nixos-store-optimize"];
in {
   # Define systemd template unit for reporting service failures via e-mail
   systemd.services = {
      "notify-email@" = {
         environment.EMAIL_ADDRESS = lib.strings.replaceStrings ["%"] ["%%"] emailAddress;
         environment.SERVICE_ID = "%i";
         path = ["/run/wrappers" "/run/current-system/sw"];
         script = ''
            {
               echo "Date: $(date -R)"
               echo "From: root (systemd notify-email)"
               echo "To: $EMAIL_ADDRESS"
               echo "Subject: [$(hostname)] service $SERVICE_ID failed"
               echo "Auto-Submitted: auto-generated"
               echo
               systemctl status "$SERVICE_ID" ||:
            } | sendmail "$EMAIL_ADDRESS"
         '';
      };

   # Merge `onFailure` attribute for all monitored services
   } // (lib.attrsets.genAttrs services (name: {
      onFailure = lib.mkBefore ["notify-email@%i.service"];
   }));
}

I wonder if something like this should perhaps be included by NixOS by default? Maybe allowing something like this:

{
   systemd.service.nixos-upgrade.notifyOnFailure = true;
   systemd.service.nixos-store-optimize.notifyOnFailure = true;
   systemd.failureNotifications = [{
       type = "email";
       mailRecipient = "postmaster@ninetailed.ninja";
   }];
}
2 Likes