I need to provide an email password to alertmanager. The setting smtp_auth_password is not safe: it would be readable to everyone with access to /nix/store. The setting smtp_auth_password_file could be used. This setting could get a path to a file that is readable only by the user running alertmanager. However the systemd service for alertmanager uses DynamicUser=true which means that before the service starts, it is not known what the uid of the user is and it is not possible to set the correct permissions on the password file.
SystemD has a mechanism for passing files with credentials: LoadCredential. This can place a password file into a temporary directory that is only accessible to the dynamic user of the service (and root). $CREDENTIALS_DIRECTORY is an environment variable that contains the directory where the password file is placed.
$CREDENTIALS_DIRECTORY is different on every start of the service. I’ve not found how have a relative path for smtp_auth_password_file that depends on an environment variable or argument to alertmanager.
A workaround might be to not use DynamicUser. That would require modifying alertmanager.nix.
At work I use the following adapted copy of alertmanager with support for loading credentials via systemd’s LoadCredential.
I’ve been meaning to turn this into a PR but haven’t gotten around to it yet. Feel free to take this code and turn it into a PR:
# This is an adapted copy of
# https://github.com/NixOS/nixpkgs/blob/0874168639713f547c05947c76124f78441ea46c/nixos/modules/services/monitoring/prometheus/alertmanager.nix
# where the check `amtool check-config` has been moved from build-time to run-time.
# We do this because we use environment variables in the config which are not allowed by check-config.
#
# The module has also been extended with a new way of passing credentials.
# See the option: services.prometheus.alertmanager.credentials
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.prometheus.alertmanager;
mkConfigFile = pkgs.writeText "alertmanager.yml" (builtins.toJSON cfg.configuration);
alertmanagerYml =
if cfg.configText != null then
pkgs.writeText "alertmanager.yml" cfg.configText
else mkConfigFile;
cmdlineArgs = cfg.extraFlags ++ [
"--config.file /tmp/alert-manager-substituted.yaml"
"--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
"--log.level ${cfg.logLevel}"
"--storage.path /var/lib/alertmanager"
(toString (map (peer: "--cluster.peer ${peer}:9094") cfg.clusterPeers))
] ++ (optional (cfg.webExternalUrl != null)
"--web.external-url ${cfg.webExternalUrl}"
) ++ (optional (cfg.logFormat != null)
"--log.format ${cfg.logFormat}"
);
in {
imports = [
(mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "user" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a user setting.")
(mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "group" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a group setting.")
(mkRemovedOptionModule [ "services" "prometheus" "alertmanagerURL" ] ''
Due to incompatibility, the alertmanagerURL option has been removed,
please use 'services.prometheus2.alertmanagers' instead.
'')
];
options = {
services.prometheus.alertmanager = {
enable = mkEnableOption "Prometheus Alertmanager";
package = mkOption {
type = types.package;
default = pkgs.prometheus-alertmanager;
defaultText = literalExpression "pkgs.alertmanager";
description = ''
Package that should be used for alertmanager.
'';
};
configuration = mkOption {
type = types.nullOr types.attrs;
default = null;
description = ''
Alertmanager configuration as nix attribute set.
'';
};
configText = mkOption {
type = types.nullOr types.lines;
default = null;
description = ''
Alertmanager configuration as YAML text. If non-null, this option
defines the text that is written to alertmanager.yml. If null, the
contents of alertmanager.yml is generated from the structured config
options.
'';
};
logFormat = mkOption {
type = types.nullOr types.str;
default = null;
description = ''
If set use a syslog logger or JSON logging.
'';
};
logLevel = mkOption {
type = types.enum ["debug" "info" "warn" "error" "fatal"];
default = "warn";
description = ''
Only log messages with the given severity or above.
'';
};
webExternalUrl = mkOption {
type = types.nullOr types.str;
default = null;
description = ''
The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy).
Used for generating relative and absolute links back to Alertmanager itself.
If the URL has a path portion, it will be used to prefix all HTTP endoints served by Alertmanager.
If omitted, relevant URL components will be derived automatically.
'';
};
listenAddress = mkOption {
type = types.str;
default = "";
description = ''
Address to listen on for the web interface and API. Empty string will listen on all interfaces.
"localhost" will listen on 127.0.0.1 (but not ::1).
'';
};
port = mkOption {
type = types.int;
default = 9093;
description = ''
Port to listen on for the web interface and API.
'';
};
openFirewall = mkOption {
type = types.bool;
default = false;
description = ''
Open port in firewall for incoming connections.
'';
};
clusterPeers = mkOption {
type = types.listOf types.str;
default = [];
description = ''
Initial peers for HA cluster.
'';
};
extraFlags = mkOption {
type = types.listOf types.str;
default = [];
description = ''
Extra commandline options when launching the Alertmanager.
'';
};
environmentFile = mkOption {
type = types.nullOr types.path;
default = null;
example = "/root/alertmanager.env";
description = ''
File to load as environment file. Environment variables
from this file will be interpolated into the config file
using envsubst with this syntax:
<literal>$ENVIRONMENT ''${VARIABLE}</literal>
'';
};
credentials = mkOption {
type = types.attrsOf types.str;
default = {};
};
};
};
config = mkMerge [
(mkIf cfg.enable {
assertions = singleton {
assertion = cfg.configuration != null || cfg.configText != null;
message = "Can not enable alertmanager without a configuration. "
+ "Set either the `configuration` or `configText` attribute.";
};
})
(mkIf cfg.enable {
networking.firewall.allowedTCPPorts = optional cfg.openFirewall cfg.port;
systemd.services.alertmanager = {
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
preStart = ''
${lib.concatMapStringsSep "\n" (name: ''export ${name}="$(<"$CREDENTIALS_DIRECTORY/${name}")"'')
(lib.attrNames cfg.credentials)}
${lib.getBin pkgs.envsubst}/bin/envsubst -o "/tmp/alert-manager-substituted.yaml" \
-i "${alertmanagerYml}"
${cfg.package}/bin/amtool check-config /tmp/alert-manager-substituted.yaml
'';
serviceConfig = {
Restart = "always";
StateDirectory = "alertmanager";
DynamicUser = true; # implies PrivateTmp
EnvironmentFile = lib.mkIf (cfg.environmentFile != null) cfg.environmentFile;
LoadCredential = lib.mapAttrsToList (name: path: "${name}:${path}") cfg.credentials;
WorkingDirectory = "/tmp";
ExecStart = "${cfg.package}/bin/alertmanager" +
optionalString (length cmdlineArgs != 0) (" \\\n " +
concatStringsSep " \\\n " cmdlineArgs);
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
};
};
})
];
}
The changed version that you use puts secrets in environment variables via ExecStartPre (preStart in NixOS). These values only live for the duration of envsubst so it’s not a big opportunity for attackers. The values in the credentials option are file paths to files with secrets, which is great because then the secrets themselves are not readable in /nix/store.
The comment that DynamicUser = true ‘implies PrivateTmp’ had escaped my notice so far. A private /tmp for the dynamic user of alertmanager makes it possible to gave an unchanging path for the secrets such as /tmp/credentials/email_password. The secrets would need to be copied over to the /tmp in ExecStartPre.
Yes this is also my conclusion. ExecStartPre runs as root and thus no further configuration is needed to let it read a secrets file because of uid=0. I personally think that this part of systemd is under-documented. My observation with a lot of services is that environment files together with envsubst are the universal tool to supply secrets to services because they either support secrets via environment variables natively (12 factor apps) or they expect them to be supplied via configuration files (legacy). Native support for LoadCredential is imho pretty rare (Miniflux is such an exception).