Hey folks,
I’m trying to implement Impermanence on my machine. I have zfs pools:
{ lib, pkgs, config, ... }:
let
mirrorBoot = { idx, device }: {
type = "disk";
device = device;
content = {
type = "gpt";
partitions = {
ESP = {
size = "2G";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot${idx}";
};
};
zfs = {
size = "100%";
content = {
type = "zfs";
pool = "zroot";
};
};
};
};
};
in
{
boot.loader.grub = {
enable = true;
efiSupport = true;
efiInstallAsRemovable = true;
mirroredBoots = [
{
path = "/boot0";
devices = [ "nodev" ];
}
{
path = "/boot1";
devices = [ "nodev" ];
}
];
};
swapDevices = [
{
device = "/swapfile";
size = 8192; # 8 GiB
randomEncryption = true;
}
];
disko.devices = {
disk = {
x = mirrorBoot {
idx = "0";
device = "/dev/disk/by-id/nvme-WD_BLACK_SN850X_1000GB_245261801360";
};
y = mirrorBoot {
idx = "1";
device = "/dev/disk/by-id/nvme-WD_BLACK_SN850X_1000GB_25097H800196";
};
};
zpool = {
zroot = {
type = "zpool";
mode = "mirror";
options = {
ashift = "12";
autotrim = "on";
};
rootFsOptions = {
canmount = "off";
checksum = "edonr";
compression = "zstd";
dnodesize = "auto";
mountpoint = "none";
normalization = "formD";
relatime = "on";
"com.sun:auto-snapshot" = "false";
};
datasets = {
"root" = {
type = "zfs_fs";
options.mountpoint = "none";
};
"root/nixos" = {
type = "zfs_fs";
mountpoint = "/";
postCreateHook = "zfs snapshot zroot/root/nixos@empty";
};
"root/nix" = {
type = "zfs_fs";
mountpoint = "/nix";
postCreateHook = "zfs snapshot zroot/root/nix@empty";
};
"root/tmp" = {
type = "zfs_fs";
mountpoint = "/tmp";
options.sync = "disabled";
};
"root/persist" = {
type = "zfs_fs";
mountpoint = "/persist";
options."com.sun:auto-snapshot" = "true";
};
"root/persist/appdata" = {
type = "zfs_fs";
mountpoint = "/persist/appdata";
};
"root/persist/microvm" = {
type = "zfs_fs";
mountpoint = "/persist/microvm";
};
};
};
};
};
}
and rollback upon boot:
boot.initrd.systemd.services.rollback = {
description = "Rollback root filesystem to a pristine state on boot";
wantedBy = [
# "zfs.target"
"initrd.target"
];
after = [
"zfs-import-zroot.service"
];
before = [
"sysroot.mount"
];
path = with pkgs; [
zfs
];
unitConfig.DefaultDependencies = "no";
serviceConfig.Type = "oneshot";
script = ''
zfs rollback -r zroot/root/nixos@empty && echo " >> >> rollback complete << <<"
'';
};
and persistence:
environment.persistence."/persist" = {
directories = [
"/etc/nixos"
"/var/lib/nixos"
"/var/lib/docker"
"/var/lib/tailscale"
];
files = [
"/etc/machine-id"
];
};
so what’s happening now, clan.lol keeps regen openssh keys every reboot. And sops templates are not working properly → they are gone after reboot, and I need manually “switch” config again for them to reappear.
I guess, I’m hitting some weird ordering issue, or… maybe not backing up something else?
[root@hommy:~]# sshd -T | grep hostkey
hostkeyagent none
hostkeyalgorithms ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
hostkey /run/secrets/vars/openssh/ssh.id_ed25519
hostkey /run/secrets/vars/openssh/ssh.id_ed25519
so hostkey is different each time: template folder is gone, and vars are re-gened
[root@hommy:~]# ls -la /run/secrets/rendered/
total 12
drwxr-x--x 2 root keys 0 Dec 31 13:17 .
drwxr-x--x 4 root keys 0 Dec 31 13:17 ..
-rw------- 1 root root 123 Dec 31 13:17 caddy.env
-rw------- 1 root root 134 Dec 31 13:17 gatus.env
-rw------- 1 root root 188 Dec 31 13:17 lldap.env
[root@hommy:~]# ls -la /run/secrets/vars/
total 0
drwxr-x--x 5 root keys 0 Dec 31 13:17 .
drwxr-x--x 4 root keys 0 Dec 31 13:17 ..
drwxr-x--x 2 root keys 0 Dec 31 13:17 borgbackup
drwxr-x--x 2 root keys 0 Dec 31 13:17 cloudflare-tunnel
drwxr-x--x 2 root keys 0 Dec 31 13:17 openssh