Hi!
I’ve got a problem that when I try to create a second generation with nixos-rebuild switch, the command fails because my entire 1 GiB boot directory is full.
With only one generation the boot directory is about 20% full. My system consists of encrypted boot, root, and home partitions and a efi partition. To generate a new generation, I’ve emptied the boot partition, except for the decryption key situated in the partition, then running nix-collect-garbage -d, and finally running the switch command. I remember even trying setting the boot partition to 10 GiB, but still the second generation filling up the partition.
Is this expected behavior? Does anybody know how to fix this?
That’s not normal, but can’t advise further without seeing your config. I have a 500 MB ESP and it can hold a fair number of generations.
1 Like
Here’s my configs:
flake.nix:
description = "NixOS configuration";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; # nixpkgs.url = "github:nixos/nixpkgs/<commit-hash>";
disko.url = "github:nix-community/disko";
disko.inputs.nixpkgs.follows = "nixpkgs";
home-manager.url = "github:nix-community/home-manager";
home-manager.inputs.nixpkgs.follows = "nixpkgs";
sops-nix.url = "github:Mic92/sops-nix";
sops-nix.inputs.nixpkgs.follows = "nixpkgs";
chaotic.url = "github:chaotic-cx/nyx/nyxpkgs-unstable";
nixvim = {
url = "github:nix-community/nixvim";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { nixpkgs, disko, home-manager, sops-nix, nixvim, chaotic, ... }:
let
system = "x86_64-linux";
overlays = builtins.map (_: (import (./overlays + _))) [
/flameshot.nix
/libreoffice-still.nix
/ncspot.nix
/neomutt.nix
/ranger.nix
/ventoy-bin.nix
/sway.nix
];
homeSettings = {
name = "name";
email = "email@email.com";
nameGit = "nameGit";
emailGit = "emailGit@emailGit.com";
terminal = "wezterm";
desktopGaps = 4;
};
in {
nixosConfigurations = {
pc = nixpkgs.lib.nixosSystem {
inherit system;
specialArgs = {
disabledModules = [
(nixpkgs + "/nixos/modules/profiles/all-hardware.nix")
(nixpkgs + "/nixos/modules/profiles/base.nix")
];
inherit overlays;
};
modules = [
./configuration.nix
(nixpkgs + "/nixos/modules/profiles/minimal.nix")
disko.nixosModules.disko
./disk-config.nix
{
_module.args.disks = [ "/dev/sda" ];
}
home-manager.nixosModules.home-manager {
home-manager.useGlobalPkgs = true;
home-manager.useUserPackages = true;
home-manager.backupFileExtension = "bak";
home-manager.users.name = import ./home.nix;
home-manager.sharedModules = [
nixvim.homeModules.nixvim
sops-nix.homeManagerModules.sops
];
home-manager.extraSpecialArgs = {
inherit homeSettings;
};
}
chaotic.nixosModules.default
];
};
};
};
}
configuration.nix:
{ lib, pkgs, disabledModules, overlays, ... }:
let
username = "username";
hostName = "hostname";
deviceIP = "192.168.1.2";
linuxVersion = "linux_6_16";
in {
imports = [
./hardware-configuration.nix # Include the results of the hardware scan.
./configuration/texlive.nix
];
inherit disabledModules;
chaotic.mesa-git.enable = true;
hardware = {
bluetooth = {
enable = true;
settings.General.FastConnectable = true;
};
cpu = {
intel.updateMicrocode = true;
x86.msr.enable = true;
};
amdgpu.overdrive.enable = true;
};
powerManagement.cpuFreqGovernor = "performance";
nixpkgs = {
inherit overlays;
config = {
rocmSupport = true; # In order for btop w/ AMDGPU support to work.
permittedInsecurePackages = [
"dotnet-runtime-7.0.20"
"libsoup-2.74.3"
"ventoy-1.1.05"
];
allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [
"discord"
"furmark"
"spotify"
"steam"
"steam-unwrapped"
"unrar"
"ventoy"
"vintagestory"
];
};
};
boot = {
kernelPackages = pkgs.linuxPackages_latest;
#extraModulePackages = with config.boot.kernelPackages; [ cpupower perf turbostat ];
#consoleLogLevel = 3; # Hide unnecessary ACPI error messages at boot
supportedFilesystems = [
"btrfs"
];
kernelModules = [
"bfq"
];
kernelParams = [
"zswap.enabled=1"
"iomem=relaxed"
#"tsc=reliable"
#"clocksource=tsc"
#"scsi_mod.use_blk_mq=1"
];
extraModprobeConfig = ''
options snd_hda_intel power_save=0
'';
# Use the GRUB 2 boot loader.
loader = {
timeout = 1;
grub = {
enableCryptodisk = true;
efiSupport = true;
configurationLimit = 100; # NOTE: Default value, should probably be
# radically decreased.
# Define on which harddrive you want to install Grub:
device = "nodev"; # or "nodev" for efi only
efiInstallAsRemovable = true;
default = "0"; # "saved" does not work w/ encrypted /boot
timeoutStyle = "hidden";
gfxmodeEfi = "1920x1080,1024x768,auto";
gfxpayloadEfi = "keep";
extraEntries = ''
menuentry "Reboot" { reboot }
menuentry "Poweroff" { halt }
menuentry "UEFI-firmware" { fwsetup }
'';
};
efi = {
efiSysMountPoint = "/efi";
canTouchEfiVariables = false;
};
};
initrd = {
compressor = "zstd";
includeDefaultModules = true;
secrets = { "/boot" = null; };
luks.devices.crypted = {
keyFile = "/boot/cryptlvm.key";
fallbackToPassword = true;
};
};
};
networking = {
inherit hostName; # Define your hostname.
extraHosts = "0.0.0.0 apresolve.spotify.com"; # TODO: Try to remove after ncspot update.
wireless.iwd = {
enable = true;
settings.General.EnableNetworkConfiguration = true; # Enable IWD's DHCP.
};
};
# Set your time zone.
time.timeZone = "<timezone>";
# Select internationalisation properties.
i18n.defaultLocale = "en_US.UTF-8";
console.useXkbConfig = true; # use xkb.options in tty.
documentation = {
enable = true;
man.enable = true;
nixos.enable = true;
};
programs = {
#ccache.enable = true;
#nix-index.enableZshIntegration = true;
#ssh.setXAuthLocation = false; # Fix conflicting values (maybe
# later try to remove).
corectrl.enable = true;
#corefreq.enable = true; # FIXME: Try to enable after new update.
firejail.enable = true;
flashrom.enable = true;
gamemode.enable = true;
gamescope.enable = true;
git.enable = true;
git.lfs.enable = true;
gnome-disks.enable = true;
htop.enable = true;
less.enable = true;
nix-ld.enable = true;
virt-manager.enable = true;
bat = {
enable = true;
extraPackages = lib.mkForce [];
};
steam = {
enable = true;
#gamescopeSession.enable = true;
extraPackages = lib.mkForce [];
extraCompatPackages = lib.mkForce [];
};
thunar = {
enable = true;
plugins = lib.mkForce [];
};
};
# Enable sound.
# PipeWire
services.pipewire = {
enable = true;
alsa.enable = true;
alsa.support32Bit = true;
pulse.enable = true;
#jack.enable = true;
wireplumber.extraConfig.bluetoothEnhancements = {
"monitor.bluez.properties" = {
"bluez5.roles" = [ "a2dp_sink" "a2dp_source" "bap_sink" "bap_source" "hsp_hs" "hsp_ag" "hfp_hf" "hfp_ag" ];
"bluez5.codecs" = [ "sbc" "sbc_xq" ];
"bluez5.enable-sbc-xq" = true;
#"bluez5.hfphsp-backend" = "native";
};
};
};
# Enable touchpad support (enabled default in most desktopManager).
# services.xserver.libinput.enable = true;
# Define a user account. Don't forget to set a password with ‘passwd’.
users.users = lib.attrsets.setAttrByPath [ username ] {
isNormalUser = true;
extraGroups = [
"wheel" # Enable ‘sudo’ for the user.
"libvirtd"
"gamemode"
"networkmanager"
"input"
"docker"
];
shell = pkgs.bash;
};
security = {
apparmor.enable = true;
rtkit.enable = true;
doas = {
enable = true;
extraRules = [
{
groups = [ "wheel" ];
runAs = "root";
persist = true;
}
];
};
fonts = {
packages = with pkgs; [
font-awesome # Waybar
jetbrains-mono
noto-fonts-emoji
#dejavu_fonts
];
};
environment = {
defaultPackages = [];
etc = {
"dict.conf".text = "server dict.org";
};
# Remove if don't needed:
#noXlibs = false;
pathsToLink = [
"/libexec"
"/share/fzf"
];
# List packages installed in system profile. To search, run:
# $ nix search wget
systemPackages = with pkgs; lib.lists.flatten [
(with linuxKernel.packages.${linuxVersion}; [ # TODO: Can the Linux version be determined programmatically?
cpupower
perf
turbostat
])
(with nodePackages; [
alex
cspell
])
(with xfce; [
ristretto
xfce4-power-manager
])
(with xorg; [
xhost
xlsclients
])
(python3.withPackages (ps: with ps; [
pdftotext
numpy
matplotlib
pip
unrardll
]))
(wl-kbptr.overrideAttrs (_: {
#mesonFlags = [ "-Dopencv=enabled" ];
#buildInputs = lib.mkOptionDefault [ opencv ];
}))
#conky
#trickle
adwaita-icon-theme
adwaita-qt
adwaita-qt6
age
qadwaitadecorations
qadwaitadecorations-qt6
android-file-transfer
atop
bash-language-server
bc
beancount
bison
bitwarden-cli
bleachbit
bluez
bluez-tools
btrfs-progs
buku
cloc
compsize
cpu-x
curl
darktable
dash
ddrescue
dialog
dict
discord
disko
dmidecode
dos2unix
du-dust
dunst
element-desktop
exiftool
ext4magic
extundelete
fastfetch
fd
fdupes
file
flex
foot
freetts # Arch linux: festival, festival-us
furmark
fzf
gcc
gimp
git
git-lfs
git-repo
gitRepo
glances
glib # Enable gio [trash]
glmark2
glxinfo
gnumake
gparted
gperf
grim
gsettings-desktop-schemas
guestfs-tools
hdparm
hledger
home-manager
htop
hypnotix
imagemagick
imgcat
inteltool # Show temperature in Conky
iotop-c
iw
jabref
jq
krita
lact
languagetool
libqalculate
liberation_ttf
libnotify
libreoffice-still
libsixel
libvirt-glib
libxml2
lsix
lua-language-server
mailcap
man-pages # Requires for 'man 7 signal' to work.
mangohud
mate.engrampa
moreutils
ncurses5
nethogs
nettools
nix-index
nix-output-monitor
nix-tree
nixd
nodejs-slim
ntfs3g
numlockx
nvtopPackages.amd
openssl
p7zip
pamixer
pass
pavucontrol
pciutils
perl
pigz
piper
playerctl
pmbootstrap
polkit_gnome
poppler_utils
procps
proselint
protontricks
protonup
protonup-qt
pstree
pv
qemu_kvm
qrencode
rclone
resources
ripgrep
ripgrep-all
rsync
rust-analyzer
schedtool
scrcpy
signal-cli
signal-desktop-bin
smartmontools
songrec
sops
sox
spectre-meltdown-checker
speedtest-cli
spotify
squashfsTools
stress
stress-ng
sway-contrib.grimshot
sysstat # iostat
swayidle
testdisk
texlab
textidote
textlint
tigervnc
tldr
tree
tree-sitter
ttyplot
ueberzugpp # Used for the cover feature in ncspot.
umu-launcher
unrar
unzip
util-linux
uuu
vale
ventoy-bin
vim-language-server
vintagestory
visidata
vlc
vulkan-tools
w3m
wavemon
wayvnc
wdiff
wev
wget
which
winetricks
wineWowPackages.waylandFull
wl-clipboard
write-good
xdg-desktop-portal-gtk
xdg-user-dirs
xdg-utils # <- xdg-open
xdotool
xz
zip
];
};
# Some programs need SUID wrappers, can be configured further or are started
# in user sessions.
# programs.mtr.enable = true;
# programs.gnupg.agent = {
# enable = true;
# enableSSHSupport = true;
# };
# List services that you want to enable:
services = {
#fstrim.enable = true;
#rsyncd.enable = true;
#smartd.enable = true;
#spotifyd.enable = true;
btrfs.autoScrub.enable = true;
envfs.enable = true;
flatpak.enable = true;
fwupd.enable = true;
getty.autologinUser = username;
minidlna.enable = false;
mpd.enable = true;
ntfy-sh.enable = true;
ntfy-sh.settings.base-url = "http://${deviceIP}";
ntp.enable = true;
power-profiles-daemon.enable = true;
ratbagd.enable = true;
tumbler.enable = true; # Enable thumbnails for images in Thunar.
udisks2.enable = true;
locate = {
enable = true;
prunePaths = [
"/tmp"
"/var/tmp"
"/var/cache"
"/var/lock"
"/var/run"
"/var/spool"
#"/nix/store"
"/nix/var/log/nix"
];
};
openssh = {
enable = true;
settings.PasswordAuthentication = false;
};
udev = {
enable = true;
extraRules = ''
# HDD
ACTION=="add|change", KERNEL=="sd[a-z]*", ATTR{queue/rotational}=="1", ATTR{queue/scheduler}="bfq"
# SSD
ACTION=="add|change", KERNEL=="sd[a-z]*|mmcblk[0-9]*", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="bfq"
# NVMe SSD
ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="none"
'';
};
clamav = {
daemon.enable = true;
updater.enable = true;
};
gvfs.enable = true; # Fix gio trash.
# Enable the X11 windowing system.
xserver = {
enable = true;
displayManager.startx.enable = true;
};
# Enable CUPS to print documents.
# printing.enable = true;
};
xdg.portal = {
enable = true;
extraPortals = [ pkgs.xdg-desktop-portal-gtk ];
config.common.default = [ "gtk" ];
};
virtualisation = {
libvirtd.enable = true;
waydroid.enable = false;
docker = {
enable = true;
extraPackages = lib.mkForce [];
};
};
networking = {
firewall = {
# Disable the firewall altogether.
enable = true;
extraPackages = lib.mkForce [];
# Open ports in the firewall.
allowedTCPPortRanges = [
{ from = 1714; to = 1764; } # KDE Connect.
];
allowedUDPPortRanges = [
{ from = 1714; to = 1764; } # KDE Connect.
{ from = 2001; to = 2001; } # ARMA Reforger.
#{ from = 2302; to = 2306; } # ARMA 3.
];
};
nftables.enable = false;
};
system = {
autoUpgrade = {
enable = false;
operation = "boot";
};
};
systemd = {
tpm2.enable = false;
oomd.enable = true;
services.lactd = {
description = "AMDGPU Control Daemon";
after = [ "multi-user.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig.ExecStart = "${lib.getExe pkgs.lact} daemon";
enable = true;
};
nix.settings = {
auto-optimise-store = true;
max-jobs = 4;
cores = 4;
experimental-features = [
"nix-command"
"flakes"
];
};
system.stateVersion = "25.05"; # Did you read the comment?
}
disk-config.nix:
let
device = "/dev/sda";
# If you want to use the key for interactive login be sure there is no
# trailing newline. For example, use: `echo -n "password" > /tmp/secret.key`
passwordFile = "/tmp/secret.key";
# For the key, use:
# dd bs=512 count=4 if=/dev/random of=/tmp/cryptlvm.key iflag=fullblock
additionalKeyFiles = [ "/tmp/cryptlvm.key" ];
EFISize = "512M";
bootSize = "1G";
rootSize = "76294M"; # 80 GigaB
swapSize = "22888M"; # 24 GigaB
in {
disko.devices = {
disk = {
sda = {
inherit device;
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
type = "EF00";
size = EFISize;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/efi";
};
};
luks = {
size = "100%";
content = {
type = "luks";
name = "crypted";
extraFormatArgs = [
"--type luks2"
];
settings.allowDiscards = true;
inherit passwordFile;
# For the key, use:
# dd bs=512 count=4 if=/dev/random of=/tmp/cryptlvm.key iflag=fullblock
inherit additionalKeyFiles;
content = {
type = "lvm_pv";
vg = "pool";
};
};
};
};
};
};
};
lvm_vg = {
pool = {
type = "lvm_vg";
lvs = {
boot = {
size = bootSize;
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/boot";
};
};
root = {
size = rootSize;
content = {
type = "filesystem";
format = "btrfs";
mountpoint = "/";
mountOptions = [
"compress=zstd:3"
];
};
};
swap = {
size = swapSize;
content = {
type = "swap";
#discardPolicy = "both";
resumeDevice = true;
};
};
home = {
size = "100%FREE";
content = {
type = "filesystem";
format = "btrfs";
mountpoint = "/home";
mountOptions = [
"compress=zstd:3"
];
};
};
};
};
};
};
}
Thanks for the help!
Would you like to share your configs, so that I could at least try to troubleshoot the issue with mine?
Well that’s not right. That’s saying, every time you nixos-rebuild, it should install the kernels / initrds in /boot and then copy the entirety of /boot into each generation’s initrd. You probably just want that to be
secrets = { "/boot/cryptlvm.key" = null; };
So that it only copies the key into the initrd.
Sidenote, encrypted /boot i.e. grub cryptodisk is strongly recommended against. It provides no added security, slows down boot a lot, requires a downgraded PBKDF, and creates an easy way to accidentally expose your LUKS key file if you’re not careful.
3 Likes
That was indeed the issue, thank you very much!
Could you provide material that explains why encrypting the boot partition is ill advised?
Well, I enumerated the main reasons before. But I’ll elaborate.
- It provides no added security: Encrypting your kernels / initrds does not prevent tampering. The only thing that does that is Secure Boot. Without Secure Boot, it is just as easy to tamper with your unencrypted boot loader as it is to tamper with unencrypted kernels or initrd. If the boot loader is tampered with, it’s free to boot whatever it likes, including tampered kernels. EDIT: I should have clarified, a good Secure Boot setup will cover the boot loader and the kernel / initrd as a UKI or something along those lines, e.g. Lanzaboote, which is why Secure Boot does not need them to be encrypted.
- It slows down boot a lot: Grub cryptodisk is insanely slow. I’ve seen it add as much as a minute and a half to boot times, depending on hardware and LUKS parameters.
- It requires a downgraded PBKDF: Grub cryptodisk is usually behind on standard LUKS features, and for a good while now that has meant you cannot use
argon2id as your PBKDF, despite this being the recommended default by cryptsetup.
- It creates an easy way to accidentally expose your LUKS key file if you’re not careful: The keyfile added to the initrd with
boot.initrd.secrets is just an easy file to accidentally leak. This one isn’t that big a deal, since hopefully you can keep a file secret. But I have seen many examples where people got this wrong and didn’t know they were leaking their disk encryption key.
Furthermore, NixOS maintainers generally recommend against grub in general. For one, see this laundry list of security vulnerabilities we had to patch. Grub just isn’t very well maintained, either in nixpkgs or upstream (AFAICT). Also, in my experience, grub is pretty buggy.
My recommendation is systemd-boot for its simplicity, or limine if you have to use legacy BIOS. A common criticism of systemd-boot is that it makes boot look less pretty with its text-only interface, but I find this is negated by setting boot.loader.timeout = 0;, which tells systemd-boot not to appear at all unless you hold spacebar during boot.
7 Likes
Thank you very much, I really appreciate your elaborate response!
I now understand why an encrypted boot makes no sense and my plan is to move to Lanzaboote.
1 Like