I want to create a raid0 for /var, but I'm unable to figure how to load mdamd on boot

Hi!

I’ve been using NixOS for 2 months now, and I have to say that I’m happy with the results so far coming from using Arch Linux. I have encounter, however, an issue after installing NixOS, and it is that NixOS was unable to mount a raid partition I created from 3 hard drivers when booting as it was assigned to /var.

On Reddit, I was told that the array needs to be automatically assemble by initramfs, but no details were given nor links to any documentation or Nix code examples. Furthermore, I dug NixOS documentation about Raid without any luck.

Running lsmod shows no sight of a raid kernel module that I can load, I think if I find one it should go into boot.kernelModules, but anyway, no sight of a raid module.

Here is my current system configuration:

# Edit this configuration file to define what should be installed on
# your system.  Help is available in the configuration.nix(5) man page
# and in the NixOS manual (accessible by running ‘nixos-help’).

{ config, pkgs, ... }:
let

  # truco de https://discourse.nixos.org/t/gdm-monitor-configuration/6356/4
  monitorsXmlContent = builtins.readFile /home/jorge/.config/monitors.xml;
  monitorsConfig = pkgs.writeText "gdm_monitors.xml" monitorsXmlContent;
in {
  imports = [ # Include the results of the hardware scan.
    ./hardware-configuration.nix
  ];

  # Bootloader.
  boot = {
    plymouth = {
      enable = true;
    };
    tmp = { cleanOnBoot = true; };
    loader = {
      systemd-boot.enable = false;
      timeout = 30;
      efi = {
        canTouchEfiVariables = true;
        efiSysMountPoint = "/boot/efi";
      };
      grub = {
        devices = [ "nodev" ];
        enable = true;
        efiSupport = true;
        theme = pkgs.nixos-grub2-theme;
        gfxmodeEfi = "1920x1080";
        extraEntries = ''
          menuentry "Reboot" {
            reboot
          }
          menuentry "Poweroff" {
            halt
          }
        '';
      };
    };
    supportedFilesystems = [ "ntfs" ];
    # Copied from virtualisation.lxd.recommendedSysctlSettings
    initrd.kernelModules = [ "vfio_pci" "vfio" "vfio_iommu_type1" ];
    kernelParams = [
      "amd_iommu=on"
      "iommu=pt"
      "kvm_amd.npt=1"
      "kvm_amd.avic=1"
      "quiet"
      "splash"
    ];
    kernel.sysctl = {
      "fs.inotify.max_queued_events" = 1048576;
      "fs.inotify.max_user_instances" = 1048576;
      "fs.inotify.max_user_watches" = 1048576;
      "vm.max_map_count" = 2147483642; # use by Steam Deck
      "kernel.dmesg_restrict" = 1;
      "net.ipv4.neigh.default.gc_thresh3" = 8192;
      "net.ipv6.neigh.default.gc_thresh3" = 8192;
      "kernel.keys.maxkeys" = 2000;
    };
  };

  systemd.tmpfiles.rules =
    [ "L+ /run/gdm/.config/monitors.xml - - - - ${monitorsConfig}" ];

  systemd.services.plantuml = {
    description = "PlantUML server";
    requires = [ "docker.service" ];
    after = [ "docker.service" ];

    serviceConfig = {
      ExecStart =
        "${pkgs.docker}/bin/docker run --rm --name plantuml -p 8686:8080 plantuml/plantuml-server:jetty";
      ExecStop = "${pkgs.docker}/bin/docker stop -t 2 plantuml";
      Restart = "always";
    };

    wantedBy = [ "default.target" ];
  };

  systemd.services.plantuml.enable = true;

  networking.hostName = "woody"; # Define your hostname.
  # networking.wireless.enable = true;  # Enables wireless support via wpa_supplicant.

  # Configure network proxy if necessary
  # networking.proxy.default = "http://user:password@proxy:port/";
  # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";

  # Enable networking
  networking.networkmanager.enable = true;

  # Set your time zone.
  time.timeZone = "America/Costa_Rica";

  # Select internationalisation properties.
  i18n.defaultLocale = "es_CR.UTF-8";

  i18n.extraLocaleSettings = {
    LC_ADDRESS = "es_CR.UTF-8";
    LC_IDENTIFICATION = "es_CR.UTF-8";
    LC_MEASUREMENT = "es_CR.UTF-8";
    LC_MONETARY = "es_CR.UTF-8";
    LC_NAME = "es_CR.UTF-8";
    LC_NUMERIC = "es_CR.UTF-8";
    LC_PAPER = "es_CR.UTF-8";
    LC_TELEPHONE = "es_CR.UTF-8";
    LC_TIME = "es_CR.UTF-8";
  };

  # Enable the X11 windowing system.
  services.xserver.enable = true;
  services.xserver.videoDrivers = [ "nvidia" ];

  # Enable the GNOME Desktop Environment.
  services.xserver.displayManager.gdm.enable = true;
  services.xserver.desktopManager.gnome.enable = true;

  # Configure keymap in X11
  services.xserver = {
    layout = "us";
    xkbVariant = "";
  };

  # Enable CUPS to print documents.
  services.printing.enable = true;
  services.printing.drivers = [ pkgs.hplip ];
  services.avahi.enable = true;
  services.avahi.nssmdns = true;
  # for a WiFi printer
  services.avahi.openFirewall = true;

  # activa accesorios para Xbox one
  hardware.xone.enable = true;

  # activa la app de Corsair
  hardware.ckb-next.enable = true;

  # Steam
  hardware.steam-hardware.enable = true;
  programs.steam = {
    enable = true;
    remotePlay.openFirewall = true;
  };

  # Enable sound with pipewire.
  sound.enable = true;
  hardware.pulseaudio.enable = false;
  security.rtkit.enable = true;
  services.pipewire = {
    enable = true;
    alsa.enable = true;
    alsa.support32Bit = true;
    pulse.enable = true;
    # If you want to use JACK applications, uncomment this
    #jack.enable = true;

    # use the example session manager (no others are packaged yet so this is enabled by default,
    # no need to redefine it in your config for now)
    #media-session.enable = true;
  };
  services.syncthing = {
    enable = true;
    user = "jorge";
    dataDir = "/home/jorge/Documentos/Syncthing";
    configDir = "/home/jorge/Documentos/Syncthing/.config/syncthing";
    extraOptions.gui = {
      user = "jorge";
      password = "<that's a secret>";
    };
  };

  nixpkgs.config.permittedInsecurePackages = [ "electron-11.5.0" ];

  # Enable touchpad support (enabled default in most desktopManager).
  # services.xserver.libinput.enable = true;

  # Ledger
  hardware.ledger.enable = true;

  # habilita ZSH
  programs.zsh.enable = true;

  # habilita kdeconnect
  programs.kdeconnect.enable = true;

  programs.droidcam.enable = true;

  # Define a user account. Don't forget to set a password with ‘passwd’.
  users.users.jorge = {
    isNormalUser = true;
    description = "Jorge";
    extraGroups = [ "networkmanager" "wheel" "docker" "libvirtd" ];
    shell = pkgs.zsh;
  };

  # Allow unfree packages
  nixpkgs.config.allowUnfree = true;

  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
    nixos-grub2-theme

    # otros
    bindfs
    pciutils

    # fuentes
    noto-fonts
    noto-fonts-cjk
    noto-fonts-emoji
    mplus-outline-fonts.githubRelease
    (nerdfonts.override { fonts = [ "FiraCode" "DroidSansMono" "Iosevka" ]; })
    iosevka
    iosevka-comfy.comfy
    iosevka-comfy.comfy-motion-duo
    fira
    fira-code

    # linea de comandos
    aspell
    aspellDicts.en
    aspellDicts.en-computers
    aspellDicts.es
    bitwarden
    bitwarden-cli
    bitwarden-menu
    butler
    ccls
    clang
    cmake
    cmake-format
    cmake-language-server
    coreutils
    curl
    d2
    dig
    emacs
    evince
    fd
    firefox
    git
    git-crypt
    gnumake
    html-tidy
    hunspell
    hunspellDicts.en_US
    hunspellDicts.es_CR
    hunspellDicts.es_ES
    jq
    libreoffice-fresh
    nixfmt
    nodePackages.prettier
    nodePackages_latest.js-beautify
    nodePackages_latest.stylelint
    plantuml
    python311Packages.grip
    ripgrep
    rust-analyzer
    shellcheck
    stow
    thefuck
    tree
    wget
    xcp
    libtool
    clipgrab
    texlive.combined.scheme-full
    texlab
    lua53Packages.digestif
    pandoc

    alacritty
    firefox
    brave
    gimp
    inkscape
    obsidian
    remnote
    signal-desktop-beta
    smplayer
    spotify
    tdesktop
    vlc
    zoom-us
    pdfarranger
    hplip
    newsflash
    avidemux
    audacity
    libsForQt5.kdenlive
    mediainfo
    #davinci-resolve
    deluge
    xsane
    obs-studio
    ffmpeg-full
    xournalpp
    syncthingtray
    nextcloud-client

    # entretenimiento y juegos
    itch
    prismlauncher
    protonup-qt
    rpcs3
    heroic
    droidcam
    linuxKernel.packages.linux_6_1.v4l2loopback

    # Gnome
    gnome.gnome-tweaks
    gnomeExtensions.appindicator
    gnomeExtensions.clipman
    gnomeExtensions.gnome-bedtime
    gnomeExtensions.memento-mori
    gnomeExtensions.no-overview
    gnomeExtensions.pop-shell
    gnomeExtensions.rounded-window-corners
    gnomeExtensions.time-awareness
    gnomeExtensions.wallpaper-switcher
    gnomeExtensions.weather
    gnomeExtensions.weather-oclock
    gnomeExtensions.auto-move-windows
    # temas de Gnome
    theme-obsidian2
    iconpack-obsidian

    # privacidad
    protonmail-bridge
    protonvpn-gui
    mu
    isync

    # docker
    # el paquete docker ya es activado como servicio más abajo
    kubecolor
    kubectl
    kubectx
    docker-compose

    # virtualization
    virt-manager

    # dependencias para construir Python
    gcc
    zlib
    libffi
    openssl
    bzip2
    readline
    sqlite
    xz

    # Android
    scrcpy

    # cripto
    ledger-live-desktop

    # otros
    gparted

    # game dev
    godot_4
    gdtoolkit

    # gaming
    ckb-next
  ];

  # Some programs need SUID wrappers, can be configured further or are
  # started in user sessions.
  # programs.mtr.enable = true;
  # programs.gnupg.agent = {
  #   enable = true;
  #   enableSSHSupport = true;
  # };

  # List services that you want to enable:

  # Enable the OpenSSH daemon.
  services.openssh.enable = true;
  virtualisation.docker = {
    enable = true;
    autoPrune.enable = true;
  };
  services.flatpak.enable = true;
  virtualisation.libvirtd.enable = true;
  programs.dconf.enable = true;

  # Open ports in the firewall.
  networking.firewall.allowedTCPPorts = [ 22 8384 22000 ];
  networking.firewall.allowedUDPPorts = [ 22000 21027 ];
  # networking.firewall.allowedUDPPorts = [ ... ];
  # Or disable the firewall altogether.
  # networking.firewall.enable = false;

  # This value determines the NixOS release from which the default
  # settings for stateful data, like file locations and database versions
  # on your system were taken. It‘s perfectly fine and recommended to leave
  # this value at the release version of the first install of this system.
  # Before changing this value read the documentation for this option
  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
  system.stateVersion = "23.05"; # Did you read the comment?

}

Looks like boot.initrd.services.swraid.mdadmConf allows you to configure a madm.conf which is inserted into the initrd.

Check the module as well: modules/tasks/swraid.nix. Have to admit I don’t understand why the module seems to hint that this only works if systemd-in-initrd is enabled.

Full usage example can be gleaned from: nixos/tests/systemd-initrd-swraid.nix

I’ve changed my configuration and ran nixos-rebuild switch:

➜  sudo nixos-rebuild switch
[sudo] contraseña para jorge:
building Nix...
building the system configuration...
these 4 derivations will be built:
  /nix/store/46zvb62zcxvyaa98l232155filqmsnsv-mdadm.conf.drv
  /nix/store/ljmwccimm9rb7gj9iiqil9b2v7ha2rmm-initrd-linux-6.1.38.drv
  /nix/store/63v1yni2sf9pmlhg5f9zk4firs1ss2f7-boot.json.drv
  /nix/store/sjsirhnpifkkg4y5a34v3bqnbkld16cd-nixos-system-woody-23.05.1935.98da3dd0de6.drv
building '/nix/store/46zvb62zcxvyaa98l232155filqmsnsv-mdadm.conf.drv'...
building '/nix/store/ljmwccimm9rb7gj9iiqil9b2v7ha2rmm-initrd-linux-6.1.38.drv'...
building '/nix/store/63v1yni2sf9pmlhg5f9zk4firs1ss2f7-boot.json.drv'...
building '/nix/store/sjsirhnpifkkg4y5a34v3bqnbkld16cd-nixos-system-woody-23.05.1935.98da3dd0de6.drv'...
updating GRUB 2 menu...
activating the configuration...
setting up /etc...
reloading user units for jorge...
setting up tmpfiles
the following new units were started: libvirtd.service, systemd-ask-password-console.path

the contents are as follow:

services.swraid = {
         mdadmConf =
           "ARRAY /dev/md0 level=raid0 num-devices=3 metadata=1.2 name=woody:0 UUID=57fafca6:b9887478:16687778:48079483 devices=/dev/sda,/dev/sdb,/dev/sdd";
};

I’ll reboot and see if the /dev/md0 device is there

Sadly, it disappeared:

NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS
sda           8:0    0   1,9T  0 disk
└─sda1        8:1    0   1,9T  0 part /home
sdb           8:16   0 298,1G  0 disk
sdc           8:32   0 111,8G  0 disk
sdd           8:48   0 931,5G  0 disk
nvme0n1     259:0    0 238,5G  0 disk
├─nvme0n1p1 259:1    0  36,9G  0 part [SWAP]
├─nvme0n1p2 259:2    0 201,1G  0 part /nix/store
│                                     /
└─nvme0n1p3 259:3    0   512M  0 part /boot/efi

am I supposed to write a /etc/mdadm.conf file?

I see that the nix module has the following option hidden(?):

enable = (lib.mkEnableOption (lib.mdDoc "swraid support using mdadm")) // {
      visible = false; # only has effect when the new stage 1 is in place
    };

I put it on my configuration with enable = true; to see what happens next.

okay, I think I have this mess resolved.

Following the Arch Wiki guide on RAID, I have to create a partition on each disk and assign it Linux RAID on fdisk or FD00 on gdisk. I changed the partition table of my 3 hard drivers to be GPT.

Then after having your partitions ready, you create the RAID as such, since I want to have a RAID0 partition for my /var system folder, I ran sudo mdadm --create /dev/md0 --level=0 --raid-devices 3 /dev/sda1 /dev/sdb1 /dev/sdc1.

You need to update the “configuration” according to the Wiki, thus I did something like this to avoid errors and speed up things a bit:

{ config, pkgs, ... }:
let

  # ...
  raid0 = ''
    DEVICE partitions
    ARRAY /dev/md0 metadata=1.2 name=woody:0 UUID=b3210ef7:588e10d0:b7ada7d6:a167b92f
  '';
in {
# ...
boot = {
    # ...
    initrd = {
      # ...
      services.swraid = {
        enable = true;
        # include this on intird
        mdadmConf = raid0;
      };
    };
# ...
};
# ...
# write the file /etc/mdadm.conf
environment.etc = { "mdadm.conf".text = raid0; };

you can then run nixos-rebuild switch.

Finally, to format /dev/md0 you have to calculate the stride and stripe width

from there, I think is a matter of entering your system from the NixOS “live CD” and move the content of /var to the mounted partition of /dev/md0 and configuring the system accordingly.

1 Like

since I want to have a RAID0 partition for my /var system folder

On a completely separate note and on the off-chance you haven’t given this any consideration - with RAID0 (striping) the loss of 1 disk will cause the entire volume to become unavailable.

Disks die (and with an alarming frequency for me at least).

I’m okay with that, if it is on /var, is not mission-critical for the functioning of my computer.

Do note that many applications store persistent data there that might be an issue if lost. If you need raid0 for performance reasons, I would suggest you leave /var alone and instead mount your raid0 somewhere else and have your speed sensitive applications use that for whatever you need.

I’m more worried about the disk space, that’s all. I have 3 hard drives of different sizes and other than RAID0 was not possible for me.

Fair enough, but the point about mounting the RAID0 disks somewhere else still stands for whatever is creating a lot of data in /var.

1 Like