Nix store gc eat the whole system!?

Hi, I have this nix script which I’m using for embedded deployment, nix is amazing on embedded the profiles and nix copy etc are just amazing, my issue is that my current repart solution for flashing the system automatically to the target machine is not registering the system init store path as a valid store path somehow and running `nix store gc` cause the system to wipe it self! (btw this do not happen on the target-vm)

{
  pkgs,
  qv,
}:

let
  commonScripts = pkgs.callPackage ../common/default.nix { };
  provisioningScripts = pkgs.callPackage ./scripts.nix { inherit commonScripts; };

  targetSystem = pkgs.nixos (
    {
      config,
      lib,
      modulesPath,
      pkgs,
      ...
    }:
    let
      efiArch = pkgs.stdenv.hostPlatform.efiArch;
    in
    {
      imports = [
        "${modulesPath}/profiles/minimal.nix"
        "${modulesPath}/image/repart.nix"
        qv.nixosModules.qvcore
      ];

      nix.settings.experimental-features = [
        "nix-command"
        "flakes"
      ];

      boot.bootspec.enable = true;
      boot.loader.systemd-boot.enable = true;
      boot.loader.systemd-boot.configurationLimit = 10;
      boot.loader.efi.canTouchEfiVariables = false;

      boot.initrd.systemd = {
        enable = true;
        emergencyAccess = true;
        extraBin = {
          lspci = "${pkgs.pciutils}/bin/lspci";
          lsusb = "${pkgs.usbutils}/bin/lsusb";
          lsblk = "${pkgs.util-linux}/bin/lsblk";
          dmesg = "${pkgs.util-linux}/bin/dmesg";
          lsmod = "${pkgs.kmod}/bin/lsmod";
          modinfo = "${pkgs.kmod}/bin/modinfo";
          strace = "${pkgs.strace}/bin/strace";
        };
      };

      boot.initrd.services.lvm.enable = true;

      boot.initrd.kernelModules = [
        "vmd"
        "thunderbolt"
        "nvme_core"
        "nvme_keyring"
        "nvme_auth"
        "hkdf"
        "nvme"
        "dm_mod"
        "dm_crypt"
      ];

      boot.initrd.availableKernelModules = [
        "xhci_pci"
        "ehci_pci"
        "ahci"
        "usb_storage"
        "sd_mod"
        "rtsx_pci_sdmmc"
        "virtio_pci"
        "virtio_blk"
        "virtio_gpu"
        "virtio_net"
        "virtio_balloon"
      ];

      boot.kernelModules = [
        "kvm-intel"
        "exfat"
      ];
      boot.extraModulePackages = [ ];

      boot.initrd.luks.devices."cryptroot" = {
        device = "/dev/disk/by-partlabel/edgebox-system";
        crypttabExtraOpts = [ "tpm2-device=auto" ];
      };

      hardware.graphics = {
        enable = true;
        extraPackages = with pkgs; [
          intel-media-driver
          libvdpau-va-gl
          vpl-gpu-rt
          intel-vaapi-driver
          intel-compute-runtime
        ];
      };

      documentation.enable = false;
      programs.command-not-found.enable = false;

      services.openssh.enable = true;
      services.desktopManager.gnome.enable = true;

      users.users.root.password = "root";

      users.users.ebx = {
        isNormalUser = true;
        extraGroups = [
          "networkmanager"
          "wheel"
          "input"
          "video"
          "render"
        ];
      };

      environment.systemPackages = with pkgs; [
        vim
        tmux
      ];

      virtualisation.docker.enable = true;

      fileSystems."/boot" = {
        device = "/dev/disk/by-label/ESP";
        fsType = "vfat";
      };

      fileSystems."/" = {
        device = "/dev/vg-edgebox/root";
        fsType = "ext4";
      };

      fileSystems."/data" = {
        device = "/dev/vg-edgebox/data";
        fsType = "ext4";
      };

      image.repart = {
        name = "edgebox-standard";
        split = true;

        partitions = {
          "esp" = {
            contents = {
              "/EFI/BOOT/BOOT${lib.toUpper efiArch}.EFI".source =
                "${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot${efiArch}.efi";

              "/loader/loader.conf".source = builtins.toFile "loader.conf" ''
                default nixos-generation-1.conf
                timeout 3
                console-mode max
              '';

              "/EFI/nixos/kernel.efi".source = "${config.system.build.toplevel}/kernel";
              "/EFI/nixos/initrd.efi".source = "${config.system.build.toplevel}/initrd";

              "/loader/entries/nixos-generation-1.conf".source = builtins.toFile "nixos-generation-1.conf" ''
                title NixOS (Generation 1)
                linux /EFI/nixos/kernel.efi
                initrd /EFI/nixos/initrd.efi
                options init=/nix/var/nix/profiles/system/init ${builtins.concatStringsSep " " config.boot.kernelParams}
              '';
            };
            repartConfig = {
              Type = "esp";
              Format = "vfat";
              Label = "ESP";
              SizeMinBytes = "512M";
              SplitName = "esp";
            };
          };

          "root" = {
            storePaths = [
              config.system.build.toplevel
              pkgs.stdenv
            ];
            repartConfig = {
              Type = "root";
              Format = "ext4";
              Label = "nixos";
              Minimize = "guess";
              SplitName = "root";
            };
          };
        };
      };

      system.stateVersion = "26.05";
    }
  );

  targetSystemVm = targetSystem.extendModules {
    modules = [
      {
        virtualisation.vmVariant = {
          virtualisation = {
            memorySize = 4096;
            cores = 4;
            forwardPorts = [
              {
                from = "host";
                host.port = 2222;
                guest.port = 22;
              }
            ];
          };
        };
      }
    ];
  };

  stubImagesDir = pkgs.runCommand "qvision-provisioning-images" { } ''
    mkdir -p $out/images
    cp ${targetSystem.config.system.build.image}/edgebox-standard.esp.raw  $out/images/esp-standard.img
    cp ${targetSystem.config.system.build.image}/edgebox-standard.root.raw $out/images/rootfs-standard.img
  '';

  provisioningImage = pkgs.nixos (
    { ... }:
    {
      imports = [ ./provisioning.nix ];
      _module.args = {
        inherit commonScripts provisioningScripts stubImagesDir;
      };
    }
  );

in
{
  packages = {
    provisioning-iso = provisioningImage.config.system.build.isoImage;
    provisioning-vm = provisioningImage.config.system.build.vm;
    disk-image = targetSystem.config.system.build.image;
    target-system = targetSystem.config.system.build.toplevel;
    target-vm = targetSystemVm.config.system.build.vm;
  };
}

the /nix/var/nix/gcroots/current-system actually exist but when query for roots i see skipping /run/current-system, point to an invalid store path and i do not know why this is an invalid store path? /nix/store/1z3l2w9zwsc2frmq83460my211fpwkkl-nixos-system-nixos-26.05pre-git/

The images you get from the systemd-repart image builder do not have a Nix DB. It’s that DB that tracks which paths are valid and what their NAR hash should be. Note that paths being “valid” is a different concept from them being “live” to the GC; “valid” just means the DB knows about it and tracks it. There’s no such thing as a live invalid path, and the repart images only have invalid paths because they don’t have a DB.

That is helpful so repart just dump store paths without tracking them, that’s why the gc wipe the system then is there any way to retrack them or maybe configure repart to at least track the system closure?

There are other image builders in nixpkgs that automatically re-register all the paths in the image in the DB when the image boots up, e.g. the ISO has this and this. You should be able to do something similar in your own repart image pretty easily. It might make sense for that to be a feature added to the image builder, though I’m less sure about that because it’s not clear if that DB should be populated from registration info during boot (like the ISO image does) or if the DB should be created at build time and included in the image.

2 Likes