Wrapping system.build.vm for checks?

I’m working on a NixOS config and I need to do some special setup for my VM when a check runs.

In practice, this is for setting up qemu tap networking in a netns.

For this example, I just want to print “Hello” on the host before the VM starts and “Goodbye” after the VM closes.

Here’s my Flake:

{
  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
  outputs = { self, nixpkgs, ... }:
    let
      system = "x86_64-linux";
      module = { pkgs, lib, ... }:
        {
          config = {
            services.getty.autologinUser = "root";
            virtualisation.vmVariant.virtualisation.graphics = false;
            boot.loader = {
              systemd-boot.enable = true;
              efi.canTouchEfiVariables = true;
            };
            nixpkgs.hostPlatform = system;
            system.stateVersion = "23.11";
            fileSystems."/" = lib.mkVMOverride { device = "/dev/vda1"; fsType = "ext4"; };
          };
        };
    in
    {
      checks.${system} =
        {
          "example-check" = (nixpkgs.lib.nixos.runTest {
            name = "example-check";
            hostPkgs = nixpkgs.legacyPackages.${system};
            node.specialArgs = { inherit self; };
            nodes = {
              machine = { pkgs, ... }: {
                imports = [ module ];
                environment.systemPackages = [ pkgs.coreutils ];
              };
            };
            testScript = ''
              start_all
              machine.wait_for_unit("multi-user.target")
              output = machine.succeed ("true")
              print(output)
            '';
          }).config.result;
        };
    };
}

This passes nix flake check -Lv just fine.

I can also build it with nix build 'path:.#checks.x86_64-linux.example-check.driver' -Lv to check the scripts generated by the build process.

When I do, this is the generated result/bin/nixos-test-driver:

#! /nix/store/5l50g7kzj7v0rdhshld1vx46rf2k5lf9-bash-5.2p26/bin/bash -e
export startScripts='/nix/store/xjzm6al5pgpnp1kgj56j3hsqvqm4cvr2-nixos-vm/bin/run-machine-vm'
export testScript='/nix/store/hkw2fx5sf586nwhinzav9ck1n3gcyvh1-nixos-test-driver-example-check/test-script'
export globalTimeout='3600'
export vlans='1'
exec -a "$0" "/nix/store/hkw2fx5sf586nwhinzav9ck1n3gcyvh1-nixos-test-driver-example-check/bin/.nixos-test-driver-wrapped"  "$@"

When I inspect run-machine-vm, I get the qemu vm launch script I expect.

I thought the solution might look something like this, but it causes infinite recursion:

nodes = {
  machine = { config, pkgs, lib, ... }: {
    imports = [ module ];
    system.build.vm = lib.mkForce (pkgs.runCommand "custom-vm" { buildInputs = [ pkgs.coreutils ]; } ''
      cat > run-vm.sh <<'EOF'
      #!/bin/sh
      echo "Hello"
      ${config.system.build.vm}/bin/run-*-vm
      echo "Goodbye"
      EOF
      chmod +x run-vm.sh
      mkdir -p $out/bin
      cp run-vm.sh $out/bin/run-custom-vm
    '');
    environment.systemPackages = [ pkgs.curl pkgs.coreutils ];
  };
};

Another idea was to add a wrapper module as nodes.machine.imports = [ module wrapVMModule ], but this too causes infinite recursion:

      wrapVMModule = { config, pkgs, lib, ... }:
        {
          system.build.vm = lib.mkForce (pkgs.writeShellApplication {
            name = "vm-wrapper";
            text = ''
              set -euxo pipefail
              echo "Hello"
              ${config.system.build.vm}/bin/run-*-vm
              echo "Goodbye"
            ''; #'
          });
        };

What is the idiomatic way to wrap the check VM launcher so I can do my setup/teardown?

I think if I could somehow wrap system.build.vm, I could do the setup I need.

If I was to fork nixpkgs, I’m sure I could add what I need, but that seems excessive (maybe not?).

I also thought I might be able to manipulate options.definitions directly, but this is not visible under checks.x86_64-linux.example-check.config.nodes.machine in the repl.

Any guidance is appreciated :pray:

1 Like

I figured it out!

Here’s a working solution:

{
  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
  outputs = { self, nixpkgs, ... }:
    let
      system = "x86_64-linux";
      module = { pkgs, lib, ... }:
        {
          config = {
            services.getty.autologinUser = "root";
            virtualisation.vmVariant.virtualisation.graphics = false;
            boot.loader = {
              systemd-boot.enable = true;
              efi.canTouchEfiVariables = true;
            };
            nixpkgs.hostPlatform = system;
            system.stateVersion = "23.11";
            fileSystems."/" = lib.mkVMOverride { device = "/dev/vda1"; fsType = "ext4"; };
          };
        };
    in
    {
      checks.${system} =
        {
          "example-check" = (nixpkgs.lib.nixos.runTest ({ config, hostPkgs, lib, ... }: {
            name = "example-check";
            hostPkgs = nixpkgs.legacyPackages.${system};
            rawTestDerivation = lib.mkForce (hostPkgs.stdenv.mkDerivation {
              name = "vm-test-run-${config.name}";
              requiredSystemFeatures = [ "nixos-test" ]
                ++ lib.optionals hostPkgs.stdenv.hostPlatform.isLinux [ "kvm" ]
                ++ lib.optionals hostPkgs.stdenv.hostPlatform.isDarwin [ "apple-virt" ];
              buildCommand = ''
                set -exuo pipefail
                mkdir -p $out

                echo "Hello"
                ${config.driver}/bin/nixos-test-driver -o $out
                echo "Goodbye"
              '';
            });
            node.specialArgs = { inherit self; };
            nodes = {
              machine = { config, options, pkgs, lib, ... }:
                {
                  imports = [ module ];
                };
            };
            testScript = ''
              start_all
              machine.wait_for_unit("multi-user.target")
              output = machine.succeed ("true")
              print(output)
            '';
          }));
        };
    };
}

This works exactly how I would expect, but I had to copy from nixos/lib/testing/run.nix and repeat attributes that I’m not changing, so it doesn’t feel particularly DRY.

I’m going to go with this for now because it works well enough, but if anyone has suggestions on how to improve the solution, please leave a comment :grin:

1 Like