The problem
Hi all! I’m trying to generate a VM image for Firecracker but am running into some issues. Specifically, the VM complains that the device /dev/vda doesn’t show up.
Setup
Here is the Nix code that I am using to generate the configuration file for Firecracker. It consists of a flake which does most of the heavy lifting: generating a rootfs, extracting the kernel binary & initrd from the guest configuration ,and assembling it all into a configuration file for Firecracker.
flake.nix
{
description = "Code execution API";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/24.05";
};
outputs = { self, nixpkgs }:
{
nixosConfigurations = {
guest = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
./guest-configuration.nix
];
};
};
packages.x86_64-linux = {
# VMLinux kernel image to be passed to Firecracker when starting a guest
guest-vmlinux = self.nixosConfigurations.guest.config.system.build.kernel.dev;
# Base rootfs to be passed to Firecracker when starting a guest
guest-rootfs =
let
nixosSystem = self.nixosConfigurations.guest;
pkgs = nixpkgs.legacyPackages.x86_64-linux;
in
import "${nixpkgs}/nixos/lib/make-disk-image.nix" {
inherit pkgs;
inherit (pkgs) lib;
diskSize = "auto";
config = nixosSystem.config;
additionalSpace = "1K";
format = "raw";
partitionTableType = "none";
installBootLoader = false;
fsType = "ext4";
copyChannel = false;
};
# A basic VM configuration for testing.
# It can be run with firecracker like so:
# firecracker --config-file XXX --no-api
firecracker-config =
let
pkgs = nixpkgs.legacyPackages.x86_64-linux;
inherit (self.packages.x86_64-linux) guest-vmlinux guest-rootfs;
config = self.nixosConfigurations.guest.config;
vm-config = {
"boot-source" = {
"kernel_image_path" = "${guest-vmlinux}/vmlinux";
"boot_args" = "${toString config.boot.kernelParams} init=${config.system.build.toplevel}/init debug boot.trace boot.shell_on_fail";
"initrd_path" = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
};
"drives" = [
{
"drive_id" = "rootfs";
"is_root_device" = false;
"path_on_host" = "${guest-rootfs}/nixos.img";
"is_read_only" = true;
}
];
"machine-config" = {
"vcpu_count" = 2;
"mem_size_mib" = 1024;
"smt" = false;
"track_dirty_pages" = false;
"huge_pages" = "None";
};
"cpu-config" = null;
"balloon" = null;
"network-interfaces" = [];
"vsock" = null;
"logger" = null;
"metrics" = null;
"mmds-config" = null;
"entropy" = null;
};
in
pkgs.writers.writeJSON "vm-config.json" vm-config;
default = self.packages.x86_64-linux.firecracker-config;
};
};
}
The actual guest confiuration is stored in guest-configuration.nix. It doesn’t do a whole lot, except configure a custom kernel.
guest-configuraiton.nix
{ pkgs, config, lib, modulesPath, ... }:
with pkgs;
with lib;
{
imports = [
"${modulesPath}/profiles/minimal.nix"
];
# Open up root for easier debugging
services.openssh = {
enable = true;
startWhenNeeded = true;
settings.PermitRootLogin = "yes";
};
users.users.root.password = "root";
boot = {
# XXX
kernelParams = [ "console=ttyS0" "reboot=k" "panic=1" "pci=off" "noapic" ];
# We are going to pass the kernel, rootfs and initrd directly to firecracker
# for execution, so there's no need to emit stuff for the boot loader.
loader.grub.enable = false;
# These are needed to read disk /dev/vda.
# FIXME: Doesn't work.
initrd.kernelModules = [
"virtio_mmio"
"virtio_blk"
];
# Load our custom kernel
kernelPackages =
let
version = "6.1.69";
majorVersion = head (splitString "." version); # e.g. 6
kernel = (linuxKernel.manualConfig {
inherit stdenv lib version;
src = fetchTarball {
url = "https://cdn.kernel.org/pub/linux/kernel/v${majorVersion}.x/linux-${version}.tar.xz";
sha256 = "sha256:1xm83zn39wks7xjvwdm5z00qi5b1r48vyywss0iilbzs4nvdr3qs";
};
configfile = ./.config;
allowImportFromDerivation = true;
});
in
linuxPackagesFor kernel;
};
fileSystems."/" = {
device = "/dev/vda";
};
system.stateVersion = "24.05";
}
The kernel config is currently just a slightly (e.g. I removed Bluetooth) stripped version of my host kernel (the default NixOS 24.05 kernel) as found under /proc/config.gz. I obviously want to use a more minimal build in the future, but I figured it was a safer bet to start with a very large kernel.
I hit the upper character limit, so see this paste for .config.
Running it
The default output of the flake is a JSON configuration file for Firecracker.
The final configuration file
$ nix build
$ jq <result
{
"balloon": null,
"boot-source": {
"boot_args": "console=ttyS0 reboot=k panic=1 pci=off noapic loglevel=4 init=/nix/store/49rnwhjfk0q0q99ywivsqlnbig4sj0rb-nixos-system-nixos-24.05.20240531.63dacb4/init debug boot.trace boot.shell_on_fail",
"initrd_path": "/nix/store/imajf1vvfp2870k1yl77xz5yxzgwzhzy-initrd-linux-6.1.69/initrd",
"kernel_image_path": "/nix/store/0ja35h6ppbvv2mn7ik5j6sk1vcs3rwf5-linux-6.1.69-dev/vmlinux"
},
"cpu-config": null,
"drives": [
{
"drive_id": "rootfs",
"is_read_only": true,
"is_root_device": false,
"path_on_host": "/nix/store/km9h00x98j76m2s0008n5zkzh0mjlmh6-nixos-disk-image/nixos.img"
}
],
"entropy": null,
"logger": null,
"machine-config": {
"huge_pages": "None",
"mem_size_mib": 1024,
"smt": false,
"track_dirty_pages": false,
"vcpu_count": 2
},
"metrics": null,
"mmds-config": null,
"network-interfaces": [],
"vsock": null
}
Then I can pass the generated configuration file to firecracker for it to run it.
$ nix build
$ firecracker --config-file result --no-api
See this paste for the logs from Firecracker and the guest kernel. I think the major takeaway from the linked logs is this line:
[ 22.152128] stage-1-init: [Wed Nov 13 12:05:43 UTC 2024] Timed out waiting for device /dev/vda, trying to mount anyway.
However, that’s where I run into the issue with init in the initramfs not finding /dev/vda. I know /dev/vdX are for virtualised drives, but I don’t know why it isn’t showing up or which system is responsible for making it show up.
Can’t mount /dev/vda
This line seems to indicate that Firecracker is injecting an argument indicating where the virtio device is located.
[ 0.000000] Command line: console=ttyS0 reboot=k panic=1 pci=off noapic loglevel=4 init=/nix/store/49rnwhjfk0q0q99ywivsqlnbig4sj0rb-nixos-system-nixos-24.05.20240531.63dacb4/init debug boot.trace boot.shell_on_fail virtio_mmio.device=4K@0xd0000000:5
But then something clearly goes wrong when trying to load that thing, as evidenced by these lines:
[ 0.405134] virtio-mmio: Registering device virtio-mmio.0 at 0xd0000000-0xd0000fff, IRQ 5.
[ 0.406100] stage-1-init: [Wed Nov 13 12:05:22 UTC 2024] + info 'loading module virtio_mmio...'
[ 0.406142] virtio-mmio virtio-mmio.0: can't request region for resource [mem 0xd0000000-0xd0000fff]
[ 0.407926] virtio-mmio: probe of virtio-mmio.0 failed with error -16
…and that’s about where I’m at. I don’t know how make the device /dev/vda appear, so I can’t mount it and start actually doing interesting stuff inside the VM.