Goal: Run an Ubuntu VM using QEMU/KVM on virtmanager.
Issue: Black Screen is the output from the GPU being passthrough.
Hardware:
CPU: Intel 12700K “Alder Lake”
Mobo: Asus Prime Z690-P D4 Wifi
Host GPU: Intel 770 UHD Integrated Graphics
Guest GPU: Vega 56 with 64 vBIOS flash
Ethernet: Realtek 2.5g
USB: 3.2gen Inatek
BIOS settings:
VT-x: Enabled
VT-d: Enabled
IOMMU: Enabled
MultiMonitor: Enabled
Primary GPU: CPU Graphics
Secure Boot: Disabled
CSM: Disabled
configuration.nix:
# Edit this configuration file to define what should be installed on
# your system. Help is available in the configuration.nix(5) man page
# and in the NixOS manual (accessible by running ‘nixos-help’).
{ config, pkgs, ... }:
{
imports =
[ # Include the results of the hardware scan.
./hardware-configuration.nix
./vfio.nix
];
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
# Setup virtualization and VFIO
virtualisation = {
libvirtd = {
enable = true;
qemu.ovmf.enable = true;
qemu.swtpm.enable = true;
onBoot = "ignore";
};
vfio = {
enable = true;
IOMMUType = "intel";
devices = [ "1002:687f" "1002:aaf8" "8086:7af0" "10ec:8125" "1b21:2142" ];
ignoreMSRs = true;
disableEFIfb = true;
blacklistAMD = true;
enableNestedVirt = true;
};
};
# Set your network preferences.
networking = {
networkmanager.enable = true;
hostName = "[redacted]"; # Define your hostname.
# The global useDHCP flag is deprecated, therefore explicitly set to false here.
# Per-interface useDHCP will be mandatory in the future, so this generated config
# replicates the default behaviour.
# useDHCP = false;
# interfaces.wlp8s0.useDHCP = true;
# wireless.enable = true; # Enables wireless support via wpa_supplicant.
# wireless.userControlled.enable = true; # Enables wpa_cli
# Configure network proxy if necessary
# proxy.default = "http://user:password@proxy:port/";
# proxy.noProxy = "127.0.0.1,localhost,internal.domain";
# Open ports in the firewall.
# firewall.allowedTCPPorts = [ ... ];
# firewall.allowedUDPPorts = [ ... ];
# Or disable the firewall altogether.
# firewall.enable = false;
};
# Set your time zone.
time.timeZone = "America/Los_Angeles";
# Select internationalisation properties.
# i18n.defaultLocale = "en_US.UTF-8";
# console = {
# font = "Lat2-Terminus16";
# keyMap = "us";
# };
# Enable the X11 windowing system.
services.xserver = {
enable = true;
displayManager.sddm.enable = true;
desktopManager.plasma5.enable = true;
# Configure keymap in X11
# layout = "us";
# xkbOptions = "eurosign:e";
# Enable touchpad support (enabled default in most desktopManager).
# libinput.enable = true;
};
# Enable CUPS to print documents.
services.printing.enable = true;
# Enable sound.
sound.enable = true;
hardware.pulseaudio = {
enable = true;
extraModules = [ pkgs.pulseaudio-modules-bt ];
extraConfig = "load-module module-switch-on-connect";
support32Bit = true;
package = pkgs.pulseaudioFull;
};
# Enable Intel accelerated graphics
hardware.opengl = {
enable = true;
extraPackages = with pkgs; [
intel-media-driver # LIBVA_DRIVER_NAME=iHD
vaapiIntel # LIBVA_DRIVER_NAME=i965 (older but works better for Firefox/Chromium)
vaapiVdpau
libvdpau-va-gl
];
};
# Enable bluetooth
hardware.bluetooth = {
enable = true;
settings = {
General = {
Enable = "Source,Sink,Media,Socket";
};
};
};
# Define a user account. Don't forget to set a password with ‘passwd’.
users.users.edgar = {
isNormalUser = true;
extraGroups = [ "wheel" "libvirtd" "kvm" "qemu-libvirtd" "audio" ]; # Enable ‘sudo’ and swtpm for the user.
};
# Configure nix packages
nixpkgs.config = {
allowUnfree = true;
pulseaudio = true;
# Intel Graphics support in nixpkgs
packageOverrides = pkgs: {
vaapiIntel = pkgs.vaapiIntel.override { enableHybridCodec = true; };
};
};
# List packages installed in system profile. To search, run:
# $ nix search wget
environment.systemPackages = with pkgs; [
firefox
git
gcc
libvirt
vim
wget
yubioath-desktop
akonadi
libsForQt5.kdeApplications.akonadiconsole
libsForQt5.kdeApplications.akonadi-search
libsForQt5.kdeApplications.kwalletmanager
libsForQt5.ark
];
environment.sessionVariables.LIBVIRT_DEFAULT_URI = [ "qemu:///system" ];
# Enable YubiKey authentication.
services.udev.packages = [
pkgs.yubikey-personalization
pkgs.libu2f-host
pkgs.crda
];
services.pcscd.enable = true;
# Some programs need SUID wrappers, can be configured further or are
# started in user sessions.
# programs.mtr.enable = true;
# programs.gnupg.agent = {
# enable = true;
# enableSSHSupport = true;
# };
programs.ssh.forwardX11 = true;
# Enable the OpenSSH daemon.
# services.openssh.enable = true;
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. It‘s perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "21.05"; # Did you read the comment?
}
vfio.nix:
{ lib, pkgs, config, ... }:
with lib;
let cfg = config.virtualisation.vfio;
in {
options.virtualisation.vfio = {
enable = mkEnableOption "VFIO Configuration";
IOMMUType = mkOption {
type = types.enum [ "intel" "amd" ];
example = "intel";
description = "Type of the IOMMU used.";
};
devices = mkOption {
type = types.listOf (types.strMatching "[0-9a-f]{4}:[0-9a-f]{4}");
default = [ ];
example = [ "10de:1b80" "10de:10f0" ];
description = "PCI IDs of devices to bind to vfio-pci.";
};
disableEFIfb = mkOption {
type = types.bool;
default = false;
description = "Disables the usage of the EFI framebuffer on boot.";
};
blacklistNvidia = mkOption {
type = types.bool;
default = false;
description = "Add Nvidia GPU modules to blacklist.";
};
blacklistAMD = mkOption {
type = types.bool;
default = false;
description = "Add AMD GPU modules to blacklist.";
};
ignoreMSRs = mkOption {
type = types.bool;
default = false;
description = "Enables or disables kvm guest access to model-specific registers.";
};
enableNestedVirt = mkOption {
type = types.bool;
default = false;
description = "Enables nested virtualization.";
};
applyACSpatch = mkOption {
type = types.bool;
default = false;
description = ''
If set, the following things will happen:
- The ACS override patch is applied
- Applies the i915-vga-arbiter patch
- Adds pcie_acs_override=downstream to the command line
'';
};
};
config = lib.mkIf cfg.enable {
boot.kernelParams = (if cfg.IOMMUType == "intel" then [
"intel_iommu=on"
"intel_iommu=igfx_off"
] else
[ "amd_iommu=on" ]) ++ (optional (builtins.length cfg.devices > 0)
("vfio-pci.ids=" + builtins.concatStringsSep "," cfg.devices))
++ (optional cfg.applyACSpatch
"pcie_acs_override=downstream,multifunction")
++ (optionals cfg.disableEFIfb [ "video=efifb:off" "video=vesafb:off" "quiet" ])
++ (optional cfg.ignoreMSRs "kvm.ignore_msrs=1");
boot.kernelModules = [ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ];
boot.initrd.kernelModules =
[ "vfio_virqfd" "vfio_pci" "vfio_iommu_type1" "vfio" ];
boot.blacklistedKernelModules =
(optionals cfg.blacklistNvidia [ "nvidia" "nouveau" ])
++ (optionals cfg.blacklistAMD [ "amdgpu" "radeon" ]);
boot.extraModprobeConfig = if cfg.enableNestedVirt then
"options kvm_${cfg.IOMMUType} nested=1"
else "";
boot.kernelPatches = optionals cfg.applyACSpatch [
{
name = "add-acs-overrides";
patch = pkgs.fetchurl {
name = "add-acs-overrides.patch";
url =
"https://aur.archlinux.org/cgit/aur.git/plain/add-acs-overrides.patch?h=linux-vfio&id=c580c0ca71ad7f64191f74606eaff7ab757f0700";
sha256 = "1hhbm9fmc69h1z75gbq311cdkvv1rcmivzxlqmmgkgk646jz8lh3";
};
}
{
name = "i915-vga-arbiter";
patch = pkgs.fetchurl {
name = "i915-vga-arbiter.patch";
url =
"https://aur.archlinux.org/cgit/aur.git/plain/i915-vga-arbiter.patch?h=linux-vfio&id=d1143eaeb4ce71590c61e8a9b281037ae5c87fa8";
sha256 = "1kscqwrjm9kxhavyq92mhxgr5jjq2id24682hkqc5kjj5f82jjh9";
};
}
];
};
}
hardware-configuration.nix:
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-intel" "wl" ];
boot.kernelPackages = pkgs.linuxPackages_5_16;
boot.extraModulePackages = [ config.boot.kernelPackages.broadcom_sta ];
boot.extraModprobeConfig = ''
softdep amdgpu pre: vfio-pci
softdep snd_hda_intel pre: vfio-pci
options vfio_pci disable_vga=1
'';
fileSystems."/" =
{ device = "/dev/disk/by-uuid/4e7bdc8e-b4e3-474f-8003-be1f2c0f9863";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/73C8-AF43";
fsType = "vfat";
};
swapDevices = [ ];
}
libvirt config:
<domain xmlns:qemu="http://libvirt.org/schemas/domain/qemu/1.0" type="kvm">
<name>ubuntu</name>
<uuid>0168cc47-9820-49ec-9005-ae96bd5670f5</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://ubuntu.com/ubuntu/20.04"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">49173504</memory>
<currentMemory unit="KiB">49173504</currentMemory>
<vcpu placement="static">16</vcpu>
<cputune>
<vcpupin vcpu="0" cpuset="0"/>
<vcpupin vcpu="1" cpuset="1"/>
<vcpupin vcpu="2" cpuset="2"/>
<vcpupin vcpu="3" cpuset="3"/>
<vcpupin vcpu="4" cpuset="4"/>
<vcpupin vcpu="5" cpuset="5"/>
<vcpupin vcpu="6" cpuset="6"/>
<vcpupin vcpu="7" cpuset="7"/>
<vcpupin vcpu="8" cpuset="8"/>
<vcpupin vcpu="9" cpuset="9"/>
<vcpupin vcpu="10" cpuset="10"/>
<vcpupin vcpu="11" cpuset="11"/>
<vcpupin vcpu="12" cpuset="12"/>
<vcpupin vcpu="13" cpuset="13"/>
<vcpupin vcpu="14" cpuset="14"/>
<vcpupin vcpu="15" cpuset="15"/>
</cputune>
<os>
<type arch="x86_64" machine="pc-q35-6.0">hvm</type>
<loader readonly="yes" type="pflash">/home/edgar/Documents/vm/EDK2s/ubuntu/OVMF_CODE.fd</loader>
<nvram>/home/edgar/Documents/vm/EDK2s/ubuntu/OVMF_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<pae/>
<hyperv>
<vpindex state="on"/>
<synic state="on"/>
</hyperv>
<kvm>
<hidden state="on"/>
</kvm>
<vmport state="off"/>
<ioapic driver="kvm"/>
</features>
<cpu mode="host-passthrough" check="none" migratable="on">
<topology sockets="1" dies="1" cores="8" threads="2"/>
<cache mode="passthrough"/>
<feature policy="disable" name="hypervisor"/>
<feature policy="require" name="vmx"/>
</cpu>
<clock offset="utc"/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/run/libvirt/nix-emulators/qemu-system-x86_64</emulator>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"/>
<source file="/home/edgar/Downloads/en-us_windows_11_consumer_editions_x64_dvd_bd3cf8df.iso"/>
<target dev="sda" bus="sata"/>
<readonly/>
<address type="drive" controller="0" bus="0" target="0" unit="0"/>
</disk>
<disk type="block" device="disk">
<driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
<source dev="/dev/nvme0n1p4"/>
<target dev="sdc" bus="sata"/>
<boot order="1"/>
<address type="drive" controller="0" bus="0" target="0" unit="2"/>
</disk>
<controller type="usb" index="0" model="ich9-ehci1">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x7"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci1">
<master startport="0"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x0" multifunction="on"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci2">
<master startport="2"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x1"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci3">
<master startport="4"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x1d" function="0x2"/>
</controller>
<controller type="sata" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="pci" index="0" model="pcie-root"/>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x10"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x11"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0x12"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0x13"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<controller type="pci" index="5" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="5" port="0x14"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
</controller>
<controller type="pci" index="6" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="6" port="0x15"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
</controller>
<controller type="pci" index="7" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="7" port="0x16"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
</controller>
<controller type="pci" index="8" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="8" port="0x17"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x7"/>
</controller>
<controller type="pci" index="9" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="9" port="0x18"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="10" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="10" port="0x19"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x1"/>
</controller>
<controller type="pci" index="11" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="11" port="0x1a"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x2"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<channel type="spicevmc">
<target type="virtio" name="com.redhat.spice.0"/>
<address type="virtio-serial" controller="0" bus="0" port="1"/>
</channel>
<input type="mouse" bus="ps2"/>
<input type="keyboard" bus="ps2"/>
<tpm model="tpm-tis">
<backend type="emulator" version="2.0"/>
</tpm>
<audio id="1" type="spice"/>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x08" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0" multifunction="on"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x1"/>
</hostdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="2"/>
</redirdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="3"/>
</redirdev>
<memballoon model="virtio">
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</memballoon>
<rng model="virtio">
<backend model="random">/dev/urandom</backend>
<address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</rng>
</devices>
<qemu:commandline>
<qemu:arg value="-monitor"/>
<qemu:arg value="stdio"/>
<qemu:arg value="-display"/>
<qemu:arg value="none"/>
</qemu:commandline>
</domain>
ubuntu.log (qemu errors):
...
audio: Could not init `spice' audio driver
audio: warning: Using timer based audio emulation
QEMU 6.1.0 monitor - type 'help' for more information
(qemu) 2022-01-31T22:42:25.961960Z qemu-system-x86_64: terminating on signal 15 from pid 1273 (/nix/store/x0jqkf81xx1fvpwrdskbjsc8bmdl36ij-libvirt-7.10.0/sbin/libvirtd)
2022-01-31 22:42:29.568+0000: shutting down, reason=destroyed
***The last error is from me using “Force Off” to shut down the vm. Everything above that log is the libvirt configuration as a qemu command.
lsiommu:
IOMMU Group 0 00:00.0 Host bridge [0600]: Intel Corporation Device [8086:4668] (rev 02)
IOMMU Group 1 00:01.0 PCI bridge [0604]: Intel Corporation Device [8086:460d] (rev 02)
IOMMU Group 2 00:06.0 PCI bridge [0604]: Intel Corporation Device [8086:464d] (rev 02)
IOMMU Group 3 00:0a.0 Signal processing controller [1180]: Intel Corporation Device [8086:467d] (rev 01)
IOMMU Group 4 00:0e.0 RAID bus controller [0104]: Intel Corporation Volume Management Device NVMe RAID Controller [8086:467f]
IOMMU Group 5 00:14.0 USB controller [0c03]: Intel Corporation Device [8086:7ae0] (rev 11)
IOMMU Group 5 00:14.2 RAM memory [0500]: Intel Corporation Device [8086:7aa7] (rev 11)
IOMMU Group 6 00:14.3 Network controller [0280]: Intel Corporation Device [8086:7af0] (rev 11)
IOMMU Group 7 00:15.0 Serial bus controller [0c80]: Intel Corporation Device [8086:7acc] (rev 11)
IOMMU Group 7 00:15.1 Serial bus controller [0c80]: Intel Corporation Device [8086:7acd] (rev 11)
IOMMU Group 7 00:15.2 Serial bus controller [0c80]: Intel Corporation Device [8086:7ace] (rev 11)
IOMMU Group 8 00:16.0 Communication controller [0780]: Intel Corporation Device [8086:7ae8] (rev 11)
IOMMU Group 9 00:17.0 SATA controller [0106]: Intel Corporation Device [8086:7ae2] (rev 11)
IOMMU Group 10 00:1a.0 PCI bridge [0604]: Intel Corporation Device [8086:7ac8] (rev 11)
IOMMU Group 11 00:1b.0 PCI bridge [0604]: Intel Corporation Device [8086:7ac0] (rev 11)
IOMMU Group 12 00:1c.0 PCI bridge [0604]: Intel Corporation Device [8086:7ab8] (rev 11)
IOMMU Group 13 00:1c.1 PCI bridge [0604]: Intel Corporation Device [8086:7ab9] (rev 11)
IOMMU Group 14 00:1c.2 PCI bridge [0604]: Intel Corporation Device [8086:7aba] (rev 11)
IOMMU Group 15 00:1d.0 PCI bridge [0604]: Intel Corporation Device [8086:7ab0] (rev 11)
IOMMU Group 16 00:1d.4 PCI bridge [0604]: Intel Corporation Device [8086:7ab4] (rev 11)
IOMMU Group 17 00:1f.0 ISA bridge [0601]: Intel Corporation Device [8086:7a84] (rev 11)
IOMMU Group 17 00:1f.3 Audio device [0403]: Intel Corporation Device [8086:7ad0] (rev 11)
IOMMU Group 17 00:1f.4 SMBus [0c05]: Intel Corporation Device [8086:7aa3] (rev 11)
IOMMU Group 17 00:1f.5 Serial bus controller [0c80]: Intel Corporation Device [8086:7aa4] (rev 11)
IOMMU Group 18 01:00.0 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Vega 10 PCIe Bridge [1022:1470] (rev c1)
IOMMU Group 19 02:00.0 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Vega 10 PCIe Bridge [1022:1471]
IOMMU Group 20 03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 10 XL/XT [Radeon RX Vega 56/64] [1002:687f] (rev c1)
IOMMU Group 21 03:00.1 Audio device [0403]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 10 HDMI Audio [Radeon Vega 56/64] [1002:aaf8]
IOMMU Group 22 04:00.0 Non-Volatile memory controller [0108]: Phison Electronics Corporation E12 NVMe Controller [1987:5012] (rev 01)
IOMMU Group 23 05:00.0 Non-Volatile memory controller [0108]: Samsung Electronics Co Ltd NVMe SSD Controller SM981/PM981/PM983 [144d:a808]
IOMMU Group 24 08:00.0 Network controller [0280]: Broadcom Inc. and subsidiaries BCM4360 802.11ac Wireless Network Adapter [14e4:43a0] (rev 03)
IOMMU Group 25 09:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd. RTL8125 2.5GbE Controller [10ec:8125] (rev 05)
IOMMU Group 26 0a:00.0 USB controller [0c03]: ASMedia Technology Inc. ASM2142 USB 3.1 Host Controller [1b21:2142]
IOMMU Group 27 0b:00.0 Non-Volatile memory controller [0108]: Samsung Electronics Co Ltd NVMe SSD Controller SM981/PM981/PM983 [144d:a808]
lspci -v (guest GPU):
03:00.0 VGA compatible controller: Advanced Micro Devices, Inc. [AMD/ATI] Vega 10 XL/XT [Radeon RX Vega 56/64] (rev c1) (prog-if 00 [VGA controller])
Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] RX Vega64
Flags: fast devsel, IRQ 16, IOMMU group 20
Memory at 6000000000 (64-bit, prefetchable) [size=256M]
Memory at 6010000000 (64-bit, prefetchable) [size=2M]
I/O ports at 5000 [size=256]
Memory at 84d00000 (32-bit, non-prefetchable) [size=512K]
Expansion ROM at 84d80000 [disabled] [size=128K]
Capabilities: <access denied>
Kernel driver in use: vfio-pci
Kernel modules: amdgpu
03:00.1 Audio device: Advanced Micro Devices, Inc. [AMD/ATI] Vega 10 HDMI Audio [Radeon Vega 56/64]
Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Vega 10 HDMI Audio [Radeon Vega 56/64]
Flags: fast devsel, IRQ 17, IOMMU group 21
Memory at 84da0000 (32-bit, non-prefetchable) [size=16K]
Capabilities: <access denied>
Kernel driver in use: vfio-pci
Kernel modules: snd_hda_intel
dmesg | grep vfio:
[ 0.000000] Command line: initrd=\efi\nixos\99m406shm28v1ip5gpcfqsjpk1j89wd8-initrd-linux-5.16.2-initrd.efi init=/nix/store/91yvr7v4fsbk986q7kab3cn84yvfmbwb-nixos-system-[redacted]-22.05pre348581.c07b471b52b/init intel_iommu=on intel_iommu=igfx_off vfio-pci.ids=1002:687f,1002:aaf8,8086:7af0,10ec:8125,1b21:2142 video=efifb:off video=vesafb:off quiet kvm.ignore_msrs=1 loglevel=4
[ 0.088378] Kernel command line: initrd=\efi\nixos\99m406shm28v1ip5gpcfqsjpk1j89wd8-initrd-linux-5.16.2-initrd.efi init=/nix/store/91yvr7v4fsbk986q7kab3cn84yvfmbwb-nixos-system-[redacted]-22.05pre348581.c07b471b52b/init intel_iommu=on intel_iommu=igfx_off vfio-pci.ids=1002:687f,1002:aaf8,8086:7af0,10ec:8125,1b21:2142 video=efifb:off video=vesafb:off quiet kvm.ignore_msrs=1 loglevel=4
[ 0.441933] stage-1-init: [Mon Jan 31 22:41:18 UTC 2022] loading module vfio_virqfd...
[ 0.448763] stage-1-init: [Mon Jan 31 22:41:18 UTC 2022] loading module vfio_pci...
[ 0.453383] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
[ 0.464780] vfio_pci: add [1002:687f[ffffffff:ffffffff]] class 0x000000/00000000
[ 0.477000] vfio_pci: add [1002:aaf8[ffffffff:ffffffff]] class 0x000000/00000000
[ 0.488918] vfio_pci: add [8086:7af0[ffffffff:ffffffff]] class 0x000000/00000000
[ 0.501038] vfio_pci: add [10ec:8125[ffffffff:ffffffff]] class 0x000000/00000000
[ 0.512920] vfio_pci: add [1b21:2142[ffffffff:ffffffff]] class 0x000000/00000000
[ 0.513883] stage-1-init: [Mon Jan 31 22:41:18 UTC 2022] loading module vfio_iommu_type1...
[ 0.514445] stage-1-init: [Mon Jan 31 22:41:18 UTC 2022] loading module vfio...
[ 2.393436] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
[ 3.258128] vfio-pci 0000:09:00.0: invalid VPD tag 0x00 (size 0) at offset 0; assume missing optional EEPROM
[ 41.223043] vfio-pci 0000:09:00.0: enabling device (0000 -> 0003)
[ 42.287328] vfio-pci 0000:09:00.0: vfio_ecap_init: hiding ecap 0x1e@0x20c
[ 43.311171] vfio-pci 0000:0a:00.0: vfio_ecap_init: hiding ecap 0x19@0x200
[ 43.311179] vfio-pci 0000:0a:00.0: vfio_ecap_init: hiding ecap 0x1e@0x400
[ 43.324971] vfio-pci 0000:03:00.0: enabling device (0002 -> 0003)
[ 43.325218] vfio-pci 0000:03:00.0: vfio_ecap_init: hiding ecap 0x19@0x270
[ 43.325226] vfio-pci 0000:03:00.0: vfio_ecap_init: hiding ecap 0x1b@0x2d0
[ 43.337998] vfio-pci 0000:03:00.1: enabling device (0000 -> 0002)
I know the other PCIe devices are being passthrough correctly after booting up the VM without the guest GPU connected. This shows that IOMMU groups and the relevant virtualization protocols are enabled and working. I passthrough solely the performance (P) cores to the VM, where the host is left with the efficiency (E) cores. I’ve also ran this NixOS setup on an AMD Ryzen Threadripper 3960X successfuly, but the only meaningful change I see between these two platforms is the use of integrated graphics over a discreet GPU (WX 7100) for the host. I’ve noticed others tinkering with Xorg and setting the driver for the iGPU explicitly. This results in NixOS booting into a shell without any graphics displayed (i.e. no desktop environment) on my end.
I’m not sure where to continue from here and would appreciate any help.