I’m using NixOS as my home router and I wanted to setup tc_cake as my network scheduler.
I see that it appears to be installed by running man tc-cake. I also added iproute to the extraPackages used by the firewall service which I use it’s extraCommands option to run the tc ... commands I want.
I found online that apparently in Fedora/RHEL if you do not have the sch_netem module enabled it will give the following error I’m getting when I try to start the firewall service:
[root@cmp-nix1:/home/cmp]# systemctl status firewall.service
● firewall.service - Firewall
Loaded: loaded (/nix/store/l4j9wccxg3jj9nyxzfag6wr58f9gh9f7-unit-firewall.service/firewall.service; enabled; vendor preset: enabled)
Active: active (exited) since Mon 2021-02-22 15:46:55 EST; 1min 35s ago
Process: 3213 ExecReload=firewall-reload (code=exited, status=1/FAILURE)
Main PID: 441 (code=exited, status=0/SUCCESS)
IP: 0B in, 0B out
CPU: 2.239s
Feb 22 15:46:53 cmp-nix1 systemd[1]: Starting Firewall...
Feb 22 15:46:55 cmp-nix1 systemd[1]: Finished Firewall.
Feb 22 15:48:19 cmp-nix1 systemd[1]: Reloading Firewall.
Feb 22 15:48:22 cmp-nix1 firewall-reload[3368]: Error: Specified qdisc not found.
Feb 22 15:48:22 cmp-nix1 firewall-reload[3213]: Failed to reload firewall... Stopping
Feb 22 15:48:22 cmp-nix1 systemd[1]: firewall.service: Control process exited, code=exited, status=1/FAILURE
Feb 22 15:48:22 cmp-nix1 systemd[1]: Reload failed for Firewall.
Specifically the error I’m searching is “Specified qdisc not found.”
This is my networking config from my configuration.nix
networking = {
hostName = "cmp-nix1"; # Define your hostname.
wireless.enable = false; # Enables wireless support via wpa_supplicant.
useDHCP = false; # Disabled because it's on each interface
bridges = {
br0.interfaces = ["enp2s0" "enp3s0" "enp4s0"];
};
interfaces = {
# WAN Port
enp1s0.useDHCP = true;
# LAN Port
br0 = {
useDHCP = false;
ipv4 = {
addresses = [
{ address = "192.168.3.1"; prefixLength = 24;}
];
};
};
};
nat = {
enable = true;
externalInterface = "enp1s0";
#externalInterface = "enp2s0";
internalInterfaces = [ "br0" "tailscale0" ];
};
firewall = {
allowPing = true;
allowedTCPPorts = [ 22 ];
allowedUDPPorts = [ 41641 ];
interfaces = {
enp1s0 = {
allowedTCPPorts = [ ];
allowedUDPPorts = [ ];
};
br0 = {
allowedTCPPorts = [ 53 80 443 8443 8080 8880 8843 6789 ];
allowedUDPPorts = [ 53 3478 10001 ];
allowedUDPPortRanges = [ { from = 67; to = 68; }];
};
};
extraPackages = [ pkgs.iproute ];
extraCommands = ''
iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
iptables -A FORWARD -i br0 -o enp1s0 -j ACCEPT
tc qdisc add dev enp1s0 root cake besteffort autorate-ingress bandwidth 1000mbit lan ingress dual-dsthost ethernet
tc qdisc add dev br0 root cake besteffort autorate-ingress bandwidth 1000mbit lan dual-dsthost ethernet
# tc qdisc add dev enp1s0 handle 1: root cake besteffort autorate-ingress bandwidth $BANDWIDTH internet ingress dual-dsthost ethernet
'';
extraStopCommands = ''
iptables -D FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT || true
iptables -D FORWARD -i br0 -o enp1s0 -j ACCEPT || true
tc qdisc del dev enp1s0 root 2> /dev/null || true
tc qdisc del dev br0 root 2> /dev/null || true
'';
};
};
I also verified after reboot that sch_netem is running
[root@cmp-nix1:/home/cmp]# lsmod | grep sch_netem
sch_netem 24576 0
I think I’m missing some package or module. I’m not really sure what tc-cake needs or how to find that. There appears to be no mention of tc, sch_netem, or tc-cake anywhere in Nixpkgs.
It really feels like I’m missing something extremely obvious, but which is not mentioned in the man pages or in any searches I’m making.
Here is what apparently the qdisc is already setup with by default
[root@cmp-nix1:/etc/systemd/system]# tc qdisc
qdisc noqueue 0: dev lo root refcnt 2
qdisc mq 0: dev enp1s0 root
qdisc fq_codel 0: dev enp1s0 parent :4 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp1s0 parent :3 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp1s0 parent :2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp1s0 parent :1 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc mq 0: dev enp2s0 root
qdisc fq_codel 0: dev enp2s0 parent :4 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp2s0 parent :3 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp2s0 parent :2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp2s0 parent :1 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc mq 0: dev enp3s0 root
qdisc fq_codel 0: dev enp3s0 parent :4 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp3s0 parent :3 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp3s0 parent :2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp3s0 parent :1 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc mq 0: dev enp4s0 root
qdisc fq_codel 0: dev enp4s0 parent :4 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp4s0 parent :3 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp4s0 parent :2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc fq_codel 0: dev enp4s0 parent :1 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
qdisc noqueue 0: dev br0 root refcnt 2
qdisc fq_codel 0: dev tailscale0 root refcnt 2 limit 10240p flows 1024 quantum 1280 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64
And I tried to replace it with a different one and still kept giving me the error from before.
[root@cmp-nix1:/etc/systemd/system]# tc qdisc replace dev enp4s0 handle 1: root cake
Error: Specified qdisc not found.
[root@cmp-nix1:/etc/systemd/system]# tc qdisc replace dev enp4s0 handle 0: root cake
Error: Specified qdisc not found.
[root@cmp-nix1:/etc/systemd/system]# tc qdisc replace dev enp4s0 root cake
Error: Specified qdisc not found.
[root@cmp-nix1:/etc/systemd/system]# tc qdisc replace dev enp4s0 root handle 1: cake
Error: Specified qdisc not found.
I do add a bunch of kernel security features copied out of the hardened.nix profile: could this be the problem?
# Security Settings from secure profile in nixos repo
boot.kernelPackages = pkgs.linuxPackages_hardened;
# environment.memoryAllocator.provider = "scudo";
# environment.variables.SCUDO_OPTIONS = "ZeroContents=1";
security.hideProcessInformation = true;
security.lockKernelModules = true;
security.protectKernelImage = true;
security.allowSimultaneousMultithreading = false;
security.forcePageTableIsolation = true;
security.virtualisation.flushL1DataCache = "always";
security.apparmor.enable = true;
boot.kernelParams = [
# Slab/slub sanity checks, redzoning, and poisoning
"slub_debug=FZP"
# Overwrite free'd memory
"page_poison=1"
# Enable page allocator randomization
"page_alloc.shuffle=1"
];
boot.kernelModules = [
"sch_netem"
];
boot.blacklistedKernelModules = [
# Obscure network protocols
"ax25"
"netrom"
"rose"
# Old or rare or insufficiently audited filesystems
"adfs"
"affs"
"bfs"
"befs"
"cramfs"
"efs"
"erofs"
"exofs"
"freevxfs"
"f2fs"
"hfs"
"hpfs"
"jfs"
"minix"
"nilfs2"
"qnx4"
"qnx6"
"sysv"
"ufs"
];
# Restrict ptrace() usage to processes with a pre-defined relationship
# (e.g., parent/child)
boot.kernel.sysctl."kernel.yama.ptrace_scope" = "1";
# Hide kptrs even for processes with CAP_SYSLOG
boot.kernel.sysctl."kernel.kptr_restrict" = "2";
# Disable bpf() JIT (to eliminate spray attacks)
boot.kernel.sysctl."net.core.bpf_jit_enable" = false;
# Disable ftrace debugging
boot.kernel.sysctl."kernel.ftrace_enabled" = false;
# Enable strict reverse path filtering (that is, do not attempt to route
# packets that "obviously" do not belong to the iface's network; dropped
# packets are logged as martians).
boot.kernel.sysctl."net.ipv4.conf.all.log_martians" = true;
boot.kernel.sysctl."net.ipv4.conf.all.rp_filter" = "1";
boot.kernel.sysctl."net.ipv4.conf.default.log_martians" = true;
boot.kernel.sysctl."net.ipv4.conf.default.rp_filter" = "1";
# Ignore broadcast ICMP (mitigate SMURF)
boot.kernel.sysctl."net.ipv4.icmp_echo_ignore_broadcasts" = true;
So it turns out that I was actually missing something very obvious.
you must enable sch_cake kernel module. also setting it as the default queue discipline helps.