I have setup a erigon etherum node running inside Native NixOS containers. my zfs mouint point are legacy and mounted by declarative config.
when set container to autostart, the caplin inside erigon always started with an old number (might be the snapshot date i have taken), and after a while, this app crashed with a long callstack about “mdbx errors”. the erigon service inside container are not autostart, even i wait for a while and manually start service, i still get this error.
I have solved it with a workaround, by delaying the bootstrap of container after zfs mounted.
- set contianer autostart = false
- make a systemd service, wait for zfs done, and manually start container
systemd.services.enableEtherumContainer = {
enable = true;
description = "enableEtherumContainer";
unitConfig = {
After = ["zfs.target" ];
};
wantedBy = [ "multi-user.target" ];
script = ''
#echo $(/run/current-system/sw/bin/zfs get mounted sn640/erigondata)
/run/current-system/sw/bin/nixos-container start etherum
/run/current-system/sw/bin/nixos-container run etherum -- systemctl start erigon
'';
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
#ExecStart = "/run/current-system/sw/bin/nixos-container start etherum";
};
};
how to adjust the startup order of Native NixOS containers?
This doesn’t particularly make sense. ZFS file systems are mounted before local-fs.target is reached, and nixos containers have default dependencies which implies they’re ordered after local-fs.target. This really shouldn’t make a difference, and there shouldn’t be a problem in the first place.
you are right, i can’t reproduce the errors…
I think I might fix this by pointing to a new bootstrap script , and manually boot the correct one.
I can partly reproduce now…
The only difference conf is wantedBy = [ "default.target" ]; make service autoboot.
after i see the serivice log
May 04 13:13:52 etherum erigon-start[186]: mdbx_setup_dxb:15946 filesize mismatch (expect 2544650682368b/310626304p, have 2544701014016b/310632448p)
i immeidately shutdown the service not to continue (it may continue works fine ),
while remove wantedBy = [ "default.target" ]; and manual start by triggering nixos-container run etherum -- systemctl start erigon don’t get this message
my conf settings:
# Edit this configuration file to define what should be installed on
# your system. Help is available in the configuration.nix(5) man page
# and in the NixOS manual (accessible by running ‘nixos-help’).
{ config, pkgs, ... }:
{
imports =
[ # Include the results of the hardware scan.
./hardware-configuration.nix
];
# Bootloader.
boot.loader.grub.enable = true;
boot.loader.grub.device = "/dev/sda";
boot.loader.grub.useOSProber = true;
boot.kernelModules = with config.boot.kernelModules; [ "msr" ];
boot.kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
boot.kernelParams = [ "nohibernate" "init_on_alloc=0" "zfs.zfs_arc_sys_free=${toString(1024*1024*1024*24)}" ];
boot.supportedFilesystems = [ "zfs" ];
boot.zfs.forceImportRoot = false;
boot.zfs.extraPools = [ "sn640" ];
fileSystems."/mnt/data" = {
device = "sn640/data";
fsType = "zfs";
};
fileSystems."/mnt/erigondata" = {
device = "sn640/erigondata";
fsType = "zfs";
};
powerManagement.enable = true;
networking.hostId = "286c56fb";
networking.hostName = "nixos"; # Define your hostname.
# networking.wireless.enable = true; # Enables wireless support via wpa_supplicant.
# Configure network proxy if necessary
# networking.proxy.default = "http://user:password@proxy:port/";
# networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
# Enable networking
networking.networkmanager.enable = true;
# Set your time zone.
time.timeZone = "Asia/Shanghai";
# Select internationalisation properties.
i18n.defaultLocale = "en_US.UTF-8";
i18n.extraLocaleSettings = {
LC_ADDRESS = "zh_CN.UTF-8";
LC_IDENTIFICATION = "zh_CN.UTF-8";
LC_MEASUREMENT = "zh_CN.UTF-8";
LC_MONETARY = "zh_CN.UTF-8";
LC_NAME = "zh_CN.UTF-8";
LC_NUMERIC = "zh_CN.UTF-8";
LC_PAPER = "zh_CN.UTF-8";
LC_TELEPHONE = "zh_CN.UTF-8";
LC_TIME = "zh_CN.UTF-8";
};
# Configure keymap in X11
services.xserver = {
layout = "us";
xkbVariant = "";
};
# Define a user account. Don't forget to set a password with ‘passwd’.
users.users.swz = {
isNormalUser = true;
description = "swz";
extraGroups = [ "networkmanager" "wheel" "docker"];
packages = with pkgs; [];
};
users.users.shiwenze = {
isNormalUser = true;
description = "for postgresql";
extraGroups = [ "networkmanager" "wheel" "docker"];
packages = with pkgs; [];
};
users.users.root.openssh.authorizedKeys.keys = [
# note: ssh-copy-id will add user@your-machine after the public key
# but we can remove the "@your-machine" part
];
# Allow unfree packages
nixpkgs.config.allowUnfree = true;
# List packages installed in system profile. To search, run:
# $ nix search wget
environment.systemPackages = with pkgs; [
vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.
wget
lm_sensors
bmon
git
iotop
stress-ng
tmux
frp
chromium
postgresql
];
environment.pathsToLink = [ "/usr/local/go/bin" ];
# Some programs need SUID wrappers, can be configured further or are
# started in user sessions.
# programs.mtr.enable = true;
# programs.gnupg.agent = {
# enable = true;
# enableSSHSupport = true;
# };
# List services that you want to enable:
# Enable the OpenSSH daemon.
services.openssh.enable = true;
# Open ports in the firewall.
# networking.firewall.allowedTCPPorts = [ ... ];
# networking.firewall.allowedUDPPorts = [ ... ];
# Or disable the firewall altogether.
networking.firewall.enable = false;
nix.settings.substituters = ["https://mirror.sjtu.edu.cn/nix-channels/store"];
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. It‘s perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "23.11"; # Did you read the comment?
virtualisation.docker.enable = true;
virtualisation.docker.daemon.settings = {
#data-root = "/mnt/data/docker_root";
registry-mirrors = [ "https://docker.mirrors.ustc.edu.cn/" ];
};
systemd.services.frpslocal = {
enable = true;
description = "frpslocal";
unitConfig = {
After = "network.target";
};
wantedBy = [ "default.target" ];
serviceConfig = {
# Type = "simple";
# KillMode = "process";
User = "swz";
Restart = "always";
RestartSec = 0;
ExecStart = "${pkgs.frp}/bin/frps -c /home/swz/frps.toml";
};
};
systemd.services.frpclocal = {
enable = true;
description = "frpclocal";
unitConfig = {
After = "network.target";
Requires = "frpslocal";
};
wantedBy = [ "default.target" ];
serviceConfig = {
# Type = "simple";
# KillMode = "process";
User = "swz";
Restart = "always";
RestartSec = 5;
ExecStart = "${pkgs.frp}/bin/frpc -c /home/swz/frpc.toml";
};
};
systemd.services.enableEtherumContainer = {
enable = false;
description = "enableEtherumContainer";
unitConfig = {
After = ["zfs.target" ];
};
wantedBy = [ "multi-user.target" ];
script = ''
#echo $(/run/current-system/sw/bin/zfs get mounted sn640/erigondata)
/run/current-system/sw/bin/nixos-container start etherum
/run/current-system/sw/bin/nixos-container run etherum -- systemctl start erigon
'';
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
#ExecStart = "/run/current-system/sw/bin/nixos-container start etherum";
};
};
systemd.services.undervolt = {
enable = true;
description = "undervolt";
# unitConfig = {
# };
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${pkgs.undervolt}/bin/undervolt --core -70 --cache -70";
};
};
services.samba = {
enable = true;
securityType = "user";
openFirewall = true;
extraConfig = ''
workgroup = WORKGROUP
server string = smbnix
netbios name = smbnix
security = user
#use sendfile = yes
#max protocol = smb2
# note: localhost is the ipv6 localhost ::1
hosts allow = 192.168. 127.0.0.1 localhost
hosts allow = 0.0.0.0/0
guest account = nobody
map to guest = bad user
'';
shares = {
share = {
path = "/mnt/data/share";
browseable = "yes";
"read only" = "no";
"guest ok" = "no";
"create mask" = "0755";
"directory mask" = "0755";
"force user" = "swz";
"force group" = "users";
};
};
};
services.samba-wsdd = {
enable = true;
openFirewall = true;
};
services.nginx = {
enable = true;
virtualHosts."_" = {
locations."/" = {
proxyPass = "https://www.baidu.com";
extraConfig = ''
proxy_read_timeout 61s;
proxy_buffering off;
'';
};
listen = [ { addr = "*"; port = 5801; } ];
};
};
virtualisation.libvirtd.enable = true;
programs.virt-manager.enable = true;
programs.chromium.enable = true;
networking.interfaces.enp5s0.ipv4.addresses = [{
address = "10.25.0.3";
prefixLength = 24;
}];
networking.nat = {
enable = true;
internalInterfaces = ["ve-+" "ve-+"];
externalInterface = "wlp4s0";
# Lazy IPv6 connectivity for the container
enableIPv6 = true;
};
networking.hosts = {
"20.205.243.166" = ["github.com"];
};
containers.etherum = {
autoStart = true;
privateNetwork = true;
hostAddress = "192.168.100.11";
localAddress = "192.168.100.12";
hostAddress6 = "fc00::1";
localAddress6 = "fc00::2";
bindMounts = {
"/mnt/data/git/erigon" = {
hostPath = "/mnt/data/git/erigon";
isReadOnly = false;
};
"/mnt/erigondata" = {
hostPath = "/mnt/erigondata";
isReadOnly = false;
};
};
config = { config, pkgs, lib, ... }: {
system.stateVersion = "23.11";
systemd.services.erigon = {
enable = true;
#path = [ pkgs.su ];
unitConfig = {
After = ["network.target" "local-fs.target"];
};
wantedBy = [ "default.target" ];
script = "/mnt/data/git/erigon/booterigon.sh";
serviceConfig = {
# Type = "simple";
# KillMode = "process";
#Restart = "always";
RestartSec = 0;
#ExecStart = "/erigon/build/bin/erigon --internalcl --txpool.disable --datadir /mnt/erigondata --http.port 8546 --authrpc.port 8552 --http.api eth,debug,net,trace,web3,erigon --torrent.port=42070 --prune=disabled --nat=stun";
#ExecStart = "/erigon/build/bin/erigon --db.size.limit=4TB --db.pagesize=4kb --injternalcl=true --txpool.disable --datadir /mnt/erigondata --http.port 8546 --authrpc.port 8552 --http.api eth,debug,net,trace,web3,erigon --torrent.port=42070 --prune=disabled --nat=stun --staticpeers enode://d04e70020071d2182f61cfe22c03ea469107089d56c91960a193652a36dd2463d6a903ec510cbaf46c4290805e54eee83154e67f2f174e700deb9c3260473ba5@10.25.0.2:30303";
};
};
networking = {
firewall = {
enable = false;
};
# Use systemd-resolved inside the container
# Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686
useHostResolvConf = lib.mkForce false;
};
services.resolved.enable = true;
services.resolved.fallbackDns = [
"223.5.5.5"
"2400:3200::1"
];
services.resolved.extraConfig = ''
DNS=2400:3200::1 223.5.5.5
'';
environment.systemPackages = with pkgs; [
dig
vim
traceroute
];
};
};
containers.cloudflared = {
autoStart = true;
#privateNetwork = true;
#hostAddress = "192.168.100.11";
#localAddress = "192.168.100.13";
#hostAddress6 = "fc00::1";
#localAddress6 = "fc00::3";
forwardPorts = [
{
containerPort = 7020;
hostPort = 7020;
protocol = "tcp";
}
];
config = { config, pkgs, lib, ... }: {
system.stateVersion = "23.11";
networking = {
firewall = {
enable = false;
};
# Use systemd-resolved inside the container
# Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686
useHostResolvConf = lib.mkForce false;
};
systemd.services.cloudflared = {
enable = true;
path = [ pkgs.su ];
unitConfig = {
After = "network.target";
};
wantedBy = [ "default.target" ];
serviceConfig = {
# Type = "simple";
# KillMode = "process";
Restart = "always";
RestartSec = 0;
ExecStart = "${pkgs.cloudflared}/bin/cloudflared tunnel --no-autoupdate --edge-ip-version 6 run --token xxxx";
};
};
services.resolved.enable = true;
services.resolved.fallbackDns = [
"2400:3200::1"
"223.5.5.5"
];
services.resolved.extraConfig = ''
DNS=2400:3200::1 223.5.5.5
'';
environment.systemPackages = with pkgs; [
cloudflared
];
};
};
services.zfs.trim.enable = true;
services.zfs.autoScrub.enable = true;
services.zfs.autoScrub.interval = "daily";
}