Local x86_64 nix builder on arm64 NixOS

runeksvendsen · December 3, 2022, 2:03pm

Hello there,

I’m running arm64 NixOS (22.11beta167.8690906c4d8) inside a Parallels Desktop VM on a MacBook Pro M1.

My goal is to get a local nix builder that can build x86_64-linux derivations. So far I’ve only tried using box64 to do this (see below) but it doesn’t work. I’d prefer using something like box64, since it has decent performance when emulating x86_64 on an arm64 CPU, as opposed to e.g. qemu which can be quite slow (in my experience at least).

How can I get this working?

[rune@nixos:/media/psf/nixos]$ box64 $(nix-build --argstr system "x86_64-linux" -A nix '<nixpkgs>')/bin/nix-build nix/test/manual-build.nix
Dynarec for ARM64, with extension: ASIMD AES CRC32 PMULL ATOMICS PageSize:4096
Box64 with Dynarec v0.1.8 nogit built on Jan  1 1980 00:00:00
Using default BOX64_LD_LIBRARY_PATH: ./:lib/:lib64/:x86_64/:bin64/:libs64/
Using default BOX64_PATH: ./:bin/
Counted 82 Env var
Looking for /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/bin/nix-build
argv[1]="nix/test/manual-build.nix"
Using emulated /nix/store/b7ac47isdc4z1ajln6adz3ppcjs1708b-libsodium-1.0.18/lib/libsodium.so.23
Using emulated /nix/store/il7dydwf1wfn9b4p4cii8q99b93i106n-editline-1.17.1/lib/libeditline.so.1
Using emulated /nix/store/zr68vx7x4bnksz4i53608d597z77kmz1-lowdown-1.0.0-lib/lib/liblowdown.so.1
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixexpr.so
Using emulated /nix/store/ibg22x7pqqp4lc1dpfyb7xb7p8xybgmg-boehm-gc-8.2.2/lib/libgc.so.1
Using native(wrapped) libpthread.so.0
Using native(wrapped) libdl.so.2
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixmain.so
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixfetchers.so
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixstore.so
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixutil.so
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixcmd.so
Using emulated /nix/store/mdck89nsfisflwjv6xv8ydj7dj0sj2pn-gcc-11.3.0-lib/lib/libstdc++.so.6
Using native(wrapped) libm.so.6
Using emulated /nix/store/4nlgxhb09sdr51nc9hdm8az5b08vzkgx-glibc-2.35-163/lib/libgcc_s.so.1
Using native(wrapped) libc.so.6
Using native(wrapped) ld-linux-x86-64.so.2
Using native(wrapped) librt.so.1
Using emulated /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libboost_context.so.1.79.0
Using emulated /nix/store/4mxnw95jcm5a27qk60z7yc0gvxp42b9a-openssl-3.0.7/lib/libcrypto.so.3
Using emulated /nix/store/9iy1ng7h1l6jdmjk157jra8n4hkrfdj1-brotli-1.0.9-lib/lib/libbrotlienc.so.1
Using emulated /nix/store/9iy1ng7h1l6jdmjk157jra8n4hkrfdj1-brotli-1.0.9-lib/lib/libbrotlidec.so.1
Using emulated /nix/store/3n9vwzn9wkq7cj93jp0s6gqpx1zbhvlj-libarchive-3.6.1-lib/lib/libarchive.so.13
Using emulated /nix/store/20d2d0jap81kxp5ng1igl7kkgnd077f3-libcpuid-0.6.2/lib/libcpuid.so.16
Using emulated /nix/store/9iy1ng7h1l6jdmjk157jra8n4hkrfdj1-brotli-1.0.9-lib/lib/libbrotlicommon.so.1
Using emulated /nix/store/7q41sbf04qcwv75j5bxis6pfjnmshy44-acl-2.3.1/lib/libacl.so.1
Error initializing native liblzma.so.5 (last dlerror is liblzma.so.5: cannot open shared object file: No such file or directory)
Using emulated /nix/store/w3sdhqiazzp4iy40wc2g85mv0grg1cx0-xz-5.2.7/lib/liblzma.so.5
Using emulated /nix/store/w10in9diaqrcqqxi5lg20n3q2jfpk6pq-zstd-1.5.2/lib/libzstd.so.1
Error initializing native libbz2.so.1 (last dlerror is libbz2.so.1: cannot open shared object file: No such file or directory)
Using emulated /nix/store/ysl6qj5r7nn63b16954dhk7x47r5yq7i-bzip2-1.0.8/lib/libbz2.so.1
Error initializing native libz.so.1 (last dlerror is libz.so.1: cannot open shared object file: No such file or directory)
Using emulated /nix/store/026hln0aq1hyshaxsdvhg0kmcm6yf45r-zlib-1.2.13/lib/libz.so.1
Error initializing native libxml2.so.2 (last dlerror is libxml2.so.2: cannot open shared object file: No such file or directory)
Using emulated /nix/store/h5slhj7gqpqh5q4jb00xdbcwbl8vqqa7-libxml2-2.10.3/lib/libxml2.so.2
Using emulated /nix/store/cr5fmwri3601s7724ayjvckhsg6cz4rv-attr-2.5.1/lib/libattr.so.1
Warning: Global Symbol ZSTD_trace_decompress_end not found, cannot apply R_X86_64_GLOB_DAT @0xffff8eedbfa8 ((nil)) in /nix/store/w10in9diaqrcqqxi5lg20n3q2jfpk6pq-zstd-1.5.2/lib/libzstd.so.1
Warning: Global Symbol ZSTD_trace_compress_begin not found, cannot apply R_X86_64_GLOB_DAT @0xffff8eedbfb0 ((nil)) in /nix/store/w10in9diaqrcqqxi5lg20n3q2jfpk6pq-zstd-1.5.2/lib/libzstd.so.1
Warning: Global Symbol ZSTD_trace_compress_end not found, cannot apply R_X86_64_GLOB_DAT @0xffff8eedbfb8 ((nil)) in /nix/store/w10in9diaqrcqqxi5lg20n3q2jfpk6pq-zstd-1.5.2/lib/libzstd.so.1
Warning: Global Symbol ZSTD_trace_decompress_begin not found, cannot apply R_X86_64_GLOB_DAT @0xffff8eedbfd0 ((nil)) in /nix/store/w10in9diaqrcqqxi5lg20n3q2jfpk6pq-zstd-1.5.2/lib/libzstd.so.1
Using emulated /nix/store/jpj9lx0p2h1vs3gkzj8jh350113bsm84-sqlite-3.39.4/lib/libsqlite3.so.0
Error initializing native libcurl.so.4 (last dlerror is libcurl-gnutls.so.4: cannot open shared object file: No such file or directory)
Using emulated /nix/store/rirzp6ijbcwnxlf0b2n286n587r3z9jw-curl-7.86.0/lib/libcurl.so.4
Using emulated /nix/store/9l3226p4p0i03whclxyp5lzps4w76hk8-aws-sdk-cpp-1.9.294/lib/libaws-cpp-sdk-transfer.so
Using emulated /nix/store/9l3226p4p0i03whclxyp5lzps4w76hk8-aws-sdk-cpp-1.9.294/lib/libaws-cpp-sdk-s3.so
Using emulated /nix/store/9l3226p4p0i03whclxyp5lzps4w76hk8-aws-sdk-cpp-1.9.294/lib/libaws-cpp-sdk-core.so
Using emulated /nix/store/c8byvs0rj8vg5cpm5mswcg5dvp7d5ir7-libseccomp-2.5.4-lib/lib/libseccomp.so.2
Error: Symbol stat64 not found, cannot apply R_X86_64_64 @0xffff8ec3e668 ((nil)) in /nix/store/jpj9lx0p2h1vs3gkzj8jh350113bsm84-sqlite-3.39.4/lib/libsqlite3.so.0
Error: Symbol fstat64 not found, cannot apply R_X86_64_64 @0xffff8ec3e680 ((nil)) in /nix/store/jpj9lx0p2h1vs3gkzj8jh350113bsm84-sqlite-3.39.4/lib/libsqlite3.so.0
Error: Symbol lstat64 not found, cannot apply R_X86_64_64 @0xffff8ec3e890 ((nil)) in /nix/store/jpj9lx0p2h1vs3gkzj8jh350113bsm84-sqlite-3.39.4/lib/libsqlite3.so.0
Using emulated /nix/store/qz400bwshaqikj5s2qyvh0c9qffgmqik-nghttp2-1.49.0-lib/lib/libnghttp2.so.14
Using emulated /nix/store/5mh5019jigj0k14rdnjam1xwk5avn1id-libidn2-2.3.2/lib/libidn2.so.0
Using emulated /nix/store/vqq9s0d6fw6kqf3sr5nrzqbys9rhygqd-libssh2-1.10.0/lib/libssh2.so.1
Using emulated /nix/store/4mxnw95jcm5a27qk60z7yc0gvxp42b9a-openssl-3.0.7/lib/libssl.so.3
Error initializing native libgssapi_krb5.so.2 (last dlerror is libgssapi_krb5.so.2: cannot open shared object file: No such file or directory)
Using emulated /nix/store/r7gl900my2fw6k33nxh2r7rzv8nv0s25-libkrb5-1.20/lib/libgssapi_krb5.so.2
Using emulated /nix/store/34xlpp3j3vy7ksn09zh44f1c04w77khf-libunistring-1.0/lib/libunistring.so.2
Error initializing native libkrb5.so.3 (last dlerror is libkrb5.so.3: cannot open shared object file: No such file or directory)
Using emulated /nix/store/r7gl900my2fw6k33nxh2r7rzv8nv0s25-libkrb5-1.20/lib/libkrb5.so.3
Using emulated /nix/store/r7gl900my2fw6k33nxh2r7rzv8nv0s25-libkrb5-1.20/lib/libk5crypto.so.3
Using emulated /nix/store/r7gl900my2fw6k33nxh2r7rzv8nv0s25-libkrb5-1.20/lib/libcom_err.so.3
Using emulated /nix/store/r7gl900my2fw6k33nxh2r7rzv8nv0s25-libkrb5-1.20/lib/libkrb5support.so.0
Using emulated /nix/store/816qwr4xy058451rbxr0ccyh1v1akhb6-keyutils-1.6.3-lib/lib/libkeyutils.so.1
Using native(wrapped) libresolv.so.2
Using emulated /nix/store/5q73izqsdasz81gma1nz6870b6220vkh-aws-crt-cpp-0.18.9/lib/libaws-crt-cpp.so
Using emulated /nix/store/1f3pdcihv4mmgfsyw4jxqjvn7j5sc2xy-aws-c-mqtt-0.7.13/lib/libaws-c-mqtt.so.1.0.0
Using emulated /nix/store/50si0kkawnfkgs8m7d8iv2zmkq0fdbm7-aws-c-event-stream-0.2.15/lib/libaws-c-event-stream.so.1.0.0
Using emulated /nix/store/84px25a6dsmdg7ni7186rmkybq1k2vrf-aws-c-s3-0.1.51/lib/libaws-c-s3.so.0unstable
Using emulated /nix/store/bwsyv47ri47ppr4ga34wd2khk89ch4n4-aws-c-auth-0.6.21/lib/libaws-c-auth.so.1.0.0
Using emulated /nix/store/a6q74vrxbsrmmw1z0fdzd0qrcm6dc0ll-aws-c-http-0.6.27/lib/libaws-c-http.so.1.0.0
Using emulated /nix/store/f116ly4fl72zc5ynb03qrwdk2bp8y61p-aws-c-io-0.13.11/lib/libaws-c-io.so.1.0.0
Using emulated /nix/store/4rkhsf7sig2lh303bygqr3ph5mfwz0ah-s2n-tls-1.3.28/lib/libs2n.so.1
Using emulated /nix/store/wnxn8173p8gj888wwgv2l9czp7zf4jl3-aws-c-compression-0.2.16/lib/libaws-c-compression.so.1.0.0
Using emulated /nix/store/6qqybxxz6636jymz8x7l2sxj4np9yzsy-aws-c-cal-0.5.20/lib/libaws-c-cal.so.1.0.0
Using emulated /nix/store/fph0r1qjhsygrb420y2zsfhjh3rssq0z-aws-c-sdkutils-0.1.7/lib/libaws-c-sdkutils.so.1.0.0
Using emulated /nix/store/3z091ijyij0mmgi3iv0mp945lm2bx4wy-aws-checksums-0.1.13/lib/libaws-checksums.so.1.0.0
Using emulated /nix/store/3j1h6psl4pzn6b3yck6rk33bpwrmihb1-aws-c-common-0.8.5/lib/libaws-c-common.so.1
Warning: Global Symbol EVP_MD_CTX_destroy not found, cannot apply R_X86_64_GLOB_DAT @0xffff8da50f88 ((nil)) in /nix/store/6qqybxxz6636jymz8x7l2sxj4np9yzsy-aws-c-cal-0.5.20/lib/libaws-c-cal.so.1.0.0
Warning: Global Symbol HMAC_CTX_init not found, cannot apply R_X86_64_GLOB_DAT @0xffff8da50fb0 ((nil)) in /nix/store/6qqybxxz6636jymz8x7l2sxj4np9yzsy-aws-c-cal-0.5.20/lib/libaws-c-cal.so.1.0.0
Warning: Global Symbol HMAC_CTX_cleanup not found, cannot apply R_X86_64_GLOB_DAT @0xffff8da50fb8 ((nil)) in /nix/store/6qqybxxz6636jymz8x7l2sxj4np9yzsy-aws-c-cal-0.5.20/lib/libaws-c-cal.so.1.0.0
Warning: Global Symbol EVP_MD_CTX_create not found, cannot apply R_X86_64_GLOB_DAT @0xffff8da50fc0 ((nil)) in /nix/store/6qqybxxz6636jymz8x7l2sxj4np9yzsy-aws-c-cal-0.5.20/lib/libaws-c-cal.so.1.0.0
Error: PltResolver: Symbol stat(ver 12: stat@GLIBC_2.33) not found, cannot apply R_X86_64_JUMP_SLOT 0xffff90cc1810 (0xffff90c00196) in /nix/store/81yi66w1q1spb27vp7blwbgb5myhwgjw-nix-2.11.0/lib/libnixutil.so
^C

Atemu · December 3, 2022, 2:29pm

I haven’t tested this but, if you’re on macOS Ventura, you should be able to pass Rosetta2 to the VM, register it via binfmt and allow it in the sandbox.

runeksvendsen · December 3, 2022, 3:03pm

I’ve tried this, but it does not seem to work. These are the steps I’ve followed: rosetta-vm.nix · GitHub. I don’t think Parallels Desktop supports it, since the given VirtioFS file system does not exist within the VM. Specifically, I don’t think Parallels Desktop has implemented the “Create the Rosetta Directory Share” step from this guide: Apple Developer Documentation.

Also, if Rosetta2 works then I believe box64 should as well (although perhaps not as fast) since they’re doing essentially the same thing. So for now I’m trying to make it work with box64 and then later switch to Rosetta2.

wamserma · December 4, 2022, 5:54pm

Why not use the powers of Nix?

You basically need NixOS on ARM - NixOS Wiki with x86_64 and aarch64 reversed.

runeksvendsen · December 10, 2022, 7:53am

The reason I’ve avoided boot.binfmt.emulatedSystems is because it uses QEMU emulation, which is really slow[2]. Compare this with binary translation like box64 and Rosetta2 which is at the very least only four times slower and very often less[1].

[1] Box86/Box64 vs QEMU vs FEX (vs Rosetta2) – Box86
[2] See e.g. the glmark2 benchmark from [1] where QEMU is ~50 times slower than box64

wamserma · December 10, 2022, 10:46am

It might be slower, but it works, which is still faster than not working.

Maybe do a test with your workload and report back on performance (and maybe a comparison once you get binary translation working).

runeksvendsen · December 17, 2022, 8:30am

Status update

First of all, I can confirm that qemu is horribly slow. The 50x slowdown seems roughly accurate.

Secondly, I got rosetta working – sort of. It works for building some x86 derivations, using the command

$(nix-build --argstr system "x86_64-linux" -A nix '<nixpkgs>')/bin/nix-build --argstr system "x86_64-linux" something.nix

However, I’m trying to build a specific derivation that fails with a segmentation fault while building a Haskell library. Relevant log output (full logs below):

[... SNIP ...]

runHook postBuild
'
++ runHook preBuild
++ local hookName=preBuild
++ shift
++ local 'hooksSlice=preBuildHooks[@]'
++ local hook
++ for hook in "_callImplicitHook 0 $hookName" ${!hooksSlice+"${!hooksSlice}"}
++ _eval '_callImplicitHook 0 preBuild'
++ declare -F '_callImplicitHook 0 preBuild'
++ eval '_callImplicitHook 0 preBuild'
+++ _callImplicitHook 0 preBuild
+++ local def=0
+++ local hookName=preBuild
+++ declare -F preBuild
+++ type -p preBuild
+++ '[' -n '' ']'
+++ return 0
++ return 0
++ ./Setup build
Preprocessing test suite 'shelly-testsuite' for shelly-1.9.0..
Building test suite 'shelly-testsuite' for shelly-1.9.0..

<no location info>: warning: [-Wmissing-home-modules]
    These modules are needed for compilation but not listed in your .cabal file's other-modules: 
        Shelly.Directory
[ 1 of 20] Compiling Shelly.Base      ( src/Shelly/Base.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/Shelly/Base.o )

src/Shelly/Base.hs:41:1: warning: [-Wunused-imports]
    The import of ‘ProcessHandle’
    from module ‘System.Process’ is redundant
   |
41 | import System.Process( ProcessHandle, StdStream(..) )
   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/Shelly/Base.hs:44:1: warning: [-Wunused-imports]
    The import of ‘liftM’ from module ‘Control.Monad’ is redundant
   |
44 | import Control.Monad (when, (>=>),
   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^...

[... SNIP ...]

[16 of 20] Compiling EnvSpec          ( test/src/EnvSpec.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/EnvSpec.o )
[17 of 20] Compiling CopySpec         ( test/src/CopySpec.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/CopySpec.o )
[18 of 20] Compiling WhichSpec        ( test/src/WhichSpec.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/WhichSpec.o )
[19 of 20] Compiling WriteSpec        ( test/src/WriteSpec.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/WriteSpec.o )
[20 of 20] Compiling Main             ( test/src/TestMain.hs, dist/build/shelly-testsuite/shelly-testsuite-tmp/Main.o )

<no location info>: warning: [-Wmissing-home-modules]
    These modules are needed for compilation but not listed in your .cabal file's other-modules: 
        Shelly.Directory
Linking dist/build/shelly-testsuite/shelly-testsuite ...
Preprocessing library for shelly-1.9.0..
Building library for shelly-1.9.0..
[1 of 7] Compiling Shelly.Base      ( src/Shelly/Base.hs, dist/build/Shelly/Base.o )
/nix/store/wgap303sj9zqz63gw7nqxvf4dqz2hgai-stdenv-linux/setup: line 1302:   258 Segmentation fault      (core dumped) ./Setup build
+ exitHandler
+ exitCode=139
+ set +e
+ '[' -n '' ']'
+ ((  139 != 0  ))
+ runHook failureHook
+ local hookName=failureHook
+ shift

I have no idea what’s going on. Full log: rosetta-nixos-segfault.log · GitHub

Link to the source code for the Haskell module that starts building right before the segfault: https://hackage.haskell.org/package/shelly-1.9.0/docs/src/Shelly.Base.html

ElvishJerricco · December 17, 2022, 4:28pm

How did you get Rosetta into the VM? One awkward quirk of Rosetta is that it needs the translated process to be in “TSO” mode, which is where the kernel sets the CPU core up with x86 style memory ordering, which is a little slower but ensures that multithreaded programs actually behave correctly. macOS puts the whole VM into TSO mode when hypervisor.framework is told to supply the VM with a rosetta binary, but not all VM softwares use hypervisor.framework in this way (e.g. qemu and probably UTM by extension). And AFAIK there’s no way to tell the linux kernel to put processes in TSO mode, even on bare metal Asahi Linux. Without TSO, multithreaded programs like GHC could end up with incoherent memory between threads, which could easily cause crashes like that.

runeksvendsen · December 18, 2022, 9:31am

Thanks a lot @ElvishJerricco! That explains it.

I’m just using the binary from here GitHub - CathyKMeow/rosetta-linux-asahi: Hacked RosettaLinux that runs on Asahi Linux, so the macOS host has no idea I’m running Rosetta inside the VM.

I will try following this guide and report back: Using Rosetta 2 in a NixOS VM - xynos space

runeksvendsen · December 24, 2022, 8:41am

So, I followed this guide and I got all the way to successfully building a test suite for an internal project at my company. However, I’m still getting a segfault when realizing a certain derivation:

[rune@nixos:~/code/smithy]$ export PATH=$(nix-build --argstr system "x86_64-linux" -A nix '<nixpkgs>')/bin:$PATH

[rune@nixos:~/code/smithy]$ file $(which nix-store)
/nix/store/1qxf5i4na4a4cdykhxki2wyal82kl0zb-nix-2.11.0/bin/nix-store: symbolic link to nix

[rune@nixos:~/code/smithy]$ file $(which nix)
/nix/store/1qxf5i4na4a4cdykhxki2wyal82kl0zb-nix-2.11.0/bin/nix: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /nix/store/4nlgxhb09sdr51nc9hdm8az5b08vzkgx-glibc-2.35-163/lib/ld-linux-x86-64.so.2, BuildID[sha1]=e85442784c8a16559b4921dcffe9d03d6fc75f14, for GNU/Linux 2.6.32, not stripped

[rune@nixos:~/code/smithy]$ nix-store --realize /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv
this derivation will be built:
  /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv
building '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv'...
error: builder for '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv' failed due to signal 11 (Segmentation fault)

[rune@nixos:~/code/smithy]$ cat /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv
Derive([("out","/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook","","")],[("/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv",["out"]),("/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv",["out"])],["/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],"x86_64-linux","/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash",["-e","/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],[("allowSubstitutes",""),("buildCommand","target=$out\nmkdir -p \"$(dirname \"$target\")\"\n\nif [ -e \"$textPath\" ]; then\n  mv \"$textPath\" \"$target\"\nelse\n  echo -n \"$text\" > \"$target\"\nfi\n\neval \"$checkPhase\"\n\n(test -n \"$executable\" && chmod +x \"$target\") || true\n"),("buildInputs",""),("builder","/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash"),("checkPhase",""),("configureFlags",""),("depsBuildBuild",""),("depsBuildBuildPropagated",""),("depsBuildTarget",""),("depsBuildTargetPropagated",""),("depsHostHost",""),("depsHostHostPropagated",""),("depsTargetTarget",""),("depsTargetTargetPropagated",""),("doCheck",""),("doInstallCheck",""),("executable",""),("name","setup-hook"),("nativeBuildInputs",""),("out","/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook"),("outputs","out"),("passAsFile","buildCommand text"),("patches",""),("preferLocalBuild","1"),("propagatedBuildInputs",""),("propagatedNativeBuildInputs",""),("stdenv","/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux"),("strictDeps",""),("system","x86_64-linux"),("text","export GZIP=\"-n\"\n")])

So first I create a shell environment where all calls to Nix executables resolve to an x86_64 executable. This executable will then be executed by Rosetta, given that I’ve followed the above-mentioned guide.

I’m not sure how to debug this further, otherwise I would provide more information.

runeksvendsen · December 25, 2022, 10:22am

I got some more interesting output from nix-store by increasing verbosity. See full logs below.

One thing that caught my eye is some aarch64-related stuff inside the build environment:

[rune@nixos:~/code/smithy]$ nix-store -vvvvvv --realize /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv 2>&1 |grep aarch64
sandbox setup: bind mounting '/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0/bin/busybox' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/bin/sh'
sandbox setup: bind mounting '/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0'

Full logs:

[rune@nixos:~/code/smithy]$ nix-store -vvvvvv --realize /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv
querying info about missing paths...
starting pool of 24 threads
reaping 1 worker threads
this derivation will be built:
  /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: created
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: woken up
querying info about missing paths...
starting pool of 24 threads
reaping 1 worker threads
entered goal loop
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: init
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: loading derivation
acquiring write lock on '/nix/var/nix/temproots/1033056'
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: have derivation
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: all outputs substituted (maybe)
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: created
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: woken up
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: created
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: woken up
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: init
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: loading derivation
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: have derivation
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: done
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: waitee 'building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file' done; 1 left
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: init
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: loading derivation
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: have derivation
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: done
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: waitee 'building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file' done; 0 left
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: woken up
building of '/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv!out' from .drv file: goal destroyed
building of '/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv!out' from .drv file: goal destroyed
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: all inputs realised
added input paths '/nix/store/0v00iapcqns4rrm2halb1lbhn9498xzw-xz-5.2.5-bin', '/nix/store/1i5y55x4b4m9qkx5dqbmr1r6bvrqbanw-multiple-outputs.sh', '/nix/store/59jmzisg8fkm9c125fw384dqq1np602l-move-docs.sh', '/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux', '/nix/store/8vrwnylyyxz2c0djkdvmccy6a580zf0k-libunistring-0.9.10', '/nix/store/8zxndz5ag0p6s526c2xyllhk1nrn4c3i-audit-tmpdir.sh', '/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh', '/nix/store/bkxq1nfi6grmww5756ynr1aph7w04lkk-strip.sh', '/nix/store/bnj8d7mvbkg3vdb07yz74yhl3g107qq5-patch-shebangs.sh', '/nix/store/c8n9kcdddp9np665xz6ri61b383nxvz8-move-systemd-user-units.sh', '/nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh', '/nix/store/fkzaww3nk3zlh9s8ssnlm48mjd370xx1-libidn2-2.3.2', '/nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh', '/nix/store/g8ai55gi1i14v33wlqj316lib8p5v2w2-xz-5.2.5', '/nix/store/hsd7gwndv436wcjj9zcshn5wwnbv570p-patchelf-0.14.3', '/nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh', '/nix/store/kxw6q8v6isaqjm702d71n2421cxamq68-make-symlinks-relative.sh', '/nix/store/lan2w3ab1mvpxj3ppiw2sizh8i7rpz7s-busybox', '/nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh', '/nix/store/ngg1cv31c8c7bcm2n8ww4g06nq7s4zhm-set-source-date-epoch-to-latest.sh', '/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools', '/nix/store/pqw9hpvlwak1cqi3gxwnbkrqwkwwkg50-bootstrap-stage0-glibc-bootstrap', '/nix/store/q29bwjibv9gi9n86203s38n0577w09sx-glibc-2.33-117', '/nix/store/wlwcf1nw2b21m4gghj70hbg1v7x53ld8-reproducible-builds.sh'
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: woken up
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: trying to build
locking path '/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook'
lock acquired on '/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook.lock'
removing invalid path '/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook'
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: woken up
found build user 'nixbld1'
found build user 'nixbld10'
found build user 'nixbld11'
found build user 'nixbld12'
found build user 'nixbld13'
found build user 'nixbld14'
found build user 'nixbld15'
found build user 'nixbld16'
found build user 'nixbld17'
found build user 'nixbld18'
found build user 'nixbld19'
found build user 'nixbld2'
found build user 'nixbld20'
found build user 'nixbld21'
found build user 'nixbld22'
found build user 'nixbld23'
found build user 'nixbld24'
found build user 'nixbld25'
found build user 'nixbld26'
found build user 'nixbld27'
found build user 'nixbld28'
found build user 'nixbld29'
found build user 'nixbld3'
found build user 'nixbld30'
found build user 'nixbld31'
found build user 'nixbld32'
found build user 'nixbld4'
found build user 'nixbld5'
found build user 'nixbld6'
found build user 'nixbld7'
found build user 'nixbld8'
found build user 'nixbld9'
trying user 'nixbld1'
killing all processes running under uid '30001'
setting up chroot environment in '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot'
executing builder '/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash'
using builder args '-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh'
setting builder env variable 'allowSubstitutes'=''
setting builder env variable 'buildCommand'='target=$out
mkdir -p "$(dirname "$target")"

if [ -e "$textPath" ]; then
  mv "$textPath" "$target"
else
  echo -n "$text" > "$target"
fi

eval "$checkPhase"

(test -n "$executable" && chmod +x "$target") || true
'
setting builder env variable 'buildInputs'=''
setting builder env variable 'builder'='/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash'
setting builder env variable 'checkPhase'=''
setting builder env variable 'configureFlags'=''
setting builder env variable 'depsBuildBuild'=''
setting builder env variable 'depsBuildBuildPropagated'=''
setting builder env variable 'depsBuildTarget'=''
setting builder env variable 'depsBuildTargetPropagated'=''
setting builder env variable 'depsHostHost'=''
setting builder env variable 'depsHostHostPropagated'=''
setting builder env variable 'depsTargetTarget'=''
setting builder env variable 'depsTargetTargetPropagated'=''
setting builder env variable 'doCheck'=''
setting builder env variable 'doInstallCheck'=''
setting builder env variable 'executable'=''
setting builder env variable 'name'='setup-hook'
setting builder env variable 'nativeBuildInputs'=''
setting builder env variable 'out'='/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook'
setting builder env variable 'outputs'='out'
setting builder env variable 'passAsFile'='buildCommand text'
setting builder env variable 'patches'=''
setting builder env variable 'preferLocalBuild'='1'
setting builder env variable 'propagatedBuildInputs'=''
setting builder env variable 'propagatedNativeBuildInputs'=''
setting builder env variable 'stdenv'='/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux'
setting builder env variable 'strictDeps'=''
setting builder env variable 'system'='x86_64-linux'
setting builder env variable 'text'='export GZIP="-n"
'
sandbox setup: bind mounting '/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0/bin/busybox' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/bin/sh'
sandbox setup: bind mounting '/tmp/nix-build-setup-hook.drv-0' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/build'
sandbox setup: bind mounting '/dev/full' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/full'
sandbox setup: bind mounting '/dev/null' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/null'
sandbox setup: bind mounting '/dev/random' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/random'
sandbox setup: bind mounting '/dev/tty' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/tty'
sandbox setup: bind mounting '/dev/urandom' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/urandom'
sandbox setup: bind mounting '/dev/zero' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/dev/zero'
sandbox setup: bind mounting '/nix/store/0v00iapcqns4rrm2halb1lbhn9498xzw-xz-5.2.5-bin' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/0v00iapcqns4rrm2halb1lbhn9498xzw-xz-5.2.5-bin'
sandbox setup: bind mounting '/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux'
sandbox setup: bind mounting '/nix/store/8vrwnylyyxz2c0djkdvmccy6a580zf0k-libunistring-0.9.10' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/8vrwnylyyxz2c0djkdvmccy6a580zf0k-libunistring-0.9.10'
sandbox setup: bind mounting '/nix/store/fkzaww3nk3zlh9s8ssnlm48mjd370xx1-libidn2-2.3.2' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/fkzaww3nk3zlh9s8ssnlm48mjd370xx1-libidn2-2.3.2'
sandbox setup: bind mounting '/nix/store/g8ai55gi1i14v33wlqj316lib8p5v2w2-xz-5.2.5' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/g8ai55gi1i14v33wlqj316lib8p5v2w2-xz-5.2.5'
sandbox setup: bind mounting '/nix/store/hsd7gwndv436wcjj9zcshn5wwnbv570p-patchelf-0.14.3' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/hsd7gwndv436wcjj9zcshn5wwnbv570p-patchelf-0.14.3'
sandbox setup: bind mounting '/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools'
sandbox setup: bind mounting '/nix/store/pqw9hpvlwak1cqi3gxwnbkrqwkwwkg50-bootstrap-stage0-glibc-bootstrap' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/pqw9hpvlwak1cqi3gxwnbkrqwkwwkg50-bootstrap-stage0-glibc-bootstrap'
sandbox setup: bind mounting '/nix/store/q29bwjibv9gi9n86203s38n0577w09sx-glibc-2.33-117' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/q29bwjibv9gi9n86203s38n0577w09sx-glibc-2.33-117'
sandbox setup: bind mounting '/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/nix/store/r4mp4iyqm7sdcxfj2a9hz108n543wnpl-busybox-static-aarch64-unknown-linux-musl-1.35.0'
sandbox setup: bind mounting '/run/binfmt' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/run/binfmt'
sandbox setup: bind mounting '/run/rosetta' to '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv.chroot/run/rosetta'
sandbox setup: closing leaked FD 3
sandbox setup: closing leaked FD 4
sandbox setup: closing leaked FD 5
sandbox setup: closing leaked FD 6
sandbox setup: closing leaked FD 7
sandbox setup: closing leaked FD 8
sandbox setup: closing leaked FD 9
sandbox setup: closing leaked FD 10
sandbox setup: closing leaked FD 11
sandbox setup: closing leaked FD 12
sandbox setup: closing leaked FD 13
sandbox setup: closing leaked FD 14
sandbox setup: closing leaked FD 15
sandbox setup: closing leaked FD 16
building '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv'...
waiting for children
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: got EOF
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: woken up
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: build done
killing process 1033062
builder process for '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv' finished
killing all processes running under uid '30001'
lock released on '/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook.lock'
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: done
building of '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv!*' from .drv file: goal destroyed
error: builder for '/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv' failed due to signal 11 (Segmentation fault)

EDIT: show-derivation output for /nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv:

{
  "/nix/store/vgw0nd9kgfs3rbvm20144zidxlc4qsax-setup-hook.drv": {
    "outputs": {
      "out": {
        "path": "/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook"
      }
    },
    "inputSrcs": [
      "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
    ],
    "inputDrvs": {
      "/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv": [
        "out"
      ],
      "/nix/store/8v3lzbyagb72zwidrh7k9ggwpsggy0hs-bootstrap-stage4-stdenv-linux.drv": [
        "out"
      ]
    },
    "system": "x86_64-linux",
    "builder": "/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash",
    "args": [
      "-e",
      "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
    ],
    "env": {
      "allowSubstitutes": "",
      "buildCommand": "target=$out\nmkdir -p \"$(dirname \"$target\")\"\n\nif [ -e \"$textPath\" ]; then\n  mv \"$textPath\" \"$target\"\nelse\n  echo -n \"$text\" > \"$target\"\nfi\n\neval \"$checkPhase\"\n\n(test -n \"$executable\" && chmod +x \"$target\") || true\n",
      "buildInputs": "",
      "builder": "/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash",
      "checkPhase": "",
      "configureFlags": "",
      "depsBuildBuild": "",
      "depsBuildBuildPropagated": "",
      "depsBuildTarget": "",
      "depsBuildTargetPropagated": "",
      "depsHostHost": "",
      "depsHostHostPropagated": "",
      "depsTargetTarget": "",
      "depsTargetTargetPropagated": "",
      "doCheck": "",
      "doInstallCheck": "",
      "executable": "",
      "name": "setup-hook",
      "nativeBuildInputs": "",
      "out": "/nix/store/j1g3pd657hvmyc25hj3syab4hg03vmyr-setup-hook",
      "outputs": "out",
      "passAsFile": "buildCommand text",
      "patches": "",
      "preferLocalBuild": "1",
      "propagatedBuildInputs": "",
      "propagatedNativeBuildInputs": "",
      "stdenv": "/nix/store/6p6gmzx72i2p0in3cclz25cjlz3d3m7l-bootstrap-stage4-stdenv-linux",
      "strictDeps": "",
      "system": "x86_64-linux",
      "text": "export GZIP=\"-n\"\n"
    }
  }
}