Cross build nixOS iso on aarch64-darwin -> x86_64-linux

Help
1

Hi all!

We have project where we develop a nixos-live-iso which runs some services. My employer gave me an old laptop which runs nixos but the build times could be better.

I own a macbook air with m2 processor and would like to build the iso there. I can build inside a VM which has virtualization.rosetta2.enable = true, and copy it into a shared folder between host and vm.

However lots of people are cross building using nix-darwin linux-builder, this works for binaries but not for the iso build.

$ nix build --system "x86_64-linux" 
...
..
.
copying path '/nix/store/l87knzbrg3w6v2qblkcgdpbkkydbdlip-python3-3.11.9' from 'https://cache.nixos.org'...
copying path '/nix/store/z7qzhs6dpp9d1lgv93cvmj4dq95v19l4-ruby3.1-msgpack-1.5.1' from 'https://cache.nixos.org'...
copying path '/nix/store/l60wm796d40rx73qjlg6kfaz1z5kkysi-xz-5.4.7-dev' from 'https://cache.nixos.org'...
error: builder for '/nix/store/81xhw0avizh74yf6m3x8czy70f7bir5i-boot.json.drv' failed with exit code 1
error: builder for '/nix/store/2a0zjzdksshv7qg5paziq31l2807y13x-loopback.cfg.drv' failed with exit code 1
error: builder for '/nix/store/ifkri444k2wqz9406cbdanyxjpgwrrgc-mounts.sh.drv' failed with exit code 1
error: 1 dependencies of derivation '/nix/store/cfrh2qi37j410n6wv35j5vqchwnygpdd-headnode.iso.drv' failed to build

where can i go from here, any ideas or explanation why this does not work?

boot.json.drv 
Derive(
  [
    ("out", "/nix/store/pf1d536dns6vsp56hivbpqd30c7gs7a8-boot.json", "", "")
  ],
  [
    ("/nix/store/8vzg5w1qxrzm5ql200dinqr6a69bv8sq-linux-6.6.59.drv", ["out"]),
    ("/nix/store/rqnkf1svc29xh9ai8wj3k783hbrb2w9j-stdenv-linux.drv", ["out"]),
    ("/nix/store/w16aryrq4w9rzq8y5sddkpsx1573l2zd-initrd-linux-6.6.59.drv", ["out"]),
    ("/nix/store/ysv6wz83jkvg7d65j0js4bml9k0yc4sv-bash-5.2p32.drv", ["out"])
  ],
  [
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  "x86_64-linux",
  "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash",
  [
    "-e",
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  [
    ("__structuredAttrs", ""),
    ("allowSubstitutes", ""),
    ("buildCommand", "target=$out\nmkdir -p \"$(dirname \"$target\")\"\n\nif [ -e \"$textPath\" ]; then\n  mv \"$textPath\" \"$target\"\nelse\n  echo -n \"$text\" > \"$target\"\nfi\n\nif [ -n \"$executable\" ]; then\n  chmod +x \"$target\"\nfi\n\neval \"$checkPhase\""),
    ("buildInputs", ""),
    ("builder", "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash"),
    ("checkPhase", ""),
    ("cmakeFlags", ""),
    ("configureFlags", ""),
    ("depsBuildBuild", ""),
    ("depsBuildBuildPropagated", ""),
    ("depsBuildTarget", ""),
    ("depsBuildTargetPropagated", ""),
    ("depsHostHost", ""),
    ("depsHostHostPropagated", ""),
    ("depsTargetTarget", ""),
    ("depsTargetTargetPropagated", ""),
    ("doCheck", ""),
    ("doInstallCheck", ""),
    ("enableParallelBuilding", "1"),
    ("enableParallelChecking", "1"),
    ("enableParallelInstalling", "1"),
    ("executable", ""),
    ("mesonFlags", ""),
    ("name", "boot.json"),
    ("nativeBuildInputs", ""),
    ("out", "/nix/store/pf1d536dns6vsp56hivbpqd30c7gs7a8-boot.json"),
    ("outputs", "out"),
    ("passAsFile", "buildCommand text"),
    ("patches", ""),
    ("preferLocalBuild", "1"),
    ("propagatedBuildInputs", ""),
    ("propagatedNativeBuildInputs", ""),
    ("stdenv", "/nix/store/6120nzd2nmphaqa2bjwmd1pdfxqyf14j-stdenv-linux"),
    ("strictDeps", ""),
    ("system", "x86_64-linux"),
    ("text", "{\"org.nixos.bootspec.v1\":{"
             "\"initrd\":\"/nix/store/8f6lljk1an6z64k536afr8xnlw0wnwxz-initrd-linux-6.6.59/initrd\","
             "\"kernel\":\"/nix/store/vnpmz816h4bmbwlli0s9i4qn0pmf36hb-linux-6.6.59/bzImage\","
             "\"kernelParams\":[\"root=LABEL=nixos-minimal-24.05-x86_64\",\"boot.shell_on_fail\",\"nohibernate\",\"loglevel=4\"],"
             "\"label\":\"NixOS Uakari 24.05.20241104.d063c1d (Linux 6.6.59)\","
             "\"system\":\"x86_64-linux\"}}")
  ]
)
loopback.cfg.drv 
Derive(
  [
    ("out", "/nix/store/i4mkn2xxkkxwf7skyq3mxc9ixpjn3bg1-loopback.cfg", "", "")
  ],
  [
    ("/nix/store/rqnkf1svc29xh9ai8wj3k783hbrb2w9j-stdenv-linux.drv", ["out"]),
    ("/nix/store/ysv6wz83jkvg7d65j0js4bml9k0yc4sv-bash-5.2p32.drv", ["out"])
  ],
  [
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  "x86_64-linux",
  "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash",
  [
    "-e",
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  [
    ("__structuredAttrs", ""),
    ("allowSubstitutes", ""),
    ("buildCommand", 
      "target=$out'/grub/loopback.cfg'\n" +
      "mkdir -p \"$(dirname \"$target\")\"\n\n" +
      "if [ -e \"$textPath\" ]; then\n" +
      "  mv \"$textPath\" \"$target\"\n" +
      "else\n" +
      "  echo -n \"$text\" > \"$target\"\n" +
      "fi\n\n" +
      "if [ -n \"$executable\" ]; then\n" +
      "  chmod +x \"$target\"\n" +
      "fi\n\n" +
      "eval \"$checkPhase\""
    ),
    ("buildInputs", ""),
    ("builder", "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash"),
    ("checkPhase", ""),
    ("cmakeFlags", ""),
    ("configureFlags", ""),
    ("depsBuildBuild", ""),
    ("depsBuildBuildPropagated", ""),
    ("depsBuildTarget", ""),
    ("depsBuildTargetPropagated", ""),
    ("depsHostHost", ""),
    ("depsHostHostPropagated", ""),
    ("depsTargetTarget", ""),
    ("depsTargetTargetPropagated", ""),
    ("doCheck", ""),
    ("doInstallCheck", ""),
    ("enableParallelBuilding", "1"),
    ("enableParallelChecking", "1"),
    ("enableParallelInstalling", "1"),
    ("exe
mounts.sh.drv 
Derive(
  [
    ("out", "/nix/store/6qzggm7322xvqp75abixvrd4jzg0x5g2-mounts.sh", "", "")
  ],
  [
    ("/nix/store/rqnkf1svc29xh9ai8wj3k783hbrb2w9j-stdenv-linux.drv", ["out"]),
    ("/nix/store/ysv6wz83jkvg7d65j0js4bml9k0yc4sv-bash-5.2p32.drv", ["out"])
  ],
  [
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  "x86_64-linux",
  "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash",
  [
    "-e",
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  [
    ("__structuredAttrs", ""),
    ("allowSubstitutes", ""),
    ("buildCommand", 
      "target=$out''\n" +
      "mkdir -p \"$(dirname \"$target\")\"\n\n" +
      "if [ -e \"$textPath\" ]; then\n" +
      "  mv \"$textPath\" \"$target\"\n" +
      "else\n" +
      "  echo -n \"$text\" > \"$target\"\n" +
      "fi\n\n" +
      "if [ -n \"$executable\" ]; then\n" +
      "  chmod +x \"$target\"\n" +
      "fi\n\n" +
      "eval \"$checkPhase\""
    ),
    ("buildInputs", ""),
    ("builder", "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash"),
    ("checkPhase", ""),
    ("cmakeFlags", ""),
    ("configureFlags", ""),
    ("depsBuildBuild", ""),
    ("depsBuildBuildPropagated", ""),
    ("depsBuildTarget", ""),
    ("depsBuildTargetPropagated", ""),
    ("depsHostHost", ""),
    ("depsHostHostPropagated", ""),
    ("depsTargetTarget", ""),
    ("depsTargetTargetPropagated", ""),
    ("doCheck", ""),
    ("doInstallCheck", ""),
    ("enableParallelBuilding", "1"),
    ("enableParallelChecking", "1"),
    ("enableParallelInstalling", "1"),
    ("executable", ""),
    ("mesonFlags", ""),
    ("name", "mounts.sh"),
    ("nativeBuildInputs", ""),
    ("out", "/nix/store/6qzggm7322xvqp75abixvrd4jzg0x5g2-mounts.sh"),
    ("outputs", "out"),
    ("passAsFile", "buildCommand text"),
    ("patches", ""),
    ("preferLocalBuild", "1"),
    ("propagatedBuildInputs", ""),
    ("propagatedNativeBuildInputs", ""),
    ("stdenv", "/nix/store/6120nzd2nmphaqa2bjwmd1pdfxqyf14j-stdenv-linux"),
    ("strictDeps", ""),
    ("system", "x86_64-linux"),
    ("text", 
      "specialMount \"devtmpfs\" \"/dev\" \"nosuid,strictatime,mode=755,size=5%\" \"devtmpfs\"\n\n" +
      "specialMount \"devpts\" \"/dev/pts\" \"nosuid,noexec,mode=620,ptmxmode=0666,gid=3\" \"devpts\"\n\n" +
      "specialMount \"tmpfs\" \"/dev/shm\" \"nosuid,nodev,strictatime,mode=1777,size=50%\" \"tmpfs\"\n\n" +
      "specialMount \"proc\" \"/proc\" \"nosuid,noexec,nodev\" \"proc\"\n\n" +
      "specialMount \"tmpfs\" \"/run\" \"nosuid,nodev,strictatime,mode=755,size=25%\" \"tmpfs\"\n\n" +
      "specialMount \"ramfs\" \"/run/keys\" \"nosuid,nodev,mode=750\" \"ramfs\"\n\n" +
      "specialMount \"tmpfs\" \"/run/wrappers\" \"nodev,mode=755,size=50%\" \"tmpfs\"\n\n" +
      "specialMount \"sysfs\" \"/sys\" \"nosuid,noexec,nodev\" \"sysfs\""
    )
  ]
)
headnode.iso.drv 
Derive(
  [
    ("out", "/nix/store/493a7c6rr1d3k75q466zwzfkf72sf44d-headnode.iso", "", "")
  ],
  [
    ("/nix/store/14pwbykg7sq9yzha4r0w20zgddprw8hg-isolinux.cfg.drv", ["out"]),
    ("/nix/store/2a0zjzdksshv7qg5paziq31l2807y13x-loopback.cfg.drv", ["out"]),
    ("/nix/store/50pclk4xlh0xslnwxik6pfjln5phssy4-libossp-uuid-1.6.2.drv", ["out"]),
    ("/nix/store/75jdkhlnz60fwbxk5033qs65hldbjsk7-syslinux-unstable-2019-02-07.drv", ["out"]),
    ("/nix/store/7la2njw3mnbxg33gzl0cxxasbzpg658g-bios-boot.png.drv", ["out"]),
    ("/nix/store/86zqjh21nf43vnbc278ywgjl7vpiyd9c-squashfs-4.6.1.drv", ["out"]),
    ("/nix/store/8vzg5w1qxrzm5ql200dinqr6a69bv8sq-linux-6.6.59.drv", ["out"]),
    ("/nix/store/i6agxjr2c5byrgr9fyc7vr8aqjpf230i-zstd-1.5.6.drv", ["dev"]),
    ("/nix/store/l67prfmm56njsh60i2dgabvwxwcxyxl7-version.drv", ["out"]),
    ("/nix/store/nzmhhhy7gmxsx2l3w5vvlsgib63cv5j6-efi-directory.drv", ["out"]),
    ("/nix/store/pyf6a99a73qf3lwwxsaabb5cwc6d5in1-efi-image_eltorito.drv", ["out"]),
    ("/nix/store/r3k7ndrph8djnhn7yinjbcpbch8336kw-closure-info.drv", ["out"]),
    ("/nix/store/rnpyxb1ak3j421fl9rsjwn09hlh1i4c0-closure-info.drv", ["out"]),
    ("/nix/store/w16aryrq4w9rzq8y5sddkpsx1573l2zd-initrd-linux-6.6.59.drv", ["out"]),
    ("/nix/store/wf4gg0vzd4j95hyhkwjqw03ka69912kq-efi-background.png.drv", ["out"]),
    ("/nix/store/wg6xr6frjqc5pdm0q6dink88j3acd91z-source.drv", ["out"]),
    ("/nix/store/wql9zbydwdr0nqxkm20crcbhn68wb4pc-stdenv-linux.drv", ["out"]),
    ("/nix/store/yfnx0m1anbgf3lpka9x03iwgg5md573h-libisoburn-1.5.6.drv", ["dev"]),
    ("/nix/store/ysv6wz83jkvg7d65j0js4bml9k0yc4sv-bash-5.2p32.drv", ["out"]),
    ("/nix/store/zli34r2d0zv5gq4rrldxzj94i62xy08z-memtest86+-7.00.drv", ["out"])
  ],
  [
    "/nix/store/36b0mymylaxg9b8ai5ixyvgliz6a50nb-make-iso9660-image.sh",
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  "x86_64-linux",
  "/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash",
  [
    "-e",
    "/nix/store/v6x3cs394jgqfbi0a42pam708flxaphh-default-builder.sh"
  ],
  [
    ("__json", 
      "{\"bootImage\":\"/isolinux/isolinux.bin\",\"bootable\":true," +
      "\"buildCommandPath\":\"/nix/store/36b0mymylaxg9b8ai5ixyvgliz6a50nb-make-iso9660-image.sh\"," +
      "\"buildInputs\":[],\"builder\":\"/nix/store/516kai7nl5dxr792c0nzq0jp8m4zvxpi-bash-5.2p32/bin/bash\"," +
      "\"closureInfo\":\"/nix/store/68mfhv90k4bjllynb34ghj20bv4307l8-closure-info\"," +
      "\"cmakeFlags\":[],\"compressImage\":false,\"configureFlags\":[],\"depsBuildBuild\":[]," +
      "\"depsBuildBuildPropagated\":[],\"depsBuildTarget\":[],\"depsBuildTargetPropagated\":[]," +
      "\"depsHostHost\":[],\"depsHostHostPropagated\":[],\"depsTargetTarget\":[]," +
      "\"depsTargetTargetPropagated\":[],\"doCheck\":false,\"doInstallCheck\":false," +
      "\"efiBootImage\":\"boot/efi.img\",\"efiBootable\":true,\"env\":{},\"isoName\":\"headnode.iso\"," +
      "\"isohybridMbrImage\":\"/nix/store/0lxv8z38mgf377h3fl0rq547dzg4zkzm-syslinux-unstable-2019-02-07/share/syslinux/isohdpfx.bin\"," +
      "\"mesonFlags\":[],\"name\":\"headnode.iso\",\"nativeBuildInputs\":[\"/nix/store/9mdghiw7xh86rpf277zvcvv2adiwmgjw-libisoburn-1.5.6-dev\"," +
      "\"/nix/store/0lxv8z38mgf377h3fl0rq547dzg4zkzm-syslinux-unstable-2019-02-07\",\"/nix/store/xhnlysbblkf7x68vymc5c6wqd9ryifqz-zstd-1.5.6-dev\"," +
      "\"/nix/store/8scnd8kwxvkm4lyqr68pdxaqag3vxgwl-libossp-uuid-1.6.2\",\"/nix/store/nxkdqys5xj6hfzxjspfk2gw4ihlvlix4-squashfs-4.6.1\"]," +
      "\"objects\":[],\"outputs\":[\"out\"],\"patches\":[],\"propagatedBuildInputs\":[]," +
      "\"propagatedNativeBuildInputs\":[],\"sources\":[\"/nix/store/vnpmz816h4bmbwlli0s9i4qn0pmf36hb-linux-6.6.59/bzImage\"," +
      "\"/nix/store/8f6lljk1an6z64k536afr8xnlw0wnwxz-initrd-linux-6.6.59/initrd\"," +
      "\"/nix/store/3kqgysbcahpj4sz5ajjz1960m292d735-version\",\"/nix/store/nvwr3w7kapjci132jrh7jrvymkq2wbwz-bios-boot.png\"," +
      "\"/nix/store/d7s7jxkh9y7fbjvzyyif95pkxb1l5swd-isolinux.cfg\",\"/nix/store/0lxv8z38mgf377h3fl0rq547dzg4zkzm-syslinux-unstable-2019-02-07/share/syslinux\"," +
      "\"/nix/store/16qh3clvbhskbgk1imlahkx6xzhz01xd-efi-image_eltorito\",\"/nix/store/m64vz1hd84ir7hjip7kx4l0ag56090cy-efi-directory/EFI\"," +
      "\"/nix/store/i4mkn2xxkkxwf7skyq3mxc9ixpjn3bg1-loopback.cfg/grub\",\"/nix/store/dxmgm5hll1889ax7ngp81syw4cs7pxfi-efi-background.png\"," +
      "\"/nix/store/n3fya23chxkhwz6v1wib30k66jhh5j7w-memtest86+-7.00/memtest.bin\",\"/nix/store/nnbd491mf2w9palppkar9z1b3r8395hi-source\"]," +
      "\"squashfsCommand\":\"closureInfo=/nix/store/gcnhj0120vs6
2

You need to share the logs for any derivations that failed with nix log.

4

Got it to work. A few things were needed.

  1. the package in the flake should be aarch64-darwin but the actual nixOSsystem should be “x86_64-linux”.
  2. GitHub - cpick/nix-rosetta-builder setup
  3. call nix build without --system "x86_64-linux" and with --option substitute false some drvs were failing: warning: error: unable to download 'https://cache.nixos.org/ssvqaik8k1b7hhbjc4ja0p1vxsrxjx5l.narinfo': Could not resolve hostname (6); retrying in 281 ms (this could also be because of my special dns setup though)
  4. nix.settings.builders-use-substitutes = lib.mkForce false;
  5. nix build --verbose --option substitute false
{
  description = "Minimal NixOS installation media";
  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
  outputs = {
    self,
    nixpkgs,
  }: {
    #1:
    packages.aarch64-darwin = rec {
      default = headnode.config.system.build.isoImage;

      headnode = nixpkgs.lib.nixosSystem {
        system = "x86_64-linux";
        modules = [
          ({
              modulesPath,
              lib,
              ...
            }:{
              imports = [
                (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix")
              ];

              isoImage = {
                makeEfiBootable = true;
                makeUsbBootable = true;
                squashfsCompression = "zstd -Xcompression-level 6"; #way faster build time
              };
            })
        ];
      };
    };
  };
}