Pkgs.dockertools built arm64 image doesn't work on arm64 Docker

I am using Nix on M1 Mac, and I am trying to build an ARM64 image for ARM64 docker. This is my derivation:

          # flake.nix
          # Some abstractions here for monoLib.mkDeno
          oak = monoLib.mkDeno {
            pname = "oak";
            src = ./apps/examples/deno/oak;
            entrypoint = "/src/index.ts";
            flag = ["--allow-net"];
          };

          oak-image = pkgs.dockerTools.buildLayeredImage {
            name = "oak";
            tag = "latest";
            contents = [ oak ];
            config = { Cmd = [ "${oak}/bin/${oak.pname}" ]; };
          };

And the definition for my derivation builder function:

# monoLib.mkDeno
{ pkgs, ... }: {
  mkDeno = ({ src, entrypoint, pname, flag ? [], buildInputs ? []}: 
  let 
    _buildInputs = with pkgs; [ deno ] ++ buildInputs;
  in pkgs.stdenv.mkDerivation {
      inherit pname src;
      name = pname;
      buildInputs = _buildInputs;
      unpackPhase = ''
        # Need to define $HOME as deno will try to cache typescript definition
        export HOME=$(pwd)/.home; mkdir -p "$HOME";
        cp -r ${src} .
      '';
      dontPatch = true;
      dontFixup = true;
      preBuild = ''
        deno lint
      '';
      buildPhase = ''
        runHook preBuild
        deno compile ${builtins.toString flag} --output ./${pname} "${src}${entrypoint}"
        runHook postBuild
      '';
      installPhase = ''
        mkdir -p $out/bin
        mv ./${pname} $out/bin
      '';
  });
}

I am getting an error like this when I run the nix built image:

exec /nix/store/r7045mbap45z4z0zm0p74a842zw34a47-oak/bin/oak: exec format error

I have checked the runtime arch for my image:

base ❯ docker inspect oak:latest 
[
    {
        "Id": "sha256:9128284f27c922a9614aed496e3ecb32475f50c3baca306a04aa40bc9338e60b",
        "RepoTags": [
            "oak:latest"
        ],
        "RepoDigests": [],
        "Parent": "",
        "Comment": "store paths: ['/nix/store/78vhgkgx819m2429z0i03vr384qvh1xh-oak-customisation-layer']",
        "Created": "1970-01-01T00:00:01Z",
        "Container": "",
        "ContainerConfig": {
            "Hostname": "",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": null,
            "Cmd": null,
            "Image": "",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": null
        },
        "DockerVersion": "",
        "Author": "",
        "Config": {
            "Hostname": "",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": null,
            "Cmd": [
                "/nix/store/r7045mbap45z4z0zm0p74a842zw34a47-oak/bin/oak"
            ],
            "Image": "",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": null
        },
        "Architecture": "arm64",
        "Os": "linux",
        "Size": 69093752,
        "VirtualSize": 69093752,
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/626c69ea88202ab97510d80d9897bc77468b20864f31b6c6c09d83fc0031744c/diff:/var/lib/docker/overlay2/d9c1404dd956ff179796cce8d7ce6a8403060637bfbfbcaa47d230204f2e6ea3/diff:/var/lib/docker/overlay2/bc8644f89dea27e5ff1e633a7f4bd0f997244131bdb62d694472946bf9490072/diff:/var/lib/docker/overlay2/84203beebe3211c99fd61d2f5e29ca725009927bc4fa1604dec3e6a814a4f791/diff:/var/lib/docker/overlay2/afb2613e78f5a9d4be3c5fb2612b6fcab2bdeb7dd35adce481923b16f5c2cf27/diff",
                "MergedDir": "/var/lib/docker/overlay2/8009f3e005a68595b299c61a5304bb7dd5e8bfe0a68427da47747da5c2c68662/merged",
                "UpperDir": "/var/lib/docker/overlay2/8009f3e005a68595b299c61a5304bb7dd5e8bfe0a68427da47747da5c2c68662/diff",
                "WorkDir": "/var/lib/docker/overlay2/8009f3e005a68595b299c61a5304bb7dd5e8bfe0a68427da47747da5c2c68662/work"
            },
            "Name": "overlay2"
        },
        "RootFS": {
            "Type": "layers",
            "Layers": [
                "sha256:feb7411d13a0341d76d6ee73c38e13f1e47971e14e45f8d2617edf1d3b404c3f",
                "sha256:c0c296a5da305b6b0d9d83b6855d78d1820c74c4ad4bb5f286acbc1f9aed1be7",
                "sha256:fbcadad7210dd974d508f1c2d193675212e88ebb69d788910bfdba014a81d9ba",
                "sha256:61dacdfed060445cd309d0c8af284bcb69350a5ae6caf6e1ce162e34583a1791",
                "sha256:126caf9639ccf46c0bc8565e332a0302ea1a50e3e4d6c3b343fceff363d8ad6e",
                "sha256:31d2fb088d02ea6f62a258ed673027d813ed6cd7215cc241bf9c20698a9a79e3"
            ]
        },
        "Metadata": {
            "LastTagTime": "0001-01-01T00:00:00Z"
        }
    }
]

so it confirmed that the arch is ARM64.

Then I checked the arch of Docker Desktop, just to make sure I am running it natively:

base ❯ docker info              
Client:
 Context:    default
 Debug Mode: false
 Plugins:
  buildx: Docker Buildx (Docker Inc., v0.8.2)
  compose: Docker Compose (Docker Inc., v2.7.0)
  extension: Manages Docker extensions (Docker Inc., v0.2.8)
  sbom: View the packaged-based Software Bill Of Materials (SBOM) for an image (Anchore Inc., 0.6.0)
  scan: Docker Scan (Docker Inc., v0.17.0)

Server:
 Containers: 2
  Running: 0
  Paused: 0
  Stopped: 2
 Images: 1
 Server Version: 20.10.17
 Storage Driver: overlay2
  Backing Filesystem: extfs
  Supports d_type: true
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: cgroupfs
 Cgroup Version: 2
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: io.containerd.runc.v2 io.containerd.runtime.v1.linux runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 10c12954828e7c7c9b6e0ea9b0c02b01407d3ae1
 runc version: v1.1.2-0-ga916309
 init version: de40ad0
 Security Options:
  seccomp
   Profile: default
  cgroupns
 Kernel Version: 5.10.104-linuxkit
 Operating System: Docker Desktop
 OSType: linux
 Architecture: aarch64
 CPUs: 4
 Total Memory: 3.841GiB
 Name: docker-desktop
 ID: XCAG:GIBI:ZRPI:FQCE:O4H6:HH6T:Q4T6:JUTU:X26S:YCI2:S7GC:SCBX
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 HTTP Proxy: http.docker.internal:3128
 HTTPS Proxy: http.docker.internal:3128
 No Proxy: hubproxy.docker.internal
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
 Insecure Registries:
  hubproxy.docker.internal:5000
  127.0.0.0/8
 Live Restore Enabled: false

It is indeed ARM64. So what can go wrong here? Why I am getting that error?

1 Like

An update, I have tried to use file to check the arch for the binary before building the image(the exec name has changed, but it is the same project):

base ❯ file ./result/bin/deno-api-dynamic-form 
./result/bin/deno-api-dynamic-form: Mach-O 64-bit executable arm64

Somehow it is in the right format, arm64. What else should I check?

Mach-O is a macOS binary. Presumably you want a Linux ELF binary for Docker.

1 Like

I see thanks! I guess I have to use cross compile then for building the binary.

Can you show the diff from applying your solution? I think I may be in the same situation. I do docker load < $(nix-build docker.nix) && docker run -p 8180:8180 foo:latest to build a docker image and run it on Linux and it works fine. But if doing the same from an M1 Mac gives that exec format error.

For my own project, it’s compiling with callcabal2nix/ghc. Does that mean that in order to build the M1 docker image on Mac, nix would need to use the Linux version of GHC on Mac?