Some substitutes for the outputs of derivation

hadilq · April 26, 2026, 1:31am

Here is my flake.nix

{
  description = "tmp";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
    fenix = {
      url = "github:nix-community/fenix";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs =
    {
      self,
      nixpkgs,
      fenix,
      ...
    }:
    let
      system = "x86_64-linux";
      pkgs = import nixpkgs {
        inherit system;
      };

      rust-toolchain = fenix.packages.${system}.fromToolchainFile {
        file = ./rust-toolchain.toml;
        sha256 = "sha256-gh/xTkxKHL4eiRXzWv8KP7vfjSk61Iq48x47BEDFgfk=";
      };
      devShell = pkgs.mkShell {
        packages = [
          rust-toolchain
        ];
      };
    in
    {
      # Development shell
      devShells.${system}.default = devShell;
    };
}

The rust-toolchain.toml file:

[toolchain]
channel = "1.94.1"
profile = "default"

The following is the logs after running nix develop.

$ nix develop
error: chmod "/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218/lib": Read-only file system
error: some substitutes for the outputs of derivation '/nix/store/f0iwc652ai39s868bxi62hzd985xqssi-bash-5.3p3.drv' failed (usually happens due to networking issues); try '--fallback' to build derivation from source
error:
       … while calling the 'derivationStrict' builtin
         at <nix/derivation-internal.nix>:37:12:
           36|
           37|   strict = derivationStrict drvAttrs;
             |            ^
           38|

       … while evaluating derivation 'nix-shell'
         whose name attribute is located at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:541:13

       … while evaluating attribute 'nativeBuildInputs' of derivation 'nix-shell'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:590:13:
          589|             depsBuildBuild = elemAt (elemAt dependencies 0) 0;
          590|             nativeBuildInputs = elemAt (elemAt dependencies 0) 1;
             |             ^
          591|             depsBuildTarget = elemAt (elemAt dependencies 0) 2;

       (stack trace truncated; use '--show-trace' to show the full, detailed trace)

       error: Cannot build '/nix/store/q3rjvy4jwr4viq77iw74icsqhz1nhx6r-channel-rust-1.94.1.toml.drv'.
       Reason: 1 dependency failed.
       Output paths:
         /nix/store/gzn3kaqa8zq05fdn8d20qw1745g90jnb-channel-rust-1.94.1.toml

After downloading a lot of data, it said it’s probably a “networking issues”! It doesn’t look like a networking issue and I don’t understand why chmod is getting run on glibc-2.40-218/lib! I have to mention, it was working this morning! Any idea what’s wrong here?

mdaniels5757 · April 26, 2026, 2:46am

What’s your flake.lock? After fixing a hash mismatch in flake.nix, when I run nix develop with the following flake.lock, it works:

{
  "nodes": {
    "fenix": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ],
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
        "lastModified": 1777102577,
        "narHash": "sha256-ycoy9svZOQgyInu/lwO7IEQtlP5liqYhEcF9m9hPRbM=",
        "owner": "nix-community",
        "repo": "fenix",
        "rev": "f37403486c59376cd285f9685a8ef8ff25c09a3c",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "repo": "fenix",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1776734388,
        "narHash": "sha256-vl3dkhlE5gzsItuHoEMVe+DlonsK+0836LIRDnm6MXQ=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "10e7ad5bbcb421fe07e3a4ad53a634b0cd57ffac",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-25.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "fenix": "fenix",
        "nixpkgs": "nixpkgs"
      }
    },
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
        "lastModified": 1776800521,
        "narHash": "sha256-f8YJfwAOsLFpIoqZuX3yF69UvMLrkx7iVzMH1pJU7cM=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
        "rev": "8954b66d43225e62c92e8bbcc8500191b5cceb1e",
        "type": "github"
      },
      "original": {
        "owner": "rust-lang",
        "ref": "nightly",
        "repo": "rust-analyzer",
        "type": "github"
      }
    }
  },
  "root": "root",
  "version": 7
}

hadilq · April 26, 2026, 3:23pm

Thank you for your feedback. mine is:

{
  "nodes": {
    "fenix": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ],
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
        "lastModified": 1777018861,
        "narHash": "sha256-l+dfxHtTq1jQM53xgYudV8ciECFmJ72PcRAqRS4ys04=",
        "owner": "nix-community",
        "repo": "fenix",
        "rev": "7b33c6466f781cd699fe250c5b69dc4193da67a7",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "repo": "fenix",
        "rev": "7b33c6466f781cd699fe250c5b69dc4193da67a7",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1776734388,
        "narHash": "sha256-vl3dkhlE5gzsItuHoEMVe+DlonsK+0836LIRDnm6MXQ=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "10e7ad5bbcb421fe07e3a4ad53a634b0cd57ffac",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-25.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "fenix": "fenix",
        "nixpkgs": "nixpkgs"
      }
    },
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
        "lastModified": 1776800521,
        "narHash": "sha256-f8YJfwAOsLFpIoqZuX3yF69UvMLrkx7iVzMH1pJU7cM=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
        "rev": "8954b66d43225e62c92e8bbcc8500191b5cceb1e",
        "type": "github"
      },
      "original": {
        "owner": "rust-lang",
        "ref": "nightly",
        "repo": "rust-analyzer",
        "type": "github"
      }
    }
  },
  "root": "root",
  "version": 7
}

unfortunately, using your flake.lock didn’t fix the issue!

mightyiam · April 26, 2026, 3:43pm

Does Nix otherwise work well?

hadilq · April 26, 2026, 4:08pm

I didn’t mention some details at first since I thought they are not important! Sorry! It fails only in one of my podman containers that has –device=nvidia.com/gpu=all flag, otherwise in other containers and the host, there’s no problem at all! I launch this container using a command like this:

podman run -td \
  --user $(id -u):$(id -g) --userns keep-id:uid=$(id -u),gid=$(id -g)\
  --device=nvidia.com/gpu=all \
  --name=container-name my-own-image:latest

Additionally, Nix works as expected. For instance, nix-shell -p hello is totally fine even in the container with --device=nvidia.com/gpu=all flag. I am investigating! It’s probably all my fault!

hadilq · April 26, 2026, 5:49pm

I stuck again! here is additional information, where I still cannot spot where chmod is getting called from!

$ nix develop --show-trace
error: chmod "/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218/lib": Read-only file system
error: some substitutes for the outputs of derivation '/nix/store/f0iwc652ai39s868bxi62hzd985xqssi-bash-5.3p3.drv' failed (usually happens due to networking issues); try '--fallback' to build derivation from source
error:
       … while calling the 'derivationStrict' builtin
         at <nix/derivation-internal.nix>:37:12:
           36|
           37|   strict = derivationStrict drvAttrs;
             |            ^
           38|

       … while evaluating derivation 'nix-shell'
         whose name attribute is located at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:541:13

       … while evaluating attribute 'nativeBuildInputs' of derivation 'nix-shell'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:590:13:
          589|             depsBuildBuild = elemAt (elemAt dependencies 0) 0;
          590|             nativeBuildInputs = elemAt (elemAt dependencies 0) 1;
             |             ^
          591|             depsBuildTarget = elemAt (elemAt dependencies 0) 2;

       … while calling the 'getAttr' builtin
         at <nix/derivation-internal.nix>:50:17:
           49|     value = commonAttrs // {
           50|       outPath = builtins.getAttr outputName strict;
             |                 ^
           51|       drvPath = strict.drvPath;

       … while calling the 'derivationStrict' builtin
         at <nix/derivation-internal.nix>:37:12:
           36|
           37|   strict = derivationStrict drvAttrs;
             |            ^
           38|

       … while evaluating derivation 'rust-1.94.1'
         whose name attribute is located at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:541:13

       … while evaluating attribute 'paths' of derivation 'rust-1.94.1'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/build-support/trivial-builders/default.nix:617:11:
          616|           inherit preferLocalBuild allowSubstitutes;
          617|           paths = mapPaths (path: "${path}${stripPrefix}") paths;
             |           ^
          618|           passAsFile = [ "paths" ];

       … from call site
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/build-support/trivial-builders/default.nix:617:19:
          616|           inherit preferLocalBuild allowSubstitutes;
          617|           paths = mapPaths (path: "${path}${stripPrefix}") paths;
             |                   ^
          618|           passAsFile = [ "paths" ];

       … while calling 'mapPaths'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/build-support/trivial-builders/default.nix:597:12:
          596|       mapPaths =
          597|         f: paths:
             |            ^
          598|         map (

       … while calling the 'map' builtin
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/build-support/trivial-builders/default.nix:598:9:
          597|         f: paths:
          598|         map (
             |         ^
          599|           path:

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:116:39:
          115|         let toolchain = fromToolchainName' target t.channel sha256; in
          116|         combine' "rust-${t.channel}" (attrVals
             |                                       ^
          117|           (filter (component: toolchain ? ${component}) (unique

       … while calling 'attrVals'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/attrsets.nix:531:24:
          530|   */
          531|   attrVals = nameList: set: map (x: set.${x}) nameList;
             |                        ^
          532|

       … while calling the 'map' builtin
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/attrsets.nix:531:29:
          530|   */
          531|   attrVals = nameList: set: map (x: set.${x}) nameList;
             |                             ^
          532|

       … while calling the 'filter' builtin
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:117:12:
          116|         combine' "rust-${t.channel}" (attrVals
          117|           (filter (component: toolchain ? ${component}) (unique
             |            ^
          118|             (toolchain.manifest.profiles.${t.profile or "default"}

       … while calling the 'foldl'' builtin
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:117:58:
          116|         combine' "rust-${t.channel}" (attrVals
          117|           (filter (component: toolchain ? ${component}) (unique
             |                                                          ^
          118|             (toolchain.manifest.profiles.${t.profile or "default"}

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:115:25:
          114|       else
          115|         let toolchain = fromToolchainName' target t.channel sha256; in
             |                         ^
          116|         combine' "rust-${t.channel}" (attrVals

       … while calling 'fromToolchainName''
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:84:38:
           83|
           84|   fromToolchainName' = target: name: sha256:
             |                                      ^
           85|     mapNullable

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:85:5:
           84|   fromToolchainName' = target: name: sha256:
           85|     mapNullable
             |     ^
           86|       (matches:

       … while calling 'mapNullable'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:385:20:
          384|   */
          385|   mapNullable = f: a: if a == null then a else f a;
             |                    ^
          386|

       … from call site
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:385:48:
          384|   */
          385|   mapNullable = f: a: if a == null then a else f a;
             |                                                ^
          386|

       … while calling anonymous lambda
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:86:8:
           85|     mapNullable
           86|       (matches:
             |        ^
           87|         let target' = elemAt matches 5; in

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:88:9:
           87|         let target' = elemAt matches 5; in
           88|         toolchainOf' (if target' == null then target else target') {
             |         ^
           89|           inherit sha256;

       … while calling 'toolchainOf''
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:71:5:
           70|   toolchainOf' = target:
           71|     { root ? "https://static.rust-lang.org/dist"
             |     ^
           72|     , channel ? "nightly"

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:79:5:
           78|     in
           79|     fromManifestFile' target "-${channel}" (if (sha256 == null) then
             |     ^
           80|       builtins.fetchurl url

       … while calling 'fromManifestFile''
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:67:37:
           66|
           67|   fromManifestFile' = target: name: file:
             |                                     ^
           68|     fromManifest' target name (importTOML file);

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:68:5:
           67|   fromManifestFile' = target: name: file:
           68|     fromManifest' target name (importTOML file);
             |     ^
           69|

       … while calling 'fromManifest''
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:47:35:
           46|
           47|   fromManifest' = target: suffix: manifest:
             |                                   ^
           48|     let

       … in the left operand of the update (//) operator
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:58:15:
           57|     in
           58|     toolchain // mapAttrs'
             |               ^
           59|       (k: v:

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:49:19:
           48|     let
           49|       toolchain = mkToolchain suffix {
             |                   ^
           50|         inherit (manifest) date;

       … while calling anonymous lambda
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/lib/mk-toolchain.nix:4:1:
            3| suffix:
            4| { date, components }:
             | ^
            5|

       … in the left operand of the update (//) operator
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/lib/mk-toolchain.nix:174:13:
          173|
          174| toolchain'' // {
             |             ^
          175|   withComponents = componentNames: combine

       … in the left operand of the update (//) operator
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/lib/mk-toolchain.nix:171:28:
          170|
          171|   toolchain'' = toolchain' // mapAttrs' (k: nameValuePair (removeSuffix "-preview" k)) toolchain';
             |                            ^
          172| in

       … in the left operand of the update (//) operator
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/lib/mk-toolchain.nix:146:26:
          145|
          146|   toolchain' = toolchain // {
             |                          ^
          147|     toolchain = combine "rust${suffix}-${date}"

       … while calling the 'mapAttrs' builtin
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/lib/mk-toolchain.nix:15:15:
           14|
           15|   toolchain = mapAttrs
             |               ^
           16|     (component: source:

       … while calling the 'mapAttrs' builtin
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:51:22:
           50|         inherit (manifest) date;
           51|         components = mapAttrs
             |                      ^
           52|           (_: src: { inherit (src) url; sha256 = src.hash; })

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:53:12:
           52|           (_: src: { inherit (src) url; sha256 = src.hash; })
           53|           (filterAttrs (_: src: src ? available && src.available) (mapAttrs
             |            ^
           54|             (_: pkg: pkg.target."*" or pkg.target.${target} or null)

       … while calling 'filterAttrs'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/attrsets.nix:663:23:
          662|   */
          663|   filterAttrs = pred: set: removeAttrs set (filter (name: !pred name set.${name}) (attrNames set));
             |                       ^
          664|

       … while calling the 'removeAttrs' builtin
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/attrsets.nix:663:28:
          662|   */
          663|   filterAttrs = pred: set: removeAttrs set (filter (name: !pred name set.${name}) (attrNames set));
             |                            ^
          664|

       … while calling the 'mapAttrs' builtin
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:53:68:
           52|           (_: src: { inherit (src) url; sha256 = src.hash; })
           53|           (filterAttrs (_: src: src ? available && src.available) (mapAttrs
             |                                                                    ^
           54|             (_: pkg: pkg.target."*" or pkg.target.${target} or null)

       … from call site
         at /nix/store/5v3r6bfx468ha861xxs7ngvq62kdshng-source/default.nix:68:32:
           67|   fromManifestFile' = target: name: file:
           68|     fromManifest' target name (importTOML file);
             |                                ^
           69|

       … while calling 'importTOML'
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:772:16:
          771|   */
          772|   importTOML = path: fromTOML (builtins.readFile path);
             |                ^
          773|

       … while calling the 'fromTOML' builtin
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:772:22:
          771|   */
          772|   importTOML = path: fromTOML (builtins.readFile path);
             |                      ^
          773|

       … while evaluating the argument passed to builtins.fromTOML

       … while calling the 'readFile' builtin
         at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:772:32:
          771|   */
          772|   importTOML = path: fromTOML (builtins.readFile path);
             |                                ^
          773|

       … while realising the context of path '/nix/store/hdvhj56j5flzhnm1wyphrii2ay2r1679-channel-rust-1.94.1.toml'

       error: Cannot build '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv'.
       Reason: 1 dependency failed.
       Output paths:
         /nix/store/hdvhj56j5flzhnm1wyphrii2ay2r1679-channel-rust-1.94.1.toml

the only observable difference between my container without –device=nvidia.com/gpu=all flag, and the one with this flag is that I have access to nvidia toolkit: nvidia-smi

$ echo $PATH
/usr/bin:/home/dev/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin

$ ls /usr/bin
env  nvidia-cuda-mps-control  nvidia-cuda-mps-server  nvidia-debugdump  nvidia-powerd  nvidia-smi

Any idea?

mdaniels5757 · April 26, 2026, 6:11pm

Maybe nix develop --show-trace -vvv will show something helpful?

hadilq · April 26, 2026, 6:54pm

the logs were too long so this is only the diff between the healthy run of nix develop and problematic one in two different containers, one with gpu flag and one without one:

$ cat nix-develop-show-trace-vvv.diff
5,9c5,9
< acquiring write lock on '/nix/var/nix/temproots/1910'
< locking path '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source'
< lock acquired on '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source.lock'
< lock released on '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source.lock'
< got tree '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source' from 'path:/home/dev/src/tmp?lastModified=1777228123'
---
> acquiring write lock on '/nix/var/nix/temproots/5792'
> locking path '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source'
> lock acquired on '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source.lock'
> lock released on '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source.lock'
> got tree '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source' from 'path:/home/dev/src/tmp?lastModified=1777228157'
13,22c13
< evaluating file '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source/flake.nix'
< performing daemon worker op: 1
< performing daemon worker op: 1
< source path '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source' is uncacheable
< copying '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source' to the store...
< performing daemon worker op: 1
< performing daemon worker op: 1
< performing daemon worker op: 1
< performing daemon worker op: 1
< performing daemon worker op: 7
---
> evaluating file '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source/flake.nix'
24a16,17
> source path '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source' is uncacheable
> copying '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source' to the store...
39,43c32,33
< performing daemon worker op: 1
< performing daemon worker op: 1
< performing daemon worker op: 1
< performing daemon worker op: 1
< copied '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source' to '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source'
---
> performing daemon worker op: 7
> copied '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source' to '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source'
45c35
< evaluating file '/nix/store/36mnv1gs63di0shhgffqpl9vscv05w3z-source/flake.nix'
---
> evaluating file '/nix/store/20v10dvp8d7gzrk5d8i5z2fds4nxs65w-source/flake.nix'
1747,1792d1736
< checking substituter 'https://cache.nixos.org' for path '/nix/store/hdvhj56j5flzhnm1wyphrii2ay2r1679-channel-rust-1.94.1.toml'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/wv8bpzriikv65xnd1vciqpq7rnr8h2q2-bash-5.3p3'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/wv8bpzriikv65xnd1vciqpq7rnr8h2q2-bash-5.3p3'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/0lz1fbrla1nm684x0xm577agg64v96v6-curl-8.19.0-dev'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/0lz1fbrla1nm684x0xm577agg64v96v6-curl-8.19.0-dev'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/0076ndvx724b4icqkmgiwfmnlp5hbw6x-zlib-1.3.2-dev'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/pjap67bv2yjgzgqzrsny1dsd54z2ca95-krb5-1.22.1-lib'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/5rx06scpk50xrlp56p8qjgb65c655id9-nghttp2-1.67.1-dev'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/ar42hw378k9qi4df74cyrp7xdbd0ssb1-curl-8.19.0'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/n84bgxb2qxhfph1ria0xxlhjhpg48740-stdenv-linux'
< checking substituter 'https://cache.nixos.org' for path '/nix/store/dxh91d5z3x3s66ki3z6q05x09k2q4wxl-openssl-3.6.1-dev'
< I removed a lot of lines due to line limit. Let me know if you need them!
< substitution of '/nix/store/xqgj6py3hb44dvinh6isa7hdr3d72a5l-gcc-14.3.0-libgcc': in final_awaiter
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': in final_awaiter
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': all references realised
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': woken up
< substitution of '/nix/store/xqgj6py3hb44dvinh6isa7hdr3d72a5l-gcc-14.3.0-libgcc': goal destroyed
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': in final_awaiter
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': trying to run
< copying path '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218' from 'https://cache.nixos.org'...
< locking path '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218'
< lock acquired on '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218.lock'
< lock released on '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218.lock'
< downloading 'https://cache.nixos.org/nar/09n07vz92rxvp8ldk5hg71l95jwbgsgq07i051avhfxzpr9iz7bg.nar.xz'...
< starting download of https://cache.nixos.org/nar/09n07vz92rxvp8ldk5hg71l95jwbgsgq07i051avhfxzpr9iz7bg.nar.xz
< finished download of 'https://cache.nixos.org/nar/09n07vz92rxvp8ldk5hg71l95jwbgsgq07i051avhfxzpr9iz7bg.nar.xz'; curl status = 0, HTTP status = 200, body = 6555708 bytes, duration = 1.97 s
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': got EOF
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': woken up
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': substitute finished
< error: chmod "/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218/lib": Read-only file system
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': in final_awaiter
< path '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218' is required, but there is no substituter that can build it
< substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218': done
< substitution of '/nix/store/wv8bpzriikv65xnd1vciqpq7rnr8h2q2-bash-5.3p3': waitee 'substitution of '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218'' done; 0 left
< substitution of '/nix/store/wv8bpzriikv65xnd1vciqpq7rnr8h2q2-bash-5.3p3': woken up
< I removed a lot of lines here! Let me know if you need them!
< building of '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv^out' from in-memory derivation: woken up
< outer obtaining drv from '/nix/store/f0iwc652ai39s868bxi62hzd985xqssi-bash-5.3p3.drv' and then building outputs 'out': goal destroyed
< outer obtaining drv from '/nix/store/250jpdy8ii986n2l5cm6ccw93kyr2967-mirrors-list.drv' and then building outputs 'out': goal destroyed
< building of '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv' from in-memory derivation: in final_awaiter
< outer obtaining drv from '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv' and then building outputs 'dev': in final_awaiter
< outer obtaining drv from '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv' and then building outputs 'dev': outer build done
< outer obtaining drv from '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv' and then building outputs 'dev': done
< building of '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv^bin,debug,dev,devdoc,man,out' from in-memory derivation: goal destroyed
< outer obtaining drv from '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv' and then building outputs 'dev': in final_awaiter
< outer obtaining drv from '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv' and then building outputs 'out': in final_awaiter
< outer obtaining drv from '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv' and then building outputs 'out': outer build done
< outer obtaining drv from '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv' and then building outputs 'out': done
< building of '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv^out' from in-memory derivation: goal destroyed
< outer obtaining drv from '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv' and then building outputs 'out': in final_awaiter
< outer obtaining drv from '/nix/store/s8ldl64hd0y1ccsg1yzqn5fyyrgp10b6-stdenv-linux.drv' and then building outputs 'out': goal destroyed
< outer obtaining drv from '/nix/store/ksfcnil7yjc8rfgfy71ygjhp1z5knxim-curl-8.19.0.drv' and then building outputs 'dev': goal destroyed
< building of '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv^out' from in-memory derivation: in final_awaiter
< building of '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv^out' from in-memory derivation: outer build done
2478d1749
< building of '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv' from in-memory derivation: goal destroyed
2486,2787c1757,2028
< error:
<        … while calling the 'derivationStrict' builtin
<          at <nix/derivation-internal.nix>:37:12:
<            36|
<            37|   strict = derivationStrict drvAttrs;
<              |            ^
<            38|
<
<        … while evaluating derivation 'nix-shell'
<          whose name attribute is located at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/stdenv/generic/make-derivation.nix:541:13
<
< I have to remove some lines that I have already posted above
<          at /nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/lib/trivial.nix:772:32:
<           771|   */
<           772|   importTOML = path: fromTOML (builtins.readFile path);
<              |                                ^
<           773|
<
<        … while realising the context of path '/nix/store/hdvhj56j5flzhnm1wyphrii2ay2r1679-channel-rust-1.94.1.toml'
<
<        error: Cannot build '/nix/store/clxq6sc4xnm89j59j9w8wil5d8x3zjqr-channel-rust-1.94.1.toml.drv'.
<        Reason: 1 dependency failed.
<        Output paths:
<          /nix/store/hdvhj56j5flzhnm1wyphrii2ay2r1679-channel-rust-1.94.1.toml
---
> performing daemon worker op: 41
> performing daemon worker op: 26
> performing daemon worker op: 7
> instantiated 'rustc-1.94.1-x86_64-unknown-linux-gnu.tar.gz' -> '/nix/store/v8wq12sccmla96khr8s107m3w569mgid-rustc-1.94.1-x86_64-unknown-linux-gnu.tar.gz.drv'
> performing daemon worker op: 7
> instantiated 'rustc-1.94.1-2026-03-26' -> '/nix/store/x9vnxpqkhrjisncmykr6vlywpgf7dbsv-rustc-1.94.1-2026-03-26.drv'
> performing daemon worker op: 7
> instantiated 'rust-std-1.94.1-x86_64-unknown-linux-gnu.tar.gz' -> '/nix/store/30vqyf0m9349hxs505sh5vkhc9vx0ibz-rust-std-1.94.1-x86_64-unknown-linux-gnu.tar.gz.drv'
> performing daemon worker op: 7
> instantiated 'rust-std-1.94.1-2026-03-26' -> '/nix/store/30f6gnwss6r91wb3mypc2g652808ybrf-rust-std-1.94.1-2026-03-26.drv'
> performing daemon worker op: 7
> I removed a lot of lines here since you probably can reproduce them as well.
> outer obtaining drv from '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv' and then building outputs 'man', 'out': in final_awaiter
> outer obtaining drv from '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv' and then building outputs 'man', 'out': outer build done
> outer obtaining drv from '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv' and then building outputs 'man', 'out': done
> building of '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv^debug,dev,doc,info,man,out' from in-memory derivation: goal destroyed
> building of '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv^debug,dev,doc,info,man,out' from in-memory derivation: goal destroyed
> outer obtaining drv from '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv' and then building outputs 'man', 'out': in final_awaiter
> outer obtaining drv from '/nix/store/hdyswmjki8ph2hq4dzf0rdg4akai6l28-bash-interactive-5.3p3.drv' and then building outputs 'man', 'out': goal destroyed

hadilq · April 26, 2026, 6:56pm

Also the nix config show in both containers, with and without gpu flag, are the same as below:

$ cat nix-config-show
abort-on-warn = false
accept-flake-config = false
access-tokens =
allow-dirty = true
allow-dirty-locks = false
allow-import-from-derivation = true
allow-new-privileges = false
allow-symlinked-store = false
allow-unsafe-native-code-during-evaluation = false
allowed-impure-host-deps =
allowed-uris =
allowed-users = *
always-allow-substitutes = false
auto-optimise-store = false
bash-prompt =
bash-prompt-prefix =
bash-prompt-suffix =
build-dir =
build-hook = /nix/store/mhs8m07mfr4jcvc6216w8v0zgfg2695r-nix-2.31.2/bin/nix __build-remote
build-poll-interval = 5
build-users-group = nixbld
builders = @/etc/nix/machines
builders-use-substitutes = false
commit-lock-file-summary =
compress-build-log = true
connect-timeout = 15
cores = 0
debugger-on-trace = false
debugger-on-warn = false
diff-hook =
download-attempts = 5
download-buffer-size = 67108864
download-speed = 0
eval-cache = true
eval-profile-file = nix.profile
eval-profiler = disabled
eval-profiler-frequency = 99
eval-system =
experimental-features = fetch-tree flakes nix-command
extra-platforms = i686-linux x86_64-v1-linux x86_64-v2-linux x86_64-v3-linux
fallback = false
filter-syscalls = true
flake-registry = https://channels.nixos.org/flake-registry.json
fsync-metadata = true
fsync-store-paths = false
gc-reserved-space = 8388608
hashed-mirrors =
http-connections = 25
http2 = true
id-count = 8388608
ignore-try = false
ignored-acls = security.csm security.selinux system.nfs4_acl
impersonate-linux-26 = false
json-log-path =
keep-build-log = true
keep-derivations = true
keep-env-derivations = false
keep-failed = false
keep-going = false
keep-outputs = false
log-lines = 25
max-build-log-size = 0
max-call-depth = 10000
max-free = 9223372036854775807
max-jobs = 1
max-silent-time = 0
max-substitution-jobs = 16
min-free = 0
min-free-check-interval = 5
nar-buffer-size = 33554432
narinfo-cache-negative-ttl = 3600
narinfo-cache-positive-ttl = 2592000
netrc-file = /etc/nix/netrc
nix-path = /nix/var/nix/profiles/per-user/dev/channels /home/dev/.nix-defexpr/channels
nix-shell-always-looks-for-shell-nix = true
nix-shell-shebang-arguments-relative-to-script = true
plugin-files =
post-build-hook =
pre-build-hook =
preallocate-contents = false
print-missing = true
pure-eval = true
require-drop-supplementary-groups = false
require-sigs = true
restrict-eval = false
run-diff-hook = false
sandbox = false
sandbox-build-dir = /build
sandbox-dev-shm-size = 50%
sandbox-fallback = true
sandbox-paths = /bin/sh=/nix/store/9v6zfa4ws3llr0k6njl6gd03rwpn644b-busybox-1.36.1/bin/busybox
secret-key-files =
show-trace = false
ssl-cert-file = /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
stalled-download-timeout = 300
start-id = 872415232
store = auto
substitute = true
substituters = https://cache.nixos.org
sync-before-registering = false
system = x86_64-linux
system-features = benchmark big-parallel nixos-test uid-range
tarball-ttl = 3600
timeout = 0
trace-function-calls = false
trace-import-from-derivation = false
trace-verbose = false
trust-tarballs-from-git-forges = true
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
trusted-substituters =
trusted-users = root
upgrade-nix-store-path-url = https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix
use-case-hack = false
use-cgroups = false
use-registries = true
use-sqlite-wal = true
use-xdg-base-directories = false
user-agent-suffix =
warn-dirty = true
warn-large-path-threshold = 0
warn-short-path-literals = false

hadilq · April 27, 2026, 12:01am

it’s getting more and more weird! I changed the flake.nix to the following to use rust-overlay instead of fenix.

{
  description = "tmp";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
    rust-overlay = {
      url = "github:oxalica/rust-overlay";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs =
    {
      self,
      nixpkgs,
      rust-overlay,
      ...
    }:
    let
      system = "x86_64-linux";
      overlays = [ (import rust-overlay) ];
      pkgs = import nixpkgs {
        inherit system overlays;
      };

      rust-toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
      devShell = pkgs.mkShell {
        packages = [
          rust-toolchain
        ];
      };
    in
    {
      # Development shell
      devShells.${system}.default = devShell;
    };
}

but still the same observations! in the host, and the container without gpu flag, it’s successful, however, in the the container with gpu flag I see the following after running nix develop --show-trace -vvv:

instantiated 'libsecret-0.21.7.tar.xz' -> '/nix/store/wlf9z8lavanjq18hdslwx42vry20n95r-libsecret-0.21.7.tar.xz.drv'
instantiated 'libsecret-0.21.7' -> '/nix/store/vwz93ribgj2krkjwh601jkrgsbghg9fr-libsecret-0.21.7.drv'
copying '/nix/store/y3ms9nc6dxcssfvni5m46zb2kv8d4r20-source/lib/cargo-miri-wrapper.sh' to the store...
copied source '/nix/store/y3ms9nc6dxcssfvni5m46zb2kv8d4r20-source/lib/cargo-miri-wrapper.sh' -> '/nix/store/nnscrmv4wx0vh2hm41sr7jc5h3sqbn89-cargo-miri-wrapper.sh'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/package.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/add-core-symbolication.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/derivation-options.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/passthru-private-frameworks.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/passthru-source-release-files.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/remove-disallowed-packages.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/process-stubs.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/propagate-inputs.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/propagate-xcrun.nix'
evaluating file '/nix/store/ds6zw8983xm46y10dq7aswqkgmry968q-source/pkgs/by-name/ap/apple-sdk/common/run-build-phase-hooks.nix'
instantiated 'rustc-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/s8cd7kmvs064whpk3ypk330ix2zkiidb-rustc-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'rustc-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/0v0045hdrywcmky19dr9y5dklfc2iv9q-rustc-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'cargo-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/bvzyzkinmzbl04zf5x5g9asnaylwa5r6-cargo-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'cargo-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/zbzdhfydqk9ijf448gamgwwiwdd4zj8z-cargo-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'rust-std-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/ykvk94jkyn31fh8dfl3filw22wf654gk-rust-std-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'rust-std-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/9rk2dq54kvllhln4ccwvc0nm9y3ahw42-rust-std-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'rust-docs-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/pflrnzhjc90yja2hbr6l14ffmq5q2cyz-rust-docs-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'rust-docs-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/clsv68s12fh5vc1h4q95szin2fzqj3zd-rust-docs-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'rustfmt-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/158vvhbjbjfawr70n1d0nimi1g501yvy-rustfmt-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'rustfmt-preview-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/3brm2c91aza5hc5cbjynj8d88j5r1gh0-rustfmt-preview-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'clippy-1.94.1-x86_64-unknown-linux-gnu.tar.xz' -> '/nix/store/ms4m04pxxmw6n5bwn3mpqknhr0kprm40-clippy-1.94.1-x86_64-unknown-linux-gnu.tar.xz.drv'
instantiated 'clippy-preview-1.94.1-x86_64-unknown-linux-gnu' -> '/nix/store/2pz29wwcd85w8c6pmwlx912fn4cv5hb0-clippy-preview-1.94.1-x86_64-unknown-linux-gnu.drv'
instantiated 'rust-default-1.94.1' -> '/nix/store/x4xnjcgxgyp8phsx54zx5s2k732bsili-rust-default-1.94.1.drv'
instantiated 'nix-shell' -> '/nix/store/1srlx7mh5pp0lxiv41afsm082pgngd2j-nix-shell.drv'
copying path '/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218' from 'https://cache.nixos.org'...
downloading 'https://cache.nixos.org/nar/09n07vz92rxvp8ldk5hg71l95jwbgsgq07i051avhfxzpr9iz7bg.nar.xz'...
error: chmod "/nix/store/pdwlyjkmc6icc0wb5gaw08m9ynqrg683-glibc-2.40-218/lib": Read-only file system
error: some substitutes for the outputs of derivation '/nix/store/f0iwc652ai39s868bxi62hzd985xqssi-bash-5.3p3.drv' failed (usually happens due to networking issues); try '--fallback' to build derivation from source
error: Cannot build '/nix/store/ghjdf5k283fli97f9wrsbl0mbhpky59y-nix-shell-env.drv'.
       Reason: 1 dependency failed.
       Output paths:
         /nix/store/kz4jqdhvqz7r8hxrlsc0qjyvc8w91ihl-nix-shell-env

Any idea?

hadilq · April 28, 2026, 3:59am

Finally! It turns out that the –device=nvidia.com/gpu=all is mounting a lot of stuff as read-only, and one of them is glibc. you can check it by running cat /proc/mounts inside the container. the stuff that will get mounted are listed in /run/cdi/nvidia-container-toolkit.json file in the host, where before this I was symlink it to /etc/cdi to be used by podman. also there’s cdi_spec_dirs variable in the /etc/containers/containers.conf file that controls it.

The solution was having a service to copy the json file from /run/cdi to /etc/cdi and remove the the glibc records. also we need to explicitly only read /etc/cdi by having a containers.conf like this

$ cat /etc/containers/containers.conf
[engine]
cdi_spec_dirs = ["/etc/cdi"]
init_path = "/nix/store/ybfcpsi84p9nd5za7yp4wkpw4xhlx533-catatonit-0.2.1/bin/catatonit"

[network]
cni_plugin_dirs = ["/nix/store/mayzfrlhn27m878dnhwsfibj10p6l4lr-cni-plugins-1.8.0/bin"]
network_backend = "netavark"

to do so I added these in the host configs:

 systemd.services.cdi-nvidia-strip-glibc = {
    description = "Copy CDI nvidia spec to /etc/cdi and strip glibc mounts";

    # Run after the toolkit CDI generator has produced the file in /run/cdi/
    after = [ "nvidia-container-toolkit-cdi-generator.service" ];
    requires = [ "nvidia-container-toolkit-cdi-generator.service" ];

    # Run on every boot so it picks up driver updates
    wantedBy = [ "multi-user.target" ];

    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
    };

    script = ''
      set -euo pipefail

      SRC="/run/cdi/nvidia-container-toolkit.json"
      DST="/etc/cdi/nvidia-container-toolkit.json"

      # Only copy if the destination does not exist yet
      if [ -f "$DST" ]; then
        echo "$DST already exists, skipping"
        exit 0
      fi

      # Wait for the source file to appear (CDI generator may be slow)
      for i in $(seq 1 30); do
        [ -f "$SRC" ] && break
        echo "Waiting for $SRC ... ($i/30)"
        sleep 1
      done

      if [ ! -f "$SRC" ]; then
        echo "ERROR: $SRC not found after waiting, aborting"
        exit 1
      fi

      echo "Copying $SRC -> $DST and stripping glibc mounts"

      mkdir -p /etc/cdi

      # Use jq to remove any mount entry where hostPath or containerPath
      # contains the string "glibc"
      ${pkgs.jq}/bin/jq '
        .containerEditions |= map(
          .mounts |= map(
            select(
              (.hostPath      | test("glibc"; "i") | not) and
              (.containerPath | test("glibc"; "i") | not)
            )
          )
        )
      ' "$SRC" > "$DST"

      echo "Done. Glibc mounts removed from $DST"
    '';
  };

  virtualisation.containers.containersConf.settings = {
    engine = {
      cdi_spec_dirs = lib.mkForce [ "/etc/cdi" ];  # drop /var/run/cdi entirely
    };
  };

hope it helps!

mightyiam · April 28, 2026, 2:13pm

Sorry for not following. Just wanted to raise a question; do your insights uncover an issue in any project?

TLATER · April 28, 2026, 2:58pm

I think the root cause here is that the nvidia container toolkit pulls in paths from the host for compatibility reasons; most likely it has to ensure that the host’s nvidia userspace libraries match up with the ones in the container, including their dependencies, and that’s why it does all this. It’s a pretty ugly hack to begin with, bind-mounting libraries into a container is suuuuper wrong, but without a generic kernel interface for this type of gpu acceleration there’s no better way (and nvidia don’t want such a thing to to exist because selling cuda is good for their wallets - no better way to abuse a dominant market position than to maintain a monopoly).

The “fix” suggested here overrides this behavior, and happens to work because the host and the container are presumably on the same OS version (or at least close enough that ABI compatibility is maintained), but in theory could result in breakage.

The reason the chown is attempted is because the nix store’s database isn’t present inside the container, so nix doesn’t know these libraries already exist within the file system, and it attempts to overbuild them.

There’s no particularly good way to solve this, it’s just the awkwardness of trying to inject some host libraries into an OCI container environment for hardware reasons while keeping others separate. The container nix shouldn’t know about the presence of those paths by design - the best you could do is some ad-hoc nix database modification at container start time, but that’d also be quite hacky.

Caveat emptor: these are educated guesses based on symptoms and outputs. Consider it conjecture.

hadilq · April 30, 2026, 1:45am

I confirm all your educated guesses here are correct. The host and container versions are the same, which is why they want exactly the same version of glibc, so this also answered Shahar’s question above that there’s nothing we can do to improve any of nix projects.