`nix-shell` is eating up all my 32 GB memory

1

I have the following shell script

let
  # Import nixpkgs
  pkgs = import <nixpkgs> { };

  # Import poetry2nix and its functions
  poetry2nixSrc = pkgs.fetchFromGitHub {
    owner = "nix-community";
    repo = "poetry2nix";
    rev = "e23218d1599e3369dfc878757e58974017e0ecc8"; # Fetch the latest master
    sha256 = "sha256-p6niqag7b4XEHvzWgG0X/xjoW/ZXbAxW8ggd8yReT3Y=";
  };

  poetry2nix = import poetry2nixSrc { };

  # Create a Python environment using poetry2nix
  finalEnv = poetry2nix.mkPoetryEnv {
    python = pkgs.python310;
    projectDir = ../../.;
    poetrylock = ../../poetry.lock; # Path to your poetry.lock file
    overrides = poetry2nix.overrides.withDefaults (self: super: {
      attrs = super.attrs.overridePythonAttrs (old: {
        buildInputs = old.buildInputs or [ ]
          ++ [ self.wheel self.hatch-vcs self.hatch-fancy-pypi-readme ];
      });
      protobuf3 = super.protobuf3.overridePythonAttrs (old: {
        buildInputs = old.buildInputs or [ ]
          ++ [ super.setuptools super.wheel ];
      });
      werkzeug = super.werkzeug.overridePythonAttrs
        (old: { buildInputs = old.buildInputs or [ ] ++ [ self.flit-core ]; });

      flask = super.flask.overridePythonAttrs
        (old: { buildInputs = old.buildInputs or [ ] ++ [ self.flit-core ]; });

      execnet = super.execnet.overridePythonAttrs (old: {
        buildInputs = old.buildInputs or [ ] ++ [
          self.hatchling
          self.wheel
          self.hatch-vcs
          self.hatch-fancy-pypi-readme
        ];
      });

      simple-websocket = super.simple-websocket.overridePythonAttrs (old: {
        buildInputs = old.buildInputs or [ ] ++ [ self.setuptools self.wheel ];
      });

      pyln-grpc-proto = super.pyln-grpc-proto.overridePythonAttrs
        (old: { buildInputs = old.buildInputs or [ ] ++ [ self.poetry ]; });

      pytest-custom-exit-code =
        super.pytest-custom-exit-code.overridePythonAttrs (old: {
          buildInputs = old.buildInputs or [ ]
            ++ [ self.setuptools self.wheel ];
        });

      pytest-test-groups = super.pytest-test-groups.overridePythonAttrs (old: {
        buildInputs = old.buildInputs or [ ] ++ [ self.setuptools self.wheel ];
      });

      cryptography = super.cryptography.overrideAttrs (oldAttrs: {
        src = pkgs.fetchurl {
          url =
            "https://pypi.io/packages/source/c/cryptography/cryptography-41.0.4.tar.gz";
          sha256 = "oXR8yBUgiA9BOfkZKBJneKWlpwHB71t/74b/5WpiKmw=";
        };
      });

      rpds-py = super.rpds-py.overrideAttrs (old: {
        src = pkgs.fetchurl {
          url =
            "https://files.pythonhosted.org/packages/9e/a8/4a4e5ef90c4c4f27683ce2bb74b9521b5b1d06ac134cd650333fdca0f52c/rpds_py-0.10.4.tar.gz";
          sha256 = "GNX/f70wWh1WQnPp6yLeg6482c1jKf3cjxL2QopxGmo=";
        };
        buildInputs = old.buildInputs ++ [ pkgs.maturin self.gevent self.mypy ];
      });
    });
  };
in with pkgs;
stdenv.mkDerivation {
  name = "git-dev-nix-env";

  buildInputs = [
    gcc
    sqlite
    autoconf
    git
    clang
    libtool
    sqlite
    autoconf
    autogen
    automake
    gnumake
    pkg-config
    gmp
    zlib
    gettext
    libsodium

    # Python dep
    poetry
    python310
    python310Packages.pip
    python310Packages.pytest
    maturin
    finalEnv # Add the poetry environment here

    # optional dev libraries
    ccache

    # debugs libraries
    valgrind

    # Indirected dependencies for running tests
    bitcoind
  ];
}

When I run it with the following command my 32 GB of RAM get eaten by the nix process

nix-shell contrib/nix/default.nix -j 4
2

See this other thread, possibly the same problem, that mentions an poetry2nix issue. If so, the only solution this far is using a not so recent version of nixpkgs.

3 Likes