Dm-tree gcc11 error in Python 3.10

Help
1

Is this a problem with gcc11 as per this PR? If so, how would I work around it?

Here is the error I’m getting:

       last 10 log lines:
       >   Running setup.py clean for dm-tree
       >   Running command python setup.py clean
       >   running clean
       >   removing 'build/lib.linux-x86_64-cpython-310' (and everything under it)
       >   'build/bdist.linux-x86_64' does not exist -- can't clean it
       >   'build/scripts-3.10' does not exist -- can't clean it
       >   removing 'build'
       > Failed to build dm-tree
       > ERROR: Failed to build one or more wheels
       > 
       For full logs, run 'nix log /nix/store/s8pkmry4my41mwdbd3bhmphndkgalcly-python3.10-dm-tree-0.1.8.drv'.
error: 1 dependencies of derivation '/nix/store/f3pp4hz0gv503c0vpn9jl2naa2k91z84-python3-3.10.10-env.drv' failed to build
error: 1 dependencies of derivation '/nix/store/y6nm295fqw3h3l1jbw232lphj6wpfza2-nix-shell-env.drv' failed to build

And here is my flake.nix file:

# flake.nix
{
  description = "A dev environment";
  inputs = {
    nixpkgs.url      = "github:NixOS/nixpkgs/nixos-unstable";
    flake-utils.url  = "github:numtide/flake-utils";
    poetry2nix = {
      url = "github:nix-community/poetry2nix";
      follows = "nixpkgs";
    };
  };
    outputs = inputs @ { self, nixpkgs, poetry2nix, flake-utils, ... }:
    flake-utils.lib.eachDefaultSystem (system:
          
      let
        pkgs = import inputs.nixpkgs 
          { 
            inherit system;
            config = { 
              permittedInsecurePackages = [
                "qtwebkit-5.212.0-alpha4"
              ];
            };
           };
        pythonEnv = pkgs.poetry2nix.mkPoetryEnv {
          python = pkgs.python310;
          poetrylock = ./poetry.lock;
          projectDir = ./.;
        };
         # pythonEnv = pkgs.python310;
      in
      with pkgs;
      {       
        devShells.default = mkShell {
          buildInputs = [
            pythonEnv
            (poetry.override { python3 = pythonEnv.python; })
            pandoc
            exa
            fd
            gdb
          ];

          shellHook = ''
            alias ls=exa
            alias find=fd
          '';
            # export OPENSSL_DIR="${openssl.dev}"
            # export OPENSSL_LIB_DIR="${openssl.out}/lib"
        };
      }
  );
}
2

Can you share these logs? We’ll need more info to really figure out what’s failing here.

3

Thanks for looking at this. I’m new to building packages and would never have looked at these logs without your advice! Am I reading this right, and I just need to include setuptools in my pyproject.toml dev dependencies?

      subprocess.check_call(['cmake', '--version'])
    File "/nix/store/fdqpyj613dr0v1l1lrzqhzay7sk4xg87-python3-3.10.10/lib/python3.10/subprocess.py", line 364, in check_call
      retcode = call(*popenargs, **kwargs)
    File "/nix/store/fdqpyj613dr0v1l1lrzqhzay7sk4xg87-python3-3.10.10/lib/python3.10/subprocess.py", line 345, in call
      with Popen(*popenargs, **kwargs) as p:
    File "/nix/store/fdqpyj613dr0v1l1lrzqhzay7sk4xg87-python3-3.10.10/lib/python3.10/subprocess.py", line 971, in __init__
      self._execute_child(args, executable, preexec_fn, close_fds,
    File "/nix/store/fdqpyj613dr0v1l1lrzqhzay7sk4xg87-python3-3.10.10/lib/python3.10/subprocess.py", line 1847, in _execute_child
      raise child_exception_type(errno_num, err_msg, err_filename)
  FileNotFoundError: [Errno 2] No such file or directory: 'cmake'

  The above exception was the direct cause of the following exception:

  Traceback (most recent call last):
    File "<string>", line 2, in <module>
    File "<pip-setuptools-caller>", line 34, in <module>
    File "/build/dm-tree-0.1.8/setup.py", line 124, in <module>
      setuptools.setup(
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/__init__.py", line 108, in setup
      return distutils.core.setup(**attrs)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/core.py", line 185, in setup
      return run_commands(dist)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/core.py", line 201, in run_comm>
      dist.run_commands()
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 969, in run_comm>
      self.run_command(cmd)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/dist.py", line 1221, in run_command
      super().run_command(command)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 988, in run_comm>
      cmd_obj.run()
    File "/nix/store/7klv29m6g3r5iw4hyjz7ayybcg4hsba4-python3.10-wheel-0.38.4/lib/python3.10/site-packages/wheel/bdist_wheel.py", line 325, in run
      self.run_command("build")
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/cmd.py", line 318, in run_comma>
      self.distribution.run_command(command)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/dist.py", line 1221, in run_command
      super().run_command(command)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 988, in run_comm>
      cmd_obj.run()
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/command/build.py", line 131, in>
      self.run_command(cmd_name)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/cmd.py", line 318, in run_comma>
      self.distribution.run_command(command)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/dist.py", line 1221, in run_command
      super().run_command(command)
    File "/nix/store/fpcah4a88pjj7jmwhrcvfb9kg6qj58vc-python3.10-setuptools-67.4.0/lib/python3.10/site-packages/setuptools/_distutils/dist.py", line 988, in run_comm>
      cmd_obj.run()
    File "/build/dm-tree-0.1.8/setup.py", line 70, in run
      self._check_build_environment()
    File "/build/dm-tree-0.1.8/setup.py", line 80, in _check_build_environment
      raise RuntimeError(
  RuntimeError: CMake must be installed to build the following extensions: _tree
  error: subprocess-exited-with-error
  
  × python setup.py bdist_wheel did not run successfully.
  │ exit code: 1
  ╰─> See above for output.
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
  full command: /nix/store/fdqpyj613dr0v1l1lrzqhzay7sk4xg87-python3-3.10.10/bin/python3.10 -u -c '
  exec(compile('"'"''"'"''"'"'
  # This is <pip-setuptools-caller> -- a caller that pip uses to run setup.py
  #
  # - It imports setuptools before invoking setup.py, to enable projects that directly
  #   import from `distutils.core` to work with newer packaging standards.
  # - It provides a clear error message when setuptools is not installed.
  # - It sets `sys.argv[0]` to the underlying `setup.py`, when invoking `setup.py` so
  #   setuptools doesn'"'"'t think the script is `-c`. This avoids the following warning:
  #     manifest_maker: standard file '"'"'-c'"'"' not found".
  # - It generates a shim setup.py, for handling setup.cfg-only projects.
  import os, sys, tokenize
  
  try:
      import setuptools
  except ImportError as error:
      print(
          "ERROR: Can not execute `setup.py` since setuptools is not available in "
          "the build environment.",
          file=sys.stderr,
      )
      sys.exit(1)
4

I tried passing setuptools and then setuptools-scm as a native build input with poetry default override but no dice. (Still get the same error.)

5

It looks like the build of dm-tree is failing. I would double check it’s build status on master, and either try to fix the derivation, or pick a commit for which it builds successfully.

6

dm-tree appears to be marked as broken as of 83c76b92372d18f49dd3666dd05a052ea83c8870:

nixpkgs on  master on ☁️  (us-west-2)
❯ git log -1
commit 83c76b92372d18f49dd3666dd05a052ea83c8870 (HEAD -> master, upstream/master, upstream/HEAD)
Merge: eccf02f84c03 019fd4553710
Author: Silvan Mosberger <contact@infinisil.com>
Date:   Fri Apr 14 17:17:46 2023 +0200

    Merge pull request #226057 from tweag/fix-function-location-revision

    doc: Fix the function locations always pointing to master

nixpkgs on  master on ☁️  (us-west-2)
❯ nix-build -A python3Packages.dm-tree
error: Package ‘python3.10-dm-tree-0.1.8’ in /Users/samuelainsworth/dev/nixpkgs/pkgs/development/python-modules/dm-tree/default.nix:52 is marked as broken, refusing to evaluate.

       a) To temporarily allow broken packages, you can use an environment variable
          for a single invocation of the nix tools.

            $ export NIXPKGS_ALLOW_BROKEN=1

        Note: For `nix shell`, `nix build`, `nix develop` or any other Nix 2.4+
        (Flake) command, `--impure` must be passed in order to read this
        environment variable.

       b) For `nixos-rebuild` you can set
         { nixpkgs.config.allowBroken = true; }
       in configuration.nix to override this.

       c) For `nix-env`, `nix-build`, `nix-shell` or any other Nix command you can add
         { allowBroken = true; }
       to ~/.config/nixpkgs/config.nix.
(use '--show-trace' to show detailed location information)
7

Oh wait, nvm that’s only on Darwin…

8

Thanks for the detective work. I think fixing the derivation is beyond my nix knowledge. I’ll try to find a version that builds, and if I succeed with that, I’ll post back here with what I did.

9

Hmm looks like the builds should be working fine. What commit of nixpkgs are you on?

Screenshot 2023-04-14 at 2.09.52 PM
Screenshot 2023-04-14 at 2.09.52 PM2720×760 180 KB

10

Ah, ok. My system is 22.11.20230410.115a96e (Raccoon), but the flake is pulling from the unstable url. I’m using poetry2nix to build a development environment, and poetry seems to put together the lock file with no problem, but then when I try to “mkPoetryEnv” the build fails with dm-tree looking for cmake.

11

I’m not familiar with poetry2nix… have you tried simply using python3Packages.dm-tree?

12

Thanks for your help! I went through some rounds of trying successive overlays to provide the required build dependencies for packages in poetry2nix, but in the end I couldn’t get cmake working for dm-tree. Based on your comment above, I ended up just installing python with a few packages and then installing all the ML libraries in a virtual environment simply with pip. It’s a lot of packages in a requirements.txt. Here’s the flake.nix I used in the end:

# flake.nix
{
  description = "A dev environment";
  inputs = {
    nixpkgs.url      = "github:NixOS/nixpkgs/nixos-unstable";
    # nixpkgs.url = "github:nixos/nixpkgs/nixos-22.11";
    flake-utils.url  = "github:numtide/flake-utils";
  };
    outputs = { self, nixpkgs, flake-utils, ... }:
    flake-utils.lib.eachDefaultSystem (system:
      let
        # overlays = [(import rust-overlay)];
        pkgs = import nixpkgs 
          { 
            inherit system;
            config = { 
            #   permittedInsecurePackages = [
            #     "qtwebkit-5.212.0-alpha4"
            #   ];
              allowUnfree = false;
            };
           };
        lib-path = with pkgs; lib.makeLibraryPath [
          libffi
          openssl
          stdenv.cc.cc
          # If you want to use CUDA, you should uncomment this line.
          # linuxPackages.nvidia_x11
        ];
          my_python = pkgs.python311.override {
            packageOverrides = self: super: {
              pylint = super.pylint.overridePythonAttrs(old: rec {
                doCheck = false;
              });
            };
          };
      python-env = my_python.withPackages(pp: with pp; [
        # mypyc-ipython
        my_python
        ipython
        pip
        virtualenv
        pylint
        black
        autoflake
        pylint
        pytest
        tkinter
        setuptools
        wheel
      ]                # ++ lib.optional vscode-debug ptvsd
    );   
      in
      with pkgs;
      {       
        devShells.default = mkShell {
          buildInputs = [
            python-env
            pandoc
            exa
            fd
            gdb
            # other packages needed for compiling python libs
            readline
            libffi
            openssl

            # unfortunately needed because of messing with LD_LIBRARY_PATH below
            # git
            openssh
            rsync
          ];
          shellHook = ''
            # Allow the use of wheels.
            SOURCE_DATE_EPOCH=$(date +%s)
            # Augment the dynamic linker path
            export "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${lib-path}"
            # Setup the virtual environment if it doesn't already exist.
            VENV=.venv
            if test ! -d $VENV; then
              virtualenv $VENV
            fi
            source ./$VENV/bin/activate
            export PYTHONPATH=`pwd`/$VENV/${my_python.sitePackages}/:$PYTHONPATH
            alias ls=exa
            alias find=fd
            export MPLBACKEND="tkAgg"
          '';
        };
      }
  );
}