devShell with python and packages not in nixpkgs

Hi,

I try to write a flake that I can use with ‘nix develop’ and that provides me a dev shell with python, some packages that are in nixpkgs and some that are not.

I tried a lot of different things found on the net, but now it still do not work and I am completely lost in the modifications I tried. I can’t find the right ‘nix way’ to achieve what I want…

I first tried to use pip, but as nix stores things in read-only folders, it does not work.
Now I’m working with mach-nix, wich seems to provide some sort of nix file I don’t really understand that builds the required packages. It produces me a nix file that I want to include in my flake, but it does not work.

Here is the flake:

Summary

{

  description = "Build Shell with any dependency of the project";

  inputs.flake-utils.url = "github:numtide/flake-utils";
  inputs.nixpkgs.url = "github:NixOs/nixpkgs";

  outputs = { self, nixpkgs, flake-utils }:
    flake-utils.lib.eachDefaultSystem
      (system:
        let pkgs = nixpkgs.legacyPackages.${system};
            python3 = pkgs.python3;
            pythonPackages = import "${self}/python-packages.nix" { inherit pkgs python3; };
            customPython = python3.override {
              packageOverrides = pythonPackages.overrides;
            };
            pyeda = customPython.pkgs.pyeda;
            
           pythonWithPackages = pkgs.python310Packages;
           pyPkgs = pythonPackages: with pythonWithPackages; [
             pygments
           ];
           pythonEnv = pythonWithPackages pyPkgs;
        in
        {
            devShells.default = pkgs.mkShell {
              nativeBuildInputs = [ 
                pyeda
                pythonEnv
                pkgs.python3
                pkgs.poetry
                pkgs.gccStdenv
                pkgs.tikzit
              ];
          };
        }
      );
}

And here is the python-packages.nix provided by mach-nix:

Summary
{ pkgs, python, ... }:
with builtins;
with pkgs.lib;
let
  pypi_fetcher_src = builtins.fetchTarball {
    name = "nix-pypi-fetcher-2";
    url = "https://github.com/DavHau/nix-pypi-fetcher-2/tarball/f83bd320cf92d2c3fcf891f16195189aab0db8fe";
    # Hash obtained using `nix-prefetch-url --unpack <url>`
    sha256 = "sha256-y7YGrGcH/RuoEG4LNLghOeecnVYS1lNhhso9NFSA2WA=";
  };
  pypiFetcher = import pypi_fetcher_src { inherit pkgs; };
  fetchPypi = pypiFetcher.fetchPypi;
  fetchPypiWheel = pypiFetcher.fetchPypiWheel;
  pkgsData = fromJSON ''{"pyeda": {"name": "pyeda", "ver": "0.28.0", "build_inputs": [], "prop_build_inputs": [], "is_root": true, "provider_info": {"provider": "sdist", "wheel_fname": null, "url": null, "hash": null}, "extras_selected": [], "removed_circular_deps": [], "build": null}}'';
  isPyModule = pkg:
    isAttrs pkg && hasAttr "pythonModule" pkg;
  normalizeName = name: (replaceStrings ["_"] ["-"] (toLower name));
  depNamesOther = [
    "depsBuildBuild"
    "depsBuildBuildPropagated"
    "nativeBuildInputs"
    "propagatedNativeBuildInputs"
    "depsBuildTarget"
    "depsBuildTargetPropagated"
    "depsHostHost"
    "depsHostHostPropagated"
    "depsTargetTarget"
    "depsTargetTargetPropagated"
    "checkInputs"
    "installCheckInputs"
  ];
  depNamesAll = depNamesOther ++ [
    "propagatedBuildInputs"
    "buildInputs"
  ];
  removeUnwantedPythonDeps = pythonSelf: pname: inputs:
    # Do not remove any deps if provider is nixpkgs and actual dependencies are unknown.
    # Otherwise we risk removing dependencies which are needed.
    if pkgsData."${pname}".provider_info.provider == "nixpkgs"
        &&
        (pkgsData."${pname}".build_inputs == null
            || pkgsData."${pname}".prop_build_inputs == null) then
      inputs
    else
      filter
        (dep:
          if ! isPyModule dep || pkgsData ? "${normalizeName (get_pname dep)}" then
            true
          else
            trace "removing dependency ${dep.name} from ${pname}" false)
        inputs;
  updatePythonDeps = newPkgs: pkg:
    if ! isPyModule pkg then pkg else
    let
      pname = normalizeName (get_pname pkg);
      newP =
        # All packages with a pname that already exists in our overrides must be replaced with our version.
        # Otherwise we will have a collision
        if newPkgs ? "${pname}" && pkg != newPkgs."${pname}" then
          trace "Updated inherited nixpkgs dep ${pname} from ${pkg.version} to ${newPkgs."${pname}".version}"
          newPkgs."${pname}"
        else
          pkg;
    in
      newP;
  updateAndRemoveDeps = pythonSelf: name: inputs:
    removeUnwantedPythonDeps pythonSelf name (map (dep: updatePythonDeps pythonSelf dep) inputs);
  cleanPythonDerivationInputs = pythonSelf: name: oldAttrs:
    mapAttrs (n: v: if elem n depNamesAll then updateAndRemoveDeps pythonSelf name v else v ) oldAttrs;
  override = pkg:
    if hasAttr "overridePythonAttrs" pkg then
        pkg.overridePythonAttrs
    else
        pkg.overrideAttrs;
  nameMap = {
    pytorch = "torch";
  };
  get_pname = pkg:
    let
      res = tryEval (
        if pkg ? src.pname then
          pkg.src.pname
        else if pkg ? pname then
          let pname = pkg.pname; in
            if nameMap ? "${pname}" then nameMap."${pname}" else pname
          else ""
      );
    in
      toString res.value;
  get_passthru = pypi_name: nix_name:
    # if pypi_name is in nixpkgs, we must pick it, otherwise risk infinite recursion.
    let
      python_pkgs = python.pkgs;
      pname = if hasAttr "${pypi_name}" python_pkgs then pypi_name else nix_name;
    in
      if hasAttr "${pname}" python_pkgs then
        let result = (tryEval
          (if isNull python_pkgs."${pname}" then
            {}
          else
            python_pkgs."${pname}".passthru));
        in
          if result.success then result.value else {}
      else {};
  allCondaDepsRec = pkg:
    let directCondaDeps =
      filter (p: p ? provider && p.provider == "conda") (pkg.propagatedBuildInputs or []);
    in
      directCondaDeps ++ filter (p: ! directCondaDeps ? p) (map (p: p.allCondaDeps) directCondaDeps);
  tests_on_off = enabled: pySelf: pySuper:
    let
      mod = {
        doCheck = enabled;
        doInstallCheck = enabled;
      };
    in
    {
      buildPythonPackage = args: pySuper.buildPythonPackage ( args // {
        doCheck = enabled;
        doInstallCheck = enabled;
      } );
      buildPythonApplication = args: pySuper.buildPythonPackage ( args // {
        doCheck = enabled;
        doInstallCheck = enabled;
      } );
    };
  pname_passthru_override = pySelf: pySuper: {
    fetchPypi = args: (pySuper.fetchPypi args).overrideAttrs (oa: {
      passthru = { inherit (args) pname; };
    });
  };
  mergeOverrides = with pkgs.lib; foldl composeExtensions (self: super: {});
  merge_with_overr = enabled: overr:
    mergeOverrides [(tests_on_off enabled) pname_passthru_override overr];
  select_pkgs = ps: [
    ps."pyeda"
  ];
  overrides' = manylinux1: autoPatchelfHook: merge_with_overr false (python-self: python-super: let all = {
    "pyeda" = python-self.buildPythonPackage {
      pname = "pyeda";
      version = "0.28.0";
      src = fetchPypi "pyeda" "0.28.0";
      passthru = (get_passthru "pyeda" "pyeda") // { provider = "sdist"; };
    };
  }; in all);
in
{
  inherit select_pkgs;
  overrides = overrides';
}

As I said before, I’m lost in the modifications I made to my flake, I don’t understand anymore what I’m doing.
What is the ‘nix way’ of building a python dev shell with packages from outside nixpkgs ?

Thanks,
JM

You make a project folder & then you initialize this flake in it…Here is the flake (which also you can copy locally):

https://github.com/MordragT/nix-templates/tree/master/python-poetry

Commands:

nix flake init -t github:MordragT/nix-templates#python-poetry

nix develop

and then you create the project with poetry:

poetry new my_app

poetry shell

and you are off to the races

Below poetry:

If you want to use packages that are not in nix, introducing overlays should be the way forward. In your flake I would recommend doing something like:

Note that this is without using flake-utils, and just ‘plain’ nix.

      # Your custom packages and modifications, exported as overlays
      overlays = import ./overlays { inherit inputs; };

      # Your custom packages
      # Acessible through 'nix build', 'nix shell', etc
      packages = forAllSystems (system:
         let pkgs = nixpkgs.legacyPackages.${system};
         in import ./pkgs { inherit pkgs; } // {
       });

      # Devshell for bootstrapping
      # Acessible through 'nix develop' or 'nix-shell' (legacy)
      devShells = forAllSystems (system:
        let
          pkgs = import nixpkgs {
            inherit system;
            overlays = [
             self.overlays.additions
             self.overlays.modifications
             self.overlays.unstable-packages
           ];
          };
        in {
          default = pkgs.mkShell ({
            # at compile time
            nativeBuildInputs = [
            ];

            # at run time
            buildInputs = with pkgs; [
              # use latest terraform version
              python310
              poetry
              python310Packages.aiohttp
            ];

            '';
          });
        }
      );

./pkgs/default.nix

# Custom packages, that can be defined similarly to ones from nixpkgs
# You can build them using 'nix build .#example' or (legacy) 'nix-build -A example'

{ pkgs }: {
  your_custom_python_package = pkgs.callPackage ./your_custom_python_package { };
}

./pkgs/your_custom_python_package/default.nix

Write your custom derivation here, I just took this example, adding required input arguments, might not work out of the box (see: Python - NixOS Wiki for some pointers)

{ buildPythonPackage
, fetchPypi
, pkgs
,
}:

buildPythonPackage rec {
      pname = "deserialize";
      version = "1.8.3";
      src = fetchPypi {
        inherit pname version;
        sha256 = "sha256-0aozmQ4Eb5zL4rtNHSFjEynfObUkYlid1PgMDVmRkwY=";
      };
      doCheck = false;
      propagatedBuildInputs = [
        # Specify dependencies
        pkgs.python3Packages.numpy
      ];
    }

./overlays/default.nix

# This file defines overlays
{ inputs, ... }:
{
  # This one brings our custom packages from the 'pkgs' directory
  additions = final: _prev: import ../pkgs { pkgs = final; };

  # This one contains whatever you want to overlay
  # You can change versions, add patches, set compilation flags, anything really.
  # https://nixos.wiki/wiki/Overlays
  modifications = final: prev: {
    # example = prev.example.overrideAttrs (oldAttrs: rec {
    # ...
    # });
  };

  # When applied, the unstable nixpkgs set (declared in the flake inputs) will
  # be accessible through 'pkgs.unstable'
  # optional, you have to add it as input tho
  unstable-packages = final: _prev: {
    unstable = import inputs.nixpkgs-unstable {
      system = final.system;
      config.allowUnfree = true;
    };
  };
}

Hope this helps.

Giving credit where it’s due, large amounts of this structure are taken from: GitHub - Misterio77/nix-starter-configs: Simple and documented config templates to help you get started with NixOS + home-manager + flakes. All the boilerplate you need! (from the standard config)

I love nix, and I hate nix at the same time… Why is this that much complicated to run a simple <50 lines python program… I wrote the code in 10 minutes and now I can’t run it for 2 days because of nix…

@superpim Your solution seems elegant, but this looks far too complicated for an usage as simple as mine…

@BriefNCounter I tried your solution: I modified my flake to use the MordragT template but now when I run ‘nix develop’, nix is eating all my RAM and SWAP in 30 seconds ans then my computer freeze and I need to force shutdown… I have 16 G Ram and 8 G swap, what is taking that much space ? ‘nix develop --verbose’ don’t display anything to help.

Okay, I finally got it to work, or at least I am able to run nix develop one time.

But now, whenever I install a package with poetry, next call to ‘nix develop’ fails because it is not able to build something. It seems it miss the ‘setuptools’ package, even if I install it with poetry.

I’m loosing my mind, how can something as simple as ‘pip install package’ become as complicated as that ?

Ok, sorry for the spam, I think I got it to work.

The trick is to specify build dependencies like ‘setuptools’ in the overrides section.

1 Like

I’ve just done this briefly, JM I’ve not used it.

Let me go over it. Don’t give up, Nix is like swimming…
at first you drown, then you learn how and it’s cool

Follow Ryan. The guy is a genius, but he will hate me for saying it.
Hook yourself up to git; then you can mess up with impunity.

Also I sometimes take ‘‘snapshots’’ as I go…back up my flake or whatever locally, then hack around in the long grass.

1 Like

Sorry JM…I am busy.

The easiest is to ask Mordrag himself if you have issues: like I said I played with this for 5 minutes, so I need to take a proper look and not talk garbage

https://github.com/MordragT/nix-templates/issues


And here are Mordrag’s instructions:

n.b. the flake name is “python-poetry”

So how do I use it ??

There are multiple ways to use the templates defined in here. To get started fast just use the following command:

The name corresponds to the different templates provided: # [deno, rust, python, tauri, jupyter-py, mdbook, slides, tex, trivial] nix flake init -t github:MordragT/nix-templates#

If you are using these templates more frequently you should probably add them to the nix registry to shorten the command above. This can be achieved by either adding them via the following command:

nix registry add templates github:MordragT/nix-templates

Or if you are using NixOS by declaratively adding the flake to youre system configuration, as shown here

After that you can run the shorter command:

nix flake init -t templates#