Hi,
I try to write a flake that I can use with ‘nix develop’ and that provides me a dev shell with python, some packages that are in nixpkgs and some that are not.
I tried a lot of different things found on the net, but now it still do not work and I am completely lost in the modifications I tried. I can’t find the right ‘nix way’ to achieve what I want…
I first tried to use pip, but as nix stores things in read-only folders, it does not work.
Now I’m working with mach-nix, wich seems to provide some sort of nix file I don’t really understand that builds the required packages. It produces me a nix file that I want to include in my flake, but it does not work.
Here is the flake:
Summary
{
description = "Build Shell with any dependency of the project";
inputs.flake-utils.url = "github:numtide/flake-utils";
inputs.nixpkgs.url = "github:NixOs/nixpkgs";
outputs = { self, nixpkgs, flake-utils }:
flake-utils.lib.eachDefaultSystem
(system:
let pkgs = nixpkgs.legacyPackages.${system};
python3 = pkgs.python3;
pythonPackages = import "${self}/python-packages.nix" { inherit pkgs python3; };
customPython = python3.override {
packageOverrides = pythonPackages.overrides;
};
pyeda = customPython.pkgs.pyeda;
pythonWithPackages = pkgs.python310Packages;
pyPkgs = pythonPackages: with pythonWithPackages; [
pygments
];
pythonEnv = pythonWithPackages pyPkgs;
in
{
devShells.default = pkgs.mkShell {
nativeBuildInputs = [
pyeda
pythonEnv
pkgs.python3
pkgs.poetry
pkgs.gccStdenv
pkgs.tikzit
];
};
}
);
}
And here is the python-packages.nix provided by mach-nix:
Summary
{ pkgs, python, ... }:
with builtins;
with pkgs.lib;
let
pypi_fetcher_src = builtins.fetchTarball {
name = "nix-pypi-fetcher-2";
url = "https://github.com/DavHau/nix-pypi-fetcher-2/tarball/f83bd320cf92d2c3fcf891f16195189aab0db8fe";
# Hash obtained using `nix-prefetch-url --unpack <url>`
sha256 = "sha256-y7YGrGcH/RuoEG4LNLghOeecnVYS1lNhhso9NFSA2WA=";
};
pypiFetcher = import pypi_fetcher_src { inherit pkgs; };
fetchPypi = pypiFetcher.fetchPypi;
fetchPypiWheel = pypiFetcher.fetchPypiWheel;
pkgsData = fromJSON ''{"pyeda": {"name": "pyeda", "ver": "0.28.0", "build_inputs": [], "prop_build_inputs": [], "is_root": true, "provider_info": {"provider": "sdist", "wheel_fname": null, "url": null, "hash": null}, "extras_selected": [], "removed_circular_deps": [], "build": null}}'';
isPyModule = pkg:
isAttrs pkg && hasAttr "pythonModule" pkg;
normalizeName = name: (replaceStrings ["_"] ["-"] (toLower name));
depNamesOther = [
"depsBuildBuild"
"depsBuildBuildPropagated"
"nativeBuildInputs"
"propagatedNativeBuildInputs"
"depsBuildTarget"
"depsBuildTargetPropagated"
"depsHostHost"
"depsHostHostPropagated"
"depsTargetTarget"
"depsTargetTargetPropagated"
"checkInputs"
"installCheckInputs"
];
depNamesAll = depNamesOther ++ [
"propagatedBuildInputs"
"buildInputs"
];
removeUnwantedPythonDeps = pythonSelf: pname: inputs:
# Do not remove any deps if provider is nixpkgs and actual dependencies are unknown.
# Otherwise we risk removing dependencies which are needed.
if pkgsData."${pname}".provider_info.provider == "nixpkgs"
&&
(pkgsData."${pname}".build_inputs == null
|| pkgsData."${pname}".prop_build_inputs == null) then
inputs
else
filter
(dep:
if ! isPyModule dep || pkgsData ? "${normalizeName (get_pname dep)}" then
true
else
trace "removing dependency ${dep.name} from ${pname}" false)
inputs;
updatePythonDeps = newPkgs: pkg:
if ! isPyModule pkg then pkg else
let
pname = normalizeName (get_pname pkg);
newP =
# All packages with a pname that already exists in our overrides must be replaced with our version.
# Otherwise we will have a collision
if newPkgs ? "${pname}" && pkg != newPkgs."${pname}" then
trace "Updated inherited nixpkgs dep ${pname} from ${pkg.version} to ${newPkgs."${pname}".version}"
newPkgs."${pname}"
else
pkg;
in
newP;
updateAndRemoveDeps = pythonSelf: name: inputs:
removeUnwantedPythonDeps pythonSelf name (map (dep: updatePythonDeps pythonSelf dep) inputs);
cleanPythonDerivationInputs = pythonSelf: name: oldAttrs:
mapAttrs (n: v: if elem n depNamesAll then updateAndRemoveDeps pythonSelf name v else v ) oldAttrs;
override = pkg:
if hasAttr "overridePythonAttrs" pkg then
pkg.overridePythonAttrs
else
pkg.overrideAttrs;
nameMap = {
pytorch = "torch";
};
get_pname = pkg:
let
res = tryEval (
if pkg ? src.pname then
pkg.src.pname
else if pkg ? pname then
let pname = pkg.pname; in
if nameMap ? "${pname}" then nameMap."${pname}" else pname
else ""
);
in
toString res.value;
get_passthru = pypi_name: nix_name:
# if pypi_name is in nixpkgs, we must pick it, otherwise risk infinite recursion.
let
python_pkgs = python.pkgs;
pname = if hasAttr "${pypi_name}" python_pkgs then pypi_name else nix_name;
in
if hasAttr "${pname}" python_pkgs then
let result = (tryEval
(if isNull python_pkgs."${pname}" then
{}
else
python_pkgs."${pname}".passthru));
in
if result.success then result.value else {}
else {};
allCondaDepsRec = pkg:
let directCondaDeps =
filter (p: p ? provider && p.provider == "conda") (pkg.propagatedBuildInputs or []);
in
directCondaDeps ++ filter (p: ! directCondaDeps ? p) (map (p: p.allCondaDeps) directCondaDeps);
tests_on_off = enabled: pySelf: pySuper:
let
mod = {
doCheck = enabled;
doInstallCheck = enabled;
};
in
{
buildPythonPackage = args: pySuper.buildPythonPackage ( args // {
doCheck = enabled;
doInstallCheck = enabled;
} );
buildPythonApplication = args: pySuper.buildPythonPackage ( args // {
doCheck = enabled;
doInstallCheck = enabled;
} );
};
pname_passthru_override = pySelf: pySuper: {
fetchPypi = args: (pySuper.fetchPypi args).overrideAttrs (oa: {
passthru = { inherit (args) pname; };
});
};
mergeOverrides = with pkgs.lib; foldl composeExtensions (self: super: {});
merge_with_overr = enabled: overr:
mergeOverrides [(tests_on_off enabled) pname_passthru_override overr];
select_pkgs = ps: [
ps."pyeda"
];
overrides' = manylinux1: autoPatchelfHook: merge_with_overr false (python-self: python-super: let all = {
"pyeda" = python-self.buildPythonPackage {
pname = "pyeda";
version = "0.28.0";
src = fetchPypi "pyeda" "0.28.0";
passthru = (get_passthru "pyeda" "pyeda") // { provider = "sdist"; };
};
}; in all);
in
{
inherit select_pkgs;
overrides = overrides';
}
As I said before, I’m lost in the modifications I made to my flake, I don’t understand anymore what I’m doing.
What is the ‘nix way’ of building a python dev shell with packages from outside nixpkgs ?
Thanks,
JM
