Dear Nixos Community,
I could very much use your expertise in installing a python package (benepar) that has among others tokenizers and torch as dependencies. I have created a demo project as a showcase and playground to reproduce the issue that I am facing. You could clone the project and either call
nix develop or if you have direnv installed run direnv allow on the project.
I am struggling with this for quite a while. I cannot get the project to build.
It would be okay for me to just use prebuild binaries by setting preferWheels = true; in mkPoetryEnv for the whole project or on some packages. I currently have
pyproject.toml:
[tool.poetry]
name = "p2nispyenv"
version = "0.1.0"
description = "Minimum demo project"
[tool.poetry.dependencies]
python = "^3.10"
benepar = "*"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
And the flake.nix:
{
description = "Application packaged using poetry2nix";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
poetry2nix = {
url = "github:robert-elles/poetry2nix?ref=mychanges";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { self, nixpkgs, flake-utils, poetry2nix }:
let
py_version = "python311";
pypkgs = "python311Packages";
in
(flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
config.allowUnfree = true;
overlays = [ poetry2nix.overlays.default ];
};
inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEnv;
in
{
packages = {
poetry = nixpkgs.legacyPackages.${system}.${pypkgs}.poetry;
myenv = mkPoetryEnv
{
projectDir = ./.;
# preferWheels = true;
python = pkgs.${py_version};
extraPackages = pypkgs: with pypkgs; [
pip
];
overrides = pkgs.poetry2nix.overrides.withDefaults (pyfinal: pyprev:
{
tokenizers = pyprev.tokenizers.override {
preferWheel = true;
};
sentencepiece = pyprev.sentencepiece.override {
preferWheel = true;
};
torch = pyprev.torch.override {
preferWheel = true;
};
torch-struct = pyprev.torch-struct.override {
preferWheel = true;
};
safetensors = pyprev.safetensors.override {
preferWheel = true;
};
nvidia-cusparse-cu12 = (pyprev.nvidia-cusparse-cu12.override {
preferWheel = true;
}).overridePythonAttrs (old: {
# autoPatchelfIgnoreMissingDeps = true;
propagatedBuildInputs = ((old.propagatedBuildInputs or [ ]) ++ [
pkgs.cudaPackages.cudatoolkit
]);
});
});
};
};
devShells.default = pkgs.mkShell {
buildInputs = [ ];
packages = [
self.packages.${system}.myenv
];
};
}
));
}
With this config I get as far as the following package collision, which I have no idea how to solve it.
direnv: loading ~/code/p2nixpyenv/.envrc
direnv: using flake . --show-trace --keep-going --verbose
warning: Git tree '/home/robert/code/p2nixpyenv' is dirty
evaluating derivation 'git+file:///home/robert/code/p2nixpyenv#devShells.x86_64-linux.default'direnv: ([/nix/store/lhcz6v59vwcdavh73aixrnzq5ppli1w5-direnv-2.34.0/bin/direnv export zsh]) is taking a while to execute. Use CTRL-C to give up.
building '/nix/store/s0br71kjyrlfdmbb23ya5bn5hj2jj8x4-python3.11-torch-2.2.1.drv'...
building '/nix/store/jw7g0pnkh69jrym1hvfkx3khc6gswj9c-python3.11-accelerate-0.28.0.drv'...
building '/nix/store/sfvhwkwll24f736dq8iyffm4lqchazss-python3.11-torch-struct-0.5.drv'...
building '/nix/store/njh3vxm6j8a0wlahbbrhsxfc4h1k1lnl-python3.11-transformers-4.39.1.drv'...
building '/nix/store/pz79sb1jfgpid0d7ij8khlrcmg0q9shn-python3.11-benepar-0.2.0.drv'...
building '/nix/store/y7arck35jj2961qj5a4dvi1x4yxcd6w0-python3-3.11.8-env.drv'...
error: builder for '/nix/store/y7arck35jj2961qj5a4dvi1x4yxcd6w0-python3-3.11.8-env.drv' failed with exit code 25;
last 1 log lines:
> error: collision between `/nix/store/s34niaxf47j9d9nsis2wv1pwad672lh7-python3.11-nvidia-cuda-cupti-cu12-12.1.105/lib/python3.11/site-packages/nvidia/__pycache__/__init__.cpython-311.pyc' and `/nix/store/zqjv7blxpswk1xarqa2b1amjmvbr191g-python3.11-nvidia-cublas-cu12-12.1.3.1/lib/python3.11/site-packages/nvidia/__pycache__/__init__.cpython-311.pyc'
For full logs, run 'nix log /nix/store/y7arck35jj2961qj5a4dvi1x4yxcd6w0-python3-3.11.8-env.drv'.
error: 1 dependencies of derivation '/nix/store/2922sjf50l1qx0f6sd71fqmjjfqlr3by-nix-shell-env.drv' failed to build
direnv: export ~PATH
It would be really fantastic if someone knows a solution, could guide or point me in the right direction.