I’m attempting to write a derivation of a python package that attempts to build a cpp extension for torch. However, it fails saying that a file does not exist.
causal_conv1d.nix
{ pkgs
, buildPythonPackage
, fetchPypi
, fetchurl
, packaging
, torchWithCuda
, cudaPackages
, which
, pybind11
, gcc11
}:
buildPythonPackage rec {
pname = "causal_conv1d";
version = "1.1.0";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-vbv7YUVyke/pB6HLLbX+RqbD7iELPKD7V7ohuNr2TwQ=";
};
nativeBuildInputs = [
which
cudaPackages.cudatoolkit
gcc11
];
preBuild = ''
export CAUSAL_CONV1D_FORCE_BUILD=TRUE
'';
buildInputs = [
packaging
pybind11
torchWithCuda
cudaPackages.cuda_cudart
];
doCheck = false;
}
The log reads as follows:
Sourcing python-remove-tests-dir-hook
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing setuptools-build-hook
Using setuptoolsBuildPhase
Using setuptoolsShellHook
Sourcing pypa-install-hook
Using pypaInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing python-namespaces-hook
Sourcing python-catch-conflicts-hook.sh
Sourcing setup-cuda-hook
@nix { "action": "setPhase", "phase": "unpackPhase" }
Running phase: unpackPhase
unpacking source archive /nix/store/jc281sm81bfachb3kihiayd46x24lyng-causal_conv1d-1.1.0.tar.gz
source root is causal_conv1d-1.1.0
setting SOURCE_DATE_EPOCH to timestamp 1702196573 of file causal_conv1d-1.1.0/setup.cfg
@nix { "action": "setPhase", "phase": "patchPhase" }
Running phase: patchPhase
@nix { "action": "setPhase", "phase": "updateAutotoolsGnuConfigScriptsPhase" }
Running phase: updateAutotoolsGnuConfigScriptsPhase
@nix { "action": "setPhase", "phase": "configurePhase" }
Running phase: configurePhase
Executing setupCUDAToolkitCompilers
no configure script, doing nothing
@nix { "action": "setPhase", "phase": "buildPhase" }
Running phase: buildPhase
Executing setuptoolsBuildPhase
No CUDA runtime is found, using CUDA_HOME='/nix/store/gg7gma5n3q9xdqgxns3d1wz0xn857m04-cudatoolkit-11.8.0'
torch.__version__ = 2.1.2
running bdist_wheel
/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/utils/cpp_extension.py:502: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.
warnings.warn(msg.format('we could not find ninja.'))
running build
running build_py
creating build
creating build/lib.linux-x86_64-cpython-311
creating build/lib.linux-x86_64-cpython-311/causal_conv1d
copying causal_conv1d/__init__.py -> build/lib.linux-x86_64-cpython-311/causal_conv1d
copying causal_conv1d/causal_conv1d_interface.py -> build/lib.linux-x86_64-cpython-311/causal_conv1d
running build_ext
/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/utils/cpp_extension.py:424: UserWarning: There are no g++ version bounds defined for CUDA version 11.8
warnings.warn(f'There are no {compiler_name} version bounds defined for CUDA version {cuda_str_version}')
building 'causal_conv1d_cuda' extension
creating build/temp.linux-x86_64-cpython-311
creating build/temp.linux-x86_64-cpython-311/csrc
g++ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -I/nix/store/jh2dj172jmmlw74kw4bsqa5ycf1blv4v-libxcrypt-4.4.36/include -fPIC -I/build/causal_conv1d-1.1.0/csrc/causal_conv1d -I/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/include -I/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/include/torch/csrc/api/include -I/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/include/TH -I/nix/store/fkw6d3hpna8rgv1v50wsfn8swh4sxh0i-python3.11-torch-2.1.2/lib/python3.11/site-packages/torch/include/THC -I/nix/store/gg7gma5n3q9xdqgxns3d1wz0xn857m04-cudatoolkit-11.8.0/include -I/nix/store/5k91mg4qjylxbfvrv748smfh51ppjq0g-python3-3.11.6/include/python3.11 -c csrc/causal_conv1d.cpp -o build/temp.linux-x86_64-cpython-311/csrc/causal_conv1d.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1017\" -DTORCH_EXTENSION_NAME=causal_conv1d_cuda -D_GLIBCXX_USE_CXX11_ABI=1 -std=c++17
cc1plus: fatal error: csrc/causal_conv1d.cpp: No such file or directory
compilation terminated.
error: command '/nix/store/psmmd1zfpijak8amj3hc5sji9axspb0a-gcc-wrapper-11.4.0/bin/g++' failed with exit code 1
/nix/store/d4jf1cbbk494zwgbqz31pxgigpsbh6w2-stdenv-linux/setup: line 1553: pop_var_context: head of shell_variables not a function context
Example flake.nix which calls the file:
{
inputs = {
flakeUtils.url = "github:numtide/flake-utils";
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
};
outputs = { self, ... } @ inputs:
inputs.flakeUtils.lib.eachSystem [ "x86_64-linux" ] (system:
let
pkgs = import inputs.nixpkgs {
inherit system;
config.allowUnfree = true;
};
causal_conv1d = with pkgs; with python3.pkgs; callPackage ./causal_conv1d.nix { };
in rec {
devShells = {
# The activation script can be used as dev-shell
default = pkgs.mkShell {
name = "env";
nativeBuildInputs = [ ];
buildInputs = with pkgs; [
cudaPackages.cudatoolkit
(python3.withPackages(ps: with ps; [
numpy
scipy
sympy
torchWithCuda
causal_conv1d
]))
];
};
};
devShell = devShells.default;
}
);
}
Reading the setup.py file from the source it seems to be farming out compilation to something else, so I’m having trouble tracing where/why it’s failing
There is clearly a csrc/causal_conv1d.cpp file in the source, and I’m assuming it’s just not getting copied to the correct place because of some nix build environment shenanigans interacting with hilariously impure builds.
Any pointers to fixing this? Alternatively, if I don’t force it to build with the preBuild hook, it attempts to download a specific wheel from the github repository, I’m OK with that if I can make it work, but I’m not sure how to provide the wheel locally.