How to install and use a python package from GitHub?

ldeck · February 20, 2020, 7:35pm

I’m trying to make use of vpn-slice for vpn split tunnelling (using openconnect).

So I’ve created the following shell.nix which is failing to build the derivation.

What might I have missed?

with import <nixpkgs> {};

( let
    vpn-slice = pkgs.python3Packages.buildPythonPackage rec {
      name = "vpn-slice";
      version = "v0.13";

      src = pkgs.fetchFromGitHub {
      	 owner = "dlenski";
	 repo = "${name}";
	 rev = "${version}";
	 sha256 = "1ibrwal80z27c2mh9hx85idmzilx6cpcmgc15z3lyz57bz0krigb";
      };

      meta = {
        homepage = "https://github.com/dlenski/vpn-slice";
        description = "vpnc-script replacement for easy and secure split-tunnel VPN setup";
        license = stdenv.lib.licenses.gpl3Plus;
        maintainers = with maintainers; [ dlenski ];
      };
    };

  in pkgs.python3.buildEnv.override rec {
    extraLibs = with pkgs.python3Packages; [ numpy toolz vpn-slice ];
  }
).env

When running nix-shell it’s failing (see log below).

these derivations will be built:
  /nix/store/x823hnk6mhfq75hl1cfn326aml14i7zd-python3.7-vpn-slice.drv
  /nix/store/05an5bj2kq8zx2rcvhzhzcddvjvz577b-python3-3.7.6-env.drv
these paths will be fetched (5.87 MiB download, 33.89 MiB unpacked):
  /nix/store/482rzsmj97b0lyrghiwjzncr2xh05ff8-python3.7-setuptools-44.0.0
  /nix/store/820xrc3wnzw0p4ny4yqkl5gpp0p291n5-python3.7-numpy-1.18.1
  /nix/store/f1w34w36cm2n8vrn4s5rnm7was2sbp39-python-catch-conflicts-hook
  /nix/store/i2355h7873dsf9jphli8i1cdws13q56x-python3.7-toolz-0.10.0
  /nix/store/ijf4r84v2kbz5dx3imxv7x0xvcld393n-python3.7-pip-19.3.1
  /nix/store/m9v9bamf3g2301b9cx4xycnmpkdwizv8-pip-install-hook
  /nix/store/nc9nf0qb6wfgyar70s52dmwhq8sq1wrw-setuptools-check-hook
  /nix/store/p73cgw6q923jbj0c9wcvw9vndaygzhnc-setuptools-setup-hook
  /nix/store/qczrwmf8byym9v2sg69ccs9slfhfwcqk-hook
  /nix/store/rh7wq0hq13x04wn3j6b32hvchmrpibgx-python-imports-check-hook.sh
  /nix/store/v8jrbnc5knxz04r9p4frsw61v3vxvhdq-python3.7-wheel-0.33.6
copying path '/nix/store/820xrc3wnzw0p4ny4yqkl5gpp0p291n5-python3.7-numpy-1.18.1' from 'https://cache.nixos.org'...
copying path '/nix/store/qczrwmf8byym9v2sg69ccs9slfhfwcqk-hook' from 'https://cache.nixos.org'...
copying path '/nix/store/rh7wq0hq13x04wn3j6b32hvchmrpibgx-python-imports-check-hook.sh' from 'https://cache.nixos.org'...
copying path '/nix/store/ijf4r84v2kbz5dx3imxv7x0xvcld393n-python3.7-pip-19.3.1' from 'https://cache.nixos.org'...
copying path '/nix/store/482rzsmj97b0lyrghiwjzncr2xh05ff8-python3.7-setuptools-44.0.0' from 'https://cache.nixos.org'...
copying path '/nix/store/m9v9bamf3g2301b9cx4xycnmpkdwizv8-pip-install-hook' from 'https://cache.nixos.org'...
copying path '/nix/store/f1w34w36cm2n8vrn4s5rnm7was2sbp39-python-catch-conflicts-hook' from 'https://cache.nixos.org'...
copying path '/nix/store/i2355h7873dsf9jphli8i1cdws13q56x-python3.7-toolz-0.10.0' from 'https://cache.nixos.org'...
copying path '/nix/store/v8jrbnc5knxz04r9p4frsw61v3vxvhdq-python3.7-wheel-0.33.6' from 'https://cache.nixos.org'...
copying path '/nix/store/nc9nf0qb6wfgyar70s52dmwhq8sq1wrw-setuptools-check-hook' from 'https://cache.nixos.org'...
copying path '/nix/store/p73cgw6q923jbj0c9wcvw9vndaygzhnc-setuptools-setup-hook' from 'https://cache.nixos.org'...
building '/nix/store/x823hnk6mhfq75hl1cfn326aml14i7zd-python3.7-vpn-slice.drv'...
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing setuptools-build-hook
Using setuptoolsBuildPhase
Using setuptoolsShellHook
Sourcing pip-install-hook
Using pipInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing setuptools-check-hook
Using setuptoolsCheckPhase
unpacking sources
unpacking source archive /nix/store/dqcbx0n2rpnwnfz4lk6vzcbpszj0y3bd-source
source root is source
setting SOURCE_DATE_EPOCH to timestamp 315619200 of file source/vpn_slice/version.py
patching sources
configuring
no configure script, doing nothing
building
Executing setuptoolsBuildPhase
running bdist_wheel
running build
running build_py
creating build
creating build/lib
creating build/lib/vpn_slice
copying vpn_slice/version.py -> build/lib/vpn_slice
copying vpn_slice/provider.py -> build/lib/vpn_slice
copying vpn_slice/util.py -> build/lib/vpn_slice
copying vpn_slice/generic.py -> build/lib/vpn_slice
copying vpn_slice/__init__.py -> build/lib/vpn_slice
copying vpn_slice/posix.py -> build/lib/vpn_slice
copying vpn_slice/linux.py -> build/lib/vpn_slice
copying vpn_slice/__main__.py -> build/lib/vpn_slice
copying vpn_slice/mac.py -> build/lib/vpn_slice
running egg_info
creating vpn_slice.egg-info
writing vpn_slice.egg-info/PKG-INFO
writing dependency_links to vpn_slice.egg-info/dependency_links.txt
writing entry points to vpn_slice.egg-info/entry_points.txt
writing requirements to vpn_slice.egg-info/requires.txt
writing top-level names to vpn_slice.egg-info/top_level.txt
writing manifest file 'vpn_slice.egg-info/SOURCES.txt'
reading manifest file 'vpn_slice.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'vpn_slice.egg-info/SOURCES.txt'
installing to build/bdist.macosx-10.6-x86_64/wheel
running install
running install_lib
creating build/bdist.macosx-10.6-x86_64
creating build/bdist.macosx-10.6-x86_64/wheel
creating build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/version.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/provider.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/util.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/generic.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/__init__.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/posix.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/linux.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/__main__.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/mac.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
running install_egg_info
Copying vpn_slice.egg-info to build/bdist.macosx-10.6-x86_64/wheel/vpn_slice-0.13-py3.7.egg-info
running install_scripts
adding license file "LICENSE.txt" (matched pattern "LICEN[CS]E*")
creating build/bdist.macosx-10.6-x86_64/wheel/vpn_slice-0.13.dist-info/WHEEL
creating 'dist/vpn_slice-0.13-py3-none-any.whl' and adding 'build/bdist.macosx-10.6-x86_64/wheel' to it
adding 'vpn_slice/__init__.py'
adding 'vpn_slice/__main__.py'
adding 'vpn_slice/generic.py'
adding 'vpn_slice/linux.py'
adding 'vpn_slice/mac.py'
adding 'vpn_slice/posix.py'
adding 'vpn_slice/provider.py'
adding 'vpn_slice/util.py'
adding 'vpn_slice/version.py'
adding 'vpn_slice-0.13.dist-info/LICENSE.txt'
adding 'vpn_slice-0.13.dist-info/METADATA'
adding 'vpn_slice-0.13.dist-info/WHEEL'
adding 'vpn_slice-0.13.dist-info/entry_points.txt'
adding 'vpn_slice-0.13.dist-info/top_level.txt'
adding 'vpn_slice-0.13.dist-info/RECORD'
removing build/bdist.macosx-10.6-x86_64/wheel
Finished executing setuptoolsBuildPhase
installing
Executing pipInstallPhase
/private/var/folders/6j/j96x43893xd3vst_44_w7pqm0000gn/T/nix-build-python3.7-vpn-slice.drv-0/source/dist /private/var/folders/6j/j96x43893xd3vst_44_w7pqm0000gn/T/nix-build-python3.7-vpn-slice.drv-0/source
Processing ./vpn_slice-0.13-py3-none-any.whl
ERROR: Could not find a version that satisfies the requirement setproctitle (from vpn-slice==0.13) (from versions: none)
ERROR: No matching distribution found for setproctitle (from vpn-slice==0.13)
builder for '/nix/store/x823hnk6mhfq75hl1cfn326aml14i7zd-python3.7-vpn-slice.drv' failed with exit code 1
cannot build derivation '/nix/store/05an5bj2kq8zx2rcvhzhzcddvjvz577b-python3-3.7.6-env.drv': 1 dependencies couldn't be built
error: build of '/nix/store/05an5bj2kq8zx2rcvhzhzcddvjvz577b-python3-3.7.6-env.drv' failed

benley · February 20, 2020, 8:46pm

It looks like you may just need to specify some dependencies for the package you’re building. Try adding something like this in your buildPythonPackage call:

{
  vpn-slice = pkgs.python3Packages.buildPythonPackage rec {
    name = "vpn-slice";
    # ...
    propagatedBuildInputs = with pkgs.python3Packages; [ setproctitle ];
  };
}

jonringer · February 20, 2020, 9:02pm

This is what I would do:

with import <nixpkgs> {};

let
    vpn-slice = with python3Packages; buildPythonPackage rec {
      name = "vpn-slice";
      version = "v0.13";

      src = pkgs.fetchFromGitHub {
      	 owner = "dlenski";
	 repo = "${name}";
	 rev = "${version}";
	 sha256 = "1ibrwal80z27c2mh9hx85idmzilx6cpcmgc15z3lyz57bz0krigb";
      };

      propagatedBuildInputs = [ numpy toolz setproctitle ];

      meta = {
        homepage = "https://github.com/dlenski/vpn-slice";
        description = "vpnc-script replacement for easy and secure split-tunnel VPN setup";
        license = stdenv.lib.licenses.gpl3Plus;
        maintainers = with maintainers; [ dlenski ];
      };
    };

in mkShell {
  name = "vpn-env";
  buildInputs = [ vpn-slice ];
  shellHook = ''
    echo "what ever you want to put here"
  '';
}

jonringer · February 20, 2020, 9:11pm

An even simpler solution would be:

with import <nixpkgs> {};

with pkgs;

mkShell {
  name = "vpn-env";
  buildInputs = with python3Packages; [ setuptools numpy toolz setproctitle venvShellHook];
  venvDir = "vpnenv";
}

and then from your shell:

$ nix-shell test.nix
Using venvShellHook
Executing venvHook
Skipping venv creation, 'vpnenv' already exists
bash: hash: hashing disabled
bash: hash: hashing disabled
Finished executing venvShellHook
(vpnenv)
[nix-shell:~/projects/nixpkgs]$ pip install vpn-slice
Collecting vpn-slice
  Using cached https://files.pythonhosted.org/packages/08/3a/5701fcd1497c9c33a58e2dd209d1deb222eec43dabf1a3b30c589ce3bb4e/vpn_slice-0.13.tar.gz
Requirement already satisfied: setproctitle in /nix/store/knywsp74mkpaxcchwqx71053qvfispb9-python3.7-setproctitle-1.1.10/lib/python3.7/site-packages (from vpn-slice) (1.1.10)
Installing collected packages: vpn-slice
  Running setup.py install for vpn-slice ... done
Successfully installed vpn-slice-0.13
WARNING: You are using pip version 19.2.3, however version 20.0.2 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
(vpnenv)
[nix-shell:~/projects/nixpkgs]$ vpn-slice --help
usage: vpn-slice [-h] [-k KILL] [-K] [--banner] [-i] [-n NAME] [-d DOMAIN]
                 [-I] [-S] [--no-host-names] [--no-short-names]
                 [--no-ns-hosts] [--nbns] [-v] [--ppid PPID] [-D] [--no-fork]
                 [-V]
                 [routes [routes ...]]
...

jonringer · February 20, 2020, 9:11pm

more info can be found https://github.com/NixOS/nixpkgs/blob/6d39b43e34aacfcf419298ebc5c162b3e4645dd9/doc/languages-frameworks/python.section.md#how-to-consume-python-modules-using-pip-in-a-virtual-environment-like-i-am-used-to-on-other-operating-systems

ldeck · February 21, 2020, 7:43am

Thanks for the suggestion @jonringer.

However, it still fails saying…

ERROR: Could not find a version that satisfies the requirement setproctitle (from vpn-slice==0.13) (from versions: none)
ERROR: No matching distribution found for setproctitle (from vpn-slice==0.13)

Full log here…

these derivations will be built:
  /nix/store/pli7a8x4v3zfib3cfr7irnd5nqx8afwj-python3.7-vpn-slice.drv
building '/nix/store/pli7a8x4v3zfib3cfr7irnd5nqx8afwj-python3.7-vpn-slice.drv'...
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing setuptools-build-hook
Using setuptoolsBuildPhase
Using setuptoolsShellHook
Sourcing pip-install-hook
Using pipInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing setuptools-check-hook
Using setuptoolsCheckPhase
unpacking sources
unpacking source archive /nix/store/dqcbx0n2rpnwnfz4lk6vzcbpszj0y3bd-source
source root is source
setting SOURCE_DATE_EPOCH to timestamp 315619200 of file source/vpn_slice/version.py
patching sources
configuring
no configure script, doing nothing
building
Executing setuptoolsBuildPhase
running bdist_wheel
running build
running build_py
creating build
creating build/lib
creating build/lib/vpn_slice
copying vpn_slice/version.py -> build/lib/vpn_slice
copying vpn_slice/provider.py -> build/lib/vpn_slice
copying vpn_slice/util.py -> build/lib/vpn_slice
copying vpn_slice/generic.py -> build/lib/vpn_slice
copying vpn_slice/__init__.py -> build/lib/vpn_slice
copying vpn_slice/posix.py -> build/lib/vpn_slice
copying vpn_slice/linux.py -> build/lib/vpn_slice
copying vpn_slice/__main__.py -> build/lib/vpn_slice
copying vpn_slice/mac.py -> build/lib/vpn_slice
running egg_info
creating vpn_slice.egg-info
writing vpn_slice.egg-info/PKG-INFO
writing dependency_links to vpn_slice.egg-info/dependency_links.txt
writing entry points to vpn_slice.egg-info/entry_points.txt
writing requirements to vpn_slice.egg-info/requires.txt
writing top-level names to vpn_slice.egg-info/top_level.txt
writing manifest file 'vpn_slice.egg-info/SOURCES.txt'
reading manifest file 'vpn_slice.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'vpn_slice.egg-info/SOURCES.txt'
installing to build/bdist.macosx-10.6-x86_64/wheel
running install
running install_lib
creating build/bdist.macosx-10.6-x86_64
creating build/bdist.macosx-10.6-x86_64/wheel
creating build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/version.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/provider.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/util.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/generic.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/__init__.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/posix.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/linux.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/__main__.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
copying build/lib/vpn_slice/mac.py -> build/bdist.macosx-10.6-x86_64/wheel/vpn_slice
running install_egg_info
Copying vpn_slice.egg-info to build/bdist.macosx-10.6-x86_64/wheel/vpn_slice-0.13-py3.7.egg-info
running install_scripts
adding license file "LICENSE.txt" (matched pattern "LICEN[CS]E*")
creating build/bdist.macosx-10.6-x86_64/wheel/vpn_slice-0.13.dist-info/WHEEL
creating 'dist/vpn_slice-0.13-py3-none-any.whl' and adding 'build/bdist.macosx-10.6-x86_64/wheel' to it
adding 'vpn_slice/__init__.py'
adding 'vpn_slice/__main__.py'
adding 'vpn_slice/generic.py'
adding 'vpn_slice/linux.py'
adding 'vpn_slice/mac.py'
adding 'vpn_slice/posix.py'
adding 'vpn_slice/provider.py'
adding 'vpn_slice/util.py'
adding 'vpn_slice/version.py'
adding 'vpn_slice-0.13.dist-info/LICENSE.txt'
adding 'vpn_slice-0.13.dist-info/METADATA'
adding 'vpn_slice-0.13.dist-info/WHEEL'
adding 'vpn_slice-0.13.dist-info/entry_points.txt'
adding 'vpn_slice-0.13.dist-info/top_level.txt'
adding 'vpn_slice-0.13.dist-info/RECORD'
removing build/bdist.macosx-10.6-x86_64/wheel
Finished executing setuptoolsBuildPhase
installing
Executing pipInstallPhase
/private/var/folders/6j/j96x43893xd3vst_44_w7pqm0000gn/T/nix-build-python3.7-vpn-slice.drv-0/source/dist /private/var/folders/6j/j96x43893xd3vst_44_w7pqm0000gn/T/nix-build-python3.7-vpn-slice.drv-0/source
Processing ./vpn_slice-0.13-py3-none-any.whl
ERROR: Could not find a version that satisfies the requirement setproctitle (from vpn-slice==0.13) (from versions: none)
ERROR: No matching distribution found for setproctitle (from vpn-slice==0.13)
builder for '/nix/store/pli7a8x4v3zfib3cfr7irnd5nqx8afwj-python3.7-vpn-slice.drv' failed with exit code 1
error: build of '/nix/store/pli7a8x4v3zfib3cfr7irnd5nqx8afwj-python3.7-vpn-slice.drv' failed

ldeck · February 21, 2020, 7:54am

Thanks @benley. That worked!

Is it preferable to use mkShell instead?

ldeck · February 21, 2020, 11:57am

Thanks to @benley and @jonringer the following works

with import <nixpkgs> {};

let
    vpn-slice = pkgs.python3Packages.buildPythonPackage rec {
      name = "vpn-slice";
      version = "v0.13";

      src = pkgs.fetchFromGitHub {
      	 owner = "dlenski";
	 repo = "${name}";
	 rev = "${version}";
	 sha256 = "1ibrwal80z27c2mh9hx85idmzilx6cpcmgc15z3lyz57bz0krigb";
      };

      propagatedBuildInputs = with pkgs.python3Packages; [ setproctitle ];

      meta = {
        homepage = "https://github.com/dlenski/vpn-slice";
        description = "vpnc-script replacement for easy and secure split-tunnel VPN setup";
        license = stdenv.lib.licenses.gpl3Plus;
        maintainers = with maintainers; [ dlenski ];
      };
    };

in mkShell {
   name = "vpn-env";
   buildInputs = with pkgs.python3Packages; [ numpy toolz vpn-slice ];
   shellHook = ''
     echo "Whatever makes this happen."
   '';
}

jonringer · February 21, 2020, 10:08pm

ldeck:

However, it still fails saying…

ERROR: Could not find a version that satisfies the requirement setproctitle (from vpn-slice==0.13) (from versions: none)
ERROR: No matching distribution found for setproctitle (from vpn-slice==0.13)

I’m tracking nixpkg-unstable, so the list of available packages might be different.