Python Package Wheel No matching distribution found

Help
1

Hi!
I’m trying to packaging GitHub - nfstream/nfstream: NFStream: a Flexible Network Data Analysis Framework.
nfstream · PyPI

But i get those errors:

this derivation will be built:
  /nix/store/b5v15x6w83xvayfprmzvdhqanz05c53z-nfstream-6.3.5.drv
building '/nix/store/b5v15x6w83xvayfprmzvdhqanz05c53z-nfstream-6.3.5.drv'...
Sourcing python-remove-tests-dir-hook
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing wheel setup hook
Using wheelUnpackPhase
Sourcing pip-install-hook
Using pipInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing python-namespaces-hook
unpacking sources
Executing wheelUnpackPhase
Finished executing wheelUnpackPhase
patching sources
configuring
no configure script, doing nothing
building
no Makefile, doing nothing
installing
Executing pipInstallPhase
/build/dist /build
Processing ./nfstream-6.3.5-cp39-cp39-manylinux1_x86_64.whl
ERROR: Could not find a version that satisfies the requirement psutil>=5.8.0 (from nfstream)
ERROR: No matching distribution found for psutil>=5.8.0
error: builder for '/nix/store/b5v15x6w83xvayfprmzvdhqanz05c53z-nfstream-6.3.5.drv' failed with exit code 1;
       last 10 log lines:
       > configuring
       > no configure script, doing nothing
       > building
       > no Makefile, doing nothing
       > installing
       > Executing pipInstallPhase
       > /build/dist /build
       > Processing ./nfstream-6.3.5-cp39-cp39-manylinux1_x86_64.whl
       > ERROR: Could not find a version that satisfies the requirement psutil>=5.8.0 (from nfstream)
       > ERROR: No matching distribution found for psutil>=5.8.0
       For full logs, run 'nix log /nix/store/b5v15x6w83xvayfprmzvdhqanz05c53z-nfstream-6.3.5.drv'.

with the following derivation:

with import <nixos-unstable> {};
with python39Packages;

assert (isPy39 || isPy38 || isPy37 || isPy36);

let urls = {
      "6.3.5" = {
        py36 = {
          url = https://files.pythonhosted.org/packages/22/c5/9159521b70d70520a66fcf6554d0d22744bfdb602dc9f3adaa8c6e5cbeb6/nfstream-6.3.5-cp36-cp36m-manylinux1_x86_64.whl;
          sha256 = "1qs8rrjaplw29gpgq4348y88nm9hfr0mn9hjak3sk4j8qikbi211";
        };
        py37 = {
          url = https://files.pythonhosted.org/packages/15/19/6fe2b23a85ddf45f689a82133605edd8e6ec908da64d0e5692c4921daa2f/nfstream-6.3.5-cp37-cp37m-manylinux1_x86_64.whl;
          sha256 = "1350hlyqv1lrvy4qc3i9lvwacx50vzqpcnjf9rqhn8zccs7rpmdz";
        };
        py38 = {
          url = https://files.pythonhosted.org/packages/19/fb/e722fe1aaf66259ce7e038362935dae948d1038744a6eb344cce92eeceb6/nfstream-6.3.5-cp38-cp38-manylinux1_x86_64.whl;
          sha256 = "0x0xbsjg5gb34dshzs5vp4f8ljk6wc4p63pw7728dnl2ashly2hz";
        };
        py39 = {
          url = https://files.pythonhosted.org/packages/b7/90/d70d47c166b6a6cf0d386e62e727276ff6b8824633432f59f10ff640e9f0/nfstream-6.3.5-cp39-cp39-manylinux1_x86_64.whl;
          sha256 = "1wqcz4vavay56gf8nn55s67wp61rmyw8s1mzki3b3zbbzwl1zbvy";
        };
      };
      "6.3.4" = {
        py36 = {
          url = https://files.pythonhosted.org/packages/cc/3a/e0058dcdb991d9730b2c1054d601105b9c5abdff1090f477af295f0129d4/nfstream-6.3.4-pp36-pypy36_pp73-manylinux1_x86_64.whl;
          sha256 = "02qcqv4m2wjhfwis09i20b5drgvas22njrmk4jihc0ybm131m7dr";
        };
        py37 = {
          url = https://files.pythonhosted.org/packages/7d/42/1c3fefa074fb506f778ab007d35e323f3c5ebd587e9c7183a497ac830825/nfstream-6.3.4-pp37-pypy37_pp73-manylinux1_x86_64.whl;
          sha256 = "1f3d5imnaajl74lbqf8grxk0bbb73j6zrbvajxfmyd6b56l4m4cd";
        };
      };
    };

in

buildPythonApplication rec {
  pname = "nfstream";
  version = "6.3.5";
  format = "wheel";

  src = builtins.fetchurl (with urls."${version}"; if isPy36 then py36 else if isPy37 then py37 else if isPy38 then py38 else py39);


  buildInputs = [ libgpgerror libgcrypt libpcap ndpi ];

  propagatedBuildInputs = [ setuptools ];

  nativeBuildInputs = [ autoPatchelfHook ];
  
  meta = with lib; {
    description = "Python Framework for network data analytics";
    homepage = "https://github.com/nfstream/nfstream";
    license = licenses.lgpl3;
    maintainers = with maintainers; [ heph2 ];
  };
}

Any ideas?

2

psutil is a runtime dependency. Adding it to propagatedBuildInputs should do the trick if I’m not missing something.

3

i’ve added psutil to “propagatedBuildInputs” and now i got this error

Processing ./nfstream-6.3.5-cp39-cp39-manylinux1_x86_64.whl
Requirement already satisfied: psutil>=5.8.0 in /nix/store/9j2pzpxpr5w9r20vc4zrrzllqmy4mssk-python3.9-psutil-5.8.0/lib/python3.9/site-packages (from nfstream==6.3.5) (5.8.0)
ERROR: Could not find a version that satisfies the requirement dpkt>=1.9.7 (from nfstream)
ERROR: No matching distribution found for dpkt>=1.9.7
error: builder for '/nix/store/3dxq3ryb7a6i6sl15q3xd3zc7w5y26bd-nfstream-6.3.5.drv' failed with exit code 1;
       last 10 log lines:
       > no configure script, doing nothing
       > building
       > no Makefile, doing nothing
       > installing
       > Executing pipInstallPhase
       > /build/dist /build
       > Processing ./nfstream-6.3.5-cp39-cp39-manylinux1_x86_64.whl
       > Requirement already satisfied: psutil>=5.8.0 in /nix/store/9j2pzpxpr5w9r20vc4zrrzllqmy4mssk-python3.9-psutil-5.8.0/lib/python3.9/site-packages (from nfstream==6.3.5) (5.8.0)
       > ERROR: Could not find a version that satisfies the requirement dpkt>=1.9.7 (from nfstream)
       > ERROR: No matching distribution found for dpkt>=1.9.7
       For full logs, run 'nix log /nix/store/3dxq3ryb7a6i6sl15q3xd3zc7w5y26bd-nfstream-6.3.5.drv'.

But AFAIK in the unstable channel dpkt is 1.9.7…

4

Have you added it to to the propagatedBuildInputs as well? also i don’t think that adding setuptools to that list is the right thing to do if it’s imported only in the setup.py…

5

That’s the first time that i try to packaging a python lib…
What should i do exactly?

ps: still error with dpkt, seems like it’s not using the unstable channel

6

can you publish your work somewhere? That way it would be much simpler for us to help you out

7

If pure-python packages aren’t packaged on nixpkgs, then you can use https://github.com/NixOS/nixpkgs/blob/f790fcd0bea2ef09f436e9d8088cfe2fea2c4c12/doc/languages-frameworks/python.section.md#how-to-consume-python-modules-using-pip-in-a-virtual-environment-like-i-am-used-to-on-other-operating-systems-how-to-consume-python-modules-using-pip-in-a-virtual-environment-like-i-am-used-to-on-other-operating-systems to use venv+nix to create a dev environment.

If you want a pure nix-environment, then you’ll need to package the missing dependencies, or use a tool like mach2nix, poetry2nix, or something similar to “bridge the gap”.

dpkt at least exists on nixpkgs master:

nix-repl> python3Packages.dpkt
«derivation /nix/store/39z4vi75ysffmirr6nklkg30pgffz1mj-python3.9-dpkt-1.9.7.2.drv»

Looking at the upstream repo, it looks like it should be pretty easy to build without resorting to wheels. I would just build it from source.

8

I’ve publish the current derivation here:
http://gist.mrkeebs.eu/#org774c8ad

9

I’m not that familiar with python and also with Nix, so my attempt to packaging usually are a bit of a mess…
I’ve tried without wheels and found this error:

Executing pipInstallPhase
/build/dist /build
ERROR: You must give at least one requirement to install (see "pip help install")
error: builder for '/nix/store/zg57m67vvqfycg1prpkwxjsmjasv7hg5-nfstream-6.3.5.drv' failed with exit code 1;
       last 10 log lines:
       > Finished executing wheelUnpackPhase
       > patching sources
       > configuring
       > no configure script, doing nothing
       > building
       > no Makefile, doing nothing
       > installing
       > Executing pipInstallPhase
       > /build/dist /build
       > ERROR: You must give at least one requirement to install (see "pip help install")
       For full logs, run 'nix log /nix/store/zg57m67vvqfycg1prpkwxjsmjasv7hg5-nfstream-6.3.5.drv'.

I’ve published what i wrote above :smiley:

10

“server gist.mrkeebs.eu is taking too long too respond” Port closed?

11

Mmm sorry, for some strange reason my DDNS script didn’t update my DNS :smiley:
Now should be reachable

12

Ok, @heph I did some more steps with this:

let
  pkgs = import <nixos-unstable> {};
  inherit (pkgs) python39Packages fetchFromGitHub lib;
  inherit (python39Packages) buildPythonApplication;

in buildPythonApplication {
  pname = "nfstream";
  version = "6.3.5";

  src = fetchFromGitHub {
    owner = "nfstream";
    repo = "nfstream";
    rev = "v6.3.5";
    sha256 = "HamSYEaCJ3z32YipjcNGwsvBxababm8OYEMM2tmdZ3A=";
  };

  buildInputs = with pkgs; [ libgpgerror libgcrypt libpcap ndpi ];

  propagatedBuildInputs = with python39Packages; [ psutil numpy dpkt cffi pandas ];

  meta = with lib; {
    description = "Python Framework for network data analytics";
    homepage = "https://github.com/nfstream/nfstream";
    license = licenses.lgpl3;
    maintainers = with maintainers; [ heph2 ];
  };
}

Now it tries to build but fails because of some hardcoded paths in https://github.com/nfstream/nfstream/blob/6d402b9435e5f8e3284ab026ac87f18a0aeea375/setup.py#L40-L48

You will have to patch it with the right ones from the store

13

Oh thanks :smiley:
I got this error in one of my attempts, but didn’t understand the cause…
So… How should i patch them from nix?

14

this is how I do patches: https://www.youtube.com/watch?v=5K_2RSjbdXc

1 Like
15

I went a bit further with this:

let
  pkgs = import <nixpkgs> {};
  inherit (pkgs) python39Packages fetchFromGitHub lib writeText
    libgpgerror libgcrypt libpcap ndpi;
  inherit (python39Packages) buildPythonApplication pytest pytest-runner;
  removeHardPaths = writeText "remove_harcoded_paths.patch" ''
    diff --git a/setup.py b/setup.py
    index 0404580..e8cac57 100644
    --- a/setup.py
    +++ b/setup.py
    @@ -37,14 +37,12 @@ def setup_engine_cc():
         if sys.platform == 'darwin':
             platform_compiler = "clang"
         print("\nSetting up engine_cc. Platform: {plat}, Byteorder: {bo}".format(plat=sys.platform, bo=sys.byteorder))
    -    subprocess.check_call([platform_compiler, '-I/usr/local/include/ndpi', '-shared', '-o',
    +    subprocess.check_call([platform_compiler, '-I${ndpi}/include/ndpi', '-shared', '-o',
                                'nfstream/engine/engine_cc.so',
                                '-g', '-fPIC', '-DPIC', '-O2', '-Wall', 'nfstream/engine/engine_cc.c',
                                # Required compiled static libs
    -                           '/usr/local/lib/libpcap.a',
    -                           '/usr/local/lib/libndpi.a',
    -                           '/usr/local/lib/libgcrypt.a',
    -                           '/usr/local/lib/libgpg-error.a'
    +                           '${libpcap}/lib/libpcap.a',
    +                           '${ndpi}/lib/libndpi.a'
                                ])
  '';
in buildPythonApplication {
  pname = "nfstream";
  version = "6.3.5";

  src = fetchFromGitHub {
    owner = "nfstream";
    repo = "nfstream";
    rev = "v6.3.5";
    sha256 = "HamSYEaCJ3z32YipjcNGwsvBxababm8OYEMM2tmdZ3A=";
  };

  buildInputs = [ libgpgerror libgcrypt libpcap ndpi pytest pytest-runner ];

  NIX_DEBUG = 1;

  patches = [ removeHardPaths ];

  propagatedBuildInputs = with python39Packages; [ psutil numpy dpkt cffi pandas ];

  meta = with lib; {
    description = "Python Framework for network data analytics";
    homepage = "https://github.com/nfstream/nfstream";
    license = licenses.lgpl3;
    maintainers = with maintainers; [ heph2 ];
  };
}

but then I hit into a wall… it seems that nfstream requires custom sources at least of libpcap and I don’t know what the best way to proceed is… I’m no really a C packaging expert…

16

I would make a patch upstream where people can specify alternate paths:

 -                           '/usr/local/lib/libpcap.a',
 +                           'os.env.get("NFSTREAM_LIB_PCAP", '/local/lib/libpcap.a'),

then in your derivation, you just need to do:

  NFSTREAM_LIB_PCAP = "${lib.getLib libpcap}/lib/libpcap.a";

It also appears that it wants the statically compiled versions of those libraries, so you will need to pull them from pkgsStatic package set.

17

I saw that libgpg-error and libgcrypt must be compiled statically. How can i specify that only those libs should be statically compiled ?
I’ll watch the video that u linked in the afternoon :smiley:

18

Uh that’s neat! At first I thought it would be easier :smiley: Probably my not familiarity with python doesn’t help me too much. And i’m also having trouble finding the resources to make my own patches…

Appreciate your help!

19

pkgsStatic.libgcrypt

[08:29:43] jon@nixos ~/projects/nixpkgs (master)
$ nix-build -A pkgsStatic.libgcrypt
/nix/store/nz97vkd5zpvps6kz38ancy9z51daa3fi-libgcrypt-static-x86_64-unknown-linux-musl-1.9.4
[08:29:52] jon@nixos ~/projects/nixpkgs (master)
$ tree ./result
./result
└── lib
    ├── libgcrypt.a
    └── libgcrypt.la

1 directory, 2 files
20

Basically adding in the diff {pkgsStatic.libcrypt} and {pkgsStatic.libgpgerror} would do the trick?