Python packages with duplicated packages in closure for dependency with poetry2nix app

Help
1

Hi,

I am currently trying to figure out how to correctly package a python app which depends on poetry2nix packaged app.

The issue is that depending on the poetry2nix app throws ‘Found duplicated packages in closure for dependency’ errors.

So currently I just remove the duplicate packages from the python packages propagatedBuildinputs and it works. However I was curious if this is the correct way to handle this issue or otherwise does one have to override the conflicting packages to use the same package for both conflicts?

Is this how people fix it or is removing the duplicates good enough?

in essence when trying to fix it without removing them I was trying to override the python input for each package with the same one from my inputs however it didn’t work. Also trying to override the attributes inside the python attribute for each package did not work. Obviously I am messing something up here.

{
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake_netboxcli = {
      url = "privateurl";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };
  outputs = { self, nixpkgs, flake_netboxcli}:
    let
      salt_overlays = import ./overlays/saltstack.nix;
      netboxcli = flake_netboxcli.packages.${system}.default;
      pkgs = import nixpkgs { inherit system; overlays = [
          overlay_salt_inputs
          salt_overlays.fix_conflicts
        ];
      };

      overlay_salt_inputs = salt_overlays.update_inputs {python3 = pkgs.python3; netboxcli = nbcli; };
      nbcli1 = netboxcli.override { python = pkgs.python3; };
      nbcli = nbcli1.overrideAttrs (new: old: {
        python = let
            packageOverrides = final: prev: {
              pynetbox = prev.pynetbox.overridePythonAttrs(old_pkg: {
                propagatedBuildInputs = (pkgs.lib.lists.subtractLists
                  [
                    netboxcli.python.pkgs.requests
                  ]
                  old_pkg.propagatedBuildInputs) ++ [pkgs.python3.pkgs.requests];
              });
            };
          in old.python.override {
            inherit packageOverrides;
        };
        propagatedBuildInputs = (pkgs.lib.lists.subtractLists
          [
            old.python.pkgs.pyyaml
            old.python.pkgs.pynetbox
          ]
          old.propagatedBuildInputs) ++ [pkgs.python3.pkgs.pyyaml new.python.pkgs.pynetbox];
      });

      system = "x86_64-linux";

    in {
      packages.${system} = {
        salt = pkgs.salt;
      };
    };
}

overlays file

{
  update_inputs = {python3, netboxcli}:
    final: prev:
    {
      salt = prev.salt.override {inherit python3; extraInputs = with prev.python3.pkgs; [pycurl cryptography netboxcli];};
    };

  fix_conflicts =
    final: prev:
    {
      salt = prev.salt.overrideAttrs ( old: {
        propagatedBuildInputs = (prev.lib.lists.subtractLists
          [
            prev.python3.pkgs.requests
            prev.python3.pkgs.pyyaml
          ]
          old.propagatedBuildInputs) ++ [final.python3.pkgs.requests final.python3.pkgs.pyyaml];
        pythonImportsCheck = [
          "netboxcli"
          "requests"
          "pyyaml"
        ];
      });
    };
}

the errors

Executing pythonRuntimeDepsCheck
...skipping...
      ...depending on: /nix/store/w5x801idypjgx2mic5nw6kdyxpq5llh9-python3.11-requests-2.31.0
      ...depending on: /nix/store/b5gcik14vvgzybbxx3xyl795fz6sz82w-python3.11-idna-3.7
Found duplicated packages in closure for dependency 'urllib3':
  urllib3 2.0.7 (/nix/store/7slmy4b47jx8xbvglx663z976r4glr1h-python3.11-urllib3-2.0.7)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/l6n107j7fh6kxzg3ma61q7zbccg9zypl-python3.11-pynetbox-6.6.2
      ...depending on: /nix/store/g9lzbvws9lafs1lj95rpwkj0vibiqs0b-python3.11-requests-2.31.0
      ...depending on: /nix/store/7slmy4b47jx8xbvglx663z976r4glr1h-python3.11-urllib3-2.0.7
  urllib3 2.2.1 (/nix/store/sq2gpzqg6l4pv0saazr3kr1sqmyr06a3-python3.11-urllib3-2.2.1)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/fs3p0xjrhxngmyvy6r5v1kl0jky41v2n-python3.11-pynetbox-7.3.3
      ...depending on: /nix/store/w5x801idypjgx2mic5nw6kdyxpq5llh9-python3.11-requests-2.31.0
      ...depending on: /nix/store/sq2gpzqg6l4pv0saazr3kr1sqmyr06a3-python3.11-urllib3-2.2.1
  urllib3 2.2.1 (/nix/store/sq2gpzqg6l4pv0saazr3kr1sqmyr06a3-python3.11-urllib3-2.2.1)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/fs3p0xjrhxngmyvy6r5v1kl0jky41v2n-python3.11-pynetbox-7.3.3
      ...depending on: /nix/store/w5x801idypjgx2mic5nw6kdyxpq5llh9-python3.11-requests-2.31.0
      ...depending on: /nix/store/sq2gpzqg6l4pv0saazr3kr1sqmyr06a3-python3.11-urllib3-2.2.1
Found duplicated packages in closure for dependency 'six':
  six 1.16.0 (/nix/store/yjdyjjjy8dkylwjh0srfbk60mhv4myxi-python3.11-six-1.16.0)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/l6n107j7fh6kxzg3ma61q7zbccg9zypl-python3.11-pynetbox-6.6.2
      ...depending on: /nix/store/yjdyjjjy8dkylwjh0srfbk60mhv4myxi-python3.11-six-1.16.0
  six 1.16.0 (/nix/store/g8dlaaa7ahvx40mb0jw8mvdxw1rib8xj-python3.11-six-1.16.0)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/fs3p0xjrhxngmyvy6r5v1kl0jky41v2n-python3.11-pynetbox-7.3.3
      ...depending on: /nix/store/g8dlaaa7ahvx40mb0jw8mvdxw1rib8xj-python3.11-six-1.16.0
Found duplicated packages in closure for dependency 'pyyaml':
  pyyaml 6.0.1 (/nix/store/izwf8h034dgqxqgkqzgmip3f3zi0qhhq-python3.11-pyyaml-6.0.1)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/izwf8h034dgqxqgkqzgmip3f3zi0qhhq-python3.11-pyyaml-6.0.1
  pyyaml 6.0.1 (/nix/store/j0qr62l3m37ss953d19k36bijf8yal9x-python3.11-pyyaml-6.0.1)
    dependency chain:
      this derivation: /nix/store/pznhr9k44grn536fxn6pdk753rbh16n3-python3.11-netboxcli-0.1.0
      ...depending on: /nix/store/j0qr62l3m37ss953d19k36bijf8yal9x-python3.11-pyyaml-6.0.1

Package duplicates found in closure, see above. Usually this happens if two packages depend on different version of the same dependency.

Here are the buildinputs

nix-repl> netboxcli.propagatedBuildInputs
[
  «derivation /nix/store/hbwrh8n2lichc1ks3q46c9ccmdh9gjzd-python3.11-attrs-23.2.0.drv»
  «derivation /nix/store/k3cv296jw1imdc9a0kkiyfq0zcy9lc0l-python3.11-cattrs-23.1.2.drv»
  «derivation /nix/store/ggqafb2s1daswqiq3cpnxbw7mpzgglw5-python3.11-click-8.1.7.drv»
  «derivation /nix/store/hjlbgy7fm1i1z5q2hcx2f7wbxjyixbw5-python3.11-pynetbox-6.6.2.drv»
  «derivation /nix/store/nhbq6mp4wm546236zwv7i7h7vrh9z2yc-python3-3.11.9.drv»
  «derivation /nix/store/x5v5l2l9ziszpkz59msmlf08idz52404-python3.11-pyyaml-6.0.1.drv»
  «derivation /nix/store/nhbq6mp4wm546236zwv7i7h7vrh9z2yc-python3-3.11.9.drv»
]

nix-repl> pkgs.salt.propagatedBuildInputs
[
  «derivation /nix/store/6yn6vm0zhvzral9wshmrmbryaz728nad-python3.11-distro-1.9.0.drv»
«derivation /nix/store/074lwqlc63q0wvm1icnhv648kxyjixg2-python3.11-jinja2-3.1.4.drv»
«derivation /nix/store/vn3yipncsqqxld2xbhg99qwdlp3ji4k2-python3.11-jmespath-1.0.1.drv»
  «derivation /nix/store/9wb32lgjnf16kl9q52san1xq7wswwmqs-python3.11-looseversion-1.3.0.drv»
  «derivation /nix/store/4spdp9q4kp28iipxckjq2zg69xai0bnz-python3.11-markupsafe-2.1.5.drv»
  «derivation /nix/store/bl9b245k87sq75n7r9im9v23cv7fs8dk-python3.11-msgpack-1.0.8.drv»
  «derivation /nix/store/flhci49mxm617nlcv6v32v9pjswp0i70-python3.11-packaging-24.0.drv»
  «derivation /nix/store/z12wmfx7rscprsziqa26qr13n7wrgjiy-python3.11-psutil-5.9.8.drv»
  «derivation /nix/store/08fiizsap06vv034ai87q9rq8bw7rd64-python3.11-pycryptodome-3.20.0.drv»
  «derivation /nix/store/yh2amyimrxn893w9j1rv1nd3nq7s7vqb-python3.11-pyyaml-6.0.1.drv»
  «derivation /nix/store/mgzwr06zr7l0096qrn4w7ndn720dknw1-python3.11-pyzmq-25.1.2.drv»
  «derivation /nix/store/89dl95704nilrk454snkgp7xlcbhzwvv-python3.11-requests-2.31.0.drv»
  «derivation /nix/store/liylqaasbwm723bka4xkj00lcg9fmyxr-python3.11-tornado-6.3.3.drv»
  «derivation /nix/store/nhbq6mp4wm546236zwv7i7h7vrh9z2yc-python3-3.11.9.drv»
]

nix-repl>

netboxcli is built with poetry2nix

using this simple overlay alone fixes the issue by just removing the conflicting packages from one

{
  fix_conflicts =
    final: prev:
    {
      salt = prev.salt.overrideAttrs ( old: {
        propagatedBuildInputs = (prev.lib.lists.subtractLists
          [
            prev.python3.pkgs.requests
            prev.python3.pkgs.pyyaml
          ]
          old.propagatedBuildInputs);
      });
    };
}

Anyone has a clue what is the correct way to handle this?

2

So after some reading and checking derivations, debugging with repl I have overridden the conflicting python packages in my included dependency with the packages given by the nixpkgs input

Currently it looks like this

{
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake_netboxcli = {
      url = "privateurl";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };
  outputs = { self, nixpkgs, flake_netboxcli}:
    let
      values = import ./values.nix { inherit nixpkgs flake_netboxcli; };
      inherit (values) system pkgs;
    in {
      packages.${system} = {
        salt = pkgs.salt;
      };
    };
}

values

{ nixpkgs, flake_netboxcli }:
rec {
      netboxcli = flake_netboxcli.packages.${system}.default;
      pkgs = import nixpkgs { inherit system; overlays = [
          (import ./overlays/pynetbox.nix)
          overlay_salt_inputs
          salt_overlays.check_imports
        ];
      };
      salt_overlays = import ./overlays/saltstack.nix;
      nbcli = (import ./overlays/netboxcli.nix).overridden {inherit pkgs netboxcli;};
      overlay_salt_inputs = salt_overlays.update_inputs {netboxcli = nbcli; };
      system = "x86_64-linux";
}

the overrides for my included custom poetry2nix package which creates the conflicts with my desired package from nixpkgs

I did the overrides in multiple steps since it was easier for me to follow than trying to nest overrides into overrides

{
  overridden = {pkgs, netboxcli}: let
    netboxcli_python_pkgs = netboxcli.python.pkgs;
    python_pkgs = pkgs.python3.pkgs;
    new_buildinputs_pynetbox = (pkgs.lib.lists.subtractLists [netboxcli_python_pkgs.requests netboxcli_python_pkgs.six] netboxcli_python_pkgs.pynetbox.propagatedBuildInputs) ++ [python_pkgs.requests python_pkgs.six];
    new_netboxcli_pynetbox = netboxcli_python_pkgs.pynetbox.overridePythonAttrs(old_pkg: {
      propagatedBuildInputs = new_buildinputs_pynetbox;
    })
    ;
    new_buildinputs_netboxcli = (pkgs.lib.lists.unique (pkgs.lib.lists.subtractLists [netboxcli_python_pkgs.pyyaml netboxcli_python_pkgs.pynetbox] netboxcli.propagatedBuildInputs)) ++ [python_pkgs.pyyaml new_netboxcli_pynetbox];
    nbcli = netboxcli.overrideAttrs (new: old: {
      propagatedBuildInputs = new_buildinputs_netboxcli;
    })
    ;
    in
    nbcli;
}

overlays for the nixpkgs package

{
  update_inputs = {netboxcli}:
    final: prev:
    {
      salt = prev.salt.override {extraInputs = with prev.python3.pkgs; [pygit2 pycurl cherrypy cryptography netboxcli];};
    };

  check_imports =
    final: prev:
    {
      salt = prev.salt.overrideAttrs ( old: {
        pythonImportsCheck = [
          "netboxcli"
          "requests"
        ];
      });
    };
}

So in conclusion I think just removing the conflicting packages in the package one is trying to include is simpler than trying to override the buildinputs of each conflict. However I think there is probably an easier way to do this.

Honestly I expected inputs.nixpkgs.follows = “nixpkgs”; to also use the python packages which are dependencies for the included custom poetry package but I guess I just don’t know how to do it correctly.