How to use custom python interpreter for a python package?

Help
1

Hello,

Does anybody know how to use custom python derivation for existing python package derivations?
At first glance pkgs.python3Packages looks pretty sealed and encapsulated. All my attempts to push in my cpython from aside failed. I don’t want to redefine all python packages from scratch.

Attempt 1: override python with overlay:

myPkgPy311 = pkgs.extend (self: super: {                                                                                 
     python3 = myPatchedPython;                                                                                              
     python38 = myPatchedPython;                                                                                             
     python39 = myPatchedPython;                                                                                             
});

Outcome: my python derivation is not picked up when I target a package.

let myPkgPy311 = ... 
in   myPkgPy311.python3Packages.send2trash

$ nix-shell
$ python3
python 3.9.5

I was expecting to see python 3.11

Attempt 2: use override of package set:

 myPkgs = pkgs.extend (self: super: {
    python39Packages = super.python39Packages.override {
      python3 = myPackedPython;                                                                                              
      python39 = myPackedPython;                                                                                             
      python = myPackedPython;
    };
  });

Outcome: for simple package it might work, but for a package with ffi breaks. So I am not sure if it is the correct way to go or just a problem with a specific package derivation:

let  pyobjc-core = myPkgs.python39Packages.buildPythonPackage rec {
    pname = "pyobjc-core";
    version = "7.3";
    adfasdf = true;
    name = "${pname}-${version}";
    src = myPkgs.python39Packages.fetchPypi {
      pname = "pyobjc-core";
      inherit version;
      sha256 = "0x3msrzvcszlmladdpl64s48l52fwk4xlnnri8daq2mliggsx0ah";
    };
    preBuild=''                                                                                                               
         export SDKROOT="/Library/Developer/CommandLineTools/SDKs/MacOSX10.12.sdk"                                            
    '';
    CFLAGS = "-iwithsysroot /usr/include -Wno-unused-argument";
  doCheck = false;
    commonPreHook = ''                                                                                                        
      export SDKROOT="/Library/Developer/CommandLineTools/SDKs/MacOSX10.12.sdk";                                              
    '';
    propagatedBuildInputs = [
      myPatchedPython
      pkgs.darwin.libobjc
      pkgs.darwin.cctools
      pkgs.darwin.apple_sdk.frameworks.Foundation
      pkgs.darwin.apple_sdk.frameworks.AppKit
      pkgs.darwin.apple_sdk.frameworks.CoreServices
      pkgs.darwin.apple_sdk.frameworks.Cocoa
      myPkgs.python39Packages.setuptools
    ];
    buildInputs = [ pkgs.libffi ];
  });

  cocoa = useMyPy (myPkgs.python39Packages.buildPythonPackage rec {
    pname = "pyobjc";
    version = "7.3";
    name = "${pname}-${version}";
    src = pkgs.python39Packages.fetchPypi {
      pname = "pyobjc-framework-Cocoa";
      inherit version;
      sha256 = "0zhbp18i06aprwfbp06l9wm3qrzsdcyy9hwis5d4b8wmlzkhb3di"; # 7.3 Cocoa                                            
    };
   CFLAGS="-Wno-unused-argument -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX10.12.sdk";
    doCheck = false;   
    propagatedBuildInputs = [
      myPatchedPython
      pyobjc-core
      pkgs.darwin.libobjc
      pkgs.darwin.apple_sdk.frameworks.Foundation
      pkgs.darwin.apple_sdk.frameworks.CoreFoundation
      pkgs.darwin.apple_sdk.frameworks.AppKit
      pkgs.darwin.apple_sdk.frameworks.Cocoa
      pkgs.darwin.apple_sdk.frameworks.CoreServices
      myPkgs.python39Packages.setuptools
    ];
  });

in pkgs.mkShell {  buildInputs = [  cocoa   ] ; }

$ nix-shell
$ python3
Python 3.11.0a0 (default, Aug 4 2021, 00:12:31) [Clang 7.1.0 (tags/RELEASE_710/final)] on darwin
Type “help”, “copyright”, “credits” or “license” for more information.

import AppKit
Traceback (most recent call last):
File “”, line 1, in
File “/nix/store/z6qb96ly5m17ciwlxvrn482d5xvmbavj-python3.9-pyobjc-7.3/lib/python3.9/site-packages/AppKit/init.py”, line 10, in
import Foundation
^^^^^^^^^^^^^^^^^
File “/nix/store/z6qb96ly5m17ciwlxvrn482d5xvmbavj-python3.9-pyobjc-7.3/lib/python3.9/site-packages/Foundation/init.py”, line 9, in
import CoreFoundation
^^^^^^^^^^^^^^^^^^^^^
File “/nix/store/z6qb96ly5m17ciwlxvrn482d5xvmbavj-python3.9-pyobjc-7.3/lib/python3.9/site-packages/CoreFoundation/init.py”, line 9, in
import objc
^^^^^^^^^^^
File “/nix/store/vq03kj72zbf6hgfr4q8wm0js5cb8fbw9-python3.9-pyobjc-core-7.3/lib/python3.9/site-packages/objc/init.py”, line 6, in
from . import _objc
^^^^^^^^^^^^^^^^^^^
ImportError: cannot import name ‘_objc’ from partially initialized module ‘objc’ (most likely due to a circular import) (/nix/store/vq03kj72zbf6hgfr4q8wm0js5cb8fbw9-python3.9-pyobjc-core-7.3/lib/python3.9/site-packages/objc/init.py)

PyObjCTools module from pyobjc-core is imported as expected and the custom cpython is in the nix-shell, but import AppKit from cocoa generates the error above.

I would like to avoid forking nixpkgs repository for introduction new cpython, because it is going to be extra bookkeeping.

1 Like
2 
mypython = python38.override {
  self = mypython;
};

Now you can use the Python packages set in mypython.pkgs that uses your custom interpreter.

1 Like
3

Thx. I made a step further, though python derivations are heavy on version number logic. master branch has number 3.11 and it spoils everything, because python derivations expect python version up to 3.10

nix-shell
error: attribute ‘python311’ missing, at /nix/store/j7rbvalwrfih1zz3a1h16k7csdy78vcl-nixpkgs-21.11pre304920.dd98b100651/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix:93:28
(use ‘–show-trace’ to show detailed location information)

    pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
    pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
    pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};

faking is not working

 .override { 
   pythonAttr = "python310";
   # or
   pkgsBuildHost = pkgs.python310;
}

lead to error

error: invalid command ‘bdist_wheel’

Right now I came up with an idea to use a hack patch to just downgrade version from 3.11 to 3.10. Fingers crossed.

1 Like
4

The expression is really not written with this in mind. You’re far better of running a patched Nixpkgs. To be honest, I don’t think it will get any better either in the future either. The expression is close to 500 lines, with a lot of exceptions for different platforms and Python versions, especially when it comes to patches.

1 Like