Jupyter notebook dependency management with Poetry

jonringer · July 14, 2020, 10:48pm

I’ve never used poetry for an environment, but this is what i do (The environment is used for many things, not just notebooks):

# shell.nix
let
  pkgs = import <nixpkgs> {};
  python = pkgs.python37;
  pythonPackages = python.pkgs;
in

with pkgs;

mkShell {
  name = "pip-env";
  buildInputs = with pythonPackages; [
    azure-cli
    kubectl

    # Python requirements (enough to get a virtualenv going).
    psutil
    #tensorflow
    #pyarrow
    pandas
    ipykernel
    jupyter
    pytest
    setuptools
    wheel
    venvShellHook

    libffi
    openssl
    gcc

    unzip
  ];
  venvDir = "venv37";
  src = null;
  postVenv = ''
    unset SOURCE_DATE_EPOCH
    ./scripts/install_local_packages.sh
  '';
  postShellHook = ''
    # Allow the use of wheels.
    unset SOURCE_DATE_EPOCH

    # get back nice looking PS1
    source ~/.bashrc
    source <(kubectl completion bash)

    PYTHONPATH=$PWD/$venvDir/${python.sitePackages}:$PYTHONPATH
  '';
}

And I can largely use it as a vanilla virtualenv environment. So if needed, i can just do something like pip install requests, and it works for most packages that don’t have native dependencies. (I guess your buildFHSUserEnv is one way around this). Another benefit is that I get to download most of my python dependency through the nixpkgs cache, which is much faster than pip install.

For running the notebook I can just do:

$ nix-shell
$ jupyter notebook

NOTE: Jupyter notebook kernels reference a python interpreter path, make sure that interpreter is able to find the necessary packages it needs. I achieve this through shell-hooks from the packages, so my PYTHONPATH contains them, but mkPoetryEnv will probably install everything into a single site-packages, which should also be fine.