Hello everybody! These past few days I’ve been struggling to get a nix shell that can run
llama 3.1 on my nvidia GPU. It always ends up having the wrong package versions, or straight up not compiling.
If anyone can share a similar config, or a different method of installing python packages, or some advice on what I generally should do, please do!
inference.py for running AI (it deviated slightly over the commits)
import transformers
import torch
from transformers import pipeline, AutoTokenizer
# Define the path to the model and the tokenizer
model_path = "./models/Meta-Llama-3.1-8B-Instruct_quantized/"
# Define rope_scaling with the required fields only
rope_scaling = {"type": "llama3", "factor": 8.0}
# Move the model to GPU if available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Initialize the pipeline with the model and tokenizer
text_generator = pipeline(
"text-generation",
model=model_path,
tokenizer=AutoTokenizer.from_pretrained(model_path),
model_kwargs={"torch_dtype": torch.bfloat16, "rope_scaling": rope_scaling},
device=device
)
# Infinite loop to take user input and generate responses
try:
while True:
prompt = input("Enter a prompt: ")
if prompt.lower() == 'exit':
break
outputs = pipe(prompt, max_new_tokens=256, do_sample=False)
print(outputs[0]['generated_text'])
except KeyboardInterrupt:
print("Program terminated.")
My attempts:
(shell.nix) Naive attempt with torchWithCuda
{ pkgs ? import (fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/nixos-24.05.tar.gz";
# I think these hashes are temporary, to get nix complaining and getting real hash
sha256 = "0rqkpdipwq1ld352sg7h2a1zc1xg3rj5ay6dlr337cysj4xsgn7b";
}) { config.allowUnfree = true; },
unstable ? import (fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz";
sha256 = "0rqkpdipwq1ld352sg7h2a1zc1xg3rj5ay6dlr337cysj4xsgn7b";
}) { config.allowUnfree = true; } }:
pkgs.mkShell {
buildInputs = with pkgs; [
# 3.12 python
python3
python312Packages.pip
python312Packages.virtualenv
unstable.python312Packages.transformers
python312Packages.torchWithCuda
python312Packages.accelerate
];
shellHook = ''
# Set up the virtual environment (optional)
if [ ! -d "venv" ]; then
python -m venv venv
echo "Virtual environment created."
fi
source venv/bin/activate
echo "Virtual environment activated."
'';
}
Problem: it takes hours to compile
(flake.nix) Flake with cachix
{
description = "Development environment flake with CUDA Cachix cache, NixOS 24.05, and basic tools";
# Cachix cache so you don't have to build cuda for 100 hours
nixConfig = {
extra-substituters = [
"https://nix-community.cachix.org"
];
extra-trusted-public-keys = [
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
"cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="
];
};
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
};
outputs = { self, nixpkgs }:
let
system = "x86_64-linux";
pkgs = import nixpkgs {
system = system;
config.allowUnfree = true;
config.cudaSupport = true;
};
in {
# Make the shell
devShell.${system} = pkgs.mkShell {
packages = with pkgs; [
python311
python311Packages.torch-bin
python311Packages.datasets
python311Packages.transformers
python311Packages.evaluate
python311Packages.accelerate
python311Packages.pip
cudatoolkit
linuxPackages.nvidia_x11
];
shellHook = ''
echo "You are now using a NIX environment"
export CUDA_PATH=${pkgs.cudatoolkit}
echo $CUDA_PATH
# Set up the virtual environment (optional)
if [ ! -d "venv" ]; then
python -m venv venv
echo "Virtual environment created."
fi
source venv/bin/activate
echo "Virtual environment activated."
'';
};
};
}
flake.lock:
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1731386116,
"narHash": "sha256-lKA770aUmjPHdTaJWnP3yQ9OI1TigenUqVC3wweqZuI=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "689fed12a013f56d4c4d3f612489634267d86529",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-24.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}
Bash error:
Traceback (most recent call last):
File "/home/syshotdev/Programming/Python/Llama-model-testing/inference.py", line 9, in <module>
model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/mk8jls52jry9pkwhcwdxdwg16z9374x1-python3.11-transformers-4.41.0/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 523, in from_pretrained
config, kwargs = AutoConfig.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/mk8jls52jry9pkwhcwdxdwg16z9374x1-python3.11-transformers-4.41.0/lib/python3.11/site-packages/transformers/models/auto/configuration_auto.py", line 958, in from_pretrained
return config_class.from_dict(config_dict, **unused_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/mk8jls52jry9pkwhcwdxdwg16z9374x1-python3.11-transformers-4.41.0/lib/python3.11/site-packages/transformers/configuration_utils.py", line 768, in from_dict
config = cls(**config_dict)
^^^^^^^^^^^^^^^^^^
File "/nix/store/mk8jls52jry9pkwhcwdxdwg16z9374x1-python3.11-transformers-4.41.0/lib/python3.11/site-packages/transformers/models/llama/configuration_llama.py", line 161, in __init__
self._rope_scaling_validation()
File "/nix/store/mk8jls52jry9pkwhcwdxdwg16z9374x1-python3.11-transformers-4.41.0/lib/python3.11/site-packages/transformers/models/llama/configuration_llama.py", line 182, in _rope_scaling_validation
raise ValueError(
ValueError: `rope_scaling` must be a dictionary with two fields, `type` and `factor`, got {'factor': 8.0, 'high_freq_factor': 4.0, 'low_freq_factor': 1.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}
(flake.nix) Updated transformers
Comes from this link: meta-llama/Llama-3.1-8B-Instruct · ValueError: `rope_scaling` must be a dictionary with two fields
{
description = "Development environment flake with CUDA Cachix cache, NixOS 24.05, and basic tools";
# Cachix cache so you don't have to build cuda for 100 hours
nixConfig = {
extra-substituters = [
"https://nix-community.cachix.org"
];
extra-trusted-public-keys = [
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
"cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="
];
};
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
# Latest nix commit at the time
nixpkgs-unstable.url = "github:nixos/nixpkgs/0bce9e80c6828de1c0af63bc96ca2059b0652a16";
};
outputs = { self, nixpkgs, nixpkgs-unstable }:
let
system = "x86_64-linux";
pkgs = import nixpkgs {
system = system;
config.allowUnfree = true;
};
unstable = import nixpkgs-unstable {
system = system;
config.allowUnfree = true;
};
in {
# Make the shell
devShell.${system} = pkgs.mkShell {
packages = with pkgs; [
python311
python311Packages.torch-bin
python311Packages.datasets
unstable.python311Packages.transformers
python311Packages.evaluate
python311Packages.accelerate
cudatoolkit
linuxPackages.nvidia_x11
];
shellHook = ''
echo "You are now using a NIX environment"
'';
};
};
}
Bash error:
Traceback (most recent call last):
File "/home/syshotdev/Programming/Python/Llama-model-testing/inference.py", line 1, in <module>
import transformers
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/__init__.py", line 26, in <module>
from . import dependency_versions_check
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/dependency_versions_check.py", line 57, in <module>
require_version_core(deps[pkg])
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 117, in require_version_core
return require_version(requirement, hint)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 111, in require_version
_compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 44, in _compare_versions
raise ImportError(
ImportError: huggingface-hub>=0.23.2,<1.0 is required for a normal functioning of this module, but found huggingface-hub==0.23.0.
Try: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main
(flake.nix) Force add huggingface-hub as package
{
description = "Development environment flake with CUDA Cachix cache, NixOS 24.05, and basic tools";
# Cachix cache so you don't have to build cuda for 100 hours
nixConfig = {
extra-substituters = [
"https://nix-community.cachix.org"
];
extra-trusted-public-keys = [
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
"cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="
];
};
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
nixpkgs-unstable.url = "github:nixos/nixpkgs/0bce9e80c6828de1c0af63bc96ca2059b0652a16";
};
outputs = { self, nixpkgs, nixpkgs-unstable }:
let
system = "x86_64-linux";
pkgs = import nixpkgs {
system = system;
config.allowUnfree = true;
};
unstable = import nixpkgs-unstable {
system = system;
config.allowUnfree = true;
};
in {
# Make the shell
devShell.${system} = pkgs.mkShell {
packages = with pkgs; [
python311
python311Packages.torch-bin
python311Packages.datasets
unstable.python311Packages.transformers
python311Packages.huggingface-hub
python311Packages.evaluate
python311Packages.accelerate
python311Packages.pip
cudatoolkit
linuxPackages.nvidia_x11
];
shellHook = ''
echo "You are now using a NIX environment"
'';
};
};
}
Bash error (exact same one as last time. The package version didn’t even change):
Traceback (most recent call last):
File "/home/syshotdev/Programming/Python/Llama-model-testing/inference.py", line 1, in <module>
import transformers
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/__init__.py", line 26, in <module>
from . import dependency_versions_check
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/dependency_versions_check.py", line 57, in <module>
require_version_core(deps[pkg])
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 117, in require_version_core
return require_version(requirement, hint)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 111, in require_version
_compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
File "/nix/store/6jq5jq1v2g1nlma811m0lrkj5ww145ml-python3.11-transformers-4.46.0/lib/python3.11/site-packages/transformers/utils/versions.py", line 44, in _compare_versions
raise ImportError(
ImportError: huggingface-hub>=0.23.2,<1.0 is required for a normal functioning of this module, but found huggingface-hub==0.23.0.
Try: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main
(shell.nix) Tried shell.nix again, pinned specific packages
{ pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/0bce9e80c6828de1c0af63bc96ca2059b0652a16.tar.gz") {}
}:
let
pythonEnv = pkgs.python311.withPackages (pythonPackages: with pythonPackages; [
(pythonPackages.buildPythonPackage rec {
pname = "transformers";
version = "4.35.0";
src = pkgs.fetchPypi {
inherit pname version;
sha256 = "sha256-5LQXY/ZRKC/JeTSNOqFIJEOH3ckWX0sYRVeYx3CuI7k=";
};
doCheck = false; # Skip tests for faster builds
})
(pythonPackages.buildPythonPackage rec {
pname = "torch";
version = "2.5.1";
src = pkgs.fetchurl {
url = "https://files.pythonhosted.org/packages/69/72/20cb30f3b39a9face296491a86adb6ff8f1a47a897e4d14667e6cf89d5c3/torch-${version}-cp313-cp313-manylinux1_x86_64.whl";
sha256 = "sha256-m2Ht87T247DgrdqLOWAma5AJ0Cs3VVlx9NHI96Ba/tc="; # Replace with the correct hash
};
format = "wheel";
doCheck = false;
})
(pythonPackages.buildPythonPackage rec {
pname = "accelerate";
version = "0.22.0";
src = pkgs.fetchPypi {
inherit pname version;
sha256 = "sha256-KwqD480HyJRIxdWpT3K8HbmNXgxJjKF5hIcfAdv4Mkc=";
};
doCheck = false;
})
pip
]);
in
pkgs.mkShell {
name = "custom-package-version-shell";
buildInputs = [
pythonEnv
];
shellHook = ''
echo "You are now using a NIX environment"
# Set up the virtual environment (optional)
if [ ! -d "venv" ]; then
python -m venv venv
echo "Virtual environment created."
fi
source venv/bin/activate
echo "Virtual environment activated."
'';
}
Bash error (the packages weren’t installed):
Traceback (most recent call last):
File "/home/syshotdev/Programming/Python/Llama-model-testing/inference.py", line 1, in <module>
import transformers
ModuleNotFoundError: No module named 'transformers'
(flake.nix) Maybe pinning packages will work in a flake?
{
description = "Development environment flake with CUDA Cachix cache, NixOS 24.05, and basic tools";
# Cachix cache so you don't have to build cuda for 100 hours
nixConfig = {
extra-substituters = [
"https://nix-community.cachix.org"
];
extra-trusted-public-keys = [
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
"cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="
];
};
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
nixpkgs-unstable.url = "github:nixos/nixpkgs/0bce9e80c6828de1c0af63bc96ca2059b0652a16";
};
outputs = { self, nixpkgs, nixpkgs-unstable }:
let
system = "x86_64-linux";
pkgs = import nixpkgs {
system = system;
config.allowUnfree = true;
};
unstable = import nixpkgs-unstable {
system = system;
config.allowUnfree = true;
};
in {
# Make the shell
devShell.${system} = pkgs.mkShell {
packages = with pkgs; [
/*
(pkgs.python311.withPackages (ps: with ps; [
torch
accelerate
pip
]))
*/
python311
python311Packages.torch-bin
python311Packages.datasets
unstable.python311Packages.transformers
(pkgs.python311Packages.buildPythonPackage rec {
pname = "huggingface-hub";
version = "0.26.2";
pyproject = true;
src = fetchFromGitHub {
owner = "huggingface";
repo = "huggingface_hub";
rev = "refs/tags/v${version}";
hash = "sha256-F2E8P0Hq3Ee+RXUEN4t2JtfBtK36aMsHQCnid9VWdLk=";
};
build-system = with pkgs; [ setuptools ];
dependencies = with pkgs; [
filelock
fsspec
packaging
pyyaml
requests
tqdm
typing-extensions
];
# Tests require network access.
doCheck = false;
pythonImportsCheck = [ "huggingface_hub" ];
})
python311Packages.evaluate
python311Packages.accelerate
cudatoolkit
linuxPackages.nvidia_x11
];
shellHook = ''
echo "You are now using a NIX environment"
'';
};
};
}
Bash error:
at /nix/store/0gil5b8wly5pg4yw42iy1vzh3d25c62z-source/flake.nix:58:39:
57|
58| build-system = with pkgs; [ setuptools ];
| ^
59|
