Building with CUDA on nixpkgs

rihardsk · October 21, 2019, 12:30pm

I’m trying to package some software that uses CUDA with Nix (i’m using nixpkgs on Ubuntu). It didn’t go well - it built successfully but crashes at runtime - so i created a simple demo program to reproduce the issue.

Sample code

add.cu

// Code from https://devblogs.nvidia.com/even-easier-introduction-cuda/

#include <iostream>
#include <math.h>

// Kernel function to add the elements of two arrays
__global__
void add(int n, float *x, float *y)
{
  int index = blockIdx.x * blockDim.x + threadIdx.x;
  int stride = blockDim.x * gridDim.x;
  for (int i = index; i < n; i += stride)
    y[i] = x[i] + y[i];
}

int main(void)
{
  int N = 1<<28;
  float *x, *y;

  // Allocate Unified Memory – accessible from CPU or GPU
  cudaMallocManaged(&x, N*sizeof(float));
  cudaMallocManaged(&y, N*sizeof(float));

  // initialize x and y arrays on the host
  for (int i = 0; i < N; i++) {
    x[i] = 1.0f;
    y[i] = 2.0f;
  }

  // Run kernel on 1M elements on the GPU
  int blockSize = 256;
  int numBlocks = (N + blockSize - 1) / blockSize;
  add<<<numBlocks, blockSize>>>(N, x, y);

  // Wait for GPU to finish before accessing on host
  cudaDeviceSynchronize();

  // Check for errors (all values should be 3.0f)
  float maxError = 0.0f;
  for (int i = 0; i < N; i++)
    maxError = fmax(maxError, fabs(y[i]-3.0f));
  std::cout << "Max error: " << maxError << std::endl;

  // Free memory
  cudaFree(x);
  cudaFree(y);

Makefile

.PHONY: build
build: add

add: add.cu
        nvcc add.cu -o add


.PHONY: install
install: add
        mkdir -p $(out)/bin
        cp add $(out)/bin/

add.nix

{ stdenv, lib, makeWrapper, linuxPackages, cudatoolkit_9_2 }:
stdenv.mkDerivation rec {
  pname = "cuda-simple-demo";
  version = "0.1.0";
  src = ./.;

  buildInputs = [ makeWrapper cudatoolkit_9_2 ];

  # tried wrapping the produced executable as seen somewhere on the net, but it didn't help either
  # installPhase = ''
  #   runHook preInstall
  #   mkdir -p $out/bin;
  #   cp add $out/bin/;
  #   wrapProgram $out/bin/add --prefix LD_LIBRARY_PATH ":" "${linuxPackages.nvidia_x11}/lib"
  # '';
}

default.nix

{ pkgs ? import <nixpkgs> {}}:
pkgs.callPackage ./add.nix {}

The issue

When building the above with nix-build, it builds fine but the result crashes with a segfault. When i instead simply invoke make to build the executable (which then uses CUDA from system’s packages), it runs as expected.

My setup

I have CUDA 10.1 installed locally from the Nvidia’s CUDA repos. I’m using Nvidia driver version 418.67 which, judging from this table (from docs.nvidia.com),

CUDA Toolkit	Linux x86_64 Driver Version	Windows x86_64 Driver Version
CUDA 10.1 (10.1.105 general release, and updates)	>= 418.39	>= 418.96
CUDA 10.0.130	>= 410.48	>= 411.31
CUDA 9.2 (9.2.148 Update 1)	>= 396.37	>= 398.26
CUDA 9.2 (9.2.88)	>= 396.26	>= 397.44
CUDA 9.1 (9.1.85)	>= 390.46	>= 391.29
CUDA 9.0 (9.0.76)	>= 384.81	>= 385.54
CUDA 8.0 (8.0.61 GA2)	>= 375.26	>= 376.51
CUDA 8.0 (8.0.44)	>= 367.48	>= 369.30
CUDA 7.5 (7.5.16)	>= 352.31	>= 353.66
CUDA 7.0 (7.0.28)	>= 346.46	>= 347.62
should be compatible with `cudatoolkit_9_2` that i’m using from nixpkgs.

I’ve checked that nvidia_uvm, nvidia_drm, nvidia_modeset and nvidia kernel modules are loaded with lsmod. I’m using the nixpkgs-unstable channel, pkgs.lib.version returns "20.03pre193309.c4196cca9ac".

What am i doing wrong here? Are there any other things one must take care of to package CLI software built with CUDA in nixpkgs?

knedlsepp · October 22, 2019, 6:33am

I’m interested if building with cudatoolkit 10.1 from nixpkgs would work on your setup. Have you tried that?

Could you provide some more information on the generated binary? What’s the output of running:

readelf -d ./result/bin/add
ldd ./result/bin/add
strace -f ./result/bin/add

Do you have LD_LIBRARY_PATH set?

rihardsk · October 22, 2019, 2:13pm

Here are the outputs:

readelf -d ./result/bin/add


Dynamic section at offset 0x6b8d0 contains 34 entries:
  Tag        Type                         Name/Value
 0x0000000000000001 (NEEDED)             Shared library: [librt.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libpthread.so.0]
 0x0000000000000001 (NEEDED)             Shared library: [libdl.so.2]
 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
 0x000000000000001d (RUNPATH)            Library runpath: [/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib:/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib]
 0x000000000000000c (INIT)               0x403000
 0x000000000000000d (FINI)               0x456f40
 0x0000000000000019 (INIT_ARRAY)         0x46a5d8
 0x000000000000001b (INIT_ARRAYSZ)       160 (bytes)
 0x000000000000001a (FINI_ARRAY)         0x46a678
 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
 0x0000000000000004 (HASH)               0x400318
 0x000000006ffffef5 (GNU_HASH)           0x400788
 0x0000000000000005 (STRTAB)             0x4015b8
 0x0000000000000006 (SYMTAB)             0x4007a8
 0x000000000000000a (STRSZ)              2254 (bytes)
 0x000000000000000b (SYMENT)             24 (bytes)
 0x0000000000000015 (DEBUG)              0x0
 0x0000000000000003 (PLTGOT)             0x46cb40
 0x0000000000000002 (PLTRELSZ)           3480 (bytes)
 0x0000000000000014 (PLTREL)             RELA
 0x0000000000000017 (JMPREL)             0x402148
 0x0000000000000007 (RELA)               0x4020e8
 0x0000000000000008 (RELASZ)             96 (bytes)
 0x0000000000000009 (RELAENT)            24 (bytes)
 0x000000000000001e (FLAGS)              BIND_NOW
 0x000000006ffffffb (FLAGS_1)            Flags: NOW
 0x000000006ffffffe (VERNEED)            0x401fb8
 0x000000006fffffff (VERNEEDNUM)         6
 0x000000006ffffff0 (VERSYM)             0x401e86
 0x0000000000000000 (NULL)               0x0

ldd ./result/bin/add

	linux-vdso.so.1 (0x00007fff1054e000)
	librt.so.1 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/librt.so.1 (0x00007fc1713af000)
	libpthread.so.0 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libpthread.so.0 (0x00007fc17138e000)
	libdl.so.2 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libdl.so.2 (0x00007fc171389000)
	libstdc++.so.6 => /nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/libstdc++.so.6 (0x00007fc1711ff000)
	libm.so.6 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libm.so.6 (0x00007fc170ffe000)
	libgcc_s.so.1 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libgcc_s.so.1 (0x00007fc170de8000)
	libc.so.6 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libc.so.6 (0x00007fc170c32000)
	/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/ld-linux-x86-64.so.2 => /lib64/ld-linux-x86-64.so.2 (0x00007fc171194000)

strace -f ./result/bin/add

execve("./result/bin/add", ["./result/bin/add"], 0x7ffd31d41b58 /* 27 vars */) = 0
brk(NULL)                               = 0x23b5000
access("/etc/ld-nix.so.preload", R_OK)  = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/x86_64", 0x7fff7f610bb0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=45848, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f976e399000
mmap(NULL, 39424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976e38f000
mmap(0x7f976e391000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f976e391000
mmap(0x7f976e395000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f976e395000
mmap(0x7f976e397000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f976e397000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300k\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=146424, ...}) = 0
mmap(NULL, 131664, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976e36e000
mmap(0x7f976e374000, 61440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f976e374000
mmap(0x7f976e383000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f976e383000
mmap(0x7f976e389000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f976e389000
mmap(0x7f976e38b000, 12880, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f976e38b000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=18520, ...}) = 0
mmap(NULL, 16528, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976e369000
mmap(0x7f976e36a000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f976e36a000
mmap(0x7f976e36b000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f976e36b000
mmap(0x7f976e36c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f976e36c000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/x86_64", 0x7fff7f610b20) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240D\t\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=1595864, ...}) = 0
mmap(NULL, 1611296, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976e1df000
mmap(0x7f976e270000, 708608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x91000) = 0x7f976e270000
mmap(0x7f976e31d000, 245760, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13e000) = 0x7f976e31d000
mmap(0x7f976e359000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x179000) = 0x7f976e359000
mmap(0x7f976e366000, 9760, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f976e366000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\302\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=1768144, ...}) = 0
mmap(NULL, 1658904, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976e049000
mmap(0x7f976e055000, 733184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x7f976e055000
mmap(0x7f976e108000, 872448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xbf000) = 0x7f976e108000
mmap(0x7f976e1dd000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x193000) = 0x7f976e1dd000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20+\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0444, st_size=100608, ...}) = 0
mmap(NULL, 2185600, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976de33000
mprotect(0x7f976de49000, 2093056, PROT_NONE) = 0
mmap(0x7f976e048000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f976e048000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260,\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=2015288, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f976de31000
mmap(NULL, 1792416, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f976dc7b000
mmap(0x7f976dc9d000, 1298432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f976dc9d000
mmap(0x7f976ddda000, 315392, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15f000) = 0x7f976ddda000
mmap(0x7f976de27000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ab000) = 0x7f976de27000
mmap(0x7f976de2d000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f976de2d000
close(3)                                = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f976dc78000
arch_prctl(ARCH_SET_FS, 0x7f976dc78740) = 0
mprotect(0x7f976de27000, 16384, PROT_READ) = 0
mprotect(0x7f976e1dd000, 4096, PROT_READ) = 0
mprotect(0x7f976e359000, 49152, PROT_READ) = 0
mprotect(0x7f976e36c000, 4096, PROT_READ) = 0
mprotect(0x7f976e389000, 4096, PROT_READ) = 0
mprotect(0x7f976e397000, 4096, PROT_READ) = 0
mprotect(0x46a000, 12288, PROT_READ)    = 0
mprotect(0x7f976e3c1000, 4096, PROT_READ) = 0
set_tid_address(0x7f976dc78a10)         = 20644
set_robust_list(0x7f976dc78a20, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f976e374640, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f976e380860}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f976e3746d0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f976e380860}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL)                               = 0x23b5000
brk(0x23d6000)                          = 0x23d6000
futex(0x7fff7f61142c, FUTEX_UNLOCK_PI)  = -1 EPERM (Operation not permitted)
futex(0x46d7f8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f976e36667c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f976e366688, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f976e36d048, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libcuda.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/libcuda.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libcuda.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
futex(0x46df40, FUTEX_WAKE_PRIVATE, 2147483647) = 0
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x7f611638} ---
+++ killed by SIGSEGV (core dumped) +++

LD_LIBRARY_PATH isn’t set.

Regarding cudatoolkit 10.1 – I would have liked to check that as well but cudatoolkit_10_0 is the latest one available. I tried to package 10.1 myself a while back but Nvidia has changed the internal structure of their package and i couldn’t really reuse the expressions of earlier cudatoolkit versions from nixpkgs. I guess, had it been that easy, somebody else would have already packaged 10.1 by now.

rihardsk · October 22, 2019, 3:22pm

Seeing the last lines of the strace output where libcuda.so.1 couldn’t be found, i decided to search for my system’s libcuda.so.1 and add it’s path to LD_LIBRARY_PATH. It worked. Running it like so:

LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/ ./result/bin/add

gives no errors. Yay!

But that raises a couple of questions – why does it need libcuda at all, seeing that it isn’t listed in the readelf or ldd outputs?

Also, why didn’t the commented out wrapper lines help in add.nix in my original post? The equivalent of those lines is setting LD_LIBRARY_PATH like so:

LD_LIBRARY_PATH=/nix/store/xs9qffpzr3zz0d0zxsv0zk913gry1sc8-nvidia-x11-435.21-4.19.72/lib ./result/bin/add

which crashes. Judging from the strace, it does indeed find libcuda.so.1, but then it crashes further down the line. I’ve uploaded the outputs here. I guess there might be a version incompatibility with libcuda.so.1 from nixpkgs and the nvidia* modules running on my system?

And finally, what is the proper way of calling ./result/bin/add then? Adding the whole /usr/lib/x86_64-linux-gnu/ to LD_LIBRARY_PATH seems wrong.

dguibert · October 22, 2019, 3:49pm

Hi,

cudatoolkit_10_1 is available here: cudatoolkit: add 10.1.243 by dguibert · Pull Request #68954 · NixOS/nixpkgs · GitHub
not merged yet

acowley · October 22, 2019, 4:38pm

I’m not sure if this is relevant, but just the other day I was sorting out a problem with an impure nix-driven build on Ubuntu that involved CUDA. The solution I ended up with is using nixGL to launch my executable. Prior to that, I wasn’t seeing a segfault, but CUDA was returning error codes due to a driver/runtime mismatch.

rihardsk · October 23, 2019, 1:42pm

I’ve seen nixGL mentioned previously but haven’t tried it yet because i wasn’t sure what it does exactly and whether it’s applicable to my CUDA setup. As far as i can tell, it wraps a program to give it access to some graphics related libs of the host system and that might include CUDA so it’s worth a try.

Though, my first try setting it up wasn’t successful because it couldn’t download the specific Nvidia driver version I have – ./nvidiaInstall.py 418.67 nixGLNvidi fails for me.

I’ll have a look at it again next week when i have more time.

knedlsepp · November 1, 2019, 12:24pm

I’m not an expert on the topic, but I’ve got the following to say.

It seems that your expression produces a binary that loads libcuda.so.1 “manually” by calling dlopen instead of letting the dynamic linker ld.so do the work on program startup.
My guess is that behaviour is not specific to NixOS, but to nvcc in general.

As to how to solve this issue, there’s a few ways on how to do this:

LD_LIBRARY_PATH
LD_PRELOAD
baking in the library search path using RUNPATH.
nixGL

There is some upsides to using RUNPATH, but as the driver is tightly coupled to the underlying kernel we canot directly make it link to the nixpkgs provided nvidia-x11. We should rather be pointing our binary to the systems libcuda.so.1. So we could encode an rpath that points to /usr/lib/…, but that would have the risk that also other libraries would be taken from that location. If you’re doing this you’re gonna have a bad time.

The most promising solution that I’ve seen is to encode the location /run/opengl-driver into the nix-built binaries and manually symlink all libcuda.so libraries from your host system there.

There is a hook in nixpkgs that should automatically do this for you. See: libglvnd, ocl-icd, vulkan-loader: Add /run/opengl-driver(-32) to RUNPATH. by ambrop72 · Pull Request #60985 · NixOS/nixpkgs · GitHub

There’s probably still quite some stuff that could go wrong here, but this looks like the most portable solution to me and only requires the machine you’re running your nix-built binaries on to create these symlinks once.

{ stdenv, lib, cudatoolkit_10, addOpenGLRunpath }:
stdenv.mkDerivation rec {
  pname = "cuda-simple-demo";
  version = "0.1.0";
  src = ./.;

  nativeBuildInputs = [ cudatoolkit_10 addOpenGLRunpath ];
  postFixup = ''
    addOpenGLRunpath $out/bin/add
  '';
}

(Minor: adding nvcc add.cu -lcuda -o add will produce readelf -d $out/bin/add: (NEEDED) Shared library: [libcuda.so.1]. Dunno if that’s needed. But it could give better error messages than just segfaulting)

rihardsk · November 5, 2019, 1:31pm

Thank you for the detailed explanation! The RUNPATH method seems close to ideal.

I symlinked libcuda.so.1 to /run/opengl-driver/lib and added the addOpenGLRunpath hook to the postFixup phase. After that, strace showed that libcuda.so.1 is successfully located in /run/opengl-driver/lib but another issue had emerged – now libnvidia-fatbinaryloader.so.418.67 was missing so i symlinked it to. It didn’t help though – libnvidia-fatbinaryloader isn’t searched for in /run/opengl-driver/lib.

Here’s the strace -f:

execve("./result/bin/add", ["./result/bin/add"], 0x7fffe86e0138 /* 27 vars */) = 0
brk(NULL)                               = 0x10d7000
access("/etc/ld-nix.so.preload", R_OK)  = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/haswell/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/haswell/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/haswell/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/haswell", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/tls/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/tls", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/haswell/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/haswell/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/haswell/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/haswell", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/opengl-driver/lib/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/run/opengl-driver/lib", {st_mode=S_IFDIR|0755, st_size=80, ...}) = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/haswell", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/tls", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/haswell", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/avx512_1", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/x86_64/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/x86_64", 0x7fff8bc3be00) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=45848, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f41b7015000
mmap(NULL, 39424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b700b000
mmap(0x7f41b700d000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f41b700d000
mmap(0x7f41b7011000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f41b7011000
mmap(0x7f41b7013000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f41b7013000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libpthread.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300k\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=146424, ...}) = 0
mmap(NULL, 131664, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b6fea000
mmap(0x7f41b6ff0000, 61440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f41b6ff0000
mmap(0x7f41b6fff000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f41b6fff000
mmap(0x7f41b7005000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f41b7005000
mmap(0x7f41b7007000, 12880, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f41b7007000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libdl.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=18520, ...}) = 0
mmap(NULL, 16528, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b6fe5000
mmap(0x7f41b6fe6000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f41b6fe6000
mmap(0x7f41b6fe7000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f41b6fe7000
mmap(0x7f41b6fe8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f41b6fe8000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/avx512_1", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/haswell", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/avx512_1", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/tls", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/avx512_1", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/haswell", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/avx512_1", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/x86_64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/x86_64", 0x7fff8bc3bd70) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240D\t\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=1595864, ...}) = 0
mmap(NULL, 1611296, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b6e5b000
mmap(0x7f41b6eec000, 708608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x91000) = 0x7f41b6eec000
mmap(0x7f41b6f99000, 245760, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13e000) = 0x7f41b6f99000
mmap(0x7f41b6fd5000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x179000) = 0x7f41b6fd5000
mmap(0x7f41b6fe2000, 9760, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f41b6fe2000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\302\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=1768144, ...}) = 0
mmap(NULL, 1658904, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b6cc5000
mmap(0x7f41b6cd1000, 733184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x7f41b6cd1000
mmap(0x7f41b6d84000, 872448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xbf000) = 0x7f41b6d84000
mmap(0x7f41b6e59000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x193000) = 0x7f41b6e59000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20+\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0444, st_size=100608, ...}) = 0
mmap(NULL, 2185600, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b6aaf000
mprotect(0x7f41b6ac5000, 2093056, PROT_NONE) = 0
mmap(0x7f41b6cc4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f41b6cc4000
close(3)                                = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260,\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0555, st_size=2015288, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f41b6aad000
mmap(NULL, 1792416, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b68f7000
mmap(0x7f41b6919000, 1298432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f41b6919000
mmap(0x7f41b6a56000, 315392, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15f000) = 0x7f41b6a56000
mmap(0x7f41b6aa3000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ab000) = 0x7f41b6aa3000
mmap(0x7f41b6aa9000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f41b6aa9000
close(3)                                = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f41b68f4000
arch_prctl(ARCH_SET_FS, 0x7f41b68f4740) = 0
mprotect(0x7f41b6aa3000, 16384, PROT_READ) = 0
mprotect(0x7f41b6e59000, 4096, PROT_READ) = 0
mprotect(0x7f41b6fd5000, 49152, PROT_READ) = 0
mprotect(0x7f41b6fe8000, 4096, PROT_READ) = 0
mprotect(0x7f41b7005000, 4096, PROT_READ) = 0
mprotect(0x7f41b7013000, 4096, PROT_READ) = 0
mprotect(0x47e000, 12288, PROT_READ)    = 0
mprotect(0x7f41b703d000, 4096, PROT_READ) = 0
set_tid_address(0x7f41b68f4a10)         = 23254
set_robust_list(0x7f41b68f4a20, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f41b6ff0640, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f41b6ffc860}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f41b6ff06d0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f41b6ffc860}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL)                               = 0x10d7000
brk(0x10f8000)                          = 0x10f8000
futex(0x7fff8bc3c67c, FUTEX_UNLOCK_PI)  = -1 EPERM (Operation not permitted)
futex(0x481990, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f41b6fe267c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f41b6fe2688, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f41b6fe9048, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/run/opengl-driver/lib/libcuda.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\240\f\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=16142096, ...}) = 0
mmap(NULL, 18303816, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f41b577f000
mprotect(0x7f41b656b000, 2093056, PROT_NONE) = 0
mmap(0x7f41b676a000, 1548288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xdeb000) = 0x7f41b676a000
mmap(0x7f41b68e4000, 64328, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f41b68e4000
close(3)                                = 0
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libnvidia-fatbinaryloader.so.418.67", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
munmap(0x7f41b577f000, 18303816)        = 0
futex(0x482270, FUTEX_WAKE_PRIVATE, 2147483647) = 0
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x8bc3c888} ---
+++ killed by SIGSEGV (core dumped) +++

Debuging further

To debug further, i ran it with LD_LIBRARY_PATH set to /run/opengl-driver/lib:

LD_LIBRARY_PATH=/run/opengl-driver/lib/ ./result/bin/add

It worked – no crash. So we see that for some reason fatbinaryloader.so.418.67 isn’t searched for in RUNPATH but it is searched for in LD_LIBRARY_PATH (i checked strace, it’s found in /run/opengl-driver/lib/ as expected). Any idea why it’s acting like that?

Here’s readelf -d ./result/bin/add output for good measure:

Dynamic section at offset 0x80890 contains 34 entries:
  Tag        Type                         Name/Value
 0x0000000000000001 (NEEDED)             Shared library: [librt.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libpthread.so.0]
 0x0000000000000001 (NEEDED)             Shared library: [libdl.so.2]
 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
 0x000000000000001d (RUNPATH)            Library runpath: [/run/opengl-driver/lib:/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib:/nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib]
 0x000000000000000c (INIT)               0x404000
 0x000000000000000d (FINI)               0x466b00
 0x0000000000000019 (INIT_ARRAY)         0x47e018
 0x000000000000001b (INIT_ARRAYSZ)       160 (bytes)
 0x000000000000001a (FINI_ARRAY)         0x47e0b8
 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
 0x0000000000000004 (HASH)               0x400b40
 0x000000006ffffef5 (GNU_HASH)           0x400b20
 0x0000000000000005 (STRTAB)             0x3ff2e0
 0x0000000000000006 (SYMTAB)             0x3ffc80
 0x000000000000000a (STRSZ)              2461 (bytes)
 0x000000000000000b (SYMENT)             24 (bytes)
 0x0000000000000015 (DEBUG)              0x0
 0x0000000000000003 (PLTGOT)             0x480b00
 0x0000000000000002 (PLTRELSZ)           3624 (bytes)
 0x0000000000000014 (PLTREL)             RELA
 0x0000000000000017 (JMPREL)             0x402238
 0x0000000000000007 (RELA)               0x4021d8
 0x0000000000000008 (RELASZ)             96 (bytes)
 0x0000000000000009 (RELAENT)            24 (bytes)
 0x000000000000001e (FLAGS)              BIND_NOW
 0x000000006ffffffb (FLAGS_1)            Flags: NOW
 0x000000006ffffffe (VERNEED)            0x4020a8
 0x000000006fffffff (VERNEEDNUM)         6
 0x000000006ffffff0 (VERSYM)             0x401f6e
 0x0000000000000000 (NULL)               0x0

(ldd output stays essentially the same.)

Passing -l flags to nvcc

Next, i tried changing the nvcc call to nvcc add.cu -lcuda -o add. I do indeed get a better error message:

$ ./result/bin/add
./result/bin/add: error while loading shared libraries: libnvidia-fatbinaryloader.so.418.67: cannot open shared object file: No such file or directory

Here are ldd outputs:

        linux-vdso.so.1 (0x00007ffdc7d8a000)
        libcuda.so.1 => /run/opengl-driver/lib/libcuda.so.1 (0x00007f5ed1da6000)
        librt.so.1 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/librt.so.1 (0x00007f5ed3136000)
        libpthread.so.0 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libpthread.so.0 (0x00007f5ed3115000)
        libdl.so.2 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libdl.so.2 (0x00007f5ed3110000)
        libstdc++.so.6 => /nix/store/xg10ap5fki3slzy9lyy8xl6zksy51s8y-gcc-7.4.0-lib/lib/libstdc++.so.6 (0x00007f5ed2f86000)
        libm.so.6 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libm.so.6 (0x00007f5ed1c10000)
        libgcc_s.so.1 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libgcc_s.so.1 (0x00007f5ed19fa000)
        libc.so.6 => /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libc.so.6 (0x00007f5ed1844000)
        libnvidia-fatbinaryloader.so.418.67 => /usr/lib/x86_64-linux-gnu/libnvidia-fatbinaryloader.so.418.67 (0x00007f5ed15f6000)
        /nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/ld-linux-x86-64.so.2 => /lib64/ld-linux-x86-64.so.2 (0x00007f5ed2f1b000)

libnvidia-fatbinaryloader.so.418.67 points to /usr/lib/x86_64-linux-gnu/ and it does exist at that location!

$ stat -t /usr/lib/x86_64-linux-gnu/libnvidia-fatbinaryloader.so.418.67
/usr/lib/x86_64-linux-gnu/libnvidia-fatbinaryloader.so.418.67 298696 584 81a4 0 0 10302 21241717 1 0 0 1572957060 1554536477 1563884453 0 4096

But when checking strace -f i see that nothing’s changed – libnvidia-fatbinaryloader is searched for in the nix store path for glibc, same as before:

openat(AT_FDCWD, "/nix/store/6yaj6n8l925xxfbcd65gzqx3dz7idrnn-glibc-2.27/lib/libnvidia-fatbinaryloader.so.418.67", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)

EDIT:
I noticed that libnvidia-fatbinaryloader is pulled in by libcuda:

$ ldd /run/opengl-driver/lib/libcuda.so.1
        linux-vdso.so.1 (0x00007fffdbbed000)
        libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f73018d8000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f73014e7000)
        libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f73012e3000)
        libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f73010c4000)
        librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f7300ebc000)
        libnvidia-fatbinaryloader.so.418.67 => /usr/lib/x86_64-linux-gnu/libnvidia-fatbinaryloader.so.418.67 (0x00007f7300c6e000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f7302deb000)

Does that explain why RPATH isn’t used when searching for it? Regardless, it still doesn’t explain why it isn’t searched for in /usr/lib/x86_64-linux-gnu/ as it’s suposed to.

knedlsepp · November 5, 2019, 3:35pm

Oh. Seems my suggestion was flawed.

Any idea why it’s acting like that?

I’m pretty sure the libnvidia-fatbinaryloader is searched via means of the rpath encoded (if any) in libcuda.so, which doesn’t know about /run/opengl-driver.

So you could probably fix this by copying said libraries and rewriting the rpath to /run/opengl-driver/lib (via patchelf), but this is an awful hack and upgrading the library on the system level doesn’t update the copies.

It might also work to encode the fatbinary-loader in the “needed” section of your “add” executable, but this might be even worse of a hack.

I’d really like to see a better solution for this, but I don’t know one.

knedlsepp · November 5, 2019, 3:43pm

Maybe some intermediate dispatching library provided by nixpkgs equivalent to libglvnd could solve this issue…

knedlsepp · November 5, 2019, 10:51pm

There is plenty of stuff to learn from reading: libGL not working on non-NixOS (without setting up) · Issue #9415 · NixOS/nixpkgs · GitHub

rihardsk · December 18, 2019, 1:22pm

What i ended up doing was symlinking a bunch of libnvidia-* and libcuda* stuff from /usr/lib/x86_64-linux-gnu/ to /run/opengl-driver/lib/ and creating a wrapper that prepends /run/opengl-driver/lib to LD_LIBRARY_PATH.

{ stdenv, lib, cudatoolkit_10, addOpenGLRunpath, makeWrapper }:
stdenv.mkDerivation rec {
  pname = "cuda-simple-demo";
  version = "0.1.0";
  src = ./.;

  nativeBuildInputs = [ cudatoolkit_10 addOpenGLRunpath makeWrapper ];
  postFixup = ''
    addOpenGLRunpath $out/bin/add  # i guess this isn't needed but whatever
    wrapProgram $out/bin/add --prefix LD_LIBRARY_PATH ":" "/run/opengl-driver/lib"
  '';
}

sudo ln -s /usr/lib/x86_64-linux-gnu/libcuda* /run/opengl-driver/
# i think only libnvidia-fatbinaryloader is needed for the demo program
# but, but when packaging another project, other libnvidia-* stuff was needed as well
sudo ln -s /usr/lib/x86_64-linux-gnu/libnvidia-* /run/opengl-driver/

I was able to package and run the sample program this way and also another real-world CUDA project.

rihardsk · February 6, 2020, 2:42pm

Ah, the above commands for creating symlinks are wrong. It should be

sudo mkdir -p /run/opengl-driver/lib
sudo ln -s /usr/lib/x86_64-linux-gnu/libcuda* /run/opengl-driver/lib/
sudo ln -s /usr/lib/x86_64-linux-gnu/libnvidia-* /run/opengl-driver/lib/

Also, on some distros, e.g., Ubuntu, /run/ is mounted to tmpfs so it might be a good idea to recreate the symlinks on system startup (also because the symlinks might become broken after a gpu driver update). One could do something along the lines of:

# A script that creates symlinks
sudo tee /usr/local/bin/nix-symlink-gpu <<EOF
#! /bin/sh
mkdir -p /run/opengl-driver/lib/
rm /run/opengl-driver/lib/*
ln -sf /usr/lib/x86_64-linux-gnu/libcuda* /run/opengl-driver/lib/
ln -sf /usr/lib/x86_64-linux-gnu/libnvidia-* /run/opengl-driver/lib/
EOF
sudo chmod a+x /usr/local/bin/nix-symlink-gpu

# A systemd unit that runs the script at startup
sudo tee /etc/systemd/system/nix-symlink-gpu.service <<EOF
[Unit]
Description=Symlink GPU drivers for use by software packaged with Nix

[Service]
Type=oneshot
ExecStart=/usr/local/bin/nix-symlink-gpu

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl enable --now nix-symlink-gpu.service