How to package a CGO program that compiles other CGO programs

Help
1

Hi.

I am trying to package hover. It is a program that allows you to build Flutter programs for desktop, and it uses Go for it.

The issue is, this program and the programs that it compiles depends on CGO dependencies (so it needs to link with C libraries, basically X11 and OpenGL). Those libraries are linked dynamically (so you can’t use CGO_ENABLED=0 to create static binaries).

This is the nix file:

{ lib, buildGoModule, buildFHSUserEnv, pkgconfig, fetchFromGitHub,
  stdenv, writeScript, xorg, libglvnd, addOpenGLRunpath, makeWrapper, gcc }:

let
  pname = "hover";
  version = "0.43.0";

  libs = with xorg; [
    libXi
    libXxf86vm
    libglvnd.dev
    libX11.dev
    libXcursor.dev
    libXinerama.dev
    libXrandr.dev
    libXrender.dev
    xorgproto
  ];

in buildGoModule rec {
  inherit pname version;

  meta = with stdenv.lib; {
    description = "A build tool to run Flutter applications on desktop";
    homepage = "https://github.com/go-flutter-desktop/hover";
    license = licenses.bsd3;
    platforms = platforms.linux ++ platforms.darwin;
  };

  subPackages = [ "." ];

  modSha256 = "0qg26bzbdmb0cl2msgg2ycxhdkhiqbriihq9f725w7a6j0mcbz3a";

  src = fetchFromGitHub {
    rev = "v${version}";
    owner = "go-flutter-desktop";
    repo = pname;
    sha256 = "0iw6sxg86wfdbihl2hxzn43ppdzl1p7g5b9wl8ac3xa9ix8759ax";
  };

  nativeBuildInputs = [ addOpenGLRunpath makeWrapper ];

  buildInputs = [ pkgconfig ] ++ libs;

  patches = [
    ./fix-assets-path.patch
  ];

  postPatch = ''
    sed -i 's|@assetsFolder@|'"''${out}/share/assets"'|g' internal/fileutils/assets.go
  '';

  postInstall = ''
    mkdir -p $out/share
    cp -r assets $out/share/assets
    chmod -R a+rx $out/share/assets

    wrapProgram "$out/bin/hover" \
      --prefix LD_LIBRARY_PATH : ${stdenv.lib.makeLibraryPath [
        xorg.libX11
        xorg.libXcursor
        xorg.libXi
        xorg.libXinerama
        xorg.libXrandr
        xorg.libXxf86vm
        xorg.libXext
      ]}
  '';

  postFixup = ''
    addOpenGLRunpath $out/bin/hover
  '';
}

The hover comand itself works fine, but when you try to use hover run or hover build linux, you get:

$ hover build linux                                                                                                            nix-shell
hover: Cleaning the build directory
hover: Missing config: file hover.yaml not found: open go/hover.yaml: no such file or directory
hover: ⚠ The go-flutter project tries to stay compatible with the beta channel of Flutter.
hover: ⚠     It's advised to use the beta channel: `flutter channel beta`
hover: Building flutter bundleFont subetting is not supported in debug mode. The --tree-shake-icons flag will be ignored.
hover: Using engine from cache
hover: Checking available release on Github
hover: Compiling 'go-flutter' and plugins
go: downloading github.com/pkg/errors v0.9.1
go: downloading github.com/go-flutter-desktop/plugins/shared_preferences v0.4.0
go: downloading github.com/go-flutter-desktop/go-flutter v0.41.2
go: downloading github.com/lucasmafra/go-flutter-share v0.0.0-20200213183323-6ca7e6570bca
go: downloading github.com/syndtr/goleveldb v1.0.0
go: downloading github.com/mitchellh/go-homedir v1.1.0
go: downloading golang.org/x/text v0.3.3
go: downloading github.com/go-gl/glfw v0.0.0-20191125211704-12ad95a8df72
go: downloading github.com/go-gl/gl v0.0.0-20190320180904-bf2b1f2f34d7
go: downloading github.com/Xuanwo/go-locale v0.3.0
go: downloading github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
go: downloading github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200707082815-5321531c36a2
# github.com/app/app/desktop/cmd
/nix/store/flb7pvdk13rm0aqz9ak5naxp4hwbvwsj-go-1.14.4/share/go/pkg/tool/linux_amd64/link: running gcc failed: exit status 1
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lGL
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lX11
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lXrandr
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lXxf86vm
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lXi
/nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/ld: cannot find -lXcursor
collect2: error: ld returned 1 exit status

However, this works:

$ nix-shell -p libGL pkgconfig glfw xorg.libX11 xorg.libXrandr xorg.libXinerama xorg.libXxf86vm xorg.libXi xorg.libXcursor xorg.libXext  --run "hover run"
nix-shell -p libGL pkgconfig glfw xorg.libX11 xorg.libXrandr xorg.libXinerama xorg.libXxf86vm xorg.libXi xorg.libXcursor xorg.libXext  --run "hover run"
hover: Missing config: file hover.yaml not found: open go/hover.yaml: no such file or directory
hover: ⚠ The go-flutter project tries to stay compatible with the beta channel of Flutter.
hover: ⚠     It's advised to use the beta channel: `flutter channel beta`
hover: Building flutter bundle
Font subetting is not supported in debug mode. The --tree-shake-icons flag will be ignored.
hover: Using engine from cache
hover: Compiling 'go-flutter' and plugins
hover: Successfully compiled executable binary for linux
hover: Build finished, starting app...
hover: Running app in debug mode

So I think the issue is that GCC or something else inside hover needs to have access to X11/OpenGL libraries, but I can’t figure out how to do it.

2 Likes
2

Changing:

buildInputs = [ pkgconfig ] ++ libs;

To:

buildInputs = [ pkgconfig gcc ] ++ libs;

Seems to do the trick. Now it is working as it should :+1:.

3

Sorry, not working actually. Testing inside a nix-shell --pure and got this:


[nix-shell:~/dev/nu/mini-meta-repo]$ hover run
hover: Missing config: file hover.yaml not found: open go/hover.yaml: no such file or directory
hover: ⚠ The go-flutter project tries to stay compatible with the beta channel of Flutter.
hover: ⚠     It's advised to use the beta channel: `flutter channel beta`
hover: Building flutter bundle
Font subetting is not supported in debug mode. The --tree-shake-icons flag will be ignored.
hover: Using engine from cache
hover: Compiling 'go-flutter' and plugins
github.com/go-gl/glfw/v3.3/glfw
github.com/go-gl/gl/v3.3-core/gl
# pkg-config --cflags  -- gl gl
pkg-config: exec: "pkg-config": executable file not found in $PATH
# github.com/go-gl/glfw/v3.3/glfw
In file included from ./glfw/src/internal.h:188,
                 from ./glfw/src/context.c:30,
                 from ../../../../go/pkg/mod/github.com/go-gl/glfw/v3.3/glfw@v0.0.0-20200707082815-5321531c36a2/c_glfw.go:4:
./glfw/src/x11_platform.h:33:10: fatal error: X11/Xlib.h: No such file or directory
   33 | #include <X11/Xlib.h>
      |          ^~~~~~~~~~~~
compilation terminated.
hover: Go build failed: exit status 2
4

With pkgs.go_1_13, this works:

{ lib, buildGoModule, buildFHSUserEnv, pkgconfig, fetchFromGitHub,
  stdenv, writeScript, xorg, libglvnd, addOpenGLRunpath, makeWrapper,
  gcc, go, flutter }:

let
  pname = "hover";
  version = "0.43.0";

  libs = with xorg; [
    libXi
    libXxf86vm
    libglvnd.dev
    libX11.dev
    libXcursor.dev
    libXinerama.dev
    libXrandr.dev
    libXrender.dev
    xorgproto
  ];
in buildGoModule rec {
  inherit pname version;

  meta = with stdenv.lib; {
    description = "A build tool to run Flutter applications on desktop";
    homepage = "https://github.com/go-flutter-desktop/hover";
    license = licenses.bsd3;
    platforms = platforms.linux ++ platforms.darwin;
  };

  subPackages = [ "." ];

  vendorSha256 = "1wr08phjm87dxim47i8449rmq5wfscvjyz65g3lxmv468x209pam";

  src = fetchFromGitHub {
    rev = "v${version}";
    owner = "go-flutter-desktop";
    repo = pname;
    sha256 = "0iw6sxg86wfdbihl2hxzn43ppdzl1p7g5b9wl8ac3xa9ix8759ax";
  };

  nativeBuildInputs = [ addOpenGLRunpath makeWrapper ];

  propagatedBuildInputs = [
    flutter
    gcc
    go
    pkgconfig
  ] ++ libs;

  checkRun = false;

  patches = [
    ./fix-assets-path.patch
  ];

  postPatch = ''
    sed -i 's|@assetsFolder@|'"''${out}/share/assets"'|g' internal/fileutils/assets.go
  '';

  postInstall = ''
    mkdir -p $out/share
    cp -r assets $out/share/assets
    chmod -R a+rx $out/share/assets

    wrapProgram "$out/bin/hover" \
      --prefix LD_LIBRARY_PATH : ${stdenv.lib.makeLibraryPath [
        xorg.libX11
        xorg.libXcursor
        xorg.libXi
        xorg.libXinerama
        xorg.libXrandr
        xorg.libXxf86vm
        xorg.libXext
      ]}
  '';

  postFixup = ''
    addOpenGLRunpath $out/bin/hover
  '';
}

However with pkgs.go_1_14, it does not:

$ nix-shell --pure           
these derivations will be built:
  /nix/store/sz5lgjkbhqikb6v3z30y09k79xpm71p8-hover-0.43.0.drv
building '/nix/store/sz5lgjkbhqikb6v3z30y09k79xpm71p8-hover-0.43.0.drv'...
unpacking sources
unpacking source archive /nix/store/mz4aq6nyg4466h9bxw709xr1icm7r0zl-source
source root is source
patching sources
applying patch /nix/store/z53l0rdj37qcryvrpavcaxgf4z9wax2y-fix-assets-path.patch
patching file internal/fileutils/assets.go
patching file internal/fileutils/file.go
configuring
building
Building subPackage ./.
github.com/Kodeworks/golang-image-ico
github.com/logrusorgru/aurora
github.com/go-flutter-desktop/hover/internal/log
github.com/go-flutter-desktop/hover/internal/androidmanifest
github.com/go-flutter-desktop/hover/internal/build
github.com/pkg/errors
gopkg.in/yaml.v2
github.com/go-flutter-desktop/hover/internal/config
github.com/GeertJohan/go.rice/embedded
runtime/cgo
net
github.com/go-flutter-desktop/hover/internal/fileutils
os/user
github.com/go-flutter-desktop/hover/internal/pubspec
github.com/otiai10/copy
github.com/go-flutter-desktop/hover/cmd/packaging
github.com/go-flutter-desktop/hover/internal/flutterversion
github.com/go-flutter-desktop/hover/internal/enginecache
github.com/go-flutter-desktop/hover/internal/logstreamer
golang.org/x/mod/internal/lazyregexp
golang.org/x/mod/semver
golang.org/x/xerrors/internal
golang.org/x/xerrors
golang.org/x/mod/module
golang.org/x/mod/modfile
github.com/go-flutter-desktop/hover/internal/modx
github.com/google/go-querystring/query
github.com/google/go-github/github
github.com/hashicorp/go-version
golang.org/x/net/html/atom
golang.org/x/net/html
github.com/tcnksm/go-latest
github.com/go-flutter-desktop/hover/internal/versioncheck
github.com/spf13/pflag
github.com/spf13/cobra
github.com/go-flutter-desktop/hover/cmd
github.com/go-flutter-desktop/hover
installing
post-installation fixup
shrinking RPATHs of ELF executables and libraries in /nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0
shrinking /nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0/bin/.hover-wrapped
strip is /nix/store/hrkc2sf2883l16d5yq3zg0y339kfw4xv-binutils-2.31.1/bin/strip
stripping (with command strip and flags -S) in /nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0/bin
patching script interpreter paths in /nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0
/nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0/share/assets/packaging/linux-appimage/AppRun.tmpl: interpreter directive changed from "/bin/sh" to "/nix/store/hrpvwkjz04s9i4nmli843hyw9z4pwhww-bash-4.4-p23/bin/sh"
/nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0/share/assets/packaging/linux/bin.tmpl: interpreter directive changed from "/bin/sh" to "/nix/store/hrpvwkjz04s9i4nmli843hyw9z4pwhww-bash-4.4-p23/bin/sh"
checking for references to /build/ in /nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0...
output '/nix/store/5wa6jzrfxcm7q78qrv4w8h72kb85a9rk-hover-0.43.0' is not allowed to refer to the following paths:
  /nix/store/flb7pvdk13rm0aqz9ak5naxp4hwbvwsj-go-1.14.4
error: build of '/nix/store/sz5lgjkbhqikb6v3z30y09k79xpm71p8-hover-0.43.0.drv' failed

Is there some reason why?

5

I doubt my nix skills would help, but maybe adding autoPatchElf to the imports:

{ lib, buildGoModule, buildFHSUserEnv, pkgconfig, fetchFromGitHub,
  stdenv, writeScript, xorg, libglvnd, addOpenGLRunpath, makeWrapper,
  gcc, go, flutter }:

and then:

  nativeBuildInputs = [ addOpenGLRunpath autoPatchelfHook makeWrapper ];
6

Sadly it didn’t make any difference for the go issue. I think for some reason pkgs.go_1_14 is broken, because this is the PATH that I got when I exported it in some other test: /nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-go-1.14.4/bin

But the major issue I have for now is that I can make it work in nix-shell --pure, but using it in environment.systemPackages as a overlay does not.

This is the current version:

{ lib, buildGoModule, buildFHSUserEnv, pkgconfig, fetchFromGitHub,
  stdenv, writeScript, xorg, libglvnd, addOpenGLRunpath, makeWrapper,
  gcc, go, flutter }:

let
  pname = "hover";
  version = "0.43.0";

  libs = with xorg; [
    libXi
    libXxf86vm
    libglvnd.dev
    libX11.dev
    libXcursor.dev
    libXinerama.dev
    libXrandr.dev
    libXrender.dev
    xorgproto
  ];
in buildGoModule rec {
  inherit pname version;

  meta = with stdenv.lib; {
    description = "A build tool to run Flutter applications on desktop";
    homepage = "https://github.com/go-flutter-desktop/hover";
    license = licenses.bsd3;
    platforms = platforms.linux ++ platforms.darwin;
  };

  subPackages = [ "." ];

  vendorSha256 = "1wr08phjm87dxim47i8449rmq5wfscvjyz65g3lxmv468x209pam";

  src = fetchFromGitHub {
    rev = "v${version}";
    owner = "go-flutter-desktop";
    repo = pname;
    sha256 = "0iw6sxg86wfdbihl2hxzn43ppdzl1p7g5b9wl8ac3xa9ix8759ax";
  };

  nativeBuildInputs = [ addOpenGLRunpath makeWrapper ];

  propagatedBuildInputs = [
    flutter
    gcc
    go
    pkgconfig
  ] ++ libs;

  checkRun = false;

  patches = [
    ./fix-assets-path.patch
  ];

  postPatch = ''
    sed -i 's|@assetsFolder@|'"''${out}/share/assets"'|g' internal/fileutils/assets.go
  '';

  postInstall = ''
    mkdir -p $out/share
    cp -r assets $out/share/assets
    chmod -R a+rx $out/share/assets

    wrapProgram "$out/bin/hover" \
      --prefix CPATH : ${xorg.libX11.dev}/include:${xorg.libXcursor.dev}/include:${xorg.libXi.dev}/include:${xorg.libXinerama.dev}/include:${xorg.libXrandr.dev}/include:${xorg.libXxf86vm.dev}/include:${xorg.libXext.dev}/include \
      --prefix LD_LIBRARY_PATH : ${stdenv.lib.makeLibraryPath [
        xorg.libX11
        xorg.libXcursor
        xorg.libXi
        xorg.libXinerama
        xorg.libXrandr
        xorg.libXxf86vm
        xorg.libXext
      ]}
  '';

  postFixup = ''
    addOpenGLRunpath $out/bin/hover
  '';
}

The difference is that I needed to wrap CPATH, or I would get a ./glfw/src/x11_platform.h:33:10: fatal error: X11/Xlib.h: No such file or directory. This works well in nix-shell --pure, but trying it in a overlay results in the same error.

This is the overlay code:

self: super:
let
  pkgs = super.pkgs;
  callPackage = super.lib.callPackageWith super;
in {
  nubank = rec {
    dart = callPackage ./pkgs/dart {};
    flutter = callPackage ./pkgs/flutter {};
    hover = callPackage ./pkgs/hover {
      inherit (super.xorg);
      # TODO: go_1_14 seems broken in 20.03
      go = pkgs.go_1_13;
      flutter = flutter;
    };
  };
}

In this repo (the idea is to upstream it once we have it working): GitHub - nubank/nixpkgs: Nix Packages collection used in Nubank

1 Like
7

Maybe this will help, this is what hover ends doing when we call hover build linux:

So it ends up calling the go binary in the system, that AFAIK ends up calling gcc to compile the C code in one of the dependencies of hover. In the end, hover is basically a wrapper for go command to pass some special instructions to build a Go program.

This is why that I think wrapping hover executable with CPATH and LD_LIBRARY_PATH works, because when hover calls go (that eventually calls gcc) everything will be available in the environment variables. But I don’t have a clue why outside a nix-shell --pure this doesn’t work anymore.

BTW, calling nix-shell -p libGL pkgconfig glfw xorg.libX11 xorg.libXrandr xorg.libXinerama xorg.libXxf86vm xorg.libXi xorg.libXcursor xorg.libXext --run "hover run" works, so it does seem that something is missing outside a nix-shell.

8

This finally seemed to do the trick:

{ lib
, buildGoModule
, buildFHSUserEnv
, pkgconfig
, fetchFromGitHub
, stdenv
, writeScript
, xorg
, libglvnd
, addOpenGLRunpath
, makeWrapper
, gcc
, go
, flutter
}:

let
  pname = "hover";
  version = "0.43.0";

  libs = with xorg; [
    libX11.dev
    libXcursor.dev
    libXext.dev
    libXi.dev
    libXinerama.dev
    libXrandr.dev
    libXrender.dev
    libXfixes.dev
    libXxf86vm
    libglvnd.dev
    xorgproto
  ];
  hover = buildGoModule rec {
    inherit pname version;

    meta = with stdenv.lib; {
      description = "A build tool to run Flutter applications on desktop";
      homepage = "https://github.com/go-flutter-desktop/hover";
      license = licenses.bsd3;
      platforms = platforms.linux ++ platforms.darwin;
    };

    subPackages = [ "." ];

    vendorSha256 = "1wr08phjm87dxim47i8449rmq5wfscvjyz65g3lxmv468x209pam";

    src = fetchFromGitHub {
      rev = "v${version}";
      owner = "go-flutter-desktop";
      repo = pname;
      sha256 = "0iw6sxg86wfdbihl2hxzn43ppdzl1p7g5b9wl8ac3xa9ix8759ax";
    };

    nativeBuildInputs = [ addOpenGLRunpath makeWrapper ];

    buildInputs = libs;

    checkRun = false;

    patches = [
      ./fix-assets-path.patch
    ];

    postPatch = ''
      sed -i 's|@assetsFolder@|'"''${out}/share/assets"'|g' internal/fileutils/assets.go
    '';

    postInstall = ''
      mkdir -p $out/share
      cp -r assets $out/share/assets
      chmod -R a+rx $out/share/assets
      wrapProgram "$out/bin/hover" \
      --prefix LD_LIBRARY_PATH : ${stdenv.lib.makeLibraryPath libs}
    '';

    postFixup = ''
      addOpenGLRunpath $out/bin/hover
    '';
  };

in
buildFHSUserEnv rec {
  name = pname;
  targetPkgs = pkgs: [
    flutter
    gcc
    go
    hover
    pkgconfig
  ] ++ libs;

  runScript = "hover";
}

I am only using buildFHSUserEnv as a workaround here to make sure that gcc can see the X11 headers. If someone knows a better solution please say so, because I am out of ideas.

1 Like