Building my first Package issue with boost and configure

Dear Community,

I am building my first package and am running into an issue with boost (presumably).

This is the flake I am using:

{
  description = "Bellmans Gapc in Nix";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  };

  outputs = {
    self,
    nixpkgs,
  }: let
    # Systems supported
    allSystems = [
      "x86_64-linux" # 64-bit Intel/AMD Linux
      "aarch64-linux" # 64-bit ARM Linux
    ];

    # Helper to provide system-specific attributes
    forAllSystems = f:
      nixpkgs.lib.genAttrs allSystems (system:
        f {
          pkgs = import nixpkgs {inherit system;};
        });
  in {
    packages = forAllSystems ({pkgs}: {
      default = let
        binName = "bellmansgapc";
        boostOverride = pkgs.boost.override {
          enableShared = false;
          enabledStatic = true;
        };

        dependencies = with pkgs; [
          gnumake
          flex
          bison
          boostOverride
          mercurial
        ];

        version = "2023.07.05";
      in
        pkgs.stdenv.mkDerivation {
          name = "gapc-${version}";

          src = pkgs.fetchFromGitHub {
            owner = "jlab";
            repo = "gapc";
            rev = "${version}";
            sha256 = "sha256-OzGf8Z4BzHEPmonlOvvvg5G1y1mtrkWSxOHPfBJU7kU=";
          };

          buildInputs = dependencies;
        };
    });
  };
}

this is the log:

@nix { "action": "setPhase", "phase": "unpackPhase" }
unpacking sources
unpacking source archive /nix/store/9fmcgbffc03qadlbzmm7f5zj0sv62djf-source
source root is source
@nix { "action": "setPhase", "phase": "patchPhase" }
patching sources
@nix { "action": "setPhase", "phase": "updateAutotoolsGnuConfigScriptsPhase" }
updateAutotoolsGnuConfigScriptsPhase
Updating Autotools / GNU config script to a newer upstream version: ./config.sub
Updating Autotools / GNU config script to a newer upstream version: ./config.guess
@nix { "action": "setPhase", "phase": "configurePhase" }
configuring
configure flags: --prefix=/nix/store/6f6a9ad86z7rpdcn3vrzaiw391x23695-gapc-2023.07.05
checking build system type... x86_64-pc-linux-gnu
checking host system type... x86_64-pc-linux-gnu
checking for bash... /nix/store/lf0wpjrj8yx4gsmw2s3xfl58ixmqk8qa-bash-5.2-p15/bin/bash
checking whether the C++ compiler works... yes
checking for C++ compiler default output file name... a.out
checking for suffix of executables... 
checking whether we are cross compiling... no
checking for suffix of object files... o
checking whether the compiler supports GNU C++... yes
checking whether g++ accepts -g... yes
checking for g++ option to enable C++11 features... none needed
checking for gcc... gcc
checking whether the compiler supports GNU C... yes
checking whether gcc accepts -g... yes
checking for gcc option to enable C11 features... none needed
checking for a BSD-compatible install... /nix/store/vwkvhj69z4qqgmpa2lwm97kabf12p26r-coreutils-9.3/bin/install -c
checking for a sed that does not truncate output... /nix/store/g5p3ky90xa05ggg5szyb0pbbl2vp7n03-gnused-4.9/bin/sed
checking whether C compiler accepts -std=c11... yes
checking whether C compiler accepts -ffast-math... yes
checking for gcc option to produce PIC... -fPIC
checking whether C++ compiler accepts -std=c++17... yes
checking whether C++ compiler accepts -D_XOPEN_SOURCE=500... yes
checking whether C++ compiler accepts -MMD -MP... yes
checking whether C++ compiler accepts -Wall -Wnon-virtual-dtor -Wno-unused-variable -Wno-parentheses... yes
checking for OpenMP flag of C++ compiler... -fopenmp
checking for flex... /nix/store/vnxzm4k659v0drwnpyqvh4ixwhr2rj9n-flex-2.6.4/bin/flex
checking for bison... /nix/store/j1d298im2my67d0skdkd9ys1y4v0dx5f-bison-3.8.2/bin/bison
checking for hg... /nix/store/rbsqynlfkqgm4qbxcl78np14lll56n6h-mercurial-6.5.2/bin/hg
checking for grep that handles long lines and -e... /nix/store/p2r51wfg9m3ga7pp7avslpfhfa7w5y83-gnugrep-3.11/bin/grep
checking for gawk... gawk
checking for bison version... 3.8.2
checking for boostlib >= 1.36... yes
checking whether the Boost::Program_Options library is available... yes
configure: error: Could not find a version of the library!

I am a little lost, as I dont understand configure too well and the error is more than cryptic in my opinion.

is there any information I am missing? Is this the right place to ask?

It’s been a long while since I used boost, but searching for:

error: Could not find a version of the library!

… lead me here: https://stackoverflow.com/a/55820620/1054322. It references the flag: --with-boost-libdir, searching nixpkgs for that gives a few hits:

❯ ag 'with-boost-libdir'
pkgs/misc/drivers/utsushi/default.nix
86:    "--with-boost-libdir=${boost}/lib"

pkgs/tools/misc/libbitcoin/libbitcoin-protocol.nix
27:    "--with-boost-libdir=${boost.out}/lib"

pkgs/tools/misc/libbitcoin/libbitcoin-explorer.nix
26:    "--with-boost-libdir=${boost.out}/lib"

pkgs/tools/misc/libbitcoin/libbitcoin-network.nix
26:    "--with-boost-libdir=${boost.out}/lib"

pkgs/tools/misc/libbitcoin/libbitcoin-client.nix
26:    "--with-boost-libdir=${boost.out}/lib"

pkgs/tools/misc/libbitcoin/libbitcoin.nix
26:    "--with-boost-libdir=${boost.out}/lib"

pkgs/tools/text/poedit/default.nix
31:    "--with-boost-libdir=${boost.out}/lib"

pkgs/development/r-modules/default.nix
1054:        "--with-boost=${pkgs.boost.dev}" "--with-boost-libdir=${pkgs.boost.out}/lib"

pkgs/development/python-modules/graph-tool/default.nix
36:    "--with-boost-libdir=${boost}/lib"

pkgs/development/libraries/nghttp2/default.nix
60:  ] ++ lib.optionals (enableAsioLib) [ "--enable-asio-lib" "--with-boost-libdir=${boost}/lib" ]

pkgs/development/libraries/libtorrent-rasterbar/1.2.nix
43:    "--with-boost-libdir=${boostPython.out}/lib"

pkgs/development/libraries/libtins/default.nix
28:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/video/mkvtoolnix/default.nix
105:    "--with-boost-libdir=${lib.getLib boost}/lib"

pkgs/applications/office/libreoffice/default.nix
400:    "--with-boost-libdir=${getLib boost}/lib"

pkgs/applications/networking/p2p/qbittorrent/default.nix
36:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/networking/compactor/default.nix
47:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/editors/codeblocks/default.nix
123:    "--with-boost-libdir=${boost}/lib"

pkgs/applications/audio/mpd-touch-screen-gui/default.nix
50:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/audio/dsf2flac/default.nix
25:  configureFlags = [ "--with-boost-libdir=${boost.out}/lib" ];

pkgs/applications/version-management/gource/default.nix
26:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/graphics/synfigstudio/default.nix
61:      "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/digibyte/default.nix
54:      "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/namecoin/default.nix
37:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/zcash/default.nix
59:    "--with-boost-libdir=${lib.getLib boost180}/lib"

pkgs/applications/blockchains/vertcoin/default.nix
56:      "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/particl-core/default.nix
33:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/bitcoin/default.nix
63:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/elements/default.nix
48:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/bitcoin-knots/default.nix
47:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/pivx/default.nix
44:  configureFlags = [ "--with-boost-libdir=${boost.out}/lib" ]

pkgs/applications/blockchains/groestlcoin/default.nix
61:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/dogecoin/default.nix
36:    "--with-boost-libdir=${boost.out}/lib"

pkgs/applications/blockchains/bitcoin-unlimited/default.nix
24:  configureFlags = [ "--with-boost-libdir=${boost.out}/lib" ]

pkgs/applications/blockchains/litecoin/default.nix
37:  configureFlags = [ "--with-boost-libdir=${boost.out}/lib" ]

pkgs/games/alephone/default.nix
42:    configureFlags = [ "--with-boost-libdir=${boost.out}/lib" ];

There’s no guarantee that those packages are actually relevant to your case, but it’s a place to start. That configureFlags attribute is showing up as an argument to pkgs.stdenv.mkDerivation, which you are already using. So perhaps something like this?

pkgs.stdenv.mkDerivation {
          name = "gapc-${version}";

          src = pkgs.fetchFromGitHub {
            owner = "jlab";
            repo = "gapc";
            rev = "${version}";
            sha256 = "sha256-OzGf8Z4BzHEPmonlOvvvg5G1y1mtrkWSxOHPfBJU7kU=";
          };

          buildInputs = dependencies;
          configureFlags = [ "--with-boost-libdir=${boostOverride.out}/lib" ];
        };

I vaguely recall having to specify the boost location in scenarios where it wasn’t in a list of hard-coded conventional locations (I think this was involved).

One of the principles of nix is that it avoids relying on conventional locations to find files (NixOS doesn’t even have /bin/bash), instead file dependencies supplied explicitly via PATH hackery and symlinks into the nix store. This prevents name collisions (you have control over which bash you’re getting). Your boost files are going to end up in some place like `/nix/store/abc123whateverhash-boost/, which is not a conventional location, so configure needs to be told where to look. I think. Maybe.

Then again, I find many packages which provide boost as a buildInput and which do not set with-boost-libdir. I doubt they’re all broken, so I’m clearly missing a part of the story.

As for this question:

Is this the right place to ask?

I don’t know. Sometimes I think it would be good to have per-language support communities: nix-python, nix-c++, nix-rust, etc… Other times it’s kind of fun to see it all in one place. I don’t think this is a bad place to ask.

1 Like

this solved my issue! Thanks your for searching my issue down to the end, i found the same start, but i didnt know where to go! Thanks for especially taking the time to explain how you came to the solution! Have a great weekend / holiday time!

1 Like

I’m very happy to hear it. It’s definitely not the first time I’ve found something that worked by copying patterns I found in nixpkgs without really knowing what’s going on :upside_down_face:

Happy holidays to you also!

2 Likes

Welcome @coru. Fair warning, this is a bit of an effortpost. I also didn’t see that your question had gotten answered in the time it took to write this. :grimacing:

Here’s how I’d write it. First, flake.nix. This is basically the same every time. :slight_smile:

{
  description = "Bellman's GAP compiler";
  inputs.nixpkgs.url = "github:nixos/nixpkgs";
  inputs.systems.url = "github:nix-systems/default";

  outputs = { self, systems, nixpkgs }@inputs:
    let
      importNixpkgs = system: import nixpkgs {
        inherit system;
      };
      systems = import inputs.systems;
      withSystemPackages = f: nixpkgs.lib.genAttrs systems (system: f (importNixpkgs system));
    in
    {
      packages = withSystemPackages (pkgs: {
        default = pkgs.callPackage ./package.nix { };
      });
    };
}

Then, package.nix. This is different each time, and will have unique issues based on what you’re packaging. I started with something like what you did, just in callPackage style.

{ stdenv
, fetchFromGitHub
, flex
, bison
, boost
}:

stdenv.mkDerivation {
  name = "gapc-unstable";
  version = "2023-07-05";

  src = fetchFromGitHub {
    owner = "jlab";
    repo = "gapc";
    rev = "2023.07.05";
    sha256 = "sha256-OzGf8Z4BzHEPmonlOvvvg5G1y1mtrkWSxOHPfBJU7kU=";
  };

  nativeBuildInputs = [ flex bison ];
  buildInputs = [ boost ];
}

Like you, I ran into this error when I ran nix build -L.

gapc-unstable> checking for boostlib >= 1.36... yes
gapc-unstable> checking whether the Boost::Program_Options library is available... yes
gapc-unstable> configure: error: Could not find a version of the library!
error: builder for '/nix/store/7cl99vqya54d7yskxi4jacnr435xr172-gapc-unstable.drv' failed with exit code 1

In this case, you’re running into an error from configure, which is generated from m4/ax_boost_program_options.m4. You can see that the test AC_CACHE_CHECK([whether the Boost::Program_Options library is available] is passing, since there’s the line checking whether the Boost::Program_Options library is available... yes.

In order to debug this, I did the following in order to get access to the build environment that stdenv.mkDerivation set up for me. Let’s reproduce the error, this time not under the nix builder, using genericBuild and nix-shell.

$ mkdir /tmp/gapc
$ cd /tmp/gapc
$ nix-shell --pure /nix/store/7cl99vqya54d7yskxi4jacnr435xr172-gapc-unstable.drv
[nix-shell:/tmp/gapc]$ type genericBuild
genericBuild is a function
genericBuild ()
{
    export GZIP_NO_TIMESTAMPS=1;
    if [ -f "${buildCommandPath:-}" ]; then
        source "$buildCommandPath";
        return;
    fi;
    if [ -n "${buildCommand:-}" ]; then
        eval "$buildCommand";
        return;
    fi;
    if [ -z "${phases[*]:-}" ]; then
        phases="${prePhases[*]:-} unpackPhase patchPhase ${preConfigurePhases[*]:-}             configurePhase ${preBuildPhases[*]:-} buildPhase checkPhase             ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase             ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}";
    fi;
    for curPhase in ${phases[*]};
    do
        runPhase "$curPhase";
    done
}

Above, we see that genericBuild calls runPhase on various different phases of the build. Let’s try it out.

[nix-shell:/tmp/gapc]$ set -ex
[nix-shell:/tmp/gapc]$ genericBuild
+ genericBuild
+ export GZIP_NO_TIMESTAMPS=1
+ GZIP_NO_TIMESTAMPS=1
+ '[' -f '' ']'
+ '[' -n '' ']'
+ '[' -z '' ']'
+ phases=' unpackPhase patchPhase  autoreconfPhase updateAutotoolsGnuConfigScriptsPhase             configurePhase  buildPhase checkPhase              installPhase  fixupPhase installCheckPhase              distPhase '
<snipped>
++ ./configure --prefix=/nix/store/0ja824nvzp3yvhfqnwcbayxxf7dzp1zk-gapc-unstable
<snipped>
checking whether the Boost::Program_Options library is available... yes
configure: error: Could not find a version of the library!
+ exitHandler
+ exitCode=1
+ set +e
+ '[' -n '' ']'
+ ((  1 != 0  ))
+ runHook failureHook
+ local hookName=failureHook
+ shift
+ local 'hooksSlice=failureHooks[@]'
+ local hook
+ for hook in "_callImplicitHook 0 $hookName" ${!hooksSlice+"${!hooksSlice}"}
+ _eval '_callImplicitHook 0 failureHook'
+ declare -F '_callImplicitHook 0 failureHook'
+ eval '_callImplicitHook 0 failureHook'
++ _callImplicitHook 0 failureHook
++ local def=0
++ local hookName=failureHook
++ declare -F failureHook
++ type -p failureHook
++ '[' -n '' ']'
++ return 0
+ return 0
+ '[' -n '' ']'
+ return 1

Note that after the command runs, you’re no longer in the nix-shell, so if you want to re-enter, you have to do nix-shell --pure <whatever>.drv again. And the build is left at the state of failure, so if it’s polluted, we’ll have to empty the directory again. But–crucially–we can run whatever we want in this environment, and grok the various build logs from the tools used to build.

We saw up at the top that there were these phases:

  • unpackPhase
  • patchPhase
  • autoreconfPhase
  • updateAutotoolsGnuConfigScriptsPhase
  • configurePhase ← this is where we’re erroring out.
  • buildPhase ← this one isn’t running since the one before it errored out.
  • checkPhase
  • installPhase
  • fixupPhase
  • installCheckPhase
  • distPhase

Let’s re-enter the nix-shell and re-run configure. We happen to know (using file configure) that it’s just a shell script, so This time, let’s run it with bash -x so that it shows what it’s doing.

$ nix-shell --pure /nix/store/1llca8smavm7gyv4bwhkknj5as5yz6br-gapc-unstable.drv
$ cd source
$ bash -x ./configure --prefix=/nix/store/0ja824nvzp3yvhfqnwcbayxxf7dzp1zk-gapc-unstable 2> out
<snipped>

Looking through out, we can see the section where it errored out:

+ printf '%s\n' '#define HAVE_BOOST_PROGRAM_OPTIONS /**/'
++ echo
++ sed -e 's/[^\/]*//'
+ BOOSTLIBDIR=
+ test x = x
++ ls '/libboost_program_options*.so*'
++ sed 's,.*/,,'
++ sed -e 's;^lib\(boost_program_options.*\)\.so.*$;\1;'
++ sed 's,.*/,,'
++ ls '/libboost_program_options*.dylib*'
++ sed -e 's;^lib\(boost_program_options.*\)\.dylib.*$;\1;'
++ sed 's,.*/,,'
++ ls '/libboost_program_options*.a*'
++ sed -e 's;^lib\(boost_program_options.*\)\.a.*$;\1;'
+ test x '!=' xyes
++ ls '/boost_program_options*.dll*'
++ sed 's,.*/,,'
++ sed -e 's;^\(boost_program_options.*\)\.dll.*$;\1;'
++ sed 's,.*/,,'
++ ls '/boost_program_options*.a*'
++ sed -e 's;^\(boost_program_options.*\)\.a.*$;\1;'
+ test x = x
+ as_fn_error 0 'Could not find a version of the library!' 6036 5
+ as_status=0
+ test 0 -eq 0
+ as_status=1
+ test 5
+ as_lineno=6036
+ as_lineno_stack=as_lineno_stack=
+ printf '%s\n' 'configure:6036: error: Could not find a version of the library!'
+ printf '%s\n' 'configure: error: Could not find a version of the library!'
configure: error: Could not find a version of the library!

So of course it can’t find the library, because it’s looking in the root, which doesn’t have it. Where is it actually?

$ find /nix/store -name libboost_program_options.so
/nix/store/5ii12aznjvbg2m427nvp9sf9shisggz3-boost-1.81.0/lib/libboost_program_options.so

And how did it try to find the BOOSTLIBDIR? From m4/ax_boost_program_options.m4:

BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`

We see in out that BOOST_LDFLAGS is empty. What value should it be set to? Let’s see if we can set it using configureFlags in package.nix.

configureFlags = [ "BOOST_LDFLAGS=-L${boost}/lib" ];

nix build -L in our flake directory, and … it works. Well, it fails a little later.

gapc-unstable> *** The gsl-config script installed by GSL could not be found
gapc-unstable> *** If GSL was installed in PREFIX, make sure PREFIX/bin is in
gapc-unstable> *** your path, or set the GSL_CONFIG environment variable to the
gapc-unstable> *** full path to gsl-config.
gapc-unstable> GSL needed to compile GapC

Let’s just add gsl to the top and to nativeBuildInputs. One nix build later:

$ nix build -L
<snipped>
gapc-unstable> stripping (with command strip and flags -S -p) in  /nix/store/lkvj2019ykd08phmzibd68x36ika03yx-gapc-unstable/lib /nix/store/lkvj2019ykd08phmzibd68x36ika03yx-gapc-unstable/bin
/nix/store/lkvj2019ykd08phmzibd68x36ika03yx-gapc-unstable/bin/gapc --help
gapc version 2023.07.05
  Copyright 2008-2011 Georg Sauthoff, GPL v3+

Usage: /nix/store/lkvj2019ykd08phmzibd68x36ika03yx-gapc-unstable/bin/gapc (OPTION)* FILE
<snipped>

Nice.

So here’s our final set of files: flake.nix from above, and package.nix like this:

{ stdenv
, fetchFromGitHub
, flex
, bison
, boost
, gsl
}:

stdenv.mkDerivation {
  name = "gapc-unstable";
  version = "2023-07-05";

  src = fetchFromGitHub {
    owner = "jlab";
    repo = "gapc";
    rev = "2023.07.05";
    sha256 = "sha256-OzGf8Z4BzHEPmonlOvvvg5G1y1mtrkWSxOHPfBJU7kU=";
  };

  nativeBuildInputs = [ flex bison gsl ];
  buildInputs = [ boost ];
  configureFlags = [ "BOOST_LDFLAGS=-L${boost}/lib" ];
}

Hope this helps.

1 Like

Heya!

Great take on this post! I love that you took the time to dig that deep into the issue! The next ‘issue’ i wanted to tackle was putting it in a form that is a little more flexible than just the flake! So thanks for guiding me through that too!

I wish you a great holiday time and hope you are having a blast!

hope you dont mind if i ask a few questions,

the ‘unstable’ tag you given the package is deliberate? is this a common suffix for self built packages?

you dropped flex and mercurial from the build, while those are named by the upstream, is this something that is automatically added by the build process? or did you deem it unnecessary, would it be terribly wrong from me to add them back in?

the ‘unstable’ tag you given the package is deliberate? is this a common suffix for self built packages?

I got this from the documentation on nixpkgs, which in your own flake doesn’t matter but if you want to contribute it to nixpkgs might.

In particular, I copied this:

you dropped flex and mercurial from the build, while those are named by the upstream, is this something that is automatically added by the build process?

flex is still there. mercurial isn’t used in the build, so it looks like a docs error to me. I expect that this project was first hosted on bitbucket or something like that using mercurial, then came over to github and started using git.

would it be terribly wrong from me to add them back in?

I think it’s best to have a minimal set of dependencies, so I would not add it back in.