Can't build SolveSpace from scratch: fetching git source fails

I try to build solvespace for my own fork GitHub - nobodyinperson/solvespace: Parametric 2d/3d CAD with this solvespace.nix mostly copied from nixpkgs:

{ lib
, stdenv
, fetchFromGitHub
, cmake
, pkg-config
, wrapGAppsHook4
, at-spi2-core
, cairo
, dbus
, eigen
, freetype
, fontconfig
, glew
, gtkmm3
, json_c
, libdatrie
, libepoxy
, libGLU
, libpng
, libselinux
, libsepol
, libspnav
, libthai
, libxkbcommon
, pangomm
, pcre
, util-linuxMinimal # provides libmount
, xorg
, zlib
}:

stdenv.mkDerivation rec {
  pname = "solvespace";
  version = "3.1";

  # from nixpkgs
  # src = fetchFromGitHub {
  #   owner = pname;
  #   repo = pname;
  #   rev = "v${version}";
  #   hash = "sha256-sSDht8pBrOG1YpsWfC/CLTTWh2cI5pn2PXGH900Z0yA=";
  #   fetchSubmodules = true;
  # };
  src = fetchFromGitHub {
    owner = "nobodyinperson"; # my fork
    repo = pname;
    rev = "b3e703c222beaae3f03c60b5d7a6e45cf14cfd29"; # master 14.03.2024
    hash = "sha256-5tALOyFVzJgUb2hnlr+2XfqZ9PZO559vCBAji6Qs2nY=";
    fetchSubmodules = true;
  };

  nativeBuildInputs = [
    cmake
    pkg-config
    wrapGAppsHook4
  ];

  buildInputs = [
    at-spi2-core
    cairo
    dbus
    eigen
    freetype
    fontconfig
    glew
    gtkmm3
    json_c
    libdatrie
    libepoxy
    libGLU
    libpng
    libselinux
    libsepol
    libspnav
    libthai
    libxkbcommon
    pangomm
    pcre
    util-linuxMinimal
    xorg.libpthreadstubs
    xorg.libXdmcp
    xorg.libXtst
    zlib
  ];

  postPatch = ''
    patch CMakeLists.txt <<EOF
    @@ -20,9 +20,9 @@
     # NOTE TO PACKAGERS: The embedded git commit hash is critical for rapid bug triage when the builds
     # can come from a variety of sources. If you are mirroring the sources or otherwise build when
     # the .git directory is not present, please comment the following line:
    -include(GetGitCommitHash)
    +# include(GetGitCommitHash)
     # and instead uncomment the following, adding the complete git hash of the checkout you are using:
    -# set(GIT_COMMIT_HASH 0000000000000000000000000000000000000000)
    +set(GIT_COMMIT_HASH $version)
    EOF
  '';

  cmakeFlags = [ "-DENABLE_OPENMP=ON" ];

  meta = with lib; {
    description = "A parametric 3d CAD program";
    license = licenses.gpl3Plus;
    maintainers = [ maintainers.edef ];
    platforms = platforms.linux;
    homepage = "https://solvespace.com";
    changelog = "https://github.com/solvespace/solvespace/raw/v${version}/CHANGELOG.md";
  };
}

I build it like this:

nix-build -E 'with import <nixpkgs> {}; callPackage ./solvespace.nix {}'
# ...
# fatal: dumb http transport does not support shallow capabilities
# ...
# Failed to clone 'extlib/freetype' a second time, aborting
# ...
# [then sha256 mismatch]

… it seems to have a problem with the extlib/freetype submodule which seems to be hosted on a dumb git remote which doesn’t support shallow clones. But even with deepClone=true it works sometimes but not all the time stopped working a while ago as one constantly runs into hash mismatches. deepClone=true is also painfully slow.

The same happens for nixpkgs’ own solvespace btw:

# prefetch all dependencies for solvespace
nix-shell -p solvespace --run ''
# delete only the source archive
nix-store --delete "$(nix eval --impure --raw --expr '(let pkgs = import <nixpkgs> {}; in "${toString pkgs.solvespace.src}")')"
# then rebuild solvespace without the cache
nix-build --no-substitute -E 'with import <nixpkgs> {}; solvespace'
# ...
# fatal: dumb http transport does not support shallow capabilities
# ...
# Failed to clone 'extlib/freetype' a second time, aborting
# ...
# [then sha256 mismatch]

So apparently Hydra (or whatever builds cache.nixos.org) has some extra magic in place that manages to build the source archive including the submodules.

I’d like to see the Hydra logs for the source archive only, but I can’t find it. Here in the log for solvespace it just says that it fetches the source from the cache, wow that doesn’t help…

unpacking source archive /nix/store/9mnamza2ixl64p5hla4q82pl8kfd0mxn-source

Seems related:

I am struggling ridiculously hard to try out that single-line patch to nix-prefetch-git in the linked PR #116289.

  • fetchTarball of that PR commit doesn’t work, gives weird pcre host not found errors
  • manually putting all the sources of fetchgit (default.nix, nix-prefetch-git, etc.) into a folder and then callPackage of it in any variant doesn’t work, I can’t build that custom fetchgit
  • git cloning nixpkgs is several gigabytes wtf
  • one apparently cannot fetchTarball or import a local downloaded tarball (with the changes in it)

It’s becoming tiring…

I managed to cherry-pick that PR 116289 onto nixos-24.05-small, but that give the same error…

{ lib
, stdenv
, fetchFromGitHub
, cmake
, pkg-config
, wrapGAppsHook4
, at-spi2-core
, cairo
, dbus
, eigen
, freetype
, fontconfig
, glew
, gtkmm3
, json_c
, libdatrie
, libepoxy
, libGLU
, libpng
, libselinux
, libsepol
, libspnav
, libthai
, libxkbcommon
, pangomm
, pcre
, util-linuxMinimal # provides libmount
, xorg
, zlib
, config
}:

let
  pr116289 = (import (fetchTarball "https://github.com/nobodyinperson/nixpkgs/archive/87c3fd7a9697a72caa339c329a10eefef6a24427.tar.gz") { } );
in
stdenv.mkDerivation rec {
  pname = "solvespace";
  version = "3.1";

  # from nixpkgs
  # src = fetchFromGitHub {
  #   owner = pname;
  #   repo = pname;
  #   rev = "v${version}";
  #   hash = "sha256-sSDht8pBrOG1YpsWfC/CLTTWh2cI5pn2PXGH900Z0yA=";
  #   fetchSubmodules = true;
  # };
  src = pr116289.fetchFromGitHub {
    owner = "nobodyinperson"; # my fork
    repo = pname;
    rev = "b3e703c222beaae3f03c60b5d7a6e45cf14cfd29"; # master 14.03.2024
    sha256 = "sha256-RtybENbE/7pGU9xqnDuXpTl3/CN8+fkihheZQszgvNM=";
    fetchSubmodules = true;
  };

  nativeBuildInputs = [
    cmake
    pkg-config
    wrapGAppsHook4
  ];

  buildInputs = [
    at-spi2-core
    cairo
    dbus
    eigen
    freetype
    fontconfig
    glew
    gtkmm3
    json_c
    libdatrie
    libepoxy
    libGLU
    libpng
    libselinux
    libsepol
    libspnav
    libthai
    libxkbcommon
    pangomm
    pcre
    util-linuxMinimal
    xorg.libpthreadstubs
    xorg.libXdmcp
    xorg.libXtst
    zlib
  ];

  postPatch = ''
    patch CMakeLists.txt <<EOF
    @@ -20,9 +20,9 @@
     # NOTE TO PACKAGERS: The embedded git commit hash is critical for rapid bug triage when the builds
     # can come from a variety of sources. If you are mirroring the sources or otherwise build when
     # the .git directory is not present, please comment the following line:
    -include(GetGitCommitHash)
    +# include(GetGitCommitHash)
     # and instead uncomment the following, adding the complete git hash of the checkout you are using:
    -# set(GIT_COMMIT_HASH 0000000000000000000000000000000000000000)
    +set(GIT_COMMIT_HASH $version)
    EOF
  '';

  cmakeFlags = [ "-DENABLE_OPENMP=ON" ];

  meta = with lib; {
    description = "A parametric 3d CAD program";
    license = licenses.gpl3Plus;
    maintainers = [ maintainers.edef ];
    platforms = platforms.linux;
    homepage = "https://solvespace.com";
    changelog = "https://github.com/solvespace/solvespace/raw/v${version}/CHANGELOG.md";
  };
}

I’m really out of ideas and frustrated by how convoluted and nested all this nix packaging is… :pensive:

I wouldn’t read too much into my old PR, I can’t remember what issue I was facing and I suspect nix-prefetch-git has moved on since then. One possible (inefficient) workaround might be to pass deepClone = true; in your fetchFromGitHub. I seem to be able to build your branch with the deepClone change and updated hash.

I haven’t verified, but it looks like it could have worked before this change last year: fetchgit: shallow clone for submodules by pineapplehunter · Pull Request #254172 · NixOS/nixpkgs · GitHub. I guess Hydra has the source cached from before that change.

Completely missed that you’ve tried deepClone, apologies for the redundant advice!

I suspect nix-prefetch-git has moved on since then

Seems like it hasn’t. --depth is still passed to dumb remotes, causing the error you tried to avoid with your PR.

I seem to be able to build your branch with the deepClone change and updated hash.

I wonder why, for me, with deepClone=true, the hash always changes. And apparently one can’t just ignore the hash, just to be able to work again :woozy_face:

Hmm, the init_submodules handling in nix-prefetch-git has definitely changed since 2021, and that’s the bit causing issues here. Definitely related to not falling back when --depth isn’t supported though.

I think I just got lucky testing last night with deepClone = true; - trying again I’m getting your hash mismatch issue more often. The hacky-feeling workaround might be to add a postFetch:

  src = fetchFromGitHub {
    owner = "nobodyinperson"; # my fork
    repo = pname;
    rev = "b3e703c222beaae3f03c60b5d7a6e45cf14cfd29"; # master 14.03.2024
    fetchSubmodules = true;
    deepClone = true;
    hash = "sha256-9nZSLXlnfEKZ+re/uJijgokKe18OmPXxuTRgbq/f+9c=";
    postFetch = ''
      echo "removing \`.git'..." >&2
      find "$out" -name .git -print0 | xargs -0 rm -rf
    '';
  };

Some packages do something similar when using deepClone or leaveDotGit. (e.g. tetrio-desktop). I guess there are some caveats to make_deterministic_repo, which sounds like it should work to avoid this issue.

1 Like

Thank you for the postFetch hint. It builds fine now with stable hashes with this much simpler solvespace.nix:

{ solvespace, fetchFromGitHub }:
solvespace.overrideAttrs (oldAttrs: rec {
  version = "${oldAttrs.version}+${src.rev}"; # Set the custom version
  src = fetchFromGitHub {
    owner = "nobodyinperson"; # my fork
    repo = oldAttrs.pname;
    rev = "c9469ae526b58966ef14e6ab774aa631fff930c9"; # Yann's changes from 14.03.2024 rebased onto solvespace master from 26.05.2024
    hash = "sha256-cQAdTZ87pqGbR/etWZVVRvqJR0SxtDyU+1V8+viAeAU=";
    fetchSubmodules = true;
    deepClone = true;
    # from https://discourse.nixos.org/t/cant-build-solvespace-from-scratch-fetching-git-source-fails/46536/8?u=nobodyinperson
    # apparently there is a bug in fetchgit (or rather its nix-prefetch-git script buried underneath),
    # that prevents it from falling back to not using --depth on dumb http remotes
    # (which is the case for solvespace's extlib/freetype submodule on) hosted on some weird GNU savannah dumb http remote
    postFetch = ''
      echo "removing \`.git'..." >&2
      find "$out" -name .git -print0 | xargs -0 rm -rf
    '';
  };
})

I played around with nix-prefetch-git a bit, but I don’t see why it doesn’t fall back to git submodule update --init without --depth if it fails with --depth. It should work, but it doesn’t. I clearly don’t understand something here. But now my build works and I’m happy, thanks! :+1: