"inconsistent vendoring" in buildGoModule when overriding source

I’m trying to build aerc from a master commit, since it has some bug fixes that I’d like to use. To do that, I have defined an overlay with overrideAttrs as follows:

(self: super: {
  # aerc 0.4.0 crashes on some emails in my account
  # use a known-good commit from master with the fix.
  aerc = super.aerc.overrideAttrs (_: {
    version = "0.4.0+master";
    src = self.fetchurl {
      url =
        "https://git.sr.ht/~sircmpwn/aerc/archive/c48f228fa5ac57984af78e19713929224874aa8b.tar.gz";
      sha256 = "0lw39c4visy91g2qy380cdqm2h1pn36b7q5zylaskd5xkfanpdgg";
    };
    # had to remake the patch because the old one didn't apply cleanly
    patches = [ ./runtime-sharedir.patch ];
  });
})

However, this doesn’t build, with the following error:

building '/nix/store/g0lvk2mn44j5npaif2q1yca0lbxb847i-aerc-0.4.0+master.drv'...
unpacking sources
unpacking source archive /nix/store/pz5wmcn90npcdcnh04q85pm5nwmr00pw-c48f228fa5ac57984af78e19713929224874aa8b.tar.gz
source root is aerc-c48f228fa5ac57984af78e19713929224874aa8b
setting SOURCE_DATE_EPOCH to timestamp 1601226058 of file aerc-c48f228fa5ac57984af78e19713929224874aa8b/worker/worker_enabled.go
patching sources
applying patch /nix/store/7wpnqjdw36r9qxnzjf5ygzn1jfczcfk5-runtime-sharedir.patch
patching file Makefile
patching file config/config.go
configuring
building
installing
go build -mod=vendor -tags=notmuch \
        -ldflags "-X main.Prefix=/nix/store/6l534iqvv7z6hsag52jfda0mczzvgrbs-aerc-0.4.0+master \
        -X main.ShareDir=/nix/store/6l534iqvv7z6hsag52jfda0mczzvgrbs-aerc-0.4.0+master/share/aerc \
        -X main.Version=0.4.0" \
        -o aerc
go: inconsistent vendoring in /build/aerc-c48f228fa5ac57984af78e19713929224874aa8b:
        github.com/emersion/go-imap@v1.0.6-0.20200914131512-88f167c1e6f7: is explicitly required in go.mod, but vendor/modules.txt indicates github.com/emersion/go-imap@v1.0.4
        github.com/emersion/go-message@v0.12.1-0.20200824204225-9094bd0b8bc0: is explicitly required in go.mod, but vendor/modules.txt indicates github.com/emersion/go-message@v0.11.1
        golang.org/x/text@v0.3.3: is explicitly required in go.mod, but vendor/modules.txt indicates golang.org/x/text@v0.3.2

run 'go mod vendor' to sync, or use -mod=mod or -mod=readonly to ignore the vendor directory
make: *** [Makefile:19: aerc] Error 1
builder for '/nix/store/g0lvk2mn44j5npaif2q1yca0lbxb847i-aerc-0.4.0+master.drv' failed with exit code 2

I found this seemingly related post, but no matter what I do with vendorSha256 (not overriding it, setting it to all 0s, setting it to null) it still gives the same error. I also tried fiddling with runVend and deleteVendor, setting them to false, true and null, but with no success.

updated runtime-sharedir.patch for convenience
From 378787b33627b7fd2906955fb1ff77e6a17b898d Mon Sep 17 00:00:00 2001
From: Tadeo Kondrak <me@tadeo.ca>
Date: Mon, 28 Oct 2019 08:36:36 -0600
Subject: [PATCH] Fix aerc breaking every time the package is rebuilt.

On NixOS, the SHAREDIR changes on every rebuild to the package, but aerc
fills it in as part of the default config and then installs that config
to the users home folder. Fix this by not substituting @SHAREDIR@ in the
default config until runtime.
---
 Makefile         | 2 +-
 config/config.go | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 458a7e3..0bdf783 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ aerc: $(GOSRC)
 		-o $@
 
 aerc.conf: config/aerc.conf.in
-	sed -e 's:@SHAREDIR@:$(SHAREDIR):g' > $@ < config/aerc.conf.in
+	cat config/aerc.conf.in > $@
 
 DOCS := \
 	aerc.1 \
diff --git a/config/config.go b/config/config.go
index 3ae26c1..34b596e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -467,6 +467,11 @@ func LoadConfigFromFile(root *string, sharedir string) (*AercConfig, error) {
 			return nil, err
 		}
 	}
+	if sec, err := file.GetSection("templates"); err == nil {
+		if key,	err := sec.GetKey("template-dirs"); err	== nil {
+			sec.NewKey("template-dirs", strings.ReplaceAll(key.String(), "@SHAREDIR@", sharedir))
+		}
+	}
 	file.NameMapper = mapName
 	config := &AercConfig{
 		Bindings: BindingConfig{
@@ -543,7 +548,9 @@ func LoadConfigFromFile(root *string, sharedir string) (*AercConfig, error) {
 	if err = config.LoadConfig(file); err != nil {
 		return nil, err
 	}
-
+	for i, filter := range config.Filters {
+		config.Filters[i].Command = strings.ReplaceAll(filter.Command, "@SHAREDIR@", sharedir)
+	}
 	if ui, err := file.GetSection("general"); err == nil {
 		if err := ui.MapTo(&config.General); err != nil {
 			return nil, err
-- 
2.28.0
2 Likes

In my setup I build using go build during active development, and then use nix-build for “release”. I’ve found that nix-build fails sometimes if I don’t first clean out my workspace.

I have the following target in my Makefile to clean up:

clean:
	go clean -modcache
	rm -fr .go-build vendor result*

I can’t swear it’ll solve your issue, of course, but it should be safe to try it out.

I did a bit more digging on this, and managed to figure it out in the end, with some confirmation from @Mic92 on IRC. What set me on the right path was after having run a GC and trying to rebuild this overlay, I spotted the following line:

copying path '/nix/store/cyqfygx7d4k6mzv0hwdph4gd2q1kci53-aerc-0.4.0-go-modules' from 'https://cache.nixos.org'...

So my overriden derivation was using the exact same vendored libraries as the upstream nixpkgs one, leading to the version mismatches I was seeing in the above. It turns out that overrideAttrs and buildGoModule aren’t aware of each other, so the -go-modules derivation there isn’t recomputed when I change the source. I got it working by defining a whole new derivation with buildGoModules as follows:

(self: super: {
  aerc = super.buildGoModule {
    inherit (super.aerc.drvAttrs)
      pname runVend doCheck nativeBuildInputs pythonPath buildInputs
      buildPhase installPhase postFixup;
    inherit (super.aerc) meta;
    version = "0.4.0+master";
    src = super.fetchurl {
      url =
        "https://git.sr.ht/~sircmpwn/aerc/archive/c48f228fa5ac57984af78e19713929224874aa8b.tar.gz";
      sha256 = "0lw39c4visy91g2qy380cdqm2h1pn36b7q5zylaskd5xkfanpdgg";
    };
    # had to remake the patch because the old one didn't apply cleanly
    patches = [ ./runtime-sharedir.patch ];
    vendorSha256 = "03f7kfri1qjqbnchcivf074w6wmg7q2nansx5lk41z6ckjkzbv2n";
  };
})

I’m not sure whether using inherit is quite the right approach (perhaps buildGoModule (super.aerc.drvAttrs // { ... }) would be better), but that now builds, installs, and the bug is fixed!

4 Likes

As an addendum, this seems to be an instance of Undocumented / impossible: override go modules · Issue #86349 · NixOS/nixpkgs · GitHub

Not very elegant but this works:

(pkgs.istioctl.overrideAttrs (old: let
  version = "1.7.1";
  src = pkgs.fetchFromGitHub {
    owner = "istio";
    repo = "istio";
    rev = version;
    sha256 = "sha256-CvYqYxb19lSc61SrvGcf/kG9e9zosHz4JbByA+bN600=";
  };
in rec {
  name = "istioct-${version}";
  inherit src;
  inherit (pkgs.buildGoModule {
    inherit name src;
    vendorSha256 = "sha256-4Z4Fgv9zmAwp3tEvHj8yLOWkFY/zFz5VfehSWCuIUcI=";
  }) go-modules;
}))
3 Likes
pkgs.rtsp-simple-server.overrideAttrs (_:
  let
    version = "0.21.2";
    src = pkgs.fetchFromGitHub {
      owner = "aler9";
      repo = "rtsp-simple-server";
      rev = "v${version}";
      sha256 = "sha256-dg+Xl3yHiWJPV+3iFcMVCPHOpHo0+pFiJv+ZVXDAK6k=";
    };
  in rec {
    name = "rtsp-simple-server-new";
    inherit src;
    inherit (pkgs.buildGoModule {     
      inherit name src;
      #vendorSha256 = pkgs.lib.fakeSha256;
      vendorSha256 = "sha256-KvG0+wxe0D+TxQmxaskTa228zUlOxCZE9rGqMYkdwzM=";
    }) go-modules;
})

I’m still getting this go: inconsistent vendoring in /build/source: error after substituting the new sha, retrieved with the help of fakeSha256.

Finally, this worked, based on this:

let                          
  version = "0.21.2";
  src = pkgs.fetchFromGitHub {
    owner = "aler9";
    repo = "rtsp-simple-server"; 
    rev = "v${version}";
    sha256 = "sha256-dg+Xl3yHiWJPV+3iFcMVCPHOpHo0+pFiJv+ZVXDAK6k=";
  };                                          
in                                              
  pkgs.rtsp-simple-server.override rec {
    buildGoModule = args: pkgs.buildGoModule ( args // {
      inherit src version;
      vendorSha256 = "sha256-KvG0+wxe0D+TxQmxaskTa228zUlOxCZE9rGqMYkdwzM=";
    }); };
8 Likes