I’m really excited about this idea.
Below is a small demo. The demo is not about lang2nix (first, to avoid language-specific objections - “rust has lock files”, “there is a sbt plugin for that”, …, and second, I have not yet implemented it for language frameworks, and by the time it happens, I will probably get rid of bash and I will have trouble showing a working demo example on that Nix we can all read).
An example about shaderc which has vendored dependencies published in a separate branch GitHub - google/shaderc at known-good shortly before or after the release and maintained in nixpkgs manually: https://github.com/NixOS/nixpkgs/blob/67c4132368dd7612d5226a99ec8a2e3c1af68b76/pkgs/development/compilers/shaderc/default.nix
The setting very similar to lang2nix, isn’t it?
{ lib, stdenv, fetchgit, cmake, python3, pkgsCurrent }:
let
version = "2021.2";
# -^^^^^^- to upgrade, just change this
# (and even that can be automated)
src = fetchgit {
url = "https://github.com/google/shaderc";
rev = "v${version}";
memoFile = ./.memo.nix; # memoFile is optional; there is a global default
#
# Look, ma, no `sha256`.
#
# There is a magic inside `fetchgit` which is explained below on example
# of `known-good`.
#
# `fetchgit` is a bit more complex, there are 2 memoization steps:
# 1. `rev` -> `fullRev`
# 2. `fullRev` -> (`sha256`, `commitTime`, `narsize`)
#
# Shortly, if `sha256` is in `import ./.memo.nix`, it is just used, without
# any IFD. Otherwise, we pause here, run `git` in sandbox and mutate
# `./.memo.nix`
#
# It lacks parallelism of @Ericson2314's .drv.drv, but has the advantage
# that the memo files are local, and can (should) be placed under version
# control, similar to the ubiquitous .lock files.
#
# Actually, `./.memo.nix`'s attrset is maintained in memory and
# flushed to disk once on Nix's exit, so this is just another
# obstacle to parallelism.
#
};
# Tolerate "known-good" branch updated within a day after the release.
# `builtins.timeToString` and `builtins.timeFromString` are guests
# from the future. Nothing magical here: just pure functions which
# could be implemened in pure Nix
commitTime-nextday =
builtins.timeToString (builtins.timeFromString src.commitTime + 86400);
# But look, ma, there is not only auto-maintained `src.sha256`,
# but also `src.commitTime`.
# and `src.fullRev`
# and could be auto-maintained `src.swhid`, `src.ipfs`, `src.magnet`, ....
known-good =
builtins.head (lib.memoize {
# `memoFile` is optional. the global default is usually a good choice
memoFile = ./.memo.nix;
# `memoFile` is a Nix file with attrs set inside.
# Here we define the keys of that attrset we are interested in
# There could be more than 1 key (e.g. `fetchurl`'ing from multiple urls)
memoKeys = [ "version=${version} fullRev=${src.fullRev}" ];
# Either Nix function (on top of functions like `builtins.fetchGit`) or
# the code to run in sandbox when `memoFile` has no requested `memoKeys`
# (`lib.memoize` also has `mode` which could be "all" or "any" to tell
# if we need values for all the `memoKeys` or for any one) or `memoKeys`
# are obsolete (there is also `memoRevision` to tell if it is desirable
# to try to calculate the value again; useful for `pkgs.geoip-database`)
calcValues =
# `pkgsCurrent` is defined next to `pkgsi686Linux`
# overriding `system=builtins.currentSystem`
# this is usualy `x86_64-linux` even if we build for/on something else.
pkgsCurrent.stdenvNoCC.mkDerivation {
# it should be actually not an IFD-derivation,
# but `builtins.sandboxedExec`, which is not yet implemented.
# Creation of derivation in Nix Store is needless and
# reuse the existing results from Nix Store is undesirable
name = "known-good-${toString builtins.currentTime}.nix";
# The derivation is not FOD, so let's allow networking explicitly
# `__allowNetworking` - another guest from the future -
# works only in IFD-derivations. again: it is actually
# `builtins.sandboxedExec` simmulated via an IFD-derivation
__allowNetworking = true;
GIT_SSL_CAINFO = "${pkgsCurrent.cacert}/etc/ssl/certs/ca-bundle.crt";
buildInputs = [ pkgsCurrent.gitMinimal ];
# Get the newest https://github.com/google/shaderc/tree/known-good
# but not newer than (`src.commitTime`+1day) and then
# store `known_good.json`'s content to `./.memo.nix`'s attrset
# under key `memoKeys`
buildCommand = ''
git init
git remote add origin ${lib.escapeShellArg src.url}
git fetch origin known-good
git checkout $(git rev-list -n1 --before=${commitTime-nextday} \
origin/known-good)
# emit a list of the same size as `memoKeys`
# each value corresponds to a key
# (with memoMode="any", it is possible to return `null` for some)
echo "[ { json = '''$(cat known_good.json)'''; # FIX: proper escape
} ]" > $out
'';
};
});
# and the rest is trivial...
in stdenv.mkDerivation rec {
pname = "shaderc";
inherit version src;
outputs = [ "out" "lib" "bin" "dev" "static" ];
patchPhase =
let
# parse JSON of
# https://github.com/google/shaderc/blob/ee00a6bc9388acbc332b1ef2290ff6481b78b2cf/known_good.json
p = lib.listToAttrs (
map (args: lib.nameValuePair args.name args)
(builtins.fromJSON known-good.json).commits
);
glslang = fetchgit {
memoFile = ./.memo.nix;
url = "https://github.com/${p.glslang .subrepo}";
rev = p.glslang .commit;
};
spirv-tools = fetchgit {
memoFile = ./.memo.nix;
url = "https://github.com/${p.spirv-tools .subrepo}";
rev = p.spirv-tools .commit;
};
spirv-headers = fetchgit {
memoFile = ./.memo.nix;
url = "https://github.com/${p.spirv-headers.subrepo}";
rev = p.spirv-headers.commit;
};
in ''
mkdir -p ${p.glslang .subdir}
mkdir -p ${p.spirv-tools .subdir}
mkdir -p ${p.spirv-headers.subdir}
#
# `fetchgit` by default produces tarballs, so `tar xf` instead of `cp`
#
tar xf ${glslang } --strip-components=1 -C ${p.glslang .subdir}
tar xf ${spirv-tools } --strip-components=1 -C ${p.spirv-tools .subdir}
tar xf ${spirv-headers} --strip-components=1 -C ${p.spirv-headers.subdir}
'';
nativeBuildInputs = [ cmake python3 ];
postInstall = ''
moveToOutput "lib/*.a" ${placeholder "static"}
'';
cmakeFlags = [ "-DSHADERC_SKIP_TESTS=ON" ];
}