bundlerEnv and gems from private Git repositories

Help
1

Hello! I continue my journey to expore Nix as a (Ruby) development environment. Now I have a problem using nix-shell for a Ruby app which depends on a gem from a private Git repository. shell.nix is trivial: it uses mkShell with ruby and gems built by bundlerEnv as buildInputs.

When I launch nix-shell, it fails because it cannot build a derivation for the private gem. Related excerpt from the output:

# ... snip ...
exporting https://gitlab.com/<my-org>/<my-gem>.git (rev <revision>) into /nix/store/3d57xxnxnfih316is8w0gynir4ir10d4-<my-gem>-7727f7a
Initialized empty Git repository in /nix/store/3d57xxnxnfih316is8w0gynir4ir10d4-<my-gem>-7727f7a/.git/
fatal: could not read Username for 'https://gitlab.com': Device not configured
fatal: could not read Username for 'https://gitlab.com': Device not configured
Unable to checkout <revision> from https://gitlab.com/<my-org>/<my-gem>.git
# ... snip ...
cannot build derivation '/nix/store/x3mvcly1n3klawih9n2a9g7qcs6zsfz7-<my-app-gems>.drv': 1 dependencies couldn't be built
error: build of '/nix/store/x3mvcly1n3klawih9n2a9g7qcs6zsfz7-<my-app-gems>.drv' failed

This discussion mentions private Git repos but it doesn’t lead to a conclusion on this topic. bundix --magic kinda works (it asks for the credentials) but it has some drawbacks:

  • it doesn’t install gems’ executables so I have to use e.g. bundle exec rails instead of rails;
  • it doesn’t seem to use pkgs.defaultGemConfig so I have to manually install native dependencies (e.g. libxml2, libiconv, libsodium, etc.), otherwise bundix --magic will fail;
  • it modifies local .bundle/config.

Is there any solution? cc @manveru, @zimbatm.

An unrelated question: my app depends on two gems, both of them install bin/console which makes nix-shell fail because of collisions. I have solved it by overriding meta.priority for one of them in bundlerEnv's gemConfig. Is this the right way?

2

The funny thing is that I have managed to build this derivation somehow, I just don’t remember how. Then I ran nix-collect-garbage and now I cannot reproduce it.

3

A possible workaround would be setting the BUNDLE_GITLAB__COM environment variable (as described in the bundle config docs). It works if I set this variable in a nix shell and manually run bundle install:

{ pkgs ? import <nixpkgs> {}
}:

pkgs.mkShell {
  buildInputs = with pkgs; [
    ruby

    libxml2
    libsodium
    libiconv
  ];

  BUNDLE_GITLAB__COM = "user:pass";
}

I expected it to work with bundlerEnv and gem config overrides:

{ pkgs ? import <nixpkgs> {}
}:

let
  ruby = pkgs.ruby_2_5;

  gems = pkgs.bundlerEnv {
    name = "my-app-gems";
    groups = [ "default" ] ++ environments;
    gemdir = ./.;

    gemConfig = pkgs.defaultGemConfig // {
      my-gem = attrs: {
        BUNDLE_GITLAB__COM = "user:pass";
      };
    };

    inherit ruby;
  };

  environments = ["development" "test" ];
in pkgs.mkShell {
  buildInputs = with pkgs; [
    ruby
    gems
    bundix
  ];
}

Unfortunately, it fails with the same error message.

4

I managed to make it work by overriding the gem’s source URL in gem config!

    # ... snip ...
    gemConfig = pkgs.defaultGemConfig // {
      my-gem = { source, ...}: {
        source = source // {
          url = "https://<user>:<pass>@gitlab.com/<my-org>/<my-gem>.git";
        };
      };
    };
   # ... snip ...

Since I didn’t want to store my Gitlab credentials in version control, I’ve come up with a solution where credentials are read from a git-ignored file:

{ pkgs ? import <nixpkgs> {}
}:

let
  ruby = pkgs.ruby_2_5;

  add-credentials-to-source = credentials-file:
    let
      inherit (pkgs.lib) removeSuffix pathExists;
      inherit (builtins) trace readFile toString replaceStrings;

      error-msg = "expected ${toString credentials-file} to exist";
      credentials =
        assert !pathExists credentials-file -> trace error-msg false;
        removeSuffix "\n" (readFile credentials-file);
    in
      host: { url, ...}@source: source // {
        url = replaceStrings ["https://${host}"] ["https://${credentials}@${host}"] url;
      };

  add-gitlab-credentials = add-credentials-to-source ./.gitlab-credentials "gitlab.com";

  gems = pkgs.bundlerEnv {
    name = "my-app-gems";
    groups = [ "default" ] ++ environments;
    gemdir = ./.;

    gemConfig = pkgs.defaultGemConfig // {
      my-gem = { source, ... }: {
        source = add-gitlab-credentials source;
      };
    };

    inherit ruby;
  };

  environments = ["development" "test" ];
in pkgs.mkShell {
  buildInputs = with pkgs; [
    ruby
    gems
    bundix
  ];
}

It’s not pretty, but it’s working. I’m marking this post as the solution until someone provides a better one.

5

I just spent some hours trying to figure how to get fetchgit use the token set in git config - until i found your solution - thanks for posting this!

Just for people trying to do the same thing, i used builtins.getEnv instead of readFile, since everyone in our team already uses direnv to setup their local environment.

So my final setup looks like this:

let
  gem_credentials = builtins.getEnv "GITHUB_ACCESS_TOKEN";
in
  gems = pkgs.bundlerEnv rec {
    ruby = pkgs.ruby_3_2;
    gemfile = ./Gemfile;
    lockfile = ./Gemfile.lock;
    gemset = ./gemset.nix;
    groups = [ "default" "development" "test" "production" ];
    gemConfig = pkgs.defaultGemConfig // {
      GEM_NAME = attrs: {
        source = attrs.source // {
          url = builtins.replaceStrings [ "https://github.com" ] [ "https://${gem_credentials}@github.com" ] attrs.source.url;
        };
      };
    };
  };