dockerTools - nix with skaffold custom builds

Having

# ./container.nix
...
{
  caddy = pkgs.dockerTools.buildImage { ... };
  web2ldap = pkgs.dockerTools.buildImage { ... };
}

nix-build ./container.nix will create

  • ./result
  • ./result-2

However for identification using automated tooling (skaffold with custom builds), I require:

  • ./result/caddy
  • ./result/web2ldap

Questions:

  • How can I work around it?
  • Where can I put the lever to provide a PR so this becomes the default?

Thanks for your help! dockerTools is really awesome, I can’t wait to start making upstream PRs to some k8s repos I care.


EDIT: I’m using tempfiles as result for skaffold now, and the linkFarm approach for produceing a nice (and git ignorable) folder hierarchy for manual builds.

There might be more elegant ways to do this, but this is a way:

pkgs.linkFarm "my-output" (                       # 4
  pkgs.lib.attrsets.mapAttrsToList                # 2
  (name: path: { inherit name path; })            # 3
  {                                               # 1
    one = pkgs.dockerTools.buildLayeredImage {
      name = "hello";
      contents = [ pkgs.hello ];
    };
    two = pkgs.dockerTools.buildLayeredImage {
      name = "other";
      contents = [ pkgs.busybox ];
    };
  }
)
$ ll result/
total 8
lrwxrwxrwx 2 root root 69 Dec 31  1969 one -> /nix/store/9174vfz18j45ashyigfh465fl0nr0hr7-docker-image-hello.tar.gz
lrwxrwxrwx 2 root root 69 Dec 31  1969 two -> /nix/store/qjxwvh5018lsh8d3c47ih1v6axlhr7zq-docker-image-other.tar.gz

EDIT: Some more details:

At (1) is the input, a bog standard attribute set, with the keys being the intended name as symlinked, and the value being any result, here it’s dockerTools uses.

At (2) we are using lib.attrsets.mapAttrsToList, with a function at (3) that shapes it into a form that linkFarm can use. I’m using kind of a dirty trick with inherits, which I can do since I named the arguments as I wanted them to be.

And well, (4) is the call to linkFarm, where I give it the name of the output (store path), and the result of mapAttrsToList.

2 Likes

(I added some more details to the original post, as it can sometimes be hard to follow what is going on in a lazily-evaluated language.)

1 Like

Thanks! Do you think a PR to bring this (as new default - after 20.09) to the docker tools would be worth attempting?

Pretty cool! For (even) better understanding, the name of the link farm is the folder itself, so I renamed it to “result-folder”.

$ ls result
result ⇒ /nix/store/mf1sbgp8fwldjwws4w3lg992y7bsn027-result-folder
$ ls result/
caddy ⇒ /nix/store/q2kia1jsgxgrnavv3djshjskx80hf0ix-docker-image-caddy.tar.gz
web2ldap ⇒ /nix/store/ijk9q6cvyqcwp6yazq4c3p5zq99i1a30-docker-image-web2ldap.tar.gz

Here is what I did, for reference:

Nixfile.nix
# Nixfile.nix

{

  nixpkgs ? (builtins.fetchGit {
    url = https://github.com/nixos/nixpkgs-channels;
    ref = "nixpkgs-unstable";
    rev = "f5db415e2f75f09048f98b96cee1a6e0d48c3a5d";
  }),

  image-tag ? "latest"

}:

let

  pkgs = import nixpkgs {config = {}; overlays = [];};
  mach-nix = import (builtins.fetchGit {
    url = "https://github.com/DavHau/mach-nix/";
    ref = "2.3.0";
    rev = "2935985fda56e4c0d73d33ffd06cb2ac7536a91c";
  });

  versions = {
     web2ldap = "1.5.94";
     gunicorn = "20.0.4";
  };

  web2ldap-env = mach-nix.mkPython {
    python = mach-nix.nixpkgs.python38;
    _.ldap0.buildInputs.add = with mach-nix.nixpkgs; [ openldap.dev cyrus_sasl.dev ];
    requirements = ''
      web2ldap==${versions.web2ldap}
      gunicorn==${versions.gunicorn}
    '';
  };

  # shared config across images
  gunicornSocketDir             = "/var/run/gunicorn";
  gunicornWeb2ldapSocket        = "${gunicornSocketDir}/web2ldap.sock";
  gunicornAEdirPwdresetSocket   = "${gunicornSocketDir}/ae-dir-pwd.sock";
  gunicornAEdirOathenrollSocket = "${gunicornSocketDir}/ae-dir-oathenroll.sock";

  images = {

    # caddy image
    "ae-dir/web-server" = let
      Caddyfile = pkgs.writeText "Caddyfile" ''
        localhost
        reverse_proxy /web2ldap/* {
            to unix/${gunicornWeb2ldapSocket}
            transport http
        }

        # reverse_proxy /pwd/* {
        #     to unix/${gunicornAEdirPwdresetSocket}
        #     transport http
        # }

        # reverse_proxy /oath/* {
        #     to unix/${gunicornAEdirOathenrollSocket}
        #     transport http
        # }

      '';
    in pkgs.dockerTools.buildImage {

      name = "ae-dir/web-server";
      tag = image-tag;
      created = "now";

      contents = [
        pkgs.caddy
      ];


    };

    # web2ldap image
    "ae-dir/web2ldap" = let
      gunicornConf = pkgs.writeText "gunicorn.conf.py" ''
        preload_app = True
        raw_env = [ "LDAPNOINIT=1" ]
        proxy_protocol = True
        bind = [ ${gunicornWeb2ldapSocket} ]
        max_requests = 5000
        threads = 10
      '';

    in pkgs.dockerTools.buildImage {

      name = "ae-dir/web2ldap";
      tag = image-tag;
      created = "now";

      contents = [
        web2ldap-env
      ];

      config = {
        Cmd = [ "${web2ldap-env}/bin/gunicorn" "-c" gunicornConf "web2ldap.wsgi" ];
      };

    };
  };

  # Explanation for the linkFarm trick:
  # https://discourse.nixos.org/t/dockertools-result-not-a-folder/8771/2
  # it yields us `./result/caddy` and `./result/web2ldap`
  link_farm = pkgs.linkFarm "result-folder" (       # 4
    pkgs.lib.attrsets.mapAttrsToList                # 2
    (name: path: { inherit name path; })            # 3
    images
  );

in {
  combined = link_farm;
  inherit images;
}
shell.nix
# in shell.nix
...
let
  skaffold-build-nix-container = pkgs.writeShellScriptBin "skaffold-build-nix-container" ''
    set -e

    repo=$(echo "$IMAGE" | cut -d':' -f 1)
    tag=$(echo "$IMAGE" | cut -d':' -f 2)
    out="$(mktemp -u)"
    nix-build "$BUILD_CONTEXT/Nixfile.nix" -A "images.$repo" -o "$out" --arg image-tag "\"$tag\""
    ${pkgs.docker}/bin/docker load -i $out

    if $PUSH_IMAGE; then
        ${pkgs.docker}/bin/docker push $IMAGE
    fi
  '';
in
pkgs.mkShell {
  buildInputs = [
    ...
    skaffold-build-nix-container
  ];
}
skaffold.yml
...

      custom:
        buildCommand: skaffold-build-nix-container
...

This is not something in dockerTools can change. The naming is handled in Nix itself so any change would need to go there.

One idea I have is that you can have the build script instead depend on the output like ${images.caddy}, which does not need to rely on the result symlink. I’m not sure if this actually applies in your case, because I’m not familiar with skaffold and how a directory is required.

Thanks a lot for the a) clarification and b) suggestion!

Finally, I resorted to using a tempfile as output for individual builds - since this is what skaffold pretends doing anyways.

out="$(mktemp -u)"
nix-build "$BUILD_CONTEXT/Nixfile.nix" -A "images.$repo" -o "$out" --arg image-tag "\"$tag\""

I’m curious what the plan is.

I’d like to get skaffold based workflow onto Open Integration Hub · GitHub and eventually GitHub - frappe/erpnext: Free and Open Source Enterprise Resource Planning (ERP) - currently I’m packaging AE-Dir for k8s.

The skaffold custom nix-builder script might be a candidate for upstreaming, not sure where exactly, though. Any precise pointer? - as part of the skaffold package?

No real ideas from me, I haven’t heard of anything in that first paragraph (not that that means anything of course!)

As for skaffold, I peeked, I was wondering about native Nix integration as well. If nothing else it might be neat for Nix visibility (I’m separately thinking of other Nix/CI integration points). I couldn’t tell if being a native integration actually got you more features than being a custom docker-based builder.

That’s indeed the motivation behind: Bring nix build to moby/buildkit (or leap-frog to a CRI shim for a rootfs composer drawing from a SAN-attached nix store)

skaffold would trivially use buildkit, if told so. So does kaniko as only option - which is used by a lot of current CIs do do in-cluster image creation (wether that’s a good thing…).

The answer might lay somewhere there: https://github.com/moby/buildkit/tree/master/executor

re: skaffold - I’m also keeping an eye on runix:

For what it’s worth, there is a nasty caveat in skaffold

https://github.com/GoogleContainerTools/skaffold/issues/4737