Nixifest: Generate Kubernetes manifests from Nix

LLM DISCLOSURE: I used GPT-5.6-Luna in Pi for much of the code and documentation, and GPT-5.6-Sol via ChatGPT for much of the planning and help with some architecture decisions

Better CRD support than Kubenix. Not tied to a single GitOps solution like Nixidy.

5 Likes

Great minds/AIs think alike! I used gpt5.6-sol to create “conditionalAttrsOf” which allows you to define config that only applies if something else defines it without mkIfExists :smiley:

They don’t do much good for evaluation time but they allow very “dynamic” configuration, I’m working on something similar as you. Using Nix to render manifests (and some related things like mounting Nix closures in Kubernetes, a terranix clone and Python bindings to Nix so I can drive it all as I want and not as Nix CLI wants).

conditionalAttrsOf
{ lib }:
let
  markerType = "ifExists";

  checkDefsForError =
    check: definitions:
    if lib.all (definition: check definition.value) definitions then
      null
    else
      {
        message = "Definition values: ${
          lib.options.showDefs (lib.filter (definition: !check definition.value) definitions)
        }";
      };

  isIfExists = value: lib.isAttrs value && (value._type or null) == markerType;

  mkIfExists = content: {
    _type = markerType;
    inherit content;
  };

  # Run the module system's normal property processing before classifying a
  # definition. This makes an ordinary `mkIf false` definition count as
  # absent and retains source locations for the real element merge.
  definitionCollector = lib.types.mkOptionType {
    name = "conditionalAttrsOf definitions";
    description = "definitions collected for conditionalAttrsOf";
    check = {
      __functor = _: _: true;
      isV2MergeCoherent = true;
    };
    merge = rec {
      __functor =
        self: loc: defs:
        (self.v2 { inherit loc defs; }).value;
      v2 =
        { defs, ... }:
        {
          headError = null;
          value = defs;
          valueMeta = { };
        };
    };
  };

  conditionalAttrsOf =
    elemType:
    let
      pushPositions = map (
        definition:
        lib.mapAttrs (name: value: {
          inherit (definition) file;
          inherit value;
        }) definition.value
      );

      mergeEntry =
        loc: definitions:
        let
          plainOrdinary = lib.all (
            definition: !(lib.isAttrs definition.value && definition.value ? _type)
          ) definitions;
          collected =
            (lib.modules.mergeDefinitions loc definitionCollector definitions).optionalValue.value or [ ];
          ordinary = lib.filter (definition: !isIfExists definition.value) collected;
          conditional = lib.filter (definition: isIfExists definition.value) collected;
          included =
            ordinary ++ map (definition: definition // { value = definition.value.content; }) conditional;
          evaluation = lib.modules.mergeDefinitions loc elemType (
            if plainOrdinary then definitions else included
          );
        in
        {
          exists = plainOrdinary || ordinary != [ ];
          value = evaluation.mergedValue;
          valueMeta = evaluation.checkedAndMerged.valueMeta;
        };
    in
    lib.types.mkOptionType rec {
      name = "conditionalAttrsOf";
      description = "attribute set of ${elemType.description} with mkIfExists support";
      check = {
        __functor = _: value: lib.isAttrs value && !isIfExists value;
        isV2MergeCoherent = true;
      };
      merge = rec {
        __functor =
          self: loc: defs:
          (self.v2 { inherit loc defs; }).value;
        v2 =
          { loc, defs }:
          let
            entries = lib.zipAttrsWith (name: entryDefinitions: mergeEntry (loc ++ [ name ]) entryDefinitions) (
              pushPositions defs
            );
            includedEntries = lib.filterAttrs (_: entry: entry.exists) entries;
          in
          {
            headError = checkDefsForError check defs;
            value = lib.mapAttrs (_: entry: entry.value) includedEntries;
            valueMeta.attrs = lib.mapAttrs (_: entry: entry.valueMeta) includedEntries;
          };
      };
      emptyValue.value = { };
      nestedTypes.elemType = elemType;
    };
in
{
  inherit conditionalAttrsOf isIfExists mkIfExists;
}

There’s also the “kubeValueType” which allows merging attrsets and lists either by index or by name.

kubeValueType
{
  conditionalAttrsOf,
  kubeLists,
  lib,
}:
let
  inherit (lib)
    any
    attrNames
    concatMap
    elem
    filter
    isList
    listToAttrs
    mkOptionType
    removeAttrs
    types
    ;

  checkDefsForError =
    check: definitions:
    if lib.all (definition: check definition.value) definitions then
      null
    else
      {
        message = "Definition values: ${
          lib.options.showDefs (lib.filter (definition: !check definition.value) definitions)
        }";
      };

  inherit (kubeLists)
    fromNamedAttrs
    fromNumberedAttrs
    isNamedList
    isNumberedList
    stripListMarker
    ;

  toNamedAttrs =
    list:
    listToAttrs (
      map (element: {
        inherit (element) name;
        value = removeAttrs element [ "name" ];
      }) list
    );

  toNumberedAttrs =
    list:
    listToAttrs (
      lib.imap0 (index: value: {
        name = toString index;
        inherit value;
      }) list
    );

  namedListOf =
    elemType:
    let
      attrsType = types.attrsOf elemType;
      listType = types.listOf elemType;
    in
    mkOptionType rec {
      name = "namedListOf";
      description = "list of ${elemType.description}, or an mkNamedList/mkNumberedList-tagged attrset";
      check = {
        __functor = _: value: listType.check value || isNamedList value || isNumberedList value;
        isV2MergeCoherent = true;
      };
      merge = rec {
        __functor =
          self: loc: defs:
          (self.v2 { inherit loc defs; }).value;
        v2 =
          { loc, defs }:
          let
            anyNamed = any (definition: isNamedList definition.value) defs;
            anyNumbered = any (definition: isNumberedList definition.value) defs;
            orderedNamedKeys = concatMap (
              definition:
              if isNamedList definition.value then
                filter (key: ((definition.value.${key} or null)._type or null) == "order") (
                  attrNames (stripListMarker definition.value)
                )
              else
                [ ]
            ) defs;
            transformDefinitions =
              transform: map (definition: definition // { value = transform definition.value; }) defs;
            attrsEvaluation =
              transform: lib.modules.mergeDefinitions loc attrsType (transformDefinitions transform);
          in
          if anyNamed && anyNumbered then
            throw ''
              The option `${lib.showOption loc}' has both an mkNamedList and an
              mkNumberedList definition. Use only one of the two for a field.
            ''
          else if anyNamed && orderedNamedKeys != [ ] then
            throw ''
              The option `${lib.showOption loc}' uses ordering properties on
              named-list entries: ${lib.concatStringsSep ", " orderedNamedKeys}.
              Use mkNumberedList when list order must be addressed explicitly.
            ''
          else if anyNamed then
            let
              evaluation = attrsEvaluation (
                value: if isList value then toNamedAttrs value else stripListMarker value
              );
              merged = evaluation.mergedValue;
              listKeys = concatMap (
                definition: if isList definition.value then map (element: element.name) definition.value else [ ]
              ) defs;
              newKeys = filter (key: !(elem key listKeys)) (attrNames merged);
              order = filter (key: merged ? ${key}) (lib.unique (listKeys ++ newKeys));
            in
            {
              headError = checkDefsForError check defs;
              value = fromNamedAttrs order merged;
              valueMeta.list = map (key: evaluation.checkedAndMerged.valueMeta.attrs.${key}) order;
            }
          else if anyNumbered then
            let
              evaluation = attrsEvaluation (
                value: if isList value then toNumberedAttrs value else stripListMarker value
              );
              merged = evaluation.mergedValue;
              order = map (entry: entry.name) (
                lib.sort (a: b: (lib.toInt a.name) < (lib.toInt b.name)) (lib.attrsToList merged)
              );
            in
            {
              headError = checkDefsForError check defs;
              value = fromNumberedAttrs merged;
              valueMeta.list = map (key: evaluation.checkedAndMerged.valueMeta.attrs.${key}) order;
            }
          else
            (lib.modules.mergeDefinitions loc listType defs).checkedAndMerged;
      };
      nestedTypes.elemType = elemType;
    };

  objectType = types.addCheck (conditionalAttrsOf valueType) (
    value: !(isNamedList value) && !(isNumberedList value)
  );

  # types.nullOr still uses the legacy merge protocol. Keeping null as a v2
  # oneOf branch prevents it from discarding nested valueMeta for the entire
  # recursive Kubernetes value type.
  nullType = mkOptionType rec {
    name = "null";
    description = "null";
    check = {
      __functor = _: value: value == null;
      isV2MergeCoherent = true;
    };
    merge = rec {
      __functor =
        self: loc: defs:
        (self.v2 { inherit loc defs; }).value;
      v2 =
        { defs, ... }:
        {
          headError = checkDefsForError check defs;
          value = null;
          valueMeta = { };
        };
    };
    emptyValue.value = null;
  };

  baseType = types.oneOf [
    nullType
    types.bool
    types.int
    types.float
    types.str
    types.path
    (namedListOf valueType)
    objectType
  ];

  valueType = baseType // {
    description = "Kubernetes-shaped JSON value with explicit mkNamedList/mkNumberedList override support";
    emptyValue.value = null;
  };
in
valueType
1 Like

I saw easykubenix! Made Nixifest because I want to use lots of CRDs scoped by apiVersion. Will check into those custom types of yours, though!

I also want to make my own Terraform JSON generator for Nix (because Tofunix requires you to input the provider list at the root eval call, instead of putting a type generator in specialArgs to IFD (or pre-bake) in imports as intended in Nixifest)! And have worked on more Nix-native container support, see:

1 Like

I actually once worked on terranix-next which was sort of like terranix but it converts a terraform provider json schema into an evalModules, but I ended up not finishing it because Nix eval is far too slow for any big provider.

EDIT: I now see that your Tofunix is rather similar to the think I was hacking on 3-4 years ago, but you actually ended up shipping it, awesome. I do wonder, if it would benefit from using Adios instead of NixOS’s evalModules.

2 Likes

so like nix-to-kubernetes/flake.nix at da68e8c0274e1eea7ce66b64ffed1b1bf3e7c5c1 · DeterminateSystems/nix-to-kubernetes · GitHub?

terraform does support raw json, so the simplest form of this (untyped) is just builtins.toJSON

if eval is your bottleneck, you may also be interested in:

1 Like

Better CRD support than Kubenix

out of curiosity, after having learned to love kubenix recently: what does better mean specifically and what was the motivation to build a solution ground-up rather than improving what’s already there in kubenix?

1 Like

Basically, to generate types for CRDs in Kubenix, I have to jump through a lot of hoops. With Nixifest, I can just put in resources.<apiVersion>.<kind>.<name> for any apiVersion and kind, and use strict validation with typegen and have everything fully typechecked.
On “improving kubenix”, I don’t like the fact that it messes with capitalization and plurals (e.g. configMaps instead of ConfigMap, which makes CRD typegen weird) and also it keeps everything toplevel, when kinds in K8S are scoped by apiVersion. Actually looking deeper into it, it looks not as bad as I thought and I could maybe be using it better, but I’d still rather have something that is always closer to the raw K8S YAML it outputs.

The problem with that is that it puts the Nix store into the image, but with some clever container-runtime-specific hacks you can mount store paths from the host INTO the container and avoid the disk space duplication. Only if your hosts are running Nix, though. Essentially you just have to figure out how to make your preferred container runtime do what nix-snapshotter does (potentially with a container-libs patch ;).

Yep, just need to hook up the Hard Part™, module system and typegen (passed through specialArgs ofc) (most likely not that hard, but relative to using just freeformType = lib.types.attrsOf lib.types.anything; and builtins.toJSON ;).

1 Like

No, more like nix-snapshotter or “nix copy with hardlinks and mounts”. Building OCI images from Nix closures is wasteful.

3 Likes

might i ask what this patch does and how it relates to the plugin you linked earlier?

1 Like

nix-snapshotter plugs into containerd, nix-storage-plugin plugs into CRI-O.

1 Like

container-libs has support for “additional layer stores” for lazy pulling (e.g. Stargz), but it requires a TOC digest, but this is too limited for supporting nix-snapshotter images (which is the goal of nix-storage-plugin). But I work around this by patching it to check more than just the TOC digest, it will check against the compressed-digest and diff-digest, which are different because nix-snapshotter makes an mostly empty tar with only the list of Nix store paths prefixed with /nix/store.
Then you build Podman/CRI-O against the patched container-libs, and configure NSP as an ALS and registry aliases for nix:0/nix/store image resolution.

See this issue:

Also “plugin” is a bit of a misnomer as there’s no real concept of a “plugin”, it’s essentially just a FUSE ALS and a container registry for resolving nix:0/nix/store/ images, I just called it that because other names would have been much longer or too unclear.

1 Like

I’m thinking of replacing the current resources."<group>/<version>".<kind> interface with resources.<group>.<version>.<kind>. Just want to make sure everyone is OK with making this change as fully breaking, or if it’s worth trying to implement backwards compat for it.

related: https://ethancedwards.com/latex/presentations/2026-def-con-nix-vegas/infra-as-dag/slides.pdf

1 Like

Nobody complained here, so I made this change:

Interesting approach on the GitOps side! Nixifest is only concerned about giving you a YAML file you can apply to a cluster however you want (kubectl apply -f, Flux, etc.).

1 Like