Podman rootless with systemd

UPDATE 2024-12-30 for Nixos 24.11

As a reminder, i want systemd user service container :

  • the container MUST be started when server start.
  • the container MUST be handle by a specific (non root) user.

The example bellow show 2 containers (postgres/pgadmin), started on server start. They can be managed by the dedicated “postgres” user (non-root)

cat podmanworkaround.nix

#
# PODMAN WORKAROUND
# when default network is pasta
# it requires the host network interfaces all setup
#
{ config, pkgs, ... }:

{

    systemd.user.services.podmanworkaround = {
        enable = true;
        description = "User-level proxy to system-level multi-user.target";
        documentation = [
            "https://github.com/containers/podman/issues/22197#issuecomment-2564715584"
            "man:podman-systemd.unit(5)"
        ];
        wantedBy = [ "default.target" ];
        after = [ "network.target" ];
        path = [ "/run/wrappers" ];
        serviceConfig = {
            Type = "oneshot";
            # Set a timeout as by default oneshot does not have one and in case network-online.target
            # never comes online we do not want to block forever, 90s is the default systemd unit timeout.
            TimeoutStartSec = 90;
            ExecStart = "${pkgs.bash}/bin/bash -c 'until systemctl is-active multi-user.target; do echo wait for multi-user.target; sleep 0.5; done; echo multi-user.target is active'";
            RemainAfterExit = "yes";
        };
    };

    systemd.user.targets.podmanworkaround = {
        enable = true;
        description = "User-level multi-user.target";
        documentation = [
            "https://github.com/containers/podman/issues/22197#issuecomment-2564715584"
            "man:podman-systemd.unit(5)"
        ];
        requires = [ "podmanworkaround.service" ];
        wants = [ "podmanworkaround.service" ];
        after = [ "podmanworkaround.service" ];
    };

}

cat postgres.nix

#
# POSTGRES service
#
{ config, pkgs, ... }:

{
    # create postgres user
    users.users."postgres" = {
        isNormalUser = true;
        linger = true; # lingering is the all tricks which enable user services with ConditionUser to be started on server start
        autoSubUidGidRange = true;
    };


    ## postgres server
    systemd.user.services.postgres = {
        enable = true;
        unitConfig = { ConditionUser = "postgres"; }; # only the opening user session "postgres" will trigger this service to start. He will be able to manage the container too
        wantedBy = [ "default.target" ];
        requires = [ "podmanworkaround.target" ]; # https://github.com/containers/podman/issues/22197#issuecomment-2564715584
        after = [ "podmanworkaround.target" ];    # https://github.com/containers/podman/issues/22197#issuecomment-2564715584
        description = "postgres container";
        path = [ "/run/wrappers" ];
        serviceConfig = 
        let
            podmancli = "${config.virtualisation.podman.package}/bin/podman";
			cid = "%t/podman/%n.cid";
            podname = "postgres";
            image = "postgres:16.4";
            options = "-p 5432:5432 " +
                      "-v postgres_data:/var/lib/postgresql/data " +
                      "-e POSTGRES_PASSWORD='toto' " +
                      "-e TZ='Europe/Paris' ";
            startpre = [
            	"${pkgs.coreutils-full}/bin/rm -f ${cid}"
            ];
            stoppost = [
                "${podmancli} rm --volumes --force --ignore --cidfile=${cid}"
            ];
        in
        {
            ExecStartPre = startpre;
            ExecStart = "${podmancli} run " +
		                "--rm " +
		                "--replace " +
		                "--name=${podname} " +
		                "--cgroups=split " +
		                "--sdnotify=conmon " +
		                "--log-driver=journald " +
		                "--name=${podname} " +
		                options +
		                "-d " +
		                "${image}";
            ExecStop = "${podmancli} stop --cidfile=${cid}";
            ExecStopPost = stoppost;
            Delegate="yes";
            Type = "notify";
            NotifyAccess = "all";
            SyslogIdentifier="%N";
            Restart = "no";
            TimeoutStopSec = 70;
            KillMode = "mixed"; # https://unix.stackexchange.com/a/714428
        };
    };

    ## pgadmin server
    systemd.user.services.pgadmin = {
        enable = true;
        unitConfig = { ConditionUser = "postgres"; }; # only the opening user session "postgres" will trigger this service to start. He will be able to manage the container too
        wantedBy = [ "default.target" ];
      	requires = [ "podmanworkaround.target" ]; # https://github.com/containers/podman/issues/22197#issuecomment-2564715584
        after = [ "podmanworkaround.target" ];    # https://github.com/containers/podman/issues/22197#issuecomment-2564715584
        description = "pgadmin container";
        path = [ "/run/wrappers" ];
        serviceConfig = 
        let
            podmancli = "${config.virtualisation.podman.package}/bin/podman";
			cid = "%t/podman/%n.cid";
            podname = "pgadmin";
            image = "dpage/pgadmin4:latest";
            options = "--network pasta:--map-guest-addr,169.254.1.2 --add-host host.containers.internal:169.254.1.2 --add-host host.docker.internal:169.254.1.2 " + # allow connecting to host from inside the container for podman v5.2.3. podman v5.3 do it by default
                      "-p 127.0.0.1:8050:80 " +
                      "-v pgadmin_data:/var/lib/pgadmin " +
                      "-e PGADMIN_DEFAULT_EMAIL='email@host.local' "+
                      "-e PGADMIN_DEFAULT_PASSWORD='toto' ";
            startpre = [
            	"${pkgs.coreutils-full}/bin/rm -f ${cid}"
            ];
            stoppost = [
                "${podmancli} rm --volumes --force --ignore --cidfile=${cid}"
            ];
        in
        {
            ExecStartPre = startpre;
            ExecStart = "${podmancli} run " +
		                "--rm " +
		                "--replace " +
		                "--name=${podname} " +
		                "--cidfile=${cid} " +
		                "--cgroups=split " +
		                "--sdnotify=conmon " +
		                "--log-driver=journald " +
		                "--name=${podname} " +
		                options +
		                "-d " +
		                "${image}";
            ExecStop = "${podmancli} stop ${podname}";
            ExecStopPost = stoppost;
            Delegate="yes";
            Type = "notify";
            NotifyAccess = "all";
            SyslogIdentifier="%N";
            Restart = "no";
            TimeoutStopSec = 70;
            KillMode = "mixed"; # https://unix.stackexchange.com/a/714428
        };
    };

}

Known issue : an issue is still existing from the beginning of nixos/podman (2021).

When the dedicated user manually call systemctl --user stop postgres everything is working as expect.

But when systemd itself call the stop (via a machine reboot/shutdown), there is sometimes a Kill which does not let podman properly stop the container. For images which ìnternaly use chown command (like postgres or pgadmin), the underlying volume is dirty with incorrect permissions. Then the start will throw error until the root chown the volume permissions to user uid.

some threads (1, 2) about this, but no workaround.

Help appreciate :grinning: