Porting VMware Workstation to NixOS

Hey folks!

I am currently working on porting VMware Workstation to NixOS and am looking for help and suggestions on how I can combat an issue I’m facing.

With this package I’m relying on a FHS user environment and have used a lot of useful code from the ArchLinux AUR package.

The problem I’m facing here though is that I need the suid bit set on the vmware binary or the pkexec binary but it looks like an FHS limitation is that this cannot be done even with security wrappers.

I did initially try to use patchelf but patching the vmware binary itself bombed out due to exceeding the maximum filesize so I feel like I’m a bit stuck on this right now.

Any help would be greatly appreciated! :slight_smile:

Overlay Code:

self: super:

let
  vmware-workstation-runtime = super.stdenv.mkDerivation rec {
    name = "vmware-workstation-runtime-${version}";
    version = "15.5.1-15018445";

    buildInputs = with super; [ bash libxslt libxml2 zlib ];
    nativeBuildInputs = with super; [ python3 sqlite ];

    src = super.fetchurl {
      url = "https://download3.vmware.com/software/wkst/file/VMware-Workstation-Full-${version}.x86_64.bundle";
      sha256 = "1ink8yhmjdk432gzlcpb9zqx9l2s0p0b5hamq40pdhklv3l8y3s9";
    };

    vmwareBundleUnpacker = super.fetchurl {
      url = https://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/eclass/vmware-bundle.eclass;
      sha256 = "d8794c22229afdeb698dae5908b7b2b3880e075b19be38e0b296bb28f4555163";
    };

    vmwareBootstrapConfig = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/vmware-bootstrap?h=vmware-workstation";
      sha256 = "0pnmfan4l6avhlsnasjn7h37iihri63biazd6a2qazldpxmb3rqj";
    };

    vmwareConfig = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/config?h=vmware-workstation";
      sha256 = "0lw4lqasrxha5dwhfakms5qfyh4046dx92qnj5skn0pdv611lfyf";
    };

    vmwareHostdConfig = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/config.xml?h=vmware-workstation";
      sha256 = "13w2v4synskp20x4wilpqgv89g2v0piw0n1izb80xpkxdj8x47yf";
    };

    vmwareHostdDatastores = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/datastores.xml?h=vmware-workstation";
      sha256 = "16ahdz36frh4qh5rjwv1g46vnx6qma55h2qfw9gbfdhd8jmd8k23";
    };

    vmwareHostdEnvironments = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/environments.xml?h=vmware-workstation";
      sha256 = "10pb5ml844i2c9nriss5z97rzc81xcwvzw3clwzy1qgw6wcfrsnc";
    };

    vmwareHostdProxy = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/proxy.xml?h=vmware-workstation";
      sha256 = "124al7ahnkr6m949yfw3qvw3zn3hdjf1v4xyxglab131c0ijb01w";
    };

    vmwareHostdVMAutostart = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/vmAutoStart.xml?h=vmware-workstation";
      sha256 = "0v8fmpcynw6q7z8ppcyfrdsiiliwa0ds0xz6wm6160v1pj62llmd";
    };

    vmwarePAMConfig = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/pam.d-vmware-authd?h=vmware-workstation";
      sha256 = "19rczamxwpm17ipfwa1brshhmdhyshc8x62xjrw520llzsis02nm";
    };

    vmwareVixBootstrapConfig = super.fetchurl {
      url = "https://aur.archlinux.org/cgit/aur.git/plain/vmware-vix-bootstrap?h=vmware-workstation";
      sha256 = "0hd1zvdpa8jkwqsrcr2n31cc9fd4xf8kijbz3in4dbkk9szrh5ns";
    };

    unpackPhase = ''
      # unpack installer
      csplit --suppress-matched -q $src /^exit$/ '{*}'
      tar xzf xx01 2>/dev/null || true
      mv install/vmware-installer vmware-installer
      rm -rf install/ xx0{0,1}

      # unpack components
      echo '
        ebegin(){
          echo "$1"
        }
        eend(){
          echo OK
        }
        source "${vmwareBundleUnpacker}"
        export T="$PWD"

        for COMP in \
          vmware-network-editor \
          vmware-network-editor-ui \
          vmware-ovftool \
          vmware-player-app \
          vmware-player-setup \
          vmware-tools-linux \
          vmware-tools-linuxPreGlibc25 \
          vmware-tools-netware \
          vmware-tools-solaris \
          vmware-tools-windows \
          vmware-tools-winPre2k \
          vmware-tools-winPreVista \
          vmware-usbarbitrator \
          vmware-virtual-printer \
          vmware-vix-core \
          vmware-vix-lib-Workstation1500 \
          vmware-vmx \
          vmware-vprobe \
          vmware-workstation \
          vmware-workstation-server; \
        do
          vmware-bundle_extract-bundle-component "$src" "$COMP" "./$COMP"
        done
      ' >./extractor.sh
       bash ./extractor.sh # this script doesnt like to work unless it's running from a file
    '';

    installPhase = ''
      # determine installer version
      local vmware_installer_version=$(cat "vmware-installer/manifest.xml" | grep -oPm1 "(?<=<version>)[^<]+")

      # create directory structure
      mkdir -p \
        $out/etc/{cups,pam.d,modprobe.d,profile.d,thnuclnt,vmware,vmware-installer} \
        $out/{bin,share} \
        $out/include/vmware-vix \
        $out/lib/{vmware/{setup,lib/libvmware-vim-cmd.so},vmware-vix,vmware-ovftool,vmware-installer/$vmware_installer_version,cups/filter,module-load.d} \
        $out/share/{doc/vmware-vix,licenses/vmware-workstation}

      # copy bin/
      cp -r \
        vmware-workstation/bin/* \
        vmware-vmx/{,s}bin/* \
        vmware-vix-core/bin/* \
        vmware-vprobe/bin/* \
        vmware-workstation-server/{vmware-hostd,vmware-vim-cmd,vmware-wssc-adminTool} \
        vmware-player-app/bin/* \
        $out/bin

      # copy etc/cups
      cp -r \
        vmware-player-app/etc/cups/* \
        $out/etc/cups

      # copy etc/thnuclnt
      cp -r \
        vmware-player-app/extras/.thnumod \
        $out/etc/thnuclnt

      # copy etc/vmware
      cp -r \
        vmware-vmx/extra/modules.xml \
        vmware-workstation-server/config/etc/vmware/* \
        vmware-workstation-server/etc/vmware/* \
        $out/etc/vmware

      # copy include/vmware-vix
      cp -r \
        vmware-vix-core/include/* \
        $out/include/vmware-vix

      # copy lib/cups/filter
      cp -r \
        vmware-player-app/extras/thnucups \
        $out/lib/cups/filter

      # copy lib/vmware
      cp -r \
        vmware-workstation/lib/* \
        vmware-player-app/lib/* \
        vmware-vmx/{lib/*,roms} \
        vmware-vprobe/lib/* \
        vmware-workstation-server/{bin,lib,hostd} \
        vmware-usbarbitrator/bin \
        vmware-network-editor/lib \
        $out/lib/vmware

      # copy lib/vmware/setup
      cp -r \
        vmware-player-setup/vmware-config \
        $out/lib/vmware/setup

      # copy lib/vmware-installer/VERSION
      cp -r \
        vmware-installer/{python,sopython,vmis,vmis-launcher,vmware-installer,vmware-installer.py} \
        $out/lib/vmware-installer/$vmware_installer_version

      # copy lib/vmware-ovftool
      cp -r \
        vmware-ovftool/* \
        $out/lib/vmware-ovftool

      # copy lib/vmware-vix
      cp -r \
        vmware-vix-lib-Workstation1500/lib/Workstation-15.0.0 \
        vmware-vix-core/{lib/*,vixwrapper-config.txt} \
        $out/lib/vmware-vix

      # copy share/
      cp -r \
        vmware-workstation/share/* \
        vmware-workstation/man \
        vmware-vmware-network-editor-ui/share/* \
        vmware-player-app/share/* \
        $out/share

      # copy share/doc/vmware-vix
      cp -r \
        vmware-vix-core/doc/* \
        $out/share/doc/vmware-vix

      # set file permissions
      chmod +x \
        $out/bin/* \
        $out/lib/vmware/bin/* \
        $out/lib/vmware/setup/* \
        $out/lib/vmware/lib/libvmware-gksu.so/gksu-run-helper \
        $out/lib/vmware-ovftool/{ovftool,ovftool.bin} \
        $out/lib/vmware-installer/$vmware_installer_version/{vmware-installer,vmis-launcher} \
        $out/lib/cups/filter/* \
        $out/lib/vmware-vix/setup/* \
        $out/etc/thnuclnt/.thnumod
      chmod 700 $out/etc/vmware/ssl
      chmod 600 $out/etc/vmware/ssl/*

      # create symlinks (replicate installer) - lib/vmware/bin
      for link in \
        licenseTool \
        vmplayer \
        vmware \
        vmware-app-control \
        vmware-enter-serial \
        vmware-fuseUI \
        vmware-gksu \
        vmware-hostd \
        vmware-modconfig \
        vmware-modconfig-console \
        vmware-mount \
        vmware-netcfg \
        vmware-setup-helper \
        vmware-tray \
        vmware-vim-cmd \
        vmware-vmblock-fuse \
        vmware-vprobe \
        vmware-virtual-printer \
        vmware-wssc-adminTool \
        vmware-zenity
      do
        ln -s $out/lib/vmware/bin/appLoader $out/lib/vmware/bin/$link
      done

      # create symlinks (replicate installer) - bin
      for link in \
        vmware-fuseUI \
        vmware-mount \
        vmware-netcfg \
        vmware-usbarbitrator
      do
        ln -s $out/lib/vmware/bin/$link $out/bin/$link
      done

      # create symlinks (replicate installer) - misc
      ln -s $out/lib/vmware/bin/appLoader $out/bin/vmrest
      ln -s $out/lib/vmware/icu $out/etc/vmware/icu
      ln -s $out/lib/vmware/lib/diskLibWrapper.so/diskLibWrapper.so $out/lib/diskLibWrapper.so
      ln -s $out/lib/vmware/lib/libvmware-hostd.so/libvmware-hostd.so $out/lib/vmware/lib/libvmware-vim-cmd.so/libvmware-vim-cmd.so
      ln -s $out/lib/vmware-ovftool/ovftool $out/bin/ovftool
      ln -s $out/lib/vmware-vix/libvixAllProducts.so $out/lib/libvixAllProducts.so

      # create database of vmware guest tools and copy them in to place (avoids vmware having to fetch them later)
      local database_filename=$out/etc/vmware-installer/database
      touch $database_filename
      sqlite3 "$database_filename" "CREATE TABLE settings(key VARCHAR PRIMARY KEY, value VARCHAR NOT NULL, component_name VARCHAR NOT NULL);"
      sqlite3 "$database_filename" "INSERT INTO settings(key,value,component_name) VALUES('db.schemaVersion','2','vmware-installer');"
      sqlite3 "$database_filename" "CREATE TABLE components(id INTEGER PRIMARY KEY, name VARCHAR NOT NULL, version VARCHAR NOT NULL, buildNumber INTEGER NOT NULL, component_core_id INTEGER NOT NULL, longName VARCHAR NOT NULL, description VARCHAR, type INTEGER NOT NULL);"      

      for isoimage in linux linuxPreGlibc25 netware solaris windows winPre2k winPreVista; do
        local isoversion=$(ls -1 vmware-tools-$isoimage/.installer)
        sqlite3 "$database_filename" "INSERT INTO components(name,version,buildNumber,component_core_id,longName,description,type) VALUES(\"vmware-tools-$isoimage\",\"$isoversion\",\"${version}\",1,\"$isoimage\",\"$isoimage\",1);"
        install -Dm 644 vmware-tools-$isoimage/$isoimage.iso $out/lib/vmware/isoimages/$isoimage.iso
      done
      for isoimage in Linux Windows; do
        install -Dm 644 vmware-virtual-printer/VirtualPrinter-$isoimage.iso $out/lib/vmware/isoimages/VirtualPrinter-$isoimage.iso
      done

      # install licenses
      install -Dm 644 vmware-workstation/doc/EULA $out/share/doc/vmware-workstation/EULA
      ln -s $out/share/doc/vmware-workstation/EULA "$out/share/licenses/vmware-workstation/VMware Workstation - EULA.txt"
      ln -s $out/lib/vmware-ovftool/vmware.eula "$out/share/licenses/vmware-workstation/VMware OVF Tool - EULA.txt"
      install -Dm 644 vmware-workstation/doc/open_source_licenses.txt "$out/share/licenses/vmware-workstation/VMware Workstation open source license.txt"
      install -Dm 644 vmware-workstation/doc/ovftool_open_source_licenses.txt "$out/share/licenses/vmware-workstation/VMware OVF Tool open source license.txt"
      install -Dm 644 vmware-vix-core/open_source_licenses.txt "$out/share/licenses/vmware-workstation/VMware VIX open source license.txt"
      rm -f $out/lib/vmware-ovftool/{vmware-eula.rtf,open_source_licenses.txt,manifest.xml}

      # install config files
      install -d -m 755 $out/lib/vmware-installer/$vmware_installer_version/{lib/lib,artwork}
      install -Dm 644 vmware-vmx/extra/modules.xml $out/lib/vmware/modules/modules.xml
      install -Dm 644 vmware-installer/bootstrap $out/etc/vmware-installer/bootstrap
      install -Dm 644 $vmwareVixBootstrapConfig $out/etc/vmware-vix/bootstrap
      install -Dm 644 $vmwareBootstrapConfig $out/etc/vmware/bootstrap
      install -Dm 644 $vmwareConfig $out/etc/vmware/config
      install -Dm 644 $vmwareHostdConfig $out/etc/vmware/hostd/config.xml
      install -Dm 644 $vmwareHostdDatastores $out/etc/vmware/hostd/datastores.xml
      install -Dm 644 $vmwareHostdEnvironments $out/etc/vmware/hostd/environments.xml
      install -Dm 644 $vmwareHostdProxy $out/etc/vmware/hostd/proxy.xml
      install -Dm 644 $vmwareHostdVMAutostart $out/etc/vmware/hostd/vmAutoStart.xml
      install -Dm 644 $vmwarePAMConfig $out/etc/pam.d/vmware-authd

      # remove unused application definitions (we create our own)
      rm -rf $out/share/{applications,icons}

      # replace placeholder vars with real paths
      for file in gtk-3.0/gdk-pixbuf.loaders; do
        sed -i 's,@@LIBCONF_DIR@@,/usr/lib/vmware/libconf,g' $out/lib/vmware/libconf/etc/$file
      done
      sed -i 's,@@AUTHD_PORT@@,902,' $out/lib/vmware/hostd/docroot/client/clients.xml
      sed \
        -e "s,@@VERSION@@,$vmware_installer_version," \
        -e "s,@@VMWARE_INSTALLER@@,/usr/lib/vmware-installer/$vmware_installer_version," \
        -i $out/etc/vmware-installer/bootstrap

      # patch out kernel module checks
      for program in vmware vmplayer vmware-tray; do
        sed -e 's,if "$BINDIR"/vmware-modconfig --appname=.*,if true ||,' -i $out/bin/$program
      done
    '';
  };
in
{
  vmware-workstation = super.buildFHSUserEnv {
    name = "vmware-workstation";

    profile = ''
      export FONTCONFIG_FILE=/etc/fonts/fonts.conf
    '';

    targetPkgs = pkgs: with super.pkgs; [
      fuse
      gcr
      glib
      gnome3.gtkmm
      gtk3
      libaio
      libcanberra
      libuuid
      pcsclite
      polkit
      vmware-workstation-runtime
      xorg.libX11
      zlib
    ];

    runScript = "${vmware-workstation-runtime}/bin/vmware";
  };
}

Do you need the setuid bit at build time or runtime?

At build time it’s definitely restricted but runtime it might be possible to poke a hole in the buildFHS sandbox. In https://github.com/NixOS/nixpkgs/blob/86ed15dcce7de9c9cac5755663b622142a89d76d/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c#L21 , /run/wrappers doesn’t seem to be listed so maybe it’s possible to add it as a bind-mount option.

Thanks for the advice @zimbatm!

I did look in to this but it looks like /run is bound by default and so that doesn’t seem to be the issue.

I see more info about this over at https://github.com/NixOS/nixpkgs/issues/69338 so this is a bit of a dead end at the minute by the looks of things.

I’m open to ideas on how I could proceed but with the maximum filesize limit on patchelf and an instability to sudo in an FHS env - I really have no idea what my next step could be here.