hardware acceleration and support for NVIDIA cards on nonfree images (#3089)

* add nvidia packages

* add nvidia deps to nonfree

* gpu_acceleration flag & nvidia hacking

* fix gpu_config & /tmp/lxc.log

* implement hardware acceleration more dynamically

* refactor OpenUI

* use mknod

* registry updates for multi-hardware-requirements

* pluralize

* handle new registry types

* remove log

* migrations and driver fixes

* wip

* misc patches

* handle nvidia-container differently

* chore: comments (#3093)

* chore: comments

* revert some sizing

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* Revert "handle nvidia-container differently"

This reverts commit d708ae53df.

* fix debian containers

* cleanup

* feat: add empty array placeholder in forms (#3095)

* fixes from testing, client side device filtering for better fingerprinting resistance

* fix mac builds

---------

Co-authored-by: Sam Sartor <me@samsartor.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>

build/dpkg-deps/depends

@@ -3,6 +3,7 @@ avahi-utils
 b3sum
 bash-completion
 beep
+binfmt-support
 bmon
 btrfs-progs
 ca-certificates
@@ -15,6 +16,7 @@ dnsutils
 dosfstools
 e2fsprogs
 ecryptfs-utils
+equivs
 exfatprogs
 flashrom
 fuse3

build/dpkg-deps/generate.sh

@@ -9,6 +9,9 @@ FEATURES+=("${ARCH}")
 if [ "$ARCH" != "$PLATFORM" ]; then
     FEATURES+=("${PLATFORM}")
 fi
+if [[ "$PLATFORM" =~ -nonfree$ ]]; then
+    FEATURES+=("nonfree")
+fi
 
 feature_file_checker='
 /^#/ { next }

build/dpkg-deps/nonfree.depends

@@ -0,0 +1,10 @@
++ firmware-amd-graphics
++ firmware-atheros
++ firmware-brcm80211
++ firmware-iwlwifi
++ firmware-libertas
++ firmware-misc-nonfree
++ firmware-realtek
++ nvidia-container-toolkit
+# + nvidia-driver
+# + nvidia-kernel-dkms

build/image-recipe/build.sh

@@ -73,7 +73,7 @@ if [ "$NON_FREE" = 1 ]; then
 	if [ "$IB_SUITE" = "bullseye" ]; then
 		ARCHIVE_AREAS="$ARCHIVE_AREAS non-free"
 	else
-		ARCHIVE_AREAS="$ARCHIVE_AREAS non-free-firmware"
+		ARCHIVE_AREAS="$ARCHIVE_AREAS non-free non-free-firmware"
 	fi
 fi
 
@@ -174,40 +174,123 @@ if [ "${IB_TARGET_PLATFORM}" = "rockchip64" ]; then
 	echo "deb https://apt.armbian.com/ ${IB_SUITE} main" > config/archives/armbian.list
 fi
 
-cat > config/archives/backports.pref <<- EOF
+if [ "$NON_FREE" = 1 ]; then
+	curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o config/archives/nvidia-container-toolkit.key
+	curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+		| sed 's#deb https://#deb [signed-by=/etc/apt/trusted.gpg.d/nvidia-container-toolkit.key.gpg] https://#g' \
+		> config/archives/nvidia-container-toolkit.list
+fi
+
+cat > config/archives/backports.pref <<-EOF
 Package: linux-image-*
 Pin: release n=${IB_SUITE}-backports
 Pin-Priority: 500
+
+Package: linux-headers-*
+Pin: release n=${IB_SUITE}-backports
+Pin-Priority: 500
+
+Package: *nvidia*
+Pin: release n=${IB_SUITE}-backports
+Pin-Priority: 500
 EOF
 
-# Dependencies
-
-## Firmware
-if [ "$NON_FREE" = 1 ]; then
-	echo 'firmware-iwlwifi firmware-misc-nonfree firmware-brcm80211 firmware-realtek firmware-atheros firmware-libertas firmware-amd-graphics' > config/package-lists/nonfree.list.chroot
-fi
+# Hooks
 
 cat > config/hooks/normal/9000-install-startos.hook.chroot << EOF
 #!/bin/bash
 
 set -e
 
+if [ "${NON_FREE}" = "1" ] && [ "${IB_TARGET_PLATFORM}" != "raspberrypi" ]; then
+    # install a specific NVIDIA driver version
+
+    # ---------------- configuration ----------------
+    NVIDIA_DRIVER_VERSION="\${NVIDIA_DRIVER_VERSION:-580.119.02}"
+
+    BASE_URL="https://download.nvidia.com/XFree86/Linux-${QEMU_ARCH}"
+
+    echo "[nvidia-hook] Using NVIDIA driver: \${NVIDIA_DRIVER_VERSION}" >&2
+
+    # ---------------- kernel version ----------------
+
+    # Determine target kernel version from newest /boot/vmlinuz-* in the chroot.
+    KVER="\$(
+        ls -1t /boot/vmlinuz-* 2>/dev/null \
+            | head -n1 \
+            | sed 's|.*/vmlinuz-||'
+    )"
+
+    if [ -z "\${KVER}" ]; then
+        echo "[nvidia-hook] ERROR: no /boot/vmlinuz-* found; cannot determine kernel version" >&2
+        exit 1
+    fi
+
+    echo "[nvidia-hook] Target kernel version: \${KVER}" >&2
+
+    # Ensure kernel headers are present
+	TEMP_APT_DEPS=(build-essential)
+    if [ ! -e "/lib/modules/\${KVER}/build" ]; then
+		TEMP_APT_DEPS+=(linux-headers-\${KVER})
+    fi
+
+    echo "[nvidia-hook] Installing build dependencies" >&2
+
+	/usr/lib/startos/scripts/install-equivs <<-EOF
+	Package: nvidia-depends
+	Version: \${NVIDIA_DRIVER_VERSION}
+	Section: unknown
+	Priority: optional
+	Depends: \${dep_list="\$(IFS=', '; echo "\${TEMP_APT_DEPS[*]}")"}
+	EOF
+
+    # ---------------- download and run installer ----------------
+
+    RUN_NAME="NVIDIA-Linux-${QEMU_ARCH}-\${NVIDIA_DRIVER_VERSION}.run"
+    RUN_PATH="/root/\${RUN_NAME}"
+    RUN_URL="\${BASE_URL}/\${NVIDIA_DRIVER_VERSION}/\${RUN_NAME}"
+
+    echo "[nvidia-hook] Downloading \${RUN_URL}" >&2
+    wget -O "\${RUN_PATH}" "\${RUN_URL}"
+    chmod +x "\${RUN_PATH}"
+
+    echo "[nvidia-hook] Running NVIDIA installer for kernel \${KVER}" >&2
+
+    sh "\${RUN_PATH}" \
+        --silent \
+        --kernel-name="\${KVER}" \
+        --no-x-check \
+        --no-nouveau-check \
+        --no-runlevel-check
+
+    # Rebuild module metadata
+    echo "[nvidia-hook] Running depmod for \${KVER}" >&2
+    depmod -a "\${KVER}"
+
+    echo "[nvidia-hook] NVIDIA \${NVIDIA_DRIVER_VERSION} installation complete for kernel \${KVER}" >&2
+
+    echo "[nvidia-hook] Removing build dependencies..." >&2
+	apt-get purge -y nvidia-depends
+	apt-get autoremove -y
+    echo "[nvidia-hook] Removed build dependencies." >&2
+fi
+
 cp /etc/resolv.conf /etc/resolv.conf.bak
 
 if [ "${IB_SUITE}" = trixie ] && [ "${IB_TARGET_ARCH}" != riscv64 ]; then
-	echo 'deb https://deb.debian.org/debian/ bookworm main' > /etc/apt/sources.list.d/bookworm.list
-	apt-get update
-	apt-get install -y postgresql-15
-	rm /etc/apt/sources.list.d/bookworm.list
-	apt-get update
-	systemctl mask postgresql
+    echo 'deb https://deb.debian.org/debian/ bookworm main' > /etc/apt/sources.list.d/bookworm.list
+    apt-get update
+    apt-get install -y postgresql-15
+    rm /etc/apt/sources.list.d/bookworm.list
+    apt-get update
+    systemctl mask postgresql
 fi
 
 if [ "${IB_TARGET_PLATFORM}" = "raspberrypi" ]; then
-	ln -sf /usr/bin/pi-beep /usr/local/bin/beep
-	KERNEL_VERSION=${RPI_KERNEL_VERSION} sh /boot/config.sh > /boot/config.txt
-	mkinitramfs -c gzip -o initrd.img-${RPI_KERNEL_VERSION}-rpi-v8 ${RPI_KERNEL_VERSION}-rpi-v8
-	mkinitramfs -c gzip -o initrd.img-${RPI_KERNEL_VERSION}-rpi-2712 ${RPI_KERNEL_VERSION}-rpi-2712
+    ln -sf /usr/bin/pi-beep /usr/local/bin/beep
+    KERNEL_VERSION=${RPI_KERNEL_VERSION} sh /boot/config.sh > /boot/config.txt
+    mkinitramfs -c gzip -o initrd.img-${RPI_KERNEL_VERSION}-rpi-v8 ${RPI_KERNEL_VERSION}-rpi-v8
+    mkinitramfs -c gzip -o initrd.img-${RPI_KERNEL_VERSION}-rpi-2712 ${RPI_KERNEL_VERSION}-rpi-2712
 fi
 
 useradd --shell /bin/bash -G startos -m start9
@@ -218,11 +301,11 @@ usermod -aG systemd-journal start9
 echo "start9 ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee "/etc/sudoers.d/010_start9-nopasswd"
 
 if [ "${IB_TARGET_PLATFORM}" != "raspberrypi" ]; then
-	/usr/lib/startos/scripts/enable-kiosk
+    /usr/lib/startos/scripts/enable-kiosk
 fi
 
 if ! [[ "${IB_OS_ENV}" =~ (^|-)dev($|-) ]]; then
-	passwd -l start9
+    passwd -l start9
 fi
 
 EOF
@@ -360,4 +443,4 @@ elif [ "${IMAGE_TYPE}" = img ]; then
 
 fi
 
-chown $IB_UID:$IB_UID $RESULTS_DIR/$IMAGE_BASENAME.*
+chown $IB_UID:$IB_UID $RESULTS_DIR/$IMAGE_BASENAME.*

build/image-recipe/raspberrypi/squashfs/boot/cmdline.txt

@@ -1 +1 @@
-usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u console=serial0,115200 console=tty1 root=PARTUUID=cb15ae4d-02 rootfstype=ext4 fsck.repair=yes rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory quiet boot=startos
+usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u console=serial0,115200 console=tty1 root=PARTUUID=cb15ae4d-02 rootfstype=ext4 fsck.repair=yes rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory boot=startos

build/lib/motd

@@ -4,7 +4,7 @@ parse_essential_db_info() {
     DB_DUMP="/tmp/startos_db.json"
 
     if command -v start-cli >/dev/null 2>&1; then
-        start-cli db dump > "$DB_DUMP" 2>/dev/null || return 1
+        timeout 30 start-cli db dump > "$DB_DUMP" 2>/dev/null || return 1
     else
         return 1
     fi

build/lib/scripts/install-equivs

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+export DEBCONF_NONINTERACTIVE_SEEN=true
+
+TMP_DIR=$(mktemp -d)
+
+(
+    set -e
+    cd $TMP_DIR
+
+    cat > control.equivs
+    equivs-build control.equivs
+    apt-get install -y ./*.deb < /dev/null
+)
+
+rm -rf $TMP_DIR
+
+echo Install complete. >&2
+exit 0

build/raspberrypi/make-image.sh

@@ -1,87 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function partition_for () {
-    if [[ "$1" =~ [0-9]+$ ]]; then
-        echo "$1p$2"
-    else
-        echo "$1$2"
-    fi
-}
-
-VERSION=$(cat VERSION.txt)
-ENVIRONMENT=$(cat ENVIRONMENT.txt)
-GIT_HASH=$(cat GIT_HASH.txt | head -c 7)
-DATE=$(date +%Y%m%d)
-
-ROOT_PART_END=7217792
-
-VERSION_FULL="$VERSION-$GIT_HASH"
-
-if [ -n "$ENVIRONMENT" ]; then
-  VERSION_FULL="$VERSION_FULL~$ENVIRONMENT"
-fi
-
-TARGET_NAME=startos-${VERSION_FULL}-${DATE}_raspberrypi.img
-TARGET_SIZE=$[($ROOT_PART_END+1)*512]
-
-rm -f $TARGET_NAME
-truncate -s $TARGET_SIZE $TARGET_NAME
-(
-    echo o
-    echo x
-    echo i
-    echo "0xcb15ae4d"
-    echo r
-    echo n
-    echo p
-    echo 1
-    echo 2048
-    echo 526335
-    echo t
-    echo c
-    echo n
-    echo p
-    echo 2
-    echo 526336
-    echo $ROOT_PART_END
-    echo a
-    echo 1
-    echo w
-) | fdisk $TARGET_NAME
-OUTPUT_DEVICE=$(sudo losetup --show -fP $TARGET_NAME)
-sudo mkfs.ext4 `partition_for ${OUTPUT_DEVICE} 2`
-sudo mkfs.vfat `partition_for ${OUTPUT_DEVICE} 1`
-
-TMPDIR=$(mktemp -d)
-
-sudo mount `partition_for ${OUTPUT_DEVICE} 2` $TMPDIR
-sudo mkdir $TMPDIR/boot
-sudo mount `partition_for ${OUTPUT_DEVICE} 1` $TMPDIR/boot
-sudo unsquashfs -f -d $TMPDIR startos.raspberrypi.squashfs
-REAL_GIT_HASH=$(cat $TMPDIR/usr/lib/startos/GIT_HASH.txt)
-REAL_VERSION=$(cat $TMPDIR/usr/lib/startos/VERSION.txt)
-REAL_ENVIRONMENT=$(cat $TMPDIR/usr/lib/startos/ENVIRONMENT.txt)
-sudo sed -i 's| boot=startos| init=/usr/lib/startos/scripts/init_resize\.sh|' $TMPDIR/boot/cmdline.txt
-sudo cp ./build/raspberrypi/fstab $TMPDIR/etc/
-sudo cp ./build/raspberrypi/init_resize.sh $TMPDIR/usr/lib/startos/scripts/init_resize.sh
-sudo umount $TMPDIR/boot
-sudo umount $TMPDIR
-sudo losetup -d $OUTPUT_DEVICE
-
-if [ "$ALLOW_VERSION_MISMATCH" != 1 ]; then
-    if [ "$(cat GIT_HASH.txt)" != "$REAL_GIT_HASH" ]; then
-        >&2 echo "startos.raspberrypi.squashfs GIT_HASH.txt mismatch"
-        >&2 echo "expected $REAL_GIT_HASH (dpkg) found $(cat GIT_HASH.txt) (repo)"
-        exit 1
-    fi
-    if [ "$(cat VERSION.txt)" != "$REAL_VERSION" ]; then
-        >&2 echo "startos.raspberrypi.squashfs VERSION.txt mismatch"
-        exit 1
-    fi
-    if [ "$(cat ENVIRONMENT.txt)" != "$REAL_ENVIRONMENT" ]; then
-        >&2 echo "startos.raspberrypi.squashfs ENVIRONMENT.txt mismatch"
-        exit 1
-    fi
-fi

build/upload-ota.sh

@@ -15,10 +15,10 @@ if [ "$SKIP_DL" != "1" ]; then
     fi
 
     if [ -n "$RUN_ID" ]; then
-        for arch in aarch64 aarch64-nonfree riscv64 x86_64 x86_64-nonfree raspberrypi; do
+        for arch in aarch64 aarch64-nonfree riscv64 riscv64-nonfree x86_64 x86_64-nonfree raspberrypi; do
             while ! gh run download -R Start9Labs/start-os $RUN_ID -n $arch.squashfs -D $(pwd); do sleep 1; done
         done
-        for arch in aarch64 aarch64-nonfree riscv64 x86_64 x86_64-nonfree; do
+        for arch in aarch64 aarch64-nonfree riscv64 riscv64-nonfree x86_64 x86_64-nonfree; do
             while ! gh run download -R Start9Labs/start-os $RUN_ID -n $arch.iso -D $(pwd); do sleep 1; done
         done
         while ! gh run download -R Start9Labs/start-os $RUN_ID -n raspberrypi.img -D $(pwd); do sleep 1; done
@@ -69,7 +69,7 @@ elif [ "$SKIP_UL" != "1" ]; then
 fi
 
 if [ "$SKIP_INDEX" != "1" ]; then
-    for arch in aarch64 aarch64-nonfree x86_64 x86_64-nonfree; do
+    for arch in aarch64 aarch64-nonfree riscv64 riscv64-nonfree x86_64 x86_64-nonfree; do
         for file in *_$arch.squashfs *_$arch.iso; do
             start-cli --registry=https://alpha-registry-x.start9.com registry os asset add --platform=$arch --version=$VERSION $file https://github.com/Start9Labs/start-os/releases/download/v$VERSION/$(echo -n "$file" | sed 's/~/./g')
         done