wip

core/src/context/rpc.rs

@@ -27,8 +27,10 @@ use crate::context::config::ServerConfig;
 use crate::db::model::Database;
 use crate::db::model::package::TaskSeverity;
 use crate::disk::OsPartitionInfo;
-use crate::disk::mount::filesystem::ReadOnly;
 use crate::disk::mount::filesystem::bind::Bind;
+use crate::disk::mount::filesystem::block_dev::BlockDev;
+use crate::disk::mount::filesystem::loop_dev::LoopDev;
+use crate::disk::mount::filesystem::{FileSystem, ReadOnly};
 use crate::disk::mount::guard::MountGuard;
 use crate::init::{InitResult, check_time_is_synchronized};
 use crate::install::PKG_ARCHIVE_DIR;
@@ -49,7 +51,7 @@ use crate::service::effects::subcontainer::NVIDIA_OVERLAY_PATH;
 use crate::shutdown::Shutdown;
 use crate::util::Invoke;
 use crate::util::future::NonDetachingJoinHandle;
-use crate::util::io::delete_file;
+use crate::util::io::{TmpDir, delete_file};
 use crate::util::lshw::LshwDevice;
 use crate::util::sync::{SyncMutex, SyncRwLock, Watch};
 use crate::{ActionId, DATA_DIR, PLATFORM, PackageId};
@@ -174,35 +176,82 @@ impl RpcContext {
         tracing::info!("Initialized Net Controller");
 
         if PLATFORM.ends_with("-nonfree") {
-            if let Err(e) = Command::new("nvidia-modprobe")
+            if let Err(e) = Command::new("nvidia-smi")
                 .invoke(ErrorKind::ParseSysInfo)
                 .await
             {
                 tracing::warn!("nvidia-modprobe: {e}");
                 tracing::info!("The above warning can be ignored if no NVIDIA card is present");
             } else {
-                if let Some(procfs) = MountGuard::mount(
-                    &Bind::new("/proc"),
-                    Path::new(NVIDIA_OVERLAY_PATH).join("proc"),
-                    ReadOnly,
-                )
-                .await
-                .log_err()
-                {
-                    Command::new("nvidia-container-cli")
-                        .arg("configure")
-                        .arg("--no-devbind")
-                        .arg("--no-cgroups")
-                        .arg("--utility")
-                        .arg("--compute")
-                        .arg("--graphics")
-                        .arg("--video")
-                        .arg(NVIDIA_OVERLAY_PATH)
-                        .invoke(ErrorKind::Unknown)
-                        .await
-                        .log_err();
-                    procfs.unmount(true).await.log_err();
+                async {
+                    let version: InternedString = String::from_utf8(
+                        Command::new("modinfo")
+                            .arg("-F")
+                            .arg("version")
+                            .arg("nvidia")
+                            .invoke(ErrorKind::ParseSysInfo)
+                            .await?,
+                    )?
+                    .trim()
+                    .into();
+                    let sqfs = Path::new("/media/startos/data/package-data/nvidia")
+                        .join(&*version)
+                        .join("container-overlay.squashfs");
+                    if tokio::fs::metadata(&sqfs).await.is_err() {
+                        let tmp = TmpDir::new().await?;
+                        let procfs = MountGuard::mount(
+                            &Bind::new("/proc"),
+                            Path::new(&*tmp).join("proc"),
+                            ReadOnly,
+                        )
+                        .await?;
+                        Command::new("nvidia-container-cli")
+                            .arg("configure")
+                            .arg("--no-devbind")
+                            .arg("--no-cgroups")
+                            .arg("--utility")
+                            .arg("--compute")
+                            .arg("--graphics")
+                            .arg("--video")
+                            .arg(&*tmp)
+                            .invoke(ErrorKind::Unknown)
+                            .await?;
+                        procfs.unmount(true).await?;
+                        Command::new("ln")
+                            .arg("-rsf")
+                            .arg(
+                                tmp.join("usr/lib64/libnvidia-ml.so")
+                                    .with_added_extension(&*version),
+                            )
+                            .arg(tmp.join("usr/lib64/libnvidia-ml.so.1"))
+                            .invoke(ErrorKind::Filesystem)
+                            .await?;
+                        Command::new("chown")
+                            .arg("-R")
+                            .arg("100000:100000")
+                            .arg(&*tmp)
+                            .invoke(ErrorKind::Filesystem)
+                            .await?;
+                        if let Some(p) = sqfs.parent() {
+                            tokio::fs::create_dir_all(p)
+                                .await
+                                .with_ctx(|_| (ErrorKind::Filesystem, format!("mkdir -p {p:?}")))?;
+                        }
+                        Command::new("mksquashfs")
+                            .arg(&*tmp)
+                            .arg(&sqfs)
+                            .invoke(ErrorKind::Filesystem)
+                            .await?;
+                        tmp.unmount_and_delete().await?;
+                    }
+                    BlockDev::new(&sqfs)
+                        .mount(NVIDIA_OVERLAY_PATH, ReadOnly)
+                        .await?;
+
+                    Ok::<_, Error>(())
                 }
+                .await
+                .log_err();
             }
         }
 

core/src/disk/mount/util.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 use tracing::instrument;
 
 use crate::Error;
+use crate::prelude::*;
 use crate::util::Invoke;
 
 pub async fn is_mountpoint(path: impl AsRef<Path>) -> Result<bool, Error> {
@@ -56,3 +57,42 @@ pub async fn unmount<P: AsRef<Path>>(mountpoint: P, lazy: bool) -> Result<(), Er
         .await?;
     Ok(())
 }
+
+/// Unmounts all mountpoints under (and including) the given path, in reverse
+/// depth order so that nested mounts are unmounted before their parents.
+#[instrument(skip_all)]
+pub async fn unmount_all_under<P: AsRef<Path>>(path: P, lazy: bool) -> Result<(), Error> {
+    let path = path.as_ref();
+    let canonical_path = tokio::fs::canonicalize(path)
+        .await
+        .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("canonicalize {path:?}")))?;
+
+    let mounts_content = tokio::fs::read_to_string("/proc/mounts")
+        .await
+        .with_ctx(|_| (ErrorKind::Filesystem, "read /proc/mounts"))?;
+
+    // Collect all mountpoints under our path
+    let mut mountpoints: Vec<&str> = mounts_content
+        .lines()
+        .filter_map(|line| {
+            let mountpoint = line.split_whitespace().nth(1)?;
+            // Check if this mountpoint is under our target path
+            let mp_path = Path::new(mountpoint);
+            if mp_path.starts_with(&canonical_path) {
+                Some(mountpoint)
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    // Sort by path length descending so we unmount deepest first
+    mountpoints.sort_by(|a, b| b.len().cmp(&a.len()));
+
+    for mountpoint in mountpoints {
+        tracing::debug!("Unmounting nested mountpoint: {}", mountpoint);
+        unmount(mountpoint, lazy).await?;
+    }
+
+    Ok(())
+}

core/src/service/effects/subcontainer/mod.rs

@@ -116,6 +116,8 @@ pub async fn create_subcontainer_fs(
         tracing::info!("Mounting overlay {guid} for {image_id}");
         let subcontainer_wrapper = Subcontainer {
             overlay: OverlayGuard::mount_layers(
+                &[],
+                image,
                 if context
                     .seed
                     .persistent_container
@@ -130,8 +132,6 @@ pub async fn create_subcontainer_fs(
                 } else {
                     &[]
                 },
-                image,
-                &[],
                 &mountpoint,
             )
             .await?,

core/src/util/io.rs

@@ -892,6 +892,16 @@ impl TmpDir {
         Ok(())
     }
 
+    pub fn leak(mut self) {
+        std::mem::take(&mut self.path);
+    }
+
+    pub async fn unmount_and_delete(self) -> Result<(), Error> {
+        crate::disk::mount::util::unmount_all_under(&self.path, false).await?;
+        tokio::fs::remove_dir_all(&self.path).await?;
+        Ok(())
+    }
+
     pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
         if let Ok(dir) = Arc::try_unwrap(self) {
             dir.delete().await