From b9c4cd2d967e443fd8f7f388957c851538d44748 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 31 Dec 2025 19:38:09 -0700 Subject: [PATCH] use mknod --- build/dpkg-deps/nonfree.depends | 2 + build/image-recipe/build.sh | 8 +- core/src/lxc/mod.rs | 295 ++++++++++++++++++++------------ core/src/util/io.rs | 28 ++- 4 files changed, 223 insertions(+), 110 deletions(-) diff --git a/build/dpkg-deps/nonfree.depends b/build/dpkg-deps/nonfree.depends index a154dcfb5..232d27df2 100644 --- a/build/dpkg-deps/nonfree.depends +++ b/build/dpkg-deps/nonfree.depends @@ -5,3 +5,5 @@ + firmware-libertas + firmware-misc-nonfree + firmware-realtek ++ nvidia-driver ++ nvidia-kernel-dkms \ No newline at end of file diff --git a/build/image-recipe/build.sh b/build/image-recipe/build.sh index 443520ce6..99cb3930b 100755 --- a/build/image-recipe/build.sh +++ b/build/image-recipe/build.sh @@ -174,14 +174,14 @@ if [ "${IB_TARGET_PLATFORM}" = "rockchip64" ]; then echo "deb https://apt.armbian.com/ ${IB_SUITE} main" > config/archives/armbian.list fi -curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o config/archives/nvidia.key -curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ - sed 's#deb https://#deb [signed-by=/etc/apt/trusted.gpg.d/nvidia.key.gpg] https://#g' > config/archives/nvidia.list - cat > config/archives/backports.pref <<- EOF Package: linux-image-* Pin: release n=${IB_SUITE}-backports Pin-Priority: 500 + +Package: nvidia-kernel-dkms +Pin: release n=${IB_SUITE}-backports +Pin-Priority: 500 EOF # Dependencies diff --git a/core/src/lxc/mod.rs b/core/src/lxc/mod.rs index bcecf9a50..d2d3aaeae 100644 --- a/core/src/lxc/mod.rs +++ b/core/src/lxc/mod.rs @@ -29,7 +29,7 @@ use crate::disk::mount::util::unmount; use crate::prelude::*; use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::service::ServiceStats; -use crate::util::io::open_file; +use crate::util::io::{open_file, write_file_owned_atomic}; use crate::util::rpc_client::UnixRpcClient; use crate::util::{FromStrParser, Invoke, new_guid}; use crate::{InvalidId, PackageId}; @@ -39,8 +39,7 @@ const RPC_DIR: &str = "media/startos/rpc"; // must not be absolute path pub const CONTAINER_RPC_SERVER_SOCKET: &str = "service.sock"; // must not be absolute path pub const HOST_RPC_SERVER_SOCKET: &str = "host.sock"; // must not be absolute path const CONTAINER_DHCP_TIMEOUT: Duration = Duration::from_secs(30); -const HARDWARE_ACCELERATION_PATHS: &[&str] = - &["/dev/dri/", "/dev/nvidia", "/dev/media", "/dev/video"]; +const HARDWARE_ACCELERATION_PATHS: &[&str] = &["/dev/dri/", "/dev/nvidia"]; #[derive( Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord, Hash, TS, @@ -159,91 +158,12 @@ impl LxcManager { } } -fn handle_devices<'a>( - guid: &'a str, - rootfs: &'a Path, - mut dir: ReadDir, - guards: &'a mut Vec, - matches: &'a [&'a str], -) -> BoxFuture<'a, Result<(), Error>> { - use std::os::linux::fs::MetadataExt; - use std::os::unix::fs::FileTypeExt; - async move { - while let Some(ent) = dir.next_entry().await? { - let path = ent.path(); - if let Some(matches) = if matches.is_empty() { - Some(Vec::new()) - } else { - let mut new_matches = Vec::new(); - for m in matches { - if if m.ends_with("/") { - path.starts_with(m) - } else { - path.to_string_lossy().starts_with(*m) - } || Path::new(*m).starts_with(&path) - { - new_matches.push(*m); - } - } - if new_matches.is_empty() { - None - } else { - Some(new_matches) - } - } { - let meta = ent.metadata().await?; - let ty = meta.file_type(); - if ty.is_dir() { - handle_devices( - guid, - rootfs, - tokio::fs::read_dir(&path) - .await - .with_ctx(|_| (ErrorKind::Filesystem, format!("readdir {path:?}")))?, - guards, - &matches, - ) - .await?; - } else { - let ty = if ty.is_char_device() { - "c" - } else if ty.is_block_device() { - "b" - } else { - continue; - }; - let rdev = meta.st_rdev(); - let maj = ((rdev >> 8) & 0xfff) as u32; - let min = ((rdev & 0xff) | ((rdev >> 12) & 0xfff00)) as u32; - Command::new("lxc-cgroup") - .arg(guid) - .arg("devices.allow") - .arg(format!("{ty} {maj}:{min} rwm")) - .invoke(ErrorKind::Lxc) - .await?; - guards.push( - MountGuard::mount( - &Bind::new(&path), - rootfs.join(path.strip_prefix("/").unwrap_or(&path)), - ReadWrite, - ) - .await?, - ); - } - } - } - Ok(()) - } - .boxed() -} - pub struct LxcContainer { manager: Weak, rootfs: OverlayGuard, pub guid: Arc, rpc_bind: TmpMountGuard, log_mount: Option, - devices: Vec, config: LxcConfig, exited: bool, } @@ -259,7 +179,6 @@ impl LxcContainer { tokio::fs::create_dir_all(&container_dir).await?; let config_str = format!(include_str!("./config.template"), guid = &*guid); tokio::fs::write(container_dir.join("config"), config_str).await?; - // TODO: append config let rootfs_dir = container_dir.join("rootfs"); let rootfs = OverlayGuard::mount( TmpMountGuard::mount( @@ -277,8 +196,25 @@ impl LxcContainer { &rootfs_dir, ) .await?; - tokio::fs::write(rootfs_dir.join("etc/machine-id"), format!("{machine_id}\n")).await?; - tokio::fs::write(rootfs_dir.join("etc/hostname"), format!("{guid}\n")).await?; + Command::new("chown") + .arg("100000:100000") + .arg(&rootfs_dir) + .invoke(ErrorKind::Filesystem) + .await?; + write_file_owned_atomic( + rootfs_dir.join("etc/machine-id"), + format!("{machine_id}\n"), + 100000, + 100000, + ) + .await?; + write_file_owned_atomic( + rootfs_dir.join("etc/hostname"), + format!("{guid}\n"), + 100000, + 100000, + ) + .await?; Command::new("sed") .arg("-i") .arg(format!("s/LXC_NAME/{guid}/g")) @@ -334,20 +270,7 @@ impl LxcContainer { .arg("DEBUG") .invoke(ErrorKind::Lxc) .await?; - let mut devices = Vec::new(); - if config.hardware_acceleration { - handle_devices( - &*guid, - rootfs.path(), - tokio::fs::read_dir("/dev") - .await - .with_ctx(|_| (ErrorKind::Filesystem, "readdir /dev"))?, - &mut devices, - HARDWARE_ACCELERATION_PATHS, - ) - .await?; - } - Ok(Self { + let res = Self { manager: Arc::downgrade(manager), rootfs, guid: Arc::new(ContainerId::try_from(&*guid)?), @@ -355,8 +278,77 @@ impl LxcContainer { config, exited: false, log_mount, - devices, - }) + }; + if res.config.hardware_acceleration { + res.handle_devices( + tokio::fs::read_dir("/dev") + .await + .with_ctx(|_| (ErrorKind::Filesystem, "readdir /dev"))?, + HARDWARE_ACCELERATION_PATHS, + ) + .await?; + } + Ok(res) + } + + fn handle_devices<'a>( + &'a self, + mut dir: ReadDir, + matches: &'a [&'a str], + ) -> BoxFuture<'a, Result<(), Error>> { + use std::os::linux::fs::MetadataExt; + use std::os::unix::fs::FileTypeExt; + async move { + while let Some(ent) = dir.next_entry().await? { + let path = ent.path(); + if let Some(matches) = if matches.is_empty() { + Some(Vec::new()) + } else { + let mut new_matches = Vec::new(); + for m in matches { + if if m.ends_with("/") { + path.starts_with(m) + } else { + path.to_string_lossy().starts_with(*m) + } || Path::new(*m).starts_with(&path) + { + new_matches.push(*m); + } + } + if new_matches.is_empty() { + None + } else { + Some(new_matches) + } + } { + let meta = ent.metadata().await?; + let ty = meta.file_type(); + if ty.is_dir() { + self.handle_devices( + tokio::fs::read_dir(&path).await.with_ctx(|_| { + (ErrorKind::Filesystem, format!("readdir {path:?}")) + })?, + &matches, + ) + .await?; + } else { + let ty = if ty.is_char_device() { + 'c' + } else if ty.is_block_device() { + 'b' + } else { + continue; + }; + let rdev = meta.st_rdev(); + let major = ((rdev >> 8) & 0xfff) as u32; + let minor = ((rdev & 0xff) | ((rdev >> 12) & 0xfff00)) as u32; + self.mknod(&path, ty, major, minor).await?; + } + } + } + Ok(()) + } + .boxed() } pub fn rootfs_dir(&self) -> &Path { @@ -429,9 +421,6 @@ impl LxcContainer { if let Some(log_mount) = self.log_mount.take() { log_mount.unmount(false).await?; } - for device in std::mem::take(&mut self.devices) { - device.unmount(false).await?; - } self.rootfs.take().unmount(true).await?; let rootfs_path = self.rootfs_dir(); if tokio::fs::metadata(&rootfs_path).await.is_ok() @@ -452,7 +441,10 @@ impl LxcContainer { .invoke(ErrorKind::Lxc) .await?; - self.exited = true; + #[allow(unused_assignments)] + { + self.exited = true; + } Ok(()) } @@ -462,6 +454,17 @@ impl LxcContainer { let sock_path = self.rpc_dir().join(CONTAINER_RPC_SERVER_SOCKET); while tokio::fs::metadata(&sock_path).await.is_err() { if timeout.map_or(false, |t| started.elapsed() > t) { + tracing::error!( + "{:?}", + Command::new("lxc-attach") + .arg(&**self.guid) + .arg("--") + .arg("systemctl") + .arg("status") + .arg("container-runtime") + .invoke(ErrorKind::Unknown) + .await + ); return Err(Error::new( eyre!("timed out waiting for socket"), ErrorKind::Timeout, @@ -472,6 +475,88 @@ impl LxcContainer { tracing::info!("Connected to socket in {:?}", started.elapsed()); Ok(UnixRpcClient::new(sock_path)) } + + pub async fn mknod(&self, path: &Path, ty: char, major: u32, minor: u32) -> Result<(), Error> { + if let Ok(dev_rel) = path.strip_prefix("/dev") { + let parent = dev_rel.parent(); + let media_dev = self.rootfs_dir().join("media/startos/dev"); + let target_path = media_dev.join(dev_rel); + if tokio::fs::metadata(&target_path).await.is_ok() { + return Ok(()); + } + if let Some(parent) = parent { + let p = media_dev.join(parent); + tokio::fs::create_dir_all(&p) + .await + .with_ctx(|_| (ErrorKind::Filesystem, format!("mkdir -p {p:?}")))?; + for p in parent.ancestors() { + Command::new("chown") + .arg("100000:100000") + .arg(media_dev.join(p)) + .invoke(ErrorKind::Filesystem) + .await?; + } + } + Command::new("mknod") + .arg(&target_path) + .arg(&*InternedString::from_display(&ty)) + .arg(&*InternedString::from_display(&major)) + .arg(&*InternedString::from_display(&minor)) + .invoke(ErrorKind::Filesystem) + .await?; + Command::new("chown") + .arg("100000:100000") + .arg(&target_path) + .invoke(ErrorKind::Filesystem) + .await?; + if let Some(parent) = parent { + Command::new("lxc-attach") + .arg(&**self.guid) + .arg("--") + .arg("mkdir") + .arg("-p") + .arg(Path::new("/dev").join(parent)) + .invoke(ErrorKind::Lxc) + .await?; + } + Command::new("lxc-attach") + .arg(&**self.guid) + .arg("--") + .arg("touch") + .arg(&path) + .invoke(ErrorKind::Lxc) + .await?; + Command::new("lxc-attach") + .arg(&**self.guid) + .arg("--") + .arg("mount") + .arg("--bind") + .arg(Path::new("/media/startos/dev").join(dev_rel)) + .arg(&path) + .invoke(ErrorKind::Lxc) + .await?; + } else { + let target_path = self + .rootfs_dir() + .join(path.strip_prefix("/").unwrap_or(&path)); + if tokio::fs::metadata(&target_path).await.is_ok() { + return Ok(()); + } + Command::new("mknod") + .arg(&target_path) + .arg(&*InternedString::from_display(&ty)) + .arg(&*InternedString::from_display(&major)) + .arg(&*InternedString::from_display(&minor)) + .invoke(ErrorKind::Filesystem) + .await?; + Command::new("chown") + .arg("100000:100000") + .arg(&target_path) + .invoke(ErrorKind::Filesystem) + .await?; + } + Ok(()) + } } impl Drop for LxcContainer { fn drop(&mut self) { diff --git a/core/src/util/io.rs b/core/src/util/io.rs index fa6ea9414..be40bf5a8 100644 --- a/core/src/util/io.rs +++ b/core/src/util/io.rs @@ -16,7 +16,7 @@ use clap::builder::ValueParserFactory; use futures::future::{BoxFuture, Fuse}; use futures::{FutureExt, Stream, TryStreamExt}; use inotify::{EventMask, EventStream, Inotify, WatchMask}; -use nix::unistd::{Gid, Uid}; +use nix::unistd::{Gid, Uid, fchown}; use serde::{Deserialize, Serialize}; use tokio::fs::{File, OpenOptions}; use tokio::io::{ @@ -1057,6 +1057,32 @@ pub async fn write_file_atomic( Ok(()) } +#[instrument(skip_all)] +pub async fn write_file_owned_atomic( + path: impl AsRef, + contents: impl AsRef<[u8]>, + uid: u32, + gid: u32, +) -> Result<(), Error> { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("mkdir -p {parent:?}")))?; + } + let mut file = AtomicFile::new(path, None::<&Path>) + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}")))?; + fchown(&*file, Some(uid.into()), Some(gid.into())).with_kind(ErrorKind::Filesystem)?; + file.write_all(contents.as_ref()) + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("write {path:?}")))?; + file.save() + .await + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("save {path:?}")))?; + Ok(()) +} + fn poll_flush_prefix( mut writer: Pin<&mut W>, cx: &mut std::task::Context<'_>,