Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-26 02:11:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

hardware acceleration and support for NVIDIA cards on nonfree images (#3089)

Browse Source 
* add nvidia packages

* add nvidia deps to nonfree

* gpu_acceleration flag & nvidia hacking

* fix gpu_config & /tmp/lxc.log

* implement hardware acceleration more dynamically

* refactor OpenUI

* use mknod

* registry updates for multi-hardware-requirements

* pluralize

* handle new registry types

* remove log

* migrations and driver fixes

* wip

* misc patches

* handle nvidia-container differently

* chore: comments (#3093)

* chore: comments

* revert some sizing

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* Revert "handle nvidia-container differently"

This reverts commit d708ae53df.

* fix debian containers

* cleanup

* feat: add empty array placeholder in forms (#3095)

* fixes from testing, client side device filtering for better fingerprinting resistance

* fix mac builds

---------

Co-authored-by: Sam Sartor <me@samsartor.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>
This commit is contained in:
Aiden McClelland
2026-01-15 11:42:17 -08:00
committed by GitHub
parent e8ef39adad
commit 99871805bd
95 changed files with 2758 additions and 1092 deletions
Download Patch File Download Diff File Expand all files Collapse all files

661
core/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

50
core/src/bins/mod.rs
View File

@@ -11,67 +11,89 @@ pub mod startd;
pub mod tunnel;
#[derive(Default)]
pub struct MultiExecutable(BTreeMap<&'static str, fn(VecDeque<OsString>)>);
pub struct MultiExecutable {
default: Option<&'static str>,
bins: BTreeMap<&'static str, fn(VecDeque<OsString>)>,
}
impl MultiExecutable {
pub fn enable_startd(&mut self) -> &mut Self {
self.0.insert("startd", startd::main);
self.0
self.bins.insert("startd", startd::main);
self.bins
.insert("embassyd", |_| deprecated::renamed("embassyd", "startd"));
self.0
self.bins
.insert("embassy-init", |_| deprecated::removed("embassy-init"));
self
}
pub fn enable_start_cli(&mut self) -> &mut Self {
self.0.insert("start-cli", start_cli::main);
self.0.insert("embassy-cli", |_| {
self.bins.insert("start-cli", start_cli::main);
self.bins.insert("embassy-cli", |_| {
deprecated::renamed("embassy-cli", "start-cli")
});
self.0
self.bins
.insert("embassy-sdk", |_| deprecated::removed("embassy-sdk"));
self
}
pub fn enable_start_container(&mut self) -> &mut Self {
self.0.insert("start-container", container_cli::main);
self.bins.insert("start-container", container_cli::main);
self
}
pub fn enable_start_registryd(&mut self) -> &mut Self {
self.0.insert("start-registryd", registry::main);
self.bins.insert("start-registryd", registry::main);
self
}
pub fn enable_start_registry(&mut self) -> &mut Self {
self.0.insert("start-registry", registry::cli);
self.bins.insert("start-registry", registry::cli);
self
}
pub fn enable_start_tunneld(&mut self) -> &mut Self {
self.0.insert("start-tunneld", tunnel::main);
self.bins.insert("start-tunneld", tunnel::main);
self
}
pub fn enable_start_tunnel(&mut self) -> &mut Self {
self.0.insert("start-tunnel", tunnel::cli);
self.bins.insert("start-tunnel", tunnel::cli);
self
}
pub fn set_default(&mut self, name: &str) -> &mut Self {
if let Some((name, _)) = self.bins.get_key_value(name) {
self.default = Some(*name);
} else {
panic!("{name} does not exist in MultiExecutable");
}
self
}
fn select_executable(&self, name: &str) -> Option<fn(VecDeque<OsString>)> {
self.0.get(&name).copied()
self.bins.get(&name).copied()
}
pub fn execute(&self) {
let mut popped = Vec::with_capacity(2);
let mut args = std::env::args_os().collect::<VecDeque<_>>();
for _ in 0..2 {
if let Some(s) = args.pop_front() {
if let Some(name) = Path::new(&*s).file_name().and_then(|s| s.to_str()) {
if name == "--contents" {
for name in self.0.keys() {
for name in self.bins.keys() {
println!("{name}");
}
return;
}
if let Some(x) = self.select_executable(&name) {
args.push_front(s);
return x(args);
}
}
popped.push(s);
}
}
if let Some(default) = self.default {
while let Some(arg) = popped.pop() {
args.push_front(arg);
}
return self.bins[default](args);
}
let args = std::env::args().collect::<VecDeque<_>>();
eprintln!(
"unknown executable: {}",

69
core/src/context/cli.rs
View File

@@ -7,6 +7,7 @@ use std::sync::Arc;
use cookie::{Cookie, Expiration, SameSite};
use cookie_store::CookieStore;
use http::HeaderMap;
use imbl::OrdMap;
use imbl_value::InternedString;
use josekit::jwk::Jwk;
use once_cell::sync::OnceCell;
@@ -238,10 +239,16 @@ impl CliContext {
where
Self: CallRemote<RemoteContext>,
{
<Self as CallRemote<RemoteContext, Empty>>::call_remote(&self, method, params, Empty {})
.await
.map_err(Error::from)
.with_ctx(|e| (e.kind, method))
<Self as CallRemote<RemoteContext, Empty>>::call_remote(
&self,
method,
OrdMap::new(),
params,
Empty {},
)
.await
.map_err(Error::from)
.with_ctx(|e| (e.kind, method))
}
pub async fn call_remote_with<RemoteContext, T>(
&self,
@@ -252,10 +259,16 @@ impl CliContext {
where
Self: CallRemote<RemoteContext, T>,
{
<Self as CallRemote<RemoteContext, T>>::call_remote(&self, method, params, extra)
.await
.map_err(Error::from)
.with_ctx(|e| (e.kind, method))
<Self as CallRemote<RemoteContext, T>>::call_remote(
&self,
method,
OrdMap::new(),
params,
extra,
)
.await
.map_err(Error::from)
.with_ctx(|e| (e.kind, method))
}
}
impl AsRef<Jwk> for CliContext {
@@ -292,7 +305,13 @@ impl AsRef<Client> for CliContext {
}
impl CallRemote<RpcContext> for CliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
if let Ok(local) = read_file_to_string(RpcContext::LOCAL_AUTH_COOKIE_PATH).await {
self.cookie_store
.lock()
@@ -319,7 +338,13 @@ impl CallRemote<RpcContext> for CliContext {
}
}
impl CallRemote<DiagnosticContext> for CliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
crate::middleware::auth::signature::call_remote(
self,
self.rpc_url.clone(),
@@ -332,7 +357,13 @@ impl CallRemote<DiagnosticContext> for CliContext {
}
}
impl CallRemote<InitContext> for CliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
crate::middleware::auth::signature::call_remote(
self,
self.rpc_url.clone(),
@@ -345,7 +376,13 @@ impl CallRemote<InitContext> for CliContext {
}
}
impl CallRemote<SetupContext> for CliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
crate::middleware::auth::signature::call_remote(
self,
self.rpc_url.clone(),
@@ -358,7 +395,13 @@ impl CallRemote<SetupContext> for CliContext {
}
}
impl CallRemote<InstallContext> for CliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
crate::middleware::auth::signature::call_remote(
self,
self.rpc_url.clone(),

148
core/src/context/rpc.rs
View File

@@ -15,6 +15,7 @@ use josekit::jwk::Jwk;
use reqwest::{Client, Proxy};
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{CallRemote, Context, Empty};
use tokio::process::Command;
use tokio::sync::{RwLock, broadcast, oneshot, watch};
use tokio::time::Instant;
use tracing::instrument;
@@ -26,6 +27,10 @@ use crate::context::config::ServerConfig;
use crate::db::model::Database;
use crate::db::model::package::TaskSeverity;
use crate::disk::OsPartitionInfo;
use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::{FileSystem, ReadOnly};
use crate::disk::mount::guard::MountGuard;
use crate::init::{InitResult, check_time_is_synchronized};
use crate::install::PKG_ARCHIVE_DIR;
use crate::lxc::LxcManager;
@@ -41,12 +46,14 @@ use crate::rpc_continuations::{Guid, OpenAuthedContinuations, RpcContinuations};
use crate::service::ServiceMap;
use crate::service::action::update_tasks;
use crate::service::effects::callbacks::ServiceCallbacks;
use crate::service::effects::subcontainer::NVIDIA_OVERLAY_PATH;
use crate::shutdown::Shutdown;
use crate::util::Invoke;
use crate::util::future::NonDetachingJoinHandle;
use crate::util::io::delete_file;
use crate::util::io::{TmpDir, delete_file};
use crate::util::lshw::LshwDevice;
use crate::util::sync::{SyncMutex, SyncRwLock, Watch};
use crate::{ActionId, DATA_DIR, PackageId};
use crate::{ActionId, DATA_DIR, PLATFORM, PackageId};
pub struct RpcContextSeed {
is_closed: AtomicBool,
@@ -167,6 +174,124 @@ impl RpcContext {
init_net_ctrl.complete();
tracing::info!("Initialized Net Controller");
if PLATFORM.ends_with("-nonfree") {
if let Err(e) = Command::new("nvidia-smi")
.invoke(ErrorKind::ParseSysInfo)
.await
{
tracing::warn!("nvidia-smi: {e}");
tracing::info!("The above warning can be ignored if no NVIDIA card is present");
} else {
async {
let version: InternedString = String::from_utf8(
Command::new("modinfo")
.arg("-F")
.arg("version")
.arg("nvidia")
.invoke(ErrorKind::ParseSysInfo)
.await?,
)?
.trim()
.into();
let nvidia_dir =
Path::new("/media/startos/data/package-data/nvidia").join(&*version);
// Generate single squashfs with both debian and generic overlays
let sqfs = nvidia_dir.join("container-overlay.squashfs");
if tokio::fs::metadata(&sqfs).await.is_err() {
let tmp = TmpDir::new().await?;
// Generate debian overlay (libs in /usr/lib/aarch64-linux-gnu/)
let debian_dir = tmp.join("debian");
tokio::fs::create_dir_all(&debian_dir).await?;
// Create /etc/debian_version to trigger debian path detection
tokio::fs::create_dir_all(debian_dir.join("etc")).await?;
tokio::fs::write(debian_dir.join("etc/debian_version"), "").await?;
let procfs = MountGuard::mount(
&Bind::new("/proc"),
debian_dir.join("proc"),
ReadOnly,
)
.await?;
Command::new("nvidia-container-cli")
.arg("configure")
.arg("--no-devbind")
.arg("--no-cgroups")
.arg("--utility")
.arg("--compute")
.arg("--graphics")
.arg("--video")
.arg(&debian_dir)
.invoke(ErrorKind::Unknown)
.await?;
procfs.unmount(true).await?;
// Run ldconfig to create proper symlinks for all NVIDIA libraries
Command::new("ldconfig")
.arg("-r")
.arg(&debian_dir)
.invoke(ErrorKind::Unknown)
.await?;
// Remove /etc/debian_version - it was only needed for nvidia-container-cli detection
tokio::fs::remove_file(debian_dir.join("etc/debian_version")).await?;
// Generate generic overlay (libs in /usr/lib64/)
let generic_dir = tmp.join("generic");
tokio::fs::create_dir_all(&generic_dir).await?;
// No /etc/debian_version - will use generic /usr/lib64 paths
let procfs = MountGuard::mount(
&Bind::new("/proc"),
generic_dir.join("proc"),
ReadOnly,
)
.await?;
Command::new("nvidia-container-cli")
.arg("configure")
.arg("--no-devbind")
.arg("--no-cgroups")
.arg("--utility")
.arg("--compute")
.arg("--graphics")
.arg("--video")
.arg(&generic_dir)
.invoke(ErrorKind::Unknown)
.await?;
procfs.unmount(true).await?;
// Run ldconfig to create proper symlinks for all NVIDIA libraries
Command::new("ldconfig")
.arg("-r")
.arg(&generic_dir)
.invoke(ErrorKind::Unknown)
.await?;
// Create squashfs with UID/GID mapping (avoids chown on readonly mounts)
if let Some(p) = sqfs.parent() {
tokio::fs::create_dir_all(p)
.await
.with_ctx(|_| (ErrorKind::Filesystem, format!("mkdir -p {p:?}")))?;
}
Command::new("mksquashfs")
.arg(&*tmp)
.arg(&sqfs)
.arg("-force-uid")
.arg("100000")
.arg("-force-gid")
.arg("100000")
.invoke(ErrorKind::Filesystem)
.await?;
tmp.unmount_and_delete().await?;
}
BlockDev::new(&sqfs)
.mount(NVIDIA_OVERLAY_PATH, ReadOnly)
.await?;
Ok::<_, Error>(())
}
.await
.log_err();
}
}
let services = ServiceMap::default();
let metrics_cache = Watch::<Option<crate::system::Metrics>>::new(None);
let socks_proxy_url = format!("socks5h://{socks_proxy}");
@@ -460,8 +585,14 @@ impl RpcContext {
where
Self: CallRemote<RemoteContext>,
{
<Self as CallRemote<RemoteContext, Empty>>::call_remote(&self, method, params, Empty {})
.await
<Self as CallRemote<RemoteContext, Empty>>::call_remote(
&self,
method,
OrdMap::new(),
params,
Empty {},
)
.await
}
pub async fn call_remote_with<RemoteContext, T>(
&self,
@@ -472,7 +603,14 @@ impl RpcContext {
where
Self: CallRemote<RemoteContext, T>,
{
<Self as CallRemote<RemoteContext, T>>::call_remote(&self, method, params, extra).await
<Self as CallRemote<RemoteContext, T>>::call_remote(
&self,
method,
OrdMap::new(),
params,
extra,
)
.await
}
}
impl AsRef<Client> for RpcContext {

45
core/src/db/prelude.rs
View File

@@ -416,6 +416,51 @@ impl<T: Map> Model<T> {
}
}
impl<T: Map> Model<T>
where
T::Key: FromStr,
Error: From<<T::Key as FromStr>::Err>,
{
/// Retains only the elements specified by the predicate.
/// The predicate can mutate the values and returns whether to keep each entry.
pub fn retain<F>(&mut self, mut f: F) -> Result<(), Error>
where
F: FnMut(&T::Key, &mut Model<T::Value>) -> Result<bool, Error>,
{
let mut to_remove = Vec::new();
match &mut self.value {
Value::Object(o) => {
for (k, v) in o.iter_mut() {
let key = T::Key::from_str(&**k)?;
if !f(&key, patch_db::ModelExt::value_as_mut(v))? {
to_remove.push(k.clone());
}
}
}
v => {
use serde::de::Error;
return Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!(
"expected object found {v}"
)),
kind: patch_db::value::ErrorKind::Deserialization,
}
.into());
}
}
// Remove entries that didn't pass the filter
if let Value::Object(o) = &mut self.value {
for k in to_remove {
o.remove(&k);
}
}
Ok(())
}
}
#[repr(transparent)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct JsonKey<T>(pub T);

59
core/src/disk/mount/filesystem/overlayfs.rs
View File

@@ -4,6 +4,7 @@ use std::path::Path;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use itertools::Itertools;
use sha2::Sha256;
use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
@@ -12,12 +13,13 @@ use crate::prelude::*;
use crate::util::io::TmpDir;
pub struct OverlayFs<P0: AsRef<Path>, P1: AsRef<Path>, P2: AsRef<Path>> {
lower: P0,
lower: Vec<P0>,
upper: P1,
work: P2,
}
impl<P0: AsRef<Path>, P1: AsRef<Path>, P2: AsRef<Path>> OverlayFs<P0, P1, P2> {
pub fn new(lower: P0, upper: P1, work: P2) -> Self {
/// layers are top to bottom
pub fn new(lower: Vec<P0>, upper: P1, work: P2) -> Self {
Self { lower, upper, work }
}
}
@@ -32,8 +34,10 @@ impl<P0: AsRef<Path> + Send + Sync, P1: AsRef<Path> + Send + Sync, P2: AsRef<Pat
}
fn mount_options(&self) -> impl IntoIterator<Item = impl Display> {
[
Box::new(lazy_format!("lowerdir={}", self.lower.as_ref().display()))
as Box<dyn Display>,
Box::new(lazy_format!(
"lowerdir={}",
self.lower.iter().map(|p| p.as_ref().display()).join(":")
)) as Box<dyn Display>,
Box::new(lazy_format!("upperdir={}", self.upper.as_ref().display())),
Box::new(lazy_format!("workdir={}", self.work.as_ref().display())),
]
@@ -51,18 +55,21 @@ impl<P0: AsRef<Path> + Send + Sync, P1: AsRef<Path> + Send + Sync, P2: AsRef<Pat
tokio::fs::create_dir_all(self.work.as_ref()).await?;
let mut sha = Sha256::new();
sha.update("OverlayFs");
sha.update(
tokio::fs::canonicalize(self.lower.as_ref())
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
self.lower.as_ref().display().to_string(),
)
})?
.as_os_str()
.as_bytes(),
);
for lower in &self.lower {
sha.update(
tokio::fs::canonicalize(lower.as_ref())
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
lower.as_ref().display().to_string(),
)
})?
.as_os_str()
.as_bytes(),
);
sha.update(b"\0");
}
sha.update(
tokio::fs::canonicalize(self.upper.as_ref())
.await
@@ -75,6 +82,7 @@ impl<P0: AsRef<Path> + Send + Sync, P1: AsRef<Path> + Send + Sync, P2: AsRef<Pat
.as_os_str()
.as_bytes(),
);
sha.update(b"\0");
sha.update(
tokio::fs::canonicalize(self.work.as_ref())
.await
@@ -87,6 +95,7 @@ impl<P0: AsRef<Path> + Send + Sync, P1: AsRef<Path> + Send + Sync, P2: AsRef<Pat
.as_os_str()
.as_bytes(),
);
sha.update(b"\0");
Ok(sha.finalize())
}
}
@@ -98,11 +107,20 @@ pub struct OverlayGuard<G: GenericMountGuard> {
inner_guard: MountGuard,
}
impl<G: GenericMountGuard> OverlayGuard<G> {
pub async fn mount(lower: G, mountpoint: impl AsRef<Path>) -> Result<Self, Error> {
pub async fn mount_layers<P: AsRef<Path>>(
pre: &[P],
guard: G,
post: &[P],
mountpoint: impl AsRef<Path>,
) -> Result<Self, Error> {
let upper = TmpDir::new().await?;
let inner_guard = MountGuard::mount(
&OverlayFs::new(
lower.path(),
std::iter::empty()
.chain(pre.into_iter().map(|p| p.as_ref()))
.chain([guard.path()])
.chain(post.into_iter().map(|p| p.as_ref()))
.collect(),
upper.as_ref().join("upper"),
upper.as_ref().join("work"),
),
@@ -111,11 +129,14 @@ impl<G: GenericMountGuard> OverlayGuard<G> {
)
.await?;
Ok(Self {
lower: Some(lower),
lower: Some(guard),
upper: Some(upper),
inner_guard,
})
}
pub async fn mount(lower: G, mountpoint: impl AsRef<Path>) -> Result<Self, Error> {
Self::mount_layers::<&Path>(&[], lower, &[], mountpoint).await
}
pub async fn unmount(mut self, delete_mountpoint: bool) -> Result<(), Error> {
self.inner_guard.take().unmount(delete_mountpoint).await?;
if let Some(lower) = self.lower.take() {

40
core/src/disk/mount/util.rs
View File

@@ -3,6 +3,7 @@ use std::path::Path;
use tracing::instrument;
use crate::Error;
use crate::prelude::*;
use crate::util::Invoke;
pub async fn is_mountpoint(path: impl AsRef<Path>) -> Result<bool, Error> {
@@ -56,3 +57,42 @@ pub async fn unmount<P: AsRef<Path>>(mountpoint: P, lazy: bool) -> Result<(), Er
.await?;
Ok(())
}
/// Unmounts all mountpoints under (and including) the given path, in reverse
/// depth order so that nested mounts are unmounted before their parents.
#[instrument(skip_all)]
pub async fn unmount_all_under<P: AsRef<Path>>(path: P, lazy: bool) -> Result<(), Error> {
let path = path.as_ref();
let canonical_path = tokio::fs::canonicalize(path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("canonicalize {path:?}")))?;
let mounts_content = tokio::fs::read_to_string("/proc/mounts")
.await
.with_ctx(|_| (ErrorKind::Filesystem, "read /proc/mounts"))?;
// Collect all mountpoints under our path
let mut mountpoints: Vec<&str> = mounts_content
.lines()
.filter_map(|line| {
let mountpoint = line.split_whitespace().nth(1)?;
// Check if this mountpoint is under our target path
let mp_path = Path::new(mountpoint);
if mp_path.starts_with(&canonical_path) {
Some(mountpoint)
} else {
None
}
})
.collect();
// Sort by path length descending so we unmount deepest first
mountpoints.sort_by(|a, b| b.len().cmp(&a.len()));
for mountpoint in mountpoints {
tracing::debug!("Unmounting nested mountpoint: {}", mountpoint);
unmount(mountpoint, lazy).await?;
}
Ok(())
}

1
core/src/init.rs
View File

@@ -4,7 +4,6 @@ use std::sync::Arc;
use std::time::{Duration, SystemTime};
use axum::extract::ws;
use const_format::formatcp;
use futures::{StreamExt, TryStreamExt};
use itertools::Itertools;
use rpc_toolkit::{Context, Empty, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};

6
core/src/install/mod.rs
View File

@@ -142,16 +142,16 @@ pub async fn install(
.await?,
)?;
let asset = &package
let (_, asset) = package
.best
.get(&version)
.and_then(|i| i.s9pks.first())
.ok_or_else(|| {
Error::new(
eyre!("{id}@{version} not found on {registry}"),
ErrorKind::NotFound,
)
})?
.s9pk;
})?;
asset.validate(SIG_CONTEXT, asset.all_signers())?;

226
core/src/lxc/mod.rs
View File

@@ -5,11 +5,13 @@ use std::sync::{Arc, Weak};
use std::time::Duration;
use clap::builder::ValueParserFactory;
use futures::StreamExt;
use futures::future::BoxFuture;
use futures::{FutureExt, StreamExt};
use imbl_value::InternedString;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{RpcRequest, RpcResponse};
use serde::{Deserialize, Serialize};
use tokio::fs::ReadDir;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command;
use tokio::sync::Mutex;
@@ -27,7 +29,7 @@ use crate::disk::mount::util::unmount;
use crate::prelude::*;
use crate::rpc_continuations::{Guid, RpcContinuation};
use crate::service::ServiceStats;
use crate::util::io::open_file;
use crate::util::io::{open_file, write_file_owned_atomic};
use crate::util::rpc_client::UnixRpcClient;
use crate::util::{FromStrParser, Invoke, new_guid};
use crate::{InvalidId, PackageId};
@@ -37,6 +39,7 @@ const RPC_DIR: &str = "media/startos/rpc"; // must not be absolute path
pub const CONTAINER_RPC_SERVER_SOCKET: &str = "service.sock"; // must not be absolute path
pub const HOST_RPC_SERVER_SOCKET: &str = "host.sock"; // must not be absolute path
const CONTAINER_DHCP_TIMEOUT: Duration = Duration::from_secs(30);
const HARDWARE_ACCELERATION_PATHS: &[&str] = &["/dev/dri", "/dev/nvidia*", "/dev/kfd"];
#[derive(
Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord, Hash, TS,
@@ -174,12 +177,8 @@ impl LxcContainer {
let machine_id = hex::encode(rand::random::<[u8; 16]>());
let container_dir = Path::new(LXC_CONTAINER_DIR).join(&*guid);
tokio::fs::create_dir_all(&container_dir).await?;
tokio::fs::write(
container_dir.join("config"),
format!(include_str!("./config.template"), guid = &*guid),
)
.await?;
// TODO: append config
let config_str = format!(include_str!("./config.template"), guid = &*guid);
tokio::fs::write(container_dir.join("config"), config_str).await?;
let rootfs_dir = container_dir.join("rootfs");
let rootfs = OverlayGuard::mount(
TmpMountGuard::mount(
@@ -197,8 +196,25 @@ impl LxcContainer {
&rootfs_dir,
)
.await?;
tokio::fs::write(rootfs_dir.join("etc/machine-id"), format!("{machine_id}\n")).await?;
tokio::fs::write(rootfs_dir.join("etc/hostname"), format!("{guid}\n")).await?;
Command::new("chown")
.arg("100000:100000")
.arg(&rootfs_dir)
.invoke(ErrorKind::Filesystem)
.await?;
write_file_owned_atomic(
rootfs_dir.join("etc/machine-id"),
format!("{machine_id}\n"),
100000,
100000,
)
.await?;
write_file_owned_atomic(
rootfs_dir.join("etc/hostname"),
format!("{guid}\n"),
100000,
100000,
)
.await?;
Command::new("sed")
.arg("-i")
.arg(format!("s/LXC_NAME/{guid}/g"))
@@ -248,9 +264,13 @@ impl LxcContainer {
.arg("-d")
.arg("--name")
.arg(&*guid)
.arg("-o")
.arg(format!("/run/startos/LXC_{guid}.log"))
.arg("-l")
.arg("DEBUG")
.invoke(ErrorKind::Lxc)
.await?;
Ok(Self {
let res = Self {
manager: Arc::downgrade(manager),
rootfs,
guid: Arc::new(ContainerId::try_from(&*guid)?),
@@ -258,7 +278,84 @@ impl LxcContainer {
config,
exited: false,
log_mount,
})
};
if res.config.hardware_acceleration {
res.handle_devices(
tokio::fs::read_dir("/dev")
.await
.with_ctx(|_| (ErrorKind::Filesystem, "readdir /dev"))?,
HARDWARE_ACCELERATION_PATHS,
)
.await?;
}
Ok(res)
}
#[cfg(not(target_os = "linux"))]
async fn handle_devices(&self, _: ReadDir, _: &[&str]) -> Result<(), Error> {
Ok(())
}
#[cfg(target_os = "linux")]
fn handle_devices<'a>(
&'a self,
mut dir: ReadDir,
matches: &'a [&'a str],
) -> BoxFuture<'a, Result<(), Error>> {
use std::os::linux::fs::MetadataExt;
use std::os::unix::fs::FileTypeExt;
async move {
while let Some(ent) = dir.next_entry().await? {
let path = ent.path();
if let Some(matches) = if matches.is_empty() {
Some(Vec::new())
} else {
let mut new_matches = Vec::new();
for mut m in matches.iter().copied() {
let could_match = if let Some(prefix) = m.strip_suffix("*") {
m = prefix;
path.to_string_lossy().starts_with(m)
} else {
path.starts_with(m)
} || Path::new(m).starts_with(&path);
if could_match {
new_matches.push(m);
}
}
if new_matches.is_empty() {
None
} else {
Some(new_matches)
}
} {
let meta = ent.metadata().await?;
let ty = meta.file_type();
if ty.is_dir() {
self.handle_devices(
tokio::fs::read_dir(&path).await.with_ctx(|_| {
(ErrorKind::Filesystem, format!("readdir {path:?}"))
})?,
&matches,
)
.await?;
} else {
let ty = if ty.is_char_device() {
'c'
} else if ty.is_block_device() {
'b'
} else {
continue;
};
let rdev = meta.st_rdev();
let major = ((rdev >> 8) & 0xfff) as u32;
let minor = ((rdev & 0xff) | ((rdev >> 12) & 0xfff00)) as u32;
self.mknod(&path, ty, major, minor).await?;
}
}
}
Ok(())
}
.boxed()
}
pub fn rootfs_dir(&self) -> &Path {
@@ -329,7 +426,7 @@ impl LxcContainer {
.await?;
self.rpc_bind.take().unmount().await?;
if let Some(log_mount) = self.log_mount.take() {
log_mount.unmount(true).await?;
log_mount.unmount(false).await?;
}
self.rootfs.take().unmount(true).await?;
let rootfs_path = self.rootfs_dir();
@@ -351,7 +448,10 @@ impl LxcContainer {
.invoke(ErrorKind::Lxc)
.await?;
self.exited = true;
#[allow(unused_assignments)]
{
self.exited = true;
}
Ok(())
}
@@ -361,6 +461,17 @@ impl LxcContainer {
let sock_path = self.rpc_dir().join(CONTAINER_RPC_SERVER_SOCKET);
while tokio::fs::metadata(&sock_path).await.is_err() {
if timeout.map_or(false, |t| started.elapsed() > t) {
tracing::error!(
"{:?}",
Command::new("lxc-attach")
.arg(&**self.guid)
.arg("--")
.arg("systemctl")
.arg("status")
.arg("container-runtime")
.invoke(ErrorKind::Unknown)
.await
);
return Err(Error::new(
eyre!("timed out waiting for socket"),
ErrorKind::Timeout,
@@ -371,6 +482,88 @@ impl LxcContainer {
tracing::info!("Connected to socket in {:?}", started.elapsed());
Ok(UnixRpcClient::new(sock_path))
}
pub async fn mknod(&self, path: &Path, ty: char, major: u32, minor: u32) -> Result<(), Error> {
if let Ok(dev_rel) = path.strip_prefix("/dev") {
let parent = dev_rel.parent();
let media_dev = self.rootfs_dir().join("media/startos/dev");
let target_path = media_dev.join(dev_rel);
if tokio::fs::metadata(&target_path).await.is_ok() {
return Ok(());
}
if let Some(parent) = parent {
let p = media_dev.join(parent);
tokio::fs::create_dir_all(&p)
.await
.with_ctx(|_| (ErrorKind::Filesystem, format!("mkdir -p {p:?}")))?;
for p in parent.ancestors() {
Command::new("chown")
.arg("100000:100000")
.arg(media_dev.join(p))
.invoke(ErrorKind::Filesystem)
.await?;
}
}
Command::new("mknod")
.arg(&target_path)
.arg(&*InternedString::from_display(&ty))
.arg(&*InternedString::from_display(&major))
.arg(&*InternedString::from_display(&minor))
.invoke(ErrorKind::Filesystem)
.await?;
Command::new("chown")
.arg("100000:100000")
.arg(&target_path)
.invoke(ErrorKind::Filesystem)
.await?;
if let Some(parent) = parent {
Command::new("lxc-attach")
.arg(&**self.guid)
.arg("--")
.arg("mkdir")
.arg("-p")
.arg(Path::new("/dev").join(parent))
.invoke(ErrorKind::Lxc)
.await?;
}
Command::new("lxc-attach")
.arg(&**self.guid)
.arg("--")
.arg("touch")
.arg(&path)
.invoke(ErrorKind::Lxc)
.await?;
Command::new("lxc-attach")
.arg(&**self.guid)
.arg("--")
.arg("mount")
.arg("--bind")
.arg(Path::new("/media/startos/dev").join(dev_rel))
.arg(&path)
.invoke(ErrorKind::Lxc)
.await?;
} else {
let target_path = self
.rootfs_dir()
.join(path.strip_prefix("/").unwrap_or(&path));
if tokio::fs::metadata(&target_path).await.is_ok() {
return Ok(());
}
Command::new("mknod")
.arg(&target_path)
.arg(&*InternedString::from_display(&ty))
.arg(&*InternedString::from_display(&major))
.arg(&*InternedString::from_display(&minor))
.invoke(ErrorKind::Filesystem)
.await?;
Command::new("chown")
.arg("100000:100000")
.arg(&target_path)
.invoke(ErrorKind::Filesystem)
.await?;
}
Ok(())
}
}
impl Drop for LxcContainer {
fn drop(&mut self) {
@@ -414,7 +607,10 @@ impl Drop for LxcContainer {
}
#[derive(Default, Serialize)]
pub struct LxcConfig {}
pub struct LxcConfig {
pub hardware_acceleration: bool,
}
pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result<Guid, Error> {
use axum::extract::ws::Message;

5
core/src/main/start-cli.rs
View File

@@ -15,5 +15,8 @@ fn main() {
}) {
PREFER_DOCKER.set(true).ok();
}
MultiExecutable::default().enable_start_cli().execute()
MultiExecutable::default()
.enable_start_cli()
.set_default("start-cli")
.execute()
}

1
core/src/main/start-container.rs
View File

@@ -3,5 +3,6 @@ use startos::bins::MultiExecutable;
fn main() {
MultiExecutable::default()
.enable_start_container()
.set_default("start-container")
.execute()
}

185
core/src/net/tls.rs
View File

@@ -151,103 +151,112 @@ where
cx: &mut std::task::Context<'_>,
) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
self.in_progress.mutate(|in_progress| {
loop {
if !in_progress.is_empty() {
if let Poll::Ready(Some((handler, res))) = in_progress.poll_next_unpin(cx) {
if let Some(res) = res.transpose() {
self.tls_handler = handler;
return Poll::Ready(res);
}
continue;
// First, check if any in-progress handshakes have completed
if !in_progress.is_empty() {
if let Poll::Ready(Some((handler, res))) = in_progress.poll_next_unpin(cx) {
if let Some(res) = res.transpose() {
self.tls_handler = handler;
return Poll::Ready(res);
}
// Connection was rejected (preprocess returned None).
// Yield to the runtime to avoid busy-looping, but wake
// immediately to continue processing.
cx.waker().wake_by_ref();
return Poll::Pending;
}
}
let (metadata, stream) = ready!(self.accept.poll_accept(cx)?);
let mut tls_handler = self.tls_handler.clone();
let mut fut = async move {
let res = async {
let mut acceptor = LazyConfigAcceptor::new(
Acceptor::default(),
BackTrackingIO::new(stream),
);
let mut mid: tokio_rustls::StartHandshake<BackTrackingIO<AcceptStream>> =
match (&mut acceptor).await {
Ok(a) => a,
Err(e) => {
let mut stream =
acceptor.take_io().or_not_found("acceptor io")?;
let (_, buf) = stream.rewind();
if std::str::from_utf8(buf)
.ok()
.and_then(|buf| {
buf.lines()
.map(|l| l.trim())
.filter(|l| !l.is_empty())
.next()
})
.map_or(false, |buf| {
regex::Regex::new("[A-Z]+ (.+) HTTP/1")
.unwrap()
.is_match(buf)
})
{
handle_http_on_https(stream).await.log_err();
// Try to accept a new connection
let (metadata, stream) = ready!(self.accept.poll_accept(cx)?);
let mut tls_handler = self.tls_handler.clone();
let mut fut = async move {
let res = async {
let mut acceptor = LazyConfigAcceptor::new(
Acceptor::default(),
BackTrackingIO::new(stream),
);
let mut mid: tokio_rustls::StartHandshake<BackTrackingIO<AcceptStream>> =
match (&mut acceptor).await {
Ok(a) => a,
Err(e) => {
let mut stream =
acceptor.take_io().or_not_found("acceptor io")?;
let (_, buf) = stream.rewind();
if std::str::from_utf8(buf)
.ok()
.and_then(|buf| {
buf.lines()
.map(|l| l.trim())
.filter(|l| !l.is_empty())
.next()
})
.map_or(false, |buf| {
regex::Regex::new("[A-Z]+ (.+) HTTP/1")
.unwrap()
.is_match(buf)
})
{
handle_http_on_https(stream).await.log_err();
return Ok(None);
} else {
return Err(e).with_kind(ErrorKind::Network);
}
return Ok(None);
} else {
return Err(e).with_kind(ErrorKind::Network);
}
};
let hello = mid.client_hello();
if let Some(cfg) = tls_handler.get_config(&hello, &metadata).await {
let buffered = mid.io.stop_buffering();
mid.io
.write_all(&buffered)
.await
.with_kind(ErrorKind::Network)?;
return Ok(match mid.into_stream(Arc::new(cfg)).await {
Ok(stream) => {
let s = stream.get_ref().1;
Some((
TlsMetadata {
inner: metadata,
tls_info: TlsHandshakeInfo {
sni: s.server_name().map(InternedString::intern),
alpn: s
.alpn_protocol()
.map(|a| MaybeUtf8String(a.to_vec())),
},
}
};
let hello = mid.client_hello();
if let Some(cfg) = tls_handler.get_config(&hello, &metadata).await {
let buffered = mid.io.stop_buffering();
mid.io
.write_all(&buffered)
.await
.with_kind(ErrorKind::Network)?;
return Ok(match mid.into_stream(Arc::new(cfg)).await {
Ok(stream) => {
let s = stream.get_ref().1;
Some((
TlsMetadata {
inner: metadata,
tls_info: TlsHandshakeInfo {
sni: s.server_name().map(InternedString::intern),
alpn: s
.alpn_protocol()
.map(|a| MaybeUtf8String(a.to_vec())),
},
Box::pin(stream) as AcceptStream,
))
}
Err(e) => {
tracing::trace!("Error completing TLS handshake: {e}");
tracing::trace!("{e:?}");
None
}
});
}
},
Box::pin(stream) as AcceptStream,
))
}
Err(e) => {
tracing::trace!("Error completing TLS handshake: {e}");
tracing::trace!("{e:?}");
None
}
});
}
Ok(None)
}
.await;
(tls_handler, res)
Ok(None)
}
.boxed();
match fut.poll_unpin(cx) {
Poll::Pending => {
in_progress.push(fut);
return Poll::Pending;
.await;
(tls_handler, res)
}
.boxed();
match fut.poll_unpin(cx) {
Poll::Pending => {
in_progress.push(fut);
Poll::Pending
}
Poll::Ready((handler, res)) => {
if let Some(res) = res.transpose() {
self.tls_handler = handler;
return Poll::Ready(res);
}
Poll::Ready((handler, res)) => {
if let Some(res) = res.transpose() {
self.tls_handler = handler;
return Poll::Ready(res);
}
}
};
// Connection was rejected (preprocess returned None).
// Yield to the runtime to avoid busy-looping, but wake
// immediately to continue processing.
cx.waker().wake_by_ref();
Poll::Pending
}
}
})
}

7
core/src/os_install/mod.rs
View File

@@ -280,8 +280,11 @@ pub async fn execute<C: Context>(
let lower = TmpMountGuard::mount(&BlockDev::new(&image_path), MountType::ReadOnly).await?;
let work = config_path.join("work");
let upper = config_path.join("overlay");
let overlay =
TmpMountGuard::mount(&OverlayFs::new(&lower.path(), &upper, &work), ReadWrite).await?;
let overlay = TmpMountGuard::mount(
&OverlayFs::new(vec![lower.path()], &upper, &work),
ReadWrite,
)
.await?;
let boot = MountGuard::mount(
&BlockDev::new(&part_info.boot),

91
core/src/registry/asset.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use reqwest::Client;
use reqwest::{Client, Response};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWrite;
use ts_rs::TS;
@@ -21,14 +21,14 @@ use crate::sign::{AnySignature, AnyVerifyingKey};
use crate::upload::UploadingFile;
use crate::util::future::NonDetachingJoinHandle;
#[derive(Debug, Deserialize, Serialize, TS)]
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RegistryAsset<Commitment> {
#[ts(type = "string")]
pub published_at: DateTime<Utc>,
#[ts(type = "string")]
pub url: Url,
#[ts(type = "string[]")]
pub urls: Vec<Url>,
pub commitment: Commitment,
pub signatures: HashMap<AnyVerifyingKey, AnySignature>,
}
@@ -42,6 +42,48 @@ impl<Commitment> RegistryAsset<Commitment> {
.collect(),
)
}
pub async fn load_http_source(&self, client: Client) -> Result<HttpSource, Error> {
for url in &self.urls {
if let Ok(source) = HttpSource::new(client.clone(), url.clone()).await {
return Ok(source);
}
}
Err(Error::new(
eyre!("Failed to load any http url"),
ErrorKind::Network,
))
}
pub async fn load_buffered_http_source(
&self,
client: Client,
progress: PhaseProgressTrackerHandle,
) -> Result<BufferedHttpSource, Error> {
for url in &self.urls {
if let Ok(response) = client.get(url.clone()).send().await {
return BufferedHttpSource::from_response(response, progress).await;
}
}
Err(Error::new(
eyre!("Failed to load any http url"),
ErrorKind::Network,
))
}
pub async fn load_buffered_http_source_with_path(
&self,
path: impl AsRef<Path>,
client: Client,
progress: PhaseProgressTrackerHandle,
) -> Result<BufferedHttpSource, Error> {
for url in &self.urls {
if let Ok(response) = client.get(url.clone()).send().await {
return BufferedHttpSource::from_response_with_path(path, response, progress).await;
}
}
Err(Error::new(
eyre!("Failed to load any http url"),
ErrorKind::Network,
))
}
}
impl<Commitment: Digestable> RegistryAsset<Commitment> {
pub fn validate(&self, context: &str, mut accept: AcceptSigners) -> Result<&Commitment, Error> {
@@ -59,7 +101,7 @@ impl<C: for<'a> Commitment<&'a HttpSource>> RegistryAsset<C> {
dst: &mut (impl AsyncWrite + Unpin + Send + ?Sized),
) -> Result<(), Error> {
self.commitment
.copy_to(&HttpSource::new(client, self.url.clone()).await?, dst)
.copy_to(&self.load_http_source(client).await?, dst)
.await
}
}
@@ -69,7 +111,7 @@ impl RegistryAsset<MerkleArchiveCommitment> {
client: Client,
) -> Result<S9pk<Section<Arc<HttpSource>>>, Error> {
S9pk::deserialize(
&Arc::new(HttpSource::new(client, self.url.clone()).await?),
&Arc::new(self.load_http_source(client).await?),
Some(&self.commitment),
)
.await
@@ -80,7 +122,7 @@ impl RegistryAsset<MerkleArchiveCommitment> {
progress: PhaseProgressTrackerHandle,
) -> Result<S9pk<Section<Arc<BufferedHttpSource>>>, Error> {
S9pk::deserialize(
&Arc::new(BufferedHttpSource::new(client, self.url.clone(), progress).await?),
&Arc::new(self.load_buffered_http_source(client, progress).await?),
Some(&self.commitment),
)
.await
@@ -98,7 +140,8 @@ impl RegistryAsset<MerkleArchiveCommitment> {
Error,
> {
let source = Arc::new(
BufferedHttpSource::with_path(path, client, self.url.clone(), progress).await?,
self.load_buffered_http_source_with_path(path, client, progress)
.await?,
);
Ok((
S9pk::deserialize(&source, Some(&self.commitment)).await?,
@@ -112,26 +155,30 @@ pub struct BufferedHttpSource {
file: UploadingFile,
}
impl BufferedHttpSource {
pub async fn with_path(
path: impl AsRef<Path>,
client: Client,
url: Url,
progress: PhaseProgressTrackerHandle,
) -> Result<Self, Error> {
let (mut handle, file) = UploadingFile::with_path(path, progress).await?;
let response = client.get(url).send().await?;
Ok(Self {
_download: tokio::spawn(async move { handle.download(response).await }).into(),
file,
})
}
pub async fn new(
client: Client,
url: Url,
progress: PhaseProgressTrackerHandle,
) -> Result<Self, Error> {
let (mut handle, file) = UploadingFile::new(progress).await?;
let response = client.get(url).send().await?;
Self::from_response(response, progress).await
}
pub async fn from_response(
response: Response,
progress: PhaseProgressTrackerHandle,
) -> Result<Self, Error> {
let (mut handle, file) = UploadingFile::new(progress).await?;
Ok(Self {
_download: tokio::spawn(async move { handle.download(response).await }).into(),
file,
})
}
pub async fn from_response_with_path(
path: impl AsRef<Path>,
response: Response,
progress: PhaseProgressTrackerHandle,
) -> Result<Self, Error> {
let (mut handle, file) = UploadingFile::with_path(path, progress).await?;
Ok(Self {
_download: tokio::spawn(async move { handle.download(response).await }).into(),
file,

29
core/src/registry/context.rs
View File

@@ -7,6 +7,7 @@ use chrono::Utc;
use clap::Parser;
use cookie::{Cookie, Expiration, SameSite};
use http::HeaderMap;
use imbl::OrdMap;
use imbl_value::InternedString;
use patch_db::PatchDb;
use patch_db::json_ptr::ROOT;
@@ -171,6 +172,7 @@ impl CallRemote<RegistryContext> for CliContext {
async fn call_remote(
&self,
mut method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
@@ -240,14 +242,21 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
async fn call_remote(
&self,
mut method: &str,
metadata: OrdMap<&'static str, Value>,
params: Value,
RegistryUrlParams { mut registry }: RegistryUrlParams,
) -> Result<Value, RpcError> {
let mut headers = HeaderMap::new();
headers.insert(
DEVICE_INFO_HEADER,
DeviceInfo::load(self).await?.to_header_value(),
);
let mut device_info = None;
if metadata
.get("get_device_info")
.and_then(|m| m.as_bool())
.unwrap_or(false)
{
let di = DeviceInfo::load(self).await?;
headers.insert(DEVICE_INFO_HEADER, di.to_header_value());
device_info = Some(di);
}
registry
.path_segments_mut()
@@ -258,15 +267,21 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
method = method.strip_prefix("registry.").unwrap_or(method);
let sig_context = registry.host_str().map(InternedString::from);
crate::middleware::auth::signature::call_remote(
let mut res = crate::middleware::auth::signature::call_remote(
self,
registry,
headers,
sig_context.as_deref(),
method,
params,
params.clone(),
)
.await
.await?;
if let Some(device_info) = device_info {
device_info.filter_for_hardware(method, params, &mut res)?;
}
Ok(res)
}
}

202
core/src/registry/device_info.rs
View File

@@ -1,5 +1,4 @@
use std::collections::BTreeMap;
use std::convert::identity;
use std::ops::Deref;
use axum::extract::Request;
@@ -7,6 +6,8 @@ use axum::response::Response;
use exver::{Version, VersionRange};
use http::HeaderValue;
use imbl_value::InternedString;
use patch_db::ModelExt;
use rpc_toolkit::yajrc::RpcMethod;
use rpc_toolkit::{Middleware, RpcRequest, RpcResponse};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
@@ -15,8 +16,13 @@ use url::Url;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::registry::context::RegistryContext;
use crate::registry::os::index::OsVersionInfoMap;
use crate::registry::package::get::{
GetPackageParams, GetPackageResponse, GetPackageResponseFull, PackageDetailLevel,
};
use crate::registry::package::index::PackageVersionInfo;
use crate::util::VersionString;
use crate::util::lshw::{LshwDevice, LshwDisplay, LshwProcessor};
use crate::util::lshw::LshwDevice;
use crate::version::VersionT;
pub const DEVICE_INFO_HEADER: &str = "X-StartOS-Device-Info";
@@ -25,13 +31,13 @@ pub const DEVICE_INFO_HEADER: &str = "X-StartOS-Device-Info";
#[serde(rename_all = "camelCase")]
pub struct DeviceInfo {
pub os: OsInfo,
pub hardware: HardwareInfo,
pub hardware: Option<HardwareInfo>,
}
impl DeviceInfo {
pub async fn load(ctx: &RpcContext) -> Result<Self, Error> {
Ok(Self {
os: OsInfo::from(ctx),
hardware: HardwareInfo::load(ctx).await?,
hardware: Some(HardwareInfo::load(ctx).await?),
})
}
}
@@ -41,21 +47,13 @@ impl DeviceInfo {
url.query_pairs_mut()
.append_pair("os.version", &self.os.version.to_string())
.append_pair("os.compat", &self.os.compat.to_string())
.append_pair("os.platform", &*self.os.platform)
.append_pair("hardware.arch", &*self.hardware.arch)
.append_pair("hardware.ram", &self.hardware.ram.to_string());
for device in &self.hardware.devices {
url.query_pairs_mut().append_pair(
&format!("hardware.device.{}", device.class()),
device.product(),
);
}
.append_pair("os.platform", &*self.os.platform);
HeaderValue::from_str(url.query().unwrap_or_default()).unwrap()
}
pub fn from_header_value(header: &HeaderValue) -> Result<Self, Error> {
let query: BTreeMap<_, _> = form_urlencoded::parse(header.as_bytes()).collect();
let has_hw_info = query.keys().any(|k| k.starts_with("hardware."));
Ok(Self {
os: OsInfo {
version: query
@@ -69,35 +67,120 @@ impl DeviceInfo {
.deref()
.into(),
},
hardware: HardwareInfo {
arch: query
.get("hardware.arch")
.or_not_found("hardware.arch")?
.parse()?,
ram: query
.get("hardware.ram")
.or_not_found("hardware.ram")?
.parse()?,
devices: identity(query)
.split_off("hardware.device.")
.into_iter()
.filter_map(|(k, v)| match k.strip_prefix("hardware.device.") {
Some("processor") => Some(LshwDevice::Processor(LshwProcessor {
product: v.into_owned(),
})),
Some("display") => Some(LshwDevice::Display(LshwDisplay {
product: v.into_owned(),
})),
Some(class) => {
tracing::warn!("unknown device class: {class}");
None
}
_ => None,
hardware: has_hw_info
.then(|| {
Ok::<_, Error>(HardwareInfo {
arch: query
.get("hardware.arch")
.or_not_found("hardware.arch")?
.parse()?,
ram: query
.get("hardware.ram")
.or_not_found("hardware.ram")?
.parse()?,
devices: None,
})
.collect(),
},
})
.transpose()?,
})
}
pub fn filter_for_hardware(
&self,
method: &str,
params: Value,
res: &mut Value,
) -> Result<(), Error> {
match method {
"package.get" => {
let params: Model<GetPackageParams> = ModelExt::from_value(params);
let other = params.as_other_versions().de()?;
if params.as_id().transpose_ref().is_some() {
if other.unwrap_or_default() == PackageDetailLevel::Full {
self.filter_package_get_full(ModelExt::value_as_mut(res))?;
} else {
self.filter_package_get(ModelExt::value_as_mut(res))?;
}
} else {
for (_, v) in res.as_object_mut().into_iter().flat_map(|o| o.iter_mut()) {
if other.unwrap_or_default() == PackageDetailLevel::Full {
self.filter_package_get_full(ModelExt::value_as_mut(v))?;
} else {
self.filter_package_get(ModelExt::value_as_mut(v))?;
}
}
}
Ok(())
}
"os.version.get" => self.filter_os_version(ModelExt::value_as_mut(res)),
_ => Ok(()),
}
}
fn filter_package_versions(
&self,
versions: &mut Model<BTreeMap<VersionString, PackageVersionInfo>>,
) -> Result<(), Error> {
let alpha_17: Version = "0.4.0-alpha.17".parse()?;
// Filter package versions using for_device
versions.retain(|_, info| info.for_device(self))?;
// Alpha.17 compatibility: add legacy fields
if self.os.version <= alpha_17 {
for (_, info) in versions.as_entries_mut()? {
let v = info.as_value_mut();
if let Some(mut tup) = v["s9pks"].get(0).cloned() {
v["s9pk"] = tup[1].take();
v["hardwareRequirements"] = tup[0].take();
v["s9pk"]["url"] = v["s9pk"]["urls"][0].clone();
}
}
}
Ok(())
}
fn filter_package_get(&self, res: &mut Model<GetPackageResponse>) -> Result<(), Error> {
self.filter_package_versions(res.as_best_mut())
}
fn filter_package_get_full(
&self,
res: &mut Model<GetPackageResponseFull>,
) -> Result<(), Error> {
self.filter_package_versions(res.as_best_mut())?;
self.filter_package_versions(res.as_other_versions_mut())
}
fn filter_os_version(&self, res: &mut Model<OsVersionInfoMap>) -> Result<(), Error> {
let alpha_17: Version = "0.4.0-alpha.17".parse()?;
// Filter OS versions based on source_version compatibility
res.retain(|_, info| {
let source_version = info.as_source_version().de()?;
Ok(self.os.version.satisfies(&source_version))
})?;
// Alpha.17 compatibility: add url field from urls array
if self.os.version <= alpha_17 {
for (_, info) in res.as_entries_mut()? {
let v = info.as_value_mut();
for asset_ty in ["iso", "squashfs", "img"] {
for (_, asset) in v[asset_ty]
.as_object_mut()
.into_iter()
.flat_map(|o| o.iter_mut())
{
asset["url"] = asset["urls"][0].clone();
}
}
}
}
Ok(())
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
@@ -127,7 +210,7 @@ pub struct HardwareInfo {
pub arch: InternedString,
#[ts(type = "number")]
pub ram: u64,
pub devices: Vec<LshwDevice>,
pub devices: Option<Vec<LshwDevice>>,
}
impl HardwareInfo {
pub async fn load(ctx: &RpcContext) -> Result<Self, Error> {
@@ -135,7 +218,7 @@ impl HardwareInfo {
Ok(Self {
arch: s.as_arch().de()?,
ram: s.as_ram().de()?,
devices: s.as_devices().de()?,
devices: Some(s.as_devices().de()?),
})
}
}
@@ -148,11 +231,17 @@ pub struct Metadata {
#[derive(Clone)]
pub struct DeviceInfoMiddleware {
device_info: Option<HeaderValue>,
device_info_header: Option<HeaderValue>,
device_info: Option<DeviceInfo>,
req: Option<RpcRequest>,
}
impl DeviceInfoMiddleware {
pub fn new() -> Self {
Self { device_info: None }
Self {
device_info_header: None,
device_info: None,
req: None,
}
}
}
@@ -163,7 +252,7 @@ impl Middleware<RegistryContext> for DeviceInfoMiddleware {
_: &RegistryContext,
request: &mut Request,
) -> Result<(), Response> {
self.device_info = request.headers_mut().remove(DEVICE_INFO_HEADER);
self.device_info_header = request.headers_mut().remove(DEVICE_INFO_HEADER);
Ok(())
}
async fn process_rpc_request(
@@ -174,9 +263,11 @@ impl Middleware<RegistryContext> for DeviceInfoMiddleware {
) -> Result<(), RpcResponse> {
async move {
if metadata.get_device_info {
if let Some(device_info) = &self.device_info {
request.params["__DeviceInfo_device_info"] =
to_value(&DeviceInfo::from_header_value(device_info)?)?;
if let Some(device_info) = &self.device_info_header {
let device_info = DeviceInfo::from_header_value(device_info)?;
request.params["__DeviceInfo_device_info"] = to_value(&device_info)?;
self.device_info = Some(device_info);
self.req = Some(request.clone());
}
}
@@ -185,4 +276,19 @@ impl Middleware<RegistryContext> for DeviceInfoMiddleware {
.await
.map_err(|e| RpcResponse::from_result(Err(e)))
}
async fn process_rpc_response(
&mut self,
_: &RegistryContext,
response: &mut RpcResponse,
) -> () {
if let (Some(req), Some(device_info), Ok(res)) =
(&self.req, &self.device_info, &mut response.result)
{
if let Err(e) =
device_info.filter_for_hardware(req.method.as_str(), req.params.clone(), res)
{
response.result = Err(e).map_err(From::from);
}
}
}
}

3
core/src/registry/migrations/m_00_package_signer_scope.rs
View File

@@ -5,9 +5,6 @@ use crate::prelude::*;
pub struct PackageSignerScopeMigration;
impl RegistryMigration for PackageSignerScopeMigration {
fn name(&self) -> &'static str {
"PackageSignerScopeMigration"
}
fn action(&self, db: &mut Value) -> Result<(), Error> {
for (_, info) in db["index"]["package"]["packages"]
.as_object_mut()

35
core/src/registry/migrations/m_01_registry_asset_array.rs Normal file
View File

@@ -0,0 +1,35 @@
use imbl::vector;
use super::RegistryMigration;
use crate::prelude::*;
pub struct RegistryAssetArray;
impl RegistryMigration for RegistryAssetArray {
fn action(&self, db: &mut Value) -> Result<(), Error> {
for (_, info) in db["index"]["package"]["packages"]
.as_object_mut()
.unwrap()
.iter_mut()
{
for (_, info) in info["versions"].as_object_mut().unwrap().iter_mut() {
let hw_req = info["hardwareRequirements"].take();
let mut s9pk = info["s9pk"].take();
s9pk["urls"] = Value::Array(vector![s9pk["url"].take()]);
info["s9pks"] = Value::Array(vector![Value::Array(vector![hw_req, s9pk])]);
}
}
for (_, info) in db["index"]["os"]["versions"]
.as_object_mut()
.unwrap()
.iter_mut()
{
for asset_ty in ["iso", "squashfs", "img"] {
for (_, info) in info[asset_ty].as_object_mut().unwrap().iter_mut() {
info["urls"] = Value::Array(vector![info["url"].take()]);
}
}
}
Ok(())
}
}

17
core/src/registry/migrations/mod.rs
View File

@@ -4,22 +4,29 @@ use crate::prelude::*;
use crate::registry::RegistryDatabase;
mod m_00_package_signer_scope;
mod m_01_registry_asset_array;
pub trait RegistryMigration {
fn name(&self) -> &'static str;
fn name(&self) -> &'static str {
let val = std::any::type_name_of_val(self);
val.rsplit_once("::").map_or(val, |v| v.1)
}
fn action(&self, db: &mut Value) -> Result<(), Error>;
}
pub const MIGRATIONS: &[&dyn RegistryMigration] =
&[&m_00_package_signer_scope::PackageSignerScopeMigration];
pub const MIGRATIONS: &[&dyn RegistryMigration] = &[
&m_00_package_signer_scope::PackageSignerScopeMigration,
&m_01_registry_asset_array::RegistryAssetArray,
];
#[instrument(skip_all)]
pub fn run_migrations(db: &mut Model<RegistryDatabase>) -> Result<(), Error> {
let mut migrations = db.as_migrations().de().unwrap_or_default();
for migration in MIGRATIONS {
if !migrations.contains(migration.name()) {
let name = migration.name();
if !migrations.contains(name) {
migration.action(ModelExt::as_value_mut(db))?;
migrations.insert(migration.name().into());
migrations.insert(name.into());
}
}
let mut db_deser = db.de()?;

5
core/src/registry/os/asset/add.rs
View File

@@ -133,7 +133,7 @@ async fn add_asset(
.upsert(&platform, || {
Ok(RegistryAsset {
published_at: Utc::now(),
url,
urls: vec![url.clone()],
commitment: commitment.clone(),
signatures: HashMap::new(),
})
@@ -146,6 +146,9 @@ async fn add_asset(
))
} else {
s.signatures.insert(signer, signature);
if !s.urls.contains(&url) {
s.urls.push(url);
}
Ok(())
}
})?;

71
core/src/registry/os/version/mod.rs
View File

@@ -187,7 +187,8 @@ pub async fn get_version(
platform,
device_info,
}: GetOsVersionParams,
) -> Result<BTreeMap<Version, OsVersionInfo>, Error> {
) -> Result<Value, Error> // BTreeMap<Version, OsVersionInfo>
{
let source = source.or_else(|| device_info.as_ref().map(|d| d.os.version.clone()));
let platform = platform.or_else(|| device_info.as_ref().map(|d| d.os.platform.clone()));
if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, &platform) {
@@ -202,33 +203,63 @@ pub async fn get_version(
.with_kind(ErrorKind::Database)?;
}
let target = target.unwrap_or(VersionRange::Any);
ctx.db
.peek()
.await
.into_index()
.into_os()
.into_versions()
.into_entries()?
.into_iter()
.map(|(v, i)| i.de().map(|i| (v, i)))
.filter_ok(|(version, info)| {
platform
.as_ref()
.map_or(true, |p| info.squashfs.contains_key(p))
&& version.satisfies(&target)
&& source
let mut res = to_value::<BTreeMap<Version, OsVersionInfo>>(
&ctx.db
.peek()
.await
.into_index()
.into_os()
.into_versions()
.into_entries()?
.into_iter()
.map(|(v, i)| i.de().map(|i| (v, i)))
.filter_ok(|(version, info)| {
platform
.as_ref()
.map_or(true, |s| s.satisfies(&info.source_version))
})
.collect()
.map_or(true, |p| info.squashfs.contains_key(p))
&& version.satisfies(&target)
&& source
.as_ref()
.map_or(true, |s| s.satisfies(&info.source_version))
})
.collect::<Result<_, _>>()?,
)?;
// TODO: remove
if device_info.map_or(false, |d| {
"0.4.0-alpha.17"
.parse::<Version>()
.map_or(false, |v| d.os.version <= v)
}) {
for (_, v) in res
.as_object_mut()
.into_iter()
.map(|v| v.iter_mut())
.flatten()
{
for asset_ty in ["iso", "squashfs", "img"] {
for (_, v) in v[asset_ty]
.as_object_mut()
.into_iter()
.map(|v| v.iter_mut())
.flatten()
{
v["url"] = v["urls"][0].clone();
}
}
}
}
Ok(res)
}
pub fn display_version_info<T>(
params: WithIoFormat<T>,
info: BTreeMap<Version, OsVersionInfo>,
info: Value, // BTreeMap<Version, OsVersionInfo>,
) -> Result<(), Error> {
use prettytable::*;
let info = from_value::<BTreeMap<Version, OsVersionInfo>>(info)?;
if let Some(format) = params.format {
return display_serializable(format, info);
}

332
core/src/registry/package/add.rs
View File

@@ -12,12 +12,11 @@ use url::Url;
use crate::PackageId;
use crate::context::CliContext;
use crate::prelude::*;
use crate::progress::{FullProgressTracker, ProgressTrackerWriter, ProgressUnits};
use crate::progress::FullProgressTracker;
use crate::registry::asset::BufferedHttpSource;
use crate::registry::context::RegistryContext;
use crate::registry::package::index::PackageVersionInfo;
use crate::s9pk::S9pk;
use crate::s9pk::merkle_archive::source::ArchiveSource;
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::sign::commitment::merkle_archive::MerkleArchiveCommitment;
@@ -25,13 +24,14 @@ use crate::sign::ed25519::Ed25519;
use crate::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
use crate::util::VersionString;
use crate::util::io::TrackingIO;
use crate::util::serde::Base64;
#[derive(Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct AddPackageParams {
#[ts(type = "string")]
pub url: Url,
#[ts(type = "string[]")]
pub urls: Vec<Url>,
#[ts(skip)]
#[serde(rename = "__Auth_signer")]
pub uploader: AnyVerifyingKey,
@@ -42,7 +42,7 @@ pub struct AddPackageParams {
pub async fn add_package(
ctx: RegistryContext,
AddPackageParams {
url,
urls,
uploader,
commitment,
signature,
@@ -53,17 +53,35 @@ pub async fn add_package(
.verify_commitment(&uploader, &commitment, SIG_CONTEXT, &signature)?;
let peek = ctx.db.peek().await;
let uploader_guid = peek.as_index().as_signers().get_signer(&uploader)?;
let Some(([url], rest)) = urls.split_at_checked(1) else {
return Err(Error::new(
eyre!("must specify at least 1 url"),
ErrorKind::InvalidRequest,
));
};
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
for url in rest {
S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
}
let manifest = s9pk.as_manifest();
let mut info = PackageVersionInfo::from_s9pk(&s9pk, url).await?;
if !info.s9pk.signatures.contains_key(&uploader) {
info.s9pk.signatures.insert(uploader.clone(), signature);
let mut info = PackageVersionInfo::from_s9pk(&s9pk, urls).await?;
for (_, s9pk) in &mut info.s9pks {
if !s9pk.signatures.contains_key(&uploader) && s9pk.commitment == commitment {
s9pk.signatures.insert(uploader.clone(), signature.clone());
}
}
ctx.db
@@ -85,7 +103,12 @@ pub async fn add_package(
.as_package_mut()
.as_packages_mut()
.upsert(&manifest.id, || Ok(Default::default()))?;
package.as_versions_mut().insert(&manifest.version, &info)?;
let v = package.as_versions_mut();
if let Some(prev) = v.as_idx_mut(&manifest.version) {
prev.mutate(|p| p.merge_with(info, true))?;
} else {
v.insert(&manifest.version, &info)?;
}
Ok(())
} else {
@@ -101,7 +124,10 @@ pub async fn add_package(
#[serde(rename_all = "camelCase")]
pub struct CliAddPackageParams {
pub file: PathBuf,
pub url: Url,
#[arg(long)]
pub url: Vec<Url>,
#[arg(long)]
pub no_verify: bool,
}
pub async fn cli_add_package(
@@ -109,7 +135,12 @@ pub async fn cli_add_package(
context: ctx,
parent_method,
method,
params: CliAddPackageParams { file, url },
params:
CliAddPackageParams {
file,
url,
no_verify,
},
..
}: HandlerArgs<CliContext, CliAddPackageParams>,
) -> Result<(), Error> {
@@ -117,7 +148,19 @@ pub async fn cli_add_package(
let progress = FullProgressTracker::new();
let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1));
let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100));
let verify = if !no_verify {
url.iter()
.map(|url| {
let phase = progress.add_phase(
InternedString::from_display(&lazy_format!("Verifying {url}")),
Some(100),
);
(url.clone(), phase)
})
.collect()
} else {
Vec::new()
};
let mut index_phase = progress.add_phase(
InternedString::intern("Adding File to Registry Index"),
Some(1),
@@ -131,11 +174,240 @@ pub async fn cli_add_package(
let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?;
sign_phase.complete();
verify_phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url.clone(), verify_phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
for (url, mut phase) in verify {
phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url, phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
}
index_phase.start();
ctx.call_remote::<RegistryContext>(
&parent_method.into_iter().chain(method).join("."),
imbl_value::json!({
"urls": &url,
"signature": AnySignature::Ed25519(signature),
"commitment": commitment,
}),
)
.await?;
index_phase.complete();
progress.complete();
progress_task.await.with_kind(ErrorKind::Unknown)?;
Ok(())
}
#[derive(Debug, Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RemovePackageParams {
pub id: PackageId,
pub version: VersionString,
#[arg(long)]
pub sighash: Option<Base64<[u8; 32]>>,
#[ts(skip)]
#[arg(skip)]
#[serde(rename = "__Auth_signer")]
pub signer: Option<AnyVerifyingKey>,
}
pub async fn remove_package(
ctx: RegistryContext,
RemovePackageParams {
id,
version,
sighash,
signer,
}: RemovePackageParams,
) -> Result<bool, Error> {
let peek = ctx.db.peek().await;
let signer =
signer.ok_or_else(|| Error::new(eyre!("missing signer"), ErrorKind::InvalidRequest))?;
let signer_guid = peek.as_index().as_signers().get_signer(&signer)?;
let rev = ctx
.db
.mutate(|db| {
if db.as_admins().de()?.contains(&signer_guid)
|| db
.as_index()
.as_package()
.as_packages()
.as_idx(&id)
.or_not_found(&id)?
.as_authorized()
.de()?
.get(&signer_guid)
.map_or(false, |v| version.satisfies(v))
{
if let Some(package) = db
.as_index_mut()
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&id)
{
if let Some(sighash) = sighash {
if if let Some(package) = package.as_versions_mut().as_idx_mut(&version) {
package.as_s9pks_mut().mutate(|s| {
s.retain(|(_, asset)| asset.commitment.root_sighash != sighash);
Ok(s.is_empty())
})?
} else {
false
} {
package.as_versions_mut().remove(&version)?;
}
} else {
package.as_versions_mut().remove(&version)?;
}
}
Ok(())
} else {
Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
}
})
.await;
rev.result.map(|_| rev.revision.is_some())
}
#[derive(Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct AddMirrorParams {
#[ts(type = "string")]
pub url: Url,
#[ts(skip)]
#[serde(rename = "__Auth_signer")]
pub uploader: AnyVerifyingKey,
pub commitment: MerkleArchiveCommitment,
pub signature: AnySignature,
}
pub async fn add_mirror(
ctx: RegistryContext,
AddMirrorParams {
url,
uploader,
commitment,
signature,
}: AddMirrorParams,
) -> Result<(), Error> {
uploader
.scheme()
.verify_commitment(&uploader, &commitment, SIG_CONTEXT, &signature)?;
let peek = ctx.db.peek().await;
let uploader_guid = peek.as_index().as_signers().get_signer(&uploader)?;
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
let manifest = s9pk.as_manifest();
let mut info = PackageVersionInfo::from_s9pk(&s9pk, vec![url]).await?;
for (_, s9pk) in &mut info.s9pks {
if !s9pk.signatures.contains_key(&uploader) && s9pk.commitment == commitment {
s9pk.signatures.insert(uploader.clone(), signature.clone());
}
}
ctx.db
.mutate(|db| {
if db.as_admins().de()?.contains(&uploader_guid)
|| db
.as_index()
.as_package()
.as_packages()
.as_idx(&manifest.id)
.or_not_found(&manifest.id)?
.as_authorized()
.de()?
.get(&uploader_guid)
.map_or(false, |v| manifest.version.satisfies(v))
{
let package = db
.as_index_mut()
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&manifest.id)
.and_then(|p| p.as_versions_mut().as_idx_mut(&manifest.version))
.or_not_found(&lazy_format!("{}@{}", &manifest.id, &manifest.version))?;
package.mutate(|p| p.merge_with(info, false))?;
Ok(())
} else {
Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
}
})
.await
.result
}
#[derive(Debug, Deserialize, Serialize, Parser)]
#[command(rename_all = "kebab-case")]
#[serde(rename_all = "camelCase")]
pub struct CliAddMirrorParams {
pub file: PathBuf,
pub url: Url,
pub no_verify: bool,
}
pub async fn cli_add_mirror(
HandlerArgs {
context: ctx,
parent_method,
method,
params:
CliAddMirrorParams {
file,
url,
no_verify,
},
..
}: HandlerArgs<CliContext, CliAddMirrorParams>,
) -> Result<(), Error> {
let s9pk = S9pk::open(&file, None).await?;
let progress = FullProgressTracker::new();
let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1));
let verify = if !no_verify {
let url = &url;
vec![(
url.clone(),
progress.add_phase(
InternedString::from_display(&lazy_format!("Verifying {url}")),
Some(100),
),
)]
} else {
Vec::new()
};
let mut index_phase = progress.add_phase(
InternedString::intern("Adding File to Registry Index"),
Some(1),
);
let progress_task =
progress.progress_bar_task(&format!("Adding {} to registry...", file.display()));
sign_phase.start();
let commitment = s9pk.as_archive().commitment().await?;
let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?;
sign_phase.complete();
for (url, mut phase) in verify {
phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url, phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
}
index_phase.start();
ctx.call_remote::<RegistryContext>(
@@ -159,22 +431,26 @@ pub async fn cli_add_package(
#[derive(Debug, Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RemovePackageParams {
pub struct RemoveMirrorParams {
pub id: PackageId,
pub version: VersionString,
#[arg(long)]
#[ts(type = "string")]
pub url: Url,
#[ts(skip)]
#[arg(skip)]
#[serde(rename = "__Auth_signer")]
pub signer: Option<AnyVerifyingKey>,
}
pub async fn remove_package(
pub async fn remove_mirror(
ctx: RegistryContext,
RemovePackageParams {
RemoveMirrorParams {
id,
version,
url,
signer,
}: RemovePackageParams,
}: RemoveMirrorParams,
) -> Result<(), Error> {
let peek = ctx.db.peek().await;
let signer =
@@ -200,8 +476,20 @@ pub async fn remove_package(
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&id)
.and_then(|p| p.as_versions_mut().as_idx_mut(&version))
{
package.as_versions_mut().remove(&version)?;
package.as_s9pks_mut().mutate(|s| {
s.iter_mut()
.for_each(|(_, asset)| asset.urls.retain(|u| u != &url));
if s.iter().any(|(_, asset)| asset.urls.is_empty()) {
Err(Error::new(
eyre!("cannot remove last mirror from an s9pk"),
ErrorKind::InvalidRequest,
))
} else {
Ok(())
}
})?;
}
Ok(())
} else {

153
core/src/registry/package/get.rs
View File

@@ -20,12 +20,12 @@ use crate::s9pk::v2::SIG_CONTEXT;
use crate::util::VersionString;
use crate::util::io::{TrackingIO, to_tmp_path};
use crate::util::serde::{WithIoFormat, display_serializable};
use crate::util::tui::choose;
use crate::util::tui::{choose, choose_custom_display};
#[derive(
Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS, ValueEnum,
)]
#[serde(rename_all = "camelCase")]
#[serde(rename_all = "kebab-case")]
#[ts(export)]
pub enum PackageDetailLevel {
None,
@@ -45,10 +45,11 @@ pub struct PackageInfoShort {
pub release_notes: String,
}
#[derive(Debug, Deserialize, Serialize, TS, Parser)]
#[derive(Debug, Deserialize, Serialize, TS, Parser, HasModel)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
#[ts(export)]
#[model = "Model<Self>"]
pub struct GetPackageParams {
pub id: Option<PackageId>,
#[ts(type = "string | null")]
@@ -60,14 +61,14 @@ pub struct GetPackageParams {
#[arg(skip)]
#[serde(rename = "__DeviceInfo_device_info")]
pub device_info: Option<DeviceInfo>,
#[serde(default)]
#[arg(default_value = "none")]
pub other_versions: PackageDetailLevel,
pub other_versions: Option<PackageDetailLevel>,
}
#[derive(Debug, Deserialize, Serialize, TS)]
#[derive(Debug, Deserialize, Serialize, TS, HasModel)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
#[model = "Model<Self>"]
pub struct GetPackageResponse {
#[ts(type = "string[]")]
pub categories: BTreeSet<InternedString>,
@@ -108,9 +109,10 @@ impl GetPackageResponse {
}
}
#[derive(Debug, Deserialize, Serialize, TS)]
#[derive(Debug, Deserialize, Serialize, TS, HasModel)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
#[model = "Model<Self>"]
pub struct GetPackageResponseFull {
#[ts(type = "string[]")]
pub categories: BTreeSet<InternedString>,
@@ -134,15 +136,15 @@ impl GetPackageResponseFull {
pub type GetPackagesResponse = BTreeMap<PackageId, GetPackageResponse>;
pub type GetPackagesResponseFull = BTreeMap<PackageId, GetPackageResponseFull>;
fn get_matching_models<'a>(
db: &'a Model<PackageIndex>,
fn get_matching_models(
db: &Model<PackageIndex>,
GetPackageParams {
id,
source_version,
device_info,
..
}: &GetPackageParams,
) -> Result<Vec<(PackageId, ExtendedVersion, &'a Model<PackageVersionInfo>)>, Error> {
) -> Result<Vec<(PackageId, ExtendedVersion, Model<PackageVersionInfo>)>, Error> {
if let Some(id) = id {
if let Some(pkg) = db.as_packages().as_idx(id) {
vec![(id.clone(), pkg)]
@@ -168,11 +170,17 @@ fn get_matching_models<'a>(
.unwrap_or(VersionRange::any()),
),
)
})? && device_info
.as_ref()
.map_or(Ok(true), |device_info| info.works_for_device(device_info))?
{
Some((k.clone(), ExtendedVersion::from(v), info))
})? {
let mut info = info.clone();
if let Some(device_info) = &device_info {
if info.for_device(device_info)? {
Some((k.clone(), ExtendedVersion::from(v), info))
} else {
None
}
} else {
Some((k.clone(), ExtendedVersion::from(v), info))
}
} else {
None
},
@@ -186,12 +194,10 @@ fn get_matching_models<'a>(
}
pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Result<Value, Error> {
use patch_db::ModelExt;
let peek = ctx.db.peek().await;
let mut best: BTreeMap<PackageId, BTreeMap<VersionString, &Model<PackageVersionInfo>>> =
let mut best: BTreeMap<PackageId, BTreeMap<VersionString, Model<PackageVersionInfo>>> =
Default::default();
let mut other: BTreeMap<PackageId, BTreeMap<VersionString, &Model<PackageVersionInfo>>> =
let mut other: BTreeMap<PackageId, BTreeMap<VersionString, Model<PackageVersionInfo>>> =
Default::default();
for (id, version, info) in get_matching_models(&peek.as_index().as_package(), &params)? {
let package_best = best.entry(id.clone()).or_default();
@@ -217,23 +223,23 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
package_other.insert(version.into(), info);
}
}
if let Some(id) = params.id {
if let Some(id) = &params.id {
let categories = peek
.as_index()
.as_package()
.as_packages()
.as_idx(&id)
.as_idx(id)
.map(|p| p.as_categories().de())
.transpose()?
.unwrap_or_default();
let best = best
.remove(&id)
let best: BTreeMap<VersionString, PackageVersionInfo> = best
.remove(id)
.unwrap_or_default()
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?;
let other = other.remove(&id).unwrap_or_default();
match params.other_versions {
let other = other.remove(id).unwrap_or_default();
match params.other_versions.unwrap_or_default() {
PackageDetailLevel::None => to_value(&GetPackageResponse {
categories,
best,
@@ -245,7 +251,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
other_versions: Some(
other
.into_iter()
.map(|(k, v)| from_value(v.as_value().clone()).map(|v| (k, v)))
.map(|(k, i)| from_value(i.into()).map(|v| (k, v)))
.try_collect()?,
),
}),
@@ -254,12 +260,12 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
best,
other_versions: other
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?,
}),
}
} else {
match params.other_versions {
match params.other_versions.unwrap_or_default() {
PackageDetailLevel::None => to_value(
&best
.into_iter()
@@ -278,7 +284,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
categories,
best: best
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?,
other_versions: None,
},
@@ -305,14 +311,12 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
categories,
best: best
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?,
other_versions: Some(
other
.into_iter()
.map(|(k, v)| {
from_value(v.as_value().clone()).map(|v| (k, v))
})
.map(|(k, i)| from_value(i.into()).map(|v| (k, v)))
.try_collect()?,
),
},
@@ -339,11 +343,11 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
categories,
best: best
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?,
other_versions: other
.into_iter()
.map(|(k, v)| v.de().map(|v| (k, v)))
.map(|(k, i)| Ok::<_, Error>((k, i.de()?)))
.try_collect()?,
},
))
@@ -363,7 +367,7 @@ pub fn display_package_info(
}
if let Some(_) = params.rest.id {
if params.rest.other_versions == PackageDetailLevel::Full {
if params.rest.other_versions.unwrap_or_default() == PackageDetailLevel::Full {
for table in from_value::<GetPackageResponseFull>(info)?.tables() {
table.print_tty(false)?;
println!();
@@ -375,7 +379,7 @@ pub fn display_package_info(
}
}
} else {
if params.rest.other_versions == PackageDetailLevel::Full {
if params.rest.other_versions.unwrap_or_default() == PackageDetailLevel::Full {
for (_, package) in from_value::<GetPackagesResponseFull>(info)? {
for table in package.tables() {
table.print_tty(false)?;
@@ -431,7 +435,9 @@ pub async fn cli_download(
)
.await?,
)?;
let PackageVersionInfo { s9pk, .. } = match res.best.len() {
let PackageVersionInfo {
s9pks: mut s9pk, ..
} = match res.best.len() {
0 => {
return Err(Error::new(
eyre!(
@@ -452,6 +458,75 @@ pub async fn cli_download(
res.best.remove(version).unwrap()
}
};
let s9pk = match s9pk.len() {
0 => {
return Err(Error::new(
eyre!(
"Could not find a version of {id} that satisfies {}",
target_version.unwrap_or(VersionRange::Any)
),
ErrorKind::NotFound,
));
}
1 => s9pk.pop().unwrap().1,
_ => {
let (_, asset) = choose_custom_display(
&format!(concat!(
"Multiple packages with different hardware requirements found. ",
"Choose a file to download:"
)),
&s9pk,
|(hw, _)| {
use std::fmt::Write;
let mut res = String::new();
if let Some(arch) = &hw.arch {
write!(
&mut res,
"{}: {}",
if arch.len() == 1 {
"Architecture"
} else {
"Architectures"
},
arch.iter().join(", ")
)
.unwrap();
}
if !hw.device.is_empty() {
if !res.is_empty() {
write!(&mut res, "; ").unwrap();
}
write!(
&mut res,
"{}: {}",
if hw.device.len() == 1 {
"Device"
} else {
"Devices"
},
hw.device.iter().map(|d| &d.description).join(", ")
)
.unwrap();
}
if let Some(ram) = hw.ram {
if !res.is_empty() {
write!(&mut res, "; ").unwrap();
}
write!(
&mut res,
"RAM >={:.2}GiB",
ram as f64 / (1024.0 * 1024.0 * 1024.0)
)
.unwrap();
}
res
},
)
.await?;
asset.clone()
}
};
s9pk.validate(SIG_CONTEXT, s9pk.all_signers())?;
fetching_progress.complete();

191
core/src/registry/package/index.rs
View File

@@ -1,8 +1,10 @@
use std::collections::{BTreeMap, BTreeSet};
use std::u32;
use chrono::Utc;
use exver::{Version, VersionRange};
use imbl_value::InternedString;
use patch_db::ModelExt;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use url::Url;
@@ -50,7 +52,7 @@ pub struct Category {
pub name: String,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[derive(Debug, Deserialize, Serialize, HasModel, TS, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
@@ -62,11 +64,10 @@ pub struct DependencyMetadata {
pub optional: bool,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[derive(Debug, Deserialize, Serialize, HasModel, TS, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct PackageVersionInfo {
pub struct PackageMetadata {
#[ts(type = "string")]
pub title: InternedString,
pub icon: DataUrl<'static>,
@@ -93,13 +94,11 @@ pub struct PackageVersionInfo {
pub os_version: Version,
#[ts(type = "string | null")]
pub sdk_version: Option<Version>,
pub hardware_requirements: HardwareRequirements,
#[ts(type = "string | null")]
pub source_version: Option<VersionRange>,
pub s9pk: RegistryAsset<MerkleArchiveCommitment>,
#[serde(default)]
pub hardware_acceleration: bool,
}
impl PackageVersionInfo {
pub async fn from_s9pk<S: FileSource + Clone>(s9pk: &S9pk<S>, url: Url) -> Result<Self, Error> {
impl PackageMetadata {
pub async fn load<S: FileSource + Clone>(s9pk: &S9pk<S>) -> Result<Self, Error> {
let manifest = s9pk.as_manifest();
let mut dependency_metadata = BTreeMap::new();
for (id, info) in &manifest.dependencies.0 {
@@ -131,67 +130,153 @@ impl PackageVersionInfo {
dependency_metadata,
os_version: manifest.os_version.clone(),
sdk_version: manifest.sdk_version.clone(),
hardware_requirements: manifest.hardware_requirements.clone(),
source_version: None, // TODO
s9pk: RegistryAsset {
published_at: Utc::now(),
url,
commitment: s9pk.as_archive().commitment().await?,
signatures: [(
AnyVerifyingKey::Ed25519(s9pk.as_archive().signer()),
AnySignature::Ed25519(s9pk.as_archive().signature().await?),
)]
.into_iter()
.collect(),
},
hardware_acceleration: manifest.hardware_acceleration.clone(),
})
}
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct PackageVersionInfo {
#[serde(flatten)]
pub metadata: PackageMetadata,
#[ts(type = "string | null")]
pub source_version: Option<VersionRange>,
pub s9pks: Vec<(HardwareRequirements, RegistryAsset<MerkleArchiveCommitment>)>,
}
impl PackageVersionInfo {
pub async fn from_s9pk<S: FileSource + Clone>(
s9pk: &S9pk<S>,
urls: Vec<Url>,
) -> Result<Self, Error> {
Ok(Self {
metadata: PackageMetadata::load(s9pk).await?,
source_version: None, // TODO
s9pks: vec![(
s9pk.as_manifest().hardware_requirements.clone(),
RegistryAsset {
published_at: Utc::now(),
urls,
commitment: s9pk.as_archive().commitment().await?,
signatures: [(
AnyVerifyingKey::Ed25519(s9pk.as_archive().signer()),
AnySignature::Ed25519(s9pk.as_archive().signature().await?),
)]
.into_iter()
.collect(),
},
)],
})
}
pub fn merge_with(&mut self, other: Self, replace_urls: bool) -> Result<(), Error> {
for (hw_req, asset) in other.s9pks {
if let Some((_, matching)) = self
.s9pks
.iter_mut()
.find(|(h, s)| s.commitment == asset.commitment && *h == hw_req)
{
if replace_urls {
matching.urls = asset.urls;
} else {
for url in asset.urls {
if matching.urls.contains(&url) {
continue;
}
matching.urls.push(url);
}
}
} else {
if let Some((h, matching)) = self.s9pks.iter_mut().find(|(h, _)| *h == hw_req) {
*matching = asset;
*h = hw_req;
} else {
self.s9pks.push((hw_req, asset));
}
}
}
self.s9pks.sort_by_key(|(h, _)| h.specificity_desc());
Ok(())
}
pub fn table(&self, version: &VersionString) -> prettytable::Table {
use prettytable::*;
let mut table = Table::new();
table.add_row(row![bc => &self.title]);
table.add_row(row![bc => &self.metadata.title]);
table.add_row(row![br -> "VERSION", AsRef::<str>::as_ref(version)]);
table.add_row(row![br -> "RELEASE NOTES", &self.release_notes]);
table.add_row(row![br -> "ABOUT", &textwrap::wrap(&self.description.short, 80).join("\n")]);
table.add_row(row![br -> "RELEASE NOTES", &self.metadata.release_notes]);
table.add_row(
row![br -> "ABOUT", &textwrap::wrap(&self.metadata.description.short, 80).join("\n")],
);
table.add_row(row![
br -> "DESCRIPTION",
&textwrap::wrap(&self.description.long, 80).join("\n")
&textwrap::wrap(&self.metadata.description.long, 80).join("\n")
]);
table.add_row(row![br -> "GIT HASH", self.git_hash.as_deref().unwrap_or("N/A")]);
table.add_row(row![br -> "LICENSE", &self.license]);
table.add_row(row![br -> "PACKAGE REPO", &self.wrapper_repo.to_string()]);
table.add_row(row![br -> "SERVICE REPO", &self.upstream_repo.to_string()]);
table.add_row(row![br -> "WEBSITE", &self.marketing_site.to_string()]);
table.add_row(row![br -> "SUPPORT", &self.support_site.to_string()]);
table.add_row(row![br -> "GIT HASH", self.metadata.git_hash.as_deref().unwrap_or("N/A")]);
table.add_row(row![br -> "LICENSE", &self.metadata.license]);
table.add_row(row![br -> "PACKAGE REPO", &self.metadata.wrapper_repo.to_string()]);
table.add_row(row![br -> "SERVICE REPO", &self.metadata.upstream_repo.to_string()]);
table.add_row(row![br -> "WEBSITE", &self.metadata.marketing_site.to_string()]);
table.add_row(row![br -> "SUPPORT", &self.metadata.support_site.to_string()]);
table
}
}
impl Model<PackageVersionInfo> {
pub fn works_for_device(&self, device_info: &DeviceInfo) -> Result<bool, Error> {
if !self.as_os_version().de()?.satisfies(&device_info.os.compat) {
/// Filters this package version for compatibility with the given device.
/// Returns false if the package is incompatible (should be removed).
/// Modifies s9pks in place to only include compatible variants.
pub fn for_device(&mut self, device_info: &DeviceInfo) -> Result<bool, Error> {
if !self
.as_metadata()
.as_os_version()
.de()?
.satisfies(&device_info.os.compat)
{
return Ok(false);
}
let hw = self.as_hardware_requirements().de()?;
if let Some(arch) = hw.arch {
if !arch.contains(&device_info.hardware.arch) {
return Ok(false);
}
}
if let Some(ram) = hw.ram {
if device_info.hardware.ram < ram {
return Ok(false);
}
}
for device_filter in hw.device {
if !device_info
.hardware
.devices
.iter()
.filter(|d| d.class() == &*device_filter.class)
.any(|d| device_filter.pattern.as_ref().is_match(d.product()))
if let Some(hw) = &device_info.hardware {
self.as_s9pks_mut().mutate(|s9pks| {
s9pks.retain(|(hw_req, _)| {
if let Some(arch) = &hw_req.arch {
if !arch.contains(&hw.arch) {
return false;
}
}
if let Some(ram) = hw_req.ram {
if hw.ram < ram {
return false;
}
}
if let Some(dev) = &hw.devices {
for device_filter in &hw_req.device {
if !dev
.iter()
.filter(|d| d.class() == &*device_filter.class)
.any(|d| device_filter.matches(d))
{
return false;
}
}
}
true
});
if hw.devices.is_some() {
s9pks.sort_by_key(|(req, _)| req.specificity_desc());
} else {
s9pks.sort_by_key(|(req, _)| {
let (dev, arch, ram) = req.specificity_desc();
(u32::MAX - dev, arch, ram)
});
}
Ok(())
})?;
if ModelExt::as_value(self.as_s9pks())
.as_array()
.map_or(true, |s| s.is_empty())
{
return Ok(false);
}

34
core/src/registry/package/mod.rs
View File

@@ -32,14 +32,46 @@ pub fn package_api<C: Context>() -> ParentHandler<C> {
.no_display()
.with_about("Add package to registry index"),
)
.subcommand(
"add-mirror",
from_fn_async(add::add_mirror)
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"add-mirror",
from_fn_async(add::cli_add_mirror)
.no_display()
.with_about("Add a mirror for an s9pk"),
)
.subcommand(
"remove",
from_fn_async(add::remove_package)
.with_metadata("get_signer", Value::Bool(true))
.no_display()
.with_custom_display_fn(|args, changed| {
if !changed {
tracing::warn!(
"{}@{}{} does not exist, so not removed",
args.params.id,
args.params.version,
args.params
.sighash
.map_or(String::new(), |h| format!("#{h}"))
);
}
Ok(())
})
.with_about("Remove package from registry index")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove-mirror",
from_fn_async(add::remove_mirror)
.with_metadata("get_signer", Value::Bool(true))
.no_display()
.with_about("Remove a mirror from a package")
.with_call_remote::<CliContext>(),
)
.subcommand(
"signer",
signer::signer_api::<C>().with_about("Add, remove, and list package signers"),

2
core/src/s9pk/git_hash.rs
View File

@@ -7,7 +7,7 @@ use ts_rs::TS;
use crate::prelude::*;
use crate::util::Invoke;
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, TS)]
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, TS, PartialEq, Eq)]
#[ts(type = "string")]
pub struct GitHash(String);

9
core/src/s9pk/v2/compat.rs
View File

@@ -242,18 +242,23 @@ impl TryFrom<ManifestV1> for Manifest {
.device
.into_iter()
.map(|(class, product)| DeviceFilter {
pattern_description: format!(
description: format!(
"a {class} device matching the expression {}",
product.as_ref()
),
class,
pattern: product,
product: Some(product),
..Default::default()
})
.collect(),
},
git_hash: value.git_hash,
os_version: value.eos_version,
sdk_version: None,
hardware_acceleration: match value.main {
PackageProcedure::Docker(d) => d.gpu_acceleration,
PackageProcedure::Script(_) => false,
},
})
}
}

120
core/src/s9pk/v2/manifest.rs
View File

@@ -15,6 +15,7 @@ use crate::s9pk::git_hash::GitHash;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::expected::{Expected, Filter};
use crate::s9pk::v2::pack::ImageConfig;
use crate::util::lshw::{LshwDevice, LshwDisplay, LshwProcessor};
use crate::util::serde::Regex;
use crate::util::{VersionString, mime};
use crate::version::{Current, VersionT};
@@ -62,6 +63,8 @@ pub struct Manifest {
pub dependencies: Dependencies,
#[serde(default)]
pub hardware_requirements: HardwareRequirements,
#[serde(default)]
pub hardware_acceleration: bool,
pub git_hash: Option<GitHash>,
#[serde(default = "current_version")]
#[ts(type = "string")]
@@ -165,7 +168,7 @@ impl Manifest {
}
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS, PartialEq)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct HardwareRequirements {
@@ -176,19 +179,122 @@ pub struct HardwareRequirements {
#[ts(type = "string[] | null")]
pub arch: Option<BTreeSet<InternedString>>,
}
impl HardwareRequirements {
/// returns a value that can be used as a sort key to get most specific requirements first
pub fn specificity_desc(&self) -> (u32, u32, u64) {
(
u32::MAX - self.device.len() as u32, // more device requirements = more specific
self.arch.as_ref().map_or(u32::MAX, |a| a.len() as u32), // more arches = less specific
self.ram.map_or(0, |r| r), // more ram = more specific
)
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct DeviceFilter {
pub description: String,
#[ts(type = "\"processor\" | \"display\"")]
pub class: InternedString,
#[ts(type = "string")]
pub pattern: Regex,
pub pattern_description: String,
#[ts(type = "string | null")]
pub product: Option<Regex>,
#[ts(type = "string | null")]
pub vendor: Option<Regex>,
#[ts(optional)]
pub capabilities: Option<BTreeSet<InternedString>>,
#[ts(optional)]
pub driver: Option<InternedString>,
}
// Omit description
impl PartialEq for DeviceFilter {
fn eq(&self, other: &Self) -> bool {
self.class == other.class
&& self.product == other.product
&& self.vendor == other.vendor
&& self.capabilities == other.capabilities
&& self.driver == other.driver
}
}
impl DeviceFilter {
pub fn matches(&self, device: &LshwDevice) -> bool {
if &*self.class != device.class() {
return false;
}
match device {
LshwDevice::Processor(LshwProcessor {
product,
vendor,
capabilities,
}) => {
if let Some(match_product) = &self.product {
if !product
.as_deref()
.map_or(false, |p| match_product.as_ref().is_match(p))
{
return false;
}
}
if let Some(match_vendor) = &self.vendor {
if !vendor
.as_deref()
.map_or(false, |v| match_vendor.as_ref().is_match(v))
{
return false;
}
}
if !self
.capabilities
.as_ref()
.map_or(true, |c| c.is_subset(capabilities))
{
return false;
}
true
}
LshwDevice::Display(LshwDisplay {
product,
vendor,
capabilities,
driver,
}) => {
if let Some(match_product) = &self.product {
if !product
.as_deref()
.map_or(false, |p| match_product.as_ref().is_match(p))
{
return false;
}
}
if let Some(match_vendor) = &self.vendor {
if !vendor
.as_deref()
.map_or(false, |v| match_vendor.as_ref().is_match(v))
{
return false;
}
}
if !self
.capabilities
.as_ref()
.map_or(true, |c| c.is_subset(capabilities))
{
return false;
}
if !self
.driver
.as_ref()
.map_or(true, |d| Some(d) == driver.as_ref())
{
return false;
}
true
}
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[derive(Clone, Debug, Deserialize, Serialize, TS, PartialEq, Eq)]
#[ts(export)]
pub struct Description {
pub short: String,
@@ -212,7 +318,7 @@ impl Description {
}
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct Alerts {

33
core/src/s9pk/v2/pack.rs
View File

@@ -265,7 +265,7 @@ impl PackParams {
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[derive(Debug, Default, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ImageConfig {
@@ -274,15 +274,8 @@ pub struct ImageConfig {
pub arch: BTreeSet<InternedString>,
#[ts(type = "string | null")]
pub emulate_missing_as: Option<InternedString>,
}
impl Default for ImageConfig {
fn default() -> Self {
Self {
source: ImageSource::Packed,
arch: BTreeSet::new(),
emulate_missing_as: None,
}
}
#[serde(default)]
pub nvidia_container: bool,
}
#[derive(Parser)]
@@ -299,6 +292,8 @@ struct CliImageConfig {
arch: Vec<InternedString>,
#[arg(long)]
emulate_missing_as: Option<InternedString>,
#[arg(long)]
nvidia_container: bool,
}
impl TryFrom<CliImageConfig> for ImageConfig {
type Error = clap::Error;
@@ -317,6 +312,7 @@ impl TryFrom<CliImageConfig> for ImageConfig {
},
arch: value.arch.into_iter().collect(),
emulate_missing_as: value.emulate_missing_as,
nvidia_container: value.nvidia_container,
};
res.emulate_missing_as
.as_ref()
@@ -379,20 +375,21 @@ pub enum ImageSource {
DockerTag(String),
// Recipe(DirRecipe),
}
impl Default for ImageSource {
fn default() -> Self {
ImageSource::Packed
}
}
impl ImageSource {
pub fn ingredients(&self) -> Vec<PathBuf> {
match self {
Self::Packed => Vec::new(),
Self::DockerBuild {
dockerfile,
workdir,
..
} => {
Self::DockerBuild { dockerfile, .. } => {
vec![
workdir
dockerfile
.as_deref()
.unwrap_or(Path::new("."))
.join(dockerfile.as_deref().unwrap_or(Path::new("Dockerfile"))),
.unwrap_or(Path::new("Dockerfile"))
.to_owned(),
]
}
Self::DockerTag(_) => Vec::new(),

9
core/src/service/cli.rs
View File

@@ -2,6 +2,7 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::Parser;
use imbl::OrdMap;
use imbl_value::Value;
use once_cell::sync::OnceCell;
use rpc_toolkit::yajrc::RpcError;
@@ -53,7 +54,13 @@ impl Context for ContainerCliContext {
}
impl CallRemote<EffectContext> for ContainerCliContext {
async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
async fn call_remote(
&self,
method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
call_remote_socket(
tokio::net::UnixStream::connect(&self.0.socket)
.await

2
core/src/service/effects/context.rs
View File

@@ -6,7 +6,7 @@ use crate::prelude::*;
use crate::service::Service;
#[derive(Clone)]
pub(in crate::service) struct EffectContext(Weak<Service>);
pub struct EffectContext(Weak<Service>);
impl EffectContext {
pub fn new(service: Weak<Service>) -> Self {
Self(service)

2
core/src/service/effects/mod.rs
View File

@@ -15,7 +15,7 @@ mod dependency;
mod health;
mod net;
mod prelude;
mod subcontainer;
pub mod subcontainer;
mod system;
mod version;

32
core/src/service/effects/subcontainer/mod.rs
View File

@@ -11,6 +11,10 @@ use crate::service::effects::prelude::*;
use crate::service::persistent_container::Subcontainer;
use crate::util::Invoke;
pub const NVIDIA_OVERLAY_PATH: &str = "/var/tmp/startos/nvidia-overlay";
pub const NVIDIA_OVERLAY_DEBIAN: &str = "/var/tmp/startos/nvidia-overlay/debian";
pub const NVIDIA_OVERLAY_GENERIC: &str = "/var/tmp/startos/nvidia-overlay/generic";
#[cfg(target_os = "linux")]
mod sync;
@@ -112,8 +116,34 @@ pub async fn create_subcontainer_fs(
.with_kind(ErrorKind::Incoherent)?,
);
tracing::info!("Mounting overlay {guid} for {image_id}");
// Determine which nvidia overlay to use based on distro detection
let nvidia_overlay: &[&str] = if context
.seed
.persistent_container
.s9pk
.as_manifest()
.images
.get(&image_id)
.map_or(false, |i| i.nvidia_container)
{
// Check if image is debian-based by looking for /etc/debian_version
let is_debian = tokio::fs::metadata(image.path().join("etc/debian_version"))
.await
.is_ok();
if is_debian && tokio::fs::metadata(NVIDIA_OVERLAY_DEBIAN).await.is_ok() {
&[NVIDIA_OVERLAY_DEBIAN]
} else if tokio::fs::metadata(NVIDIA_OVERLAY_GENERIC).await.is_ok() {
&[NVIDIA_OVERLAY_GENERIC]
} else {
&[]
}
} else {
&[]
};
let subcontainer_wrapper = Subcontainer {
overlay: OverlayGuard::mount(image, &mountpoint).await?,
overlay: OverlayGuard::mount_layers(&[], image, nvidia_overlay, &mountpoint).await?,
name: name
.unwrap_or_else(|| InternedString::intern(format!("subcontainer-{}", image_id))),
image_id: image_id.clone(),

2
core/src/service/mod.rs
View File

@@ -9,7 +9,6 @@ use std::sync::{Arc, Weak};
use std::time::Duration;
use axum::extract::ws::Utf8Bytes;
use crate::util::net::WebSocket;
use clap::Parser;
use futures::future::BoxFuture;
use futures::stream::FusedStream;
@@ -48,6 +47,7 @@ use crate::util::Never;
use crate::util::actor::concurrent::ConcurrentActor;
use crate::util::future::NonDetachingJoinHandle;
use crate::util::io::{AsyncReadStream, AtomicFile, TermSize, delete_file};
use crate::util::net::WebSocket;
use crate::util::serde::Pem;
use crate::util::sync::SyncMutex;
use crate::volume::data_dir;

4
core/src/service/persistent_container.rs
View File

@@ -96,7 +96,9 @@ impl PersistentContainer {
.join("logs")
.join(&s9pk.as_manifest().id),
),
LxcConfig::default(),
LxcConfig {
hardware_acceleration: s9pk.manifest.hardware_acceleration,
},
)
.await?;
let rpc_client = lxc_container.connect_rpc(Some(RPC_CONNECT_TIMEOUT)).await?;

2
core/src/sign/commitment/merkle_archive.rs
View File

@@ -11,7 +11,7 @@ use crate::sign::commitment::{Commitment, Digestable};
use crate::util::io::TrackingIO;
use crate::util::serde::Base64;
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[derive(Clone, Copy, Debug, Deserialize, Serialize, HasModel, TS, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]

3
core/src/status/mod.rs
View File

@@ -51,6 +51,7 @@ impl Model<StatusInfo> {
}
pub fn stopped(&mut self) -> Result<(), Error> {
self.as_started_mut().ser(&None)?;
self.as_health_mut().ser(&Default::default())?;
Ok(())
}
pub fn restart(&mut self) -> Result<(), Error> {
@@ -59,7 +60,7 @@ impl Model<StatusInfo> {
Ok(())
}
pub fn init(&mut self) -> Result<(), Error> {
self.as_started_mut().ser(&None)?;
self.stopped()?;
self.as_desired_mut().map_mutate(|s| {
Ok(match s {
DesiredStatus::BackingUp {

3
core/src/tunnel/context.rs
View File

@@ -251,6 +251,8 @@ impl CallRemote<TunnelContext> for CliContext {
async fn call_remote(
&self,
mut method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
_: Empty,
) -> Result<Value, RpcError> {
@@ -315,6 +317,7 @@ impl CallRemote<TunnelContext, TunnelUrlParams> for RpcContext {
async fn call_remote(
&self,
mut method: &str,
_: OrdMap<&'static str, Value>,
params: Value,
TunnelUrlParams { tunnel }: TunnelUrlParams,
) -> Result<Value, RpcError> {

2
core/src/util/data_url.rs
View File

@@ -13,7 +13,7 @@ use ts_rs::TS;
use crate::util::mime::{mime, unmime};
use crate::{Error, ErrorKind, ResultExt};
#[derive(Clone, TS)]
#[derive(Clone, TS, PartialEq, Eq)]
#[ts(type = "string")]
pub struct DataUrl<'a> {
pub mime: InternedString,

38
core/src/util/io.rs
View File

@@ -16,7 +16,7 @@ use clap::builder::ValueParserFactory;
use futures::future::{BoxFuture, Fuse};
use futures::{FutureExt, Stream, TryStreamExt};
use inotify::{EventMask, EventStream, Inotify, WatchMask};
use nix::unistd::{Gid, Uid};
use nix::unistd::{Gid, Uid, fchown};
use serde::{Deserialize, Serialize};
use tokio::fs::{File, OpenOptions};
use tokio::io::{
@@ -892,6 +892,16 @@ impl TmpDir {
Ok(())
}
pub fn leak(mut self) {
std::mem::take(&mut self.path);
}
pub async fn unmount_and_delete(self) -> Result<(), Error> {
crate::disk::mount::util::unmount_all_under(&self.path, false).await?;
tokio::fs::remove_dir_all(&self.path).await?;
Ok(())
}
pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
if let Ok(dir) = Arc::try_unwrap(self) {
dir.delete().await
@@ -1057,6 +1067,32 @@ pub async fn write_file_atomic(
Ok(())
}
#[instrument(skip_all)]
pub async fn write_file_owned_atomic(
path: impl AsRef<Path>,
contents: impl AsRef<[u8]>,
uid: u32,
gid: u32,
) -> Result<(), Error> {
let path = path.as_ref();
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("mkdir -p {parent:?}")))?;
}
let mut file = AtomicFile::new(path, None::<&Path>)
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}")))?;
fchown(&*file, Some(uid.into()), Some(gid.into())).with_kind(ErrorKind::Filesystem)?;
file.write_all(contents.as_ref())
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("write {path:?}")))?;
file.save()
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("save {path:?}")))?;
Ok(())
}
fn poll_flush_prefix<W: AsyncWrite>(
mut writer: Pin<&mut W>,
cx: &mut std::task::Context<'_>,

71
core/src/util/lshw.rs
View File

@@ -1,9 +1,12 @@
use std::collections::BTreeSet;
use imbl_value::InternedString;
use serde::{Deserialize, Serialize};
use tokio::process::Command;
use ts_rs::TS;
use crate::prelude::*;
use crate::util::Invoke;
use crate::{Error, ResultExt};
const KNOWN_CLASSES: &[&str] = &["processor", "display"];
@@ -22,22 +25,57 @@ impl LshwDevice {
Self::Display(_) => "display",
}
}
pub fn product(&self) -> &str {
match self {
Self::Processor(hw) => hw.product.as_str(),
Self::Display(hw) => hw.product.as_str(),
pub fn from_value(value: &Value) -> Option<Self> {
match value["class"].as_str() {
Some("processor") => Some(LshwDevice::Processor(LshwProcessor::from_value(value))),
Some("display") => Some(LshwDevice::Display(LshwDisplay::from_value(value))),
_ => None,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct LshwProcessor {
pub product: String,
pub product: Option<InternedString>,
pub vendor: Option<InternedString>,
pub capabilities: BTreeSet<InternedString>,
}
impl LshwProcessor {
fn from_value(value: &Value) -> Self {
Self {
product: value["product"].as_str().map(From::from),
vendor: value["vendor"].as_str().map(From::from),
capabilities: value["capabilities"]
.as_object()
.into_iter()
.flat_map(|o| o.keys())
.map(|k| k.clone())
.collect(),
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct LshwDisplay {
pub product: String,
pub product: Option<InternedString>,
pub vendor: Option<InternedString>,
pub capabilities: BTreeSet<InternedString>,
pub driver: Option<InternedString>,
}
impl LshwDisplay {
fn from_value(value: &Value) -> Self {
Self {
product: value["product"].as_str().map(From::from),
vendor: value["vendor"].as_str().map(From::from),
capabilities: value["capabilities"]
.as_object()
.into_iter()
.flat_map(|o| o.keys())
.map(|k| k.clone())
.collect(),
driver: value["configuration"]["driver"].as_str().map(From::from),
}
}
}
pub async fn lshw() -> Result<Vec<LshwDevice>, Error> {
@@ -47,19 +85,10 @@ pub async fn lshw() -> Result<Vec<LshwDevice>, Error> {
cmd.arg("-class").arg(*class);
}
Ok(
serde_json::from_slice::<Vec<serde_json::Value>>(
&cmd.invoke(crate::ErrorKind::Lshw).await?,
)
.with_kind(crate::ErrorKind::Deserialization)?
.into_iter()
.filter_map(|v| match serde_json::from_value(v) {
Ok(a) => Some(a),
Err(e) => {
tracing::error!("Failed to parse lshw output: {e}");
tracing::debug!("{e:?}");
None
}
})
.collect(),
serde_json::from_slice::<Vec<Value>>(&cmd.invoke(crate::ErrorKind::Lshw).await?)
.with_kind(crate::ErrorKind::Deserialization)?
.iter()
.filter_map(LshwDevice::from_value)
.collect(),
)
}

5
core/src/util/serde.rs
View File

@@ -1127,6 +1127,11 @@ impl Serialize for Regex {
serialize_display(&self.0, serializer)
}
}
impl PartialEq for Regex {
fn eq(&self, other: &Self) -> bool {
InternedString::from_display(self.as_ref()) == InternedString::from_display(other.as_ref())
}
}
// TODO: make this not allocate
#[derive(Debug)]

6
core/src/util/tui.rs
View File

@@ -95,7 +95,7 @@ pub async fn prompt_multiline<
Ok(res)
}
pub async fn choose_custom_display<'t, T: std::fmt::Display>(
pub async fn choose_custom_display<'t, T>(
prompt: &str,
choices: &'t [T],
mut display: impl FnMut(&T) -> String,
@@ -121,7 +121,7 @@ pub async fn choose_custom_display<'t, T: std::fmt::Display>(
if choice.len() < 1 {
return Err(Error::new(eyre!("Aborted"), ErrorKind::Cancelled));
}
let (idx, _) = string_choices
let (idx, choice_str) = string_choices
.iter()
.enumerate()
.find(|(_, s)| s.as_str() == choice[0].as_str())
@@ -132,7 +132,7 @@ pub async fn choose_custom_display<'t, T: std::fmt::Display>(
)
})?;
let choice = &choices[idx];
println!("{prompt} {choice}");
println!("{prompt} {choice_str}");
Ok(&choice)
}

27
core/src/version/v0_3_6_alpha_8.rs
View File

@@ -1,6 +1,7 @@
use std::path::Path;
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::json;
use tokio::fs::File;
use super::v0_3_5::V0_3_0_COMPAT;
@@ -10,7 +11,7 @@ use crate::context::RpcContext;
use crate::install::PKG_ARCHIVE_DIR;
use crate::prelude::*;
use crate::s9pk::S9pk;
use crate::s9pk::manifest::{DeviceFilter, Manifest};
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::MerkleArchive;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::v2::SIG_CONTEXT;
@@ -84,28 +85,8 @@ impl VersionT for Version {
let mut manifest = previous_manifest.clone();
if let Some(device) =
previous_manifest["hardwareRequirements"]["device"].as_object()
{
manifest["hardwareRequirements"]["device"] = to_value(
&device
.into_iter()
.map(|(class, product)| {
Ok::<_, Error>(DeviceFilter {
pattern_description: format!(
"a {class} device matching the expression {}",
&product
),
class: class.clone(),
pattern: from_value(product.clone())?,
})
})
.fold(Ok::<_, Error>(Vec::new()), |acc, value| {
let mut acc = acc?;
acc.push(value?);
Ok(acc)
})?,
)?;
if let Some(_) = previous_manifest["hardwareRequirements"]["device"].as_object() {
manifest["hardwareRequirements"]["device"] = json!([]);
}
if previous_manifest != manifest {