Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-04-02 05:23:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

reorganize package data and write dependencies rpc (#2571)

Browse Source 
* wip

* finish dependencies

* minor fixes
This commit is contained in:
Aiden McClelland
2024-03-15 13:02:47 -06:00
committed by GitHub
parent e604c914d1
commit 1a396cfc7b
30 changed files with 1045 additions and 897 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
core/startos/src/service/mod.rs
View File

@@ -16,9 +16,8 @@ use crate::action::ActionResult;
use crate::config::action::ConfigRes;
use crate::context::{CliContext, RpcContext};
use crate::core::rpc_continuations::RequestGuid;
use crate::db::model::{
InstalledPackageInfo, PackageDataEntry, PackageDataEntryInstalled, PackageDataEntryMatchModel,
StaticFiles,
use crate::db::model::package::{
InstalledState, PackageDataEntry, PackageState, PackageStateMatchModelRef, UpdatingState,
};
use crate::disk::mount::guard::GenericMountGuard;
use crate::install::PKG_ARCHIVE_DIR;
@@ -28,7 +27,7 @@ use crate::s9pk::S9pk;
use crate::service::service_map::InstallProgressHandles;
use crate::service::transition::TransitionKind;
use crate::status::health_check::HealthCheckResult;
use crate::status::{MainStatus, Status};
use crate::status::MainStatus;
use crate::util::actor::{Actor, BackgroundJobs, SimpleActor};
use crate::volume::data_dir;
@@ -100,7 +99,7 @@ impl Service {
) -> Result<Option<Self>, Error> {
let handle_installed = {
let ctx = ctx.clone();
move |s9pk: S9pk, i: Model<InstalledPackageInfo>| async move {
move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
for volume_id in &s9pk.as_manifest().volumes {
let tmp_path =
data_dir(&ctx.datadir, &s9pk.as_manifest().id.clone(), volume_id);
@@ -118,16 +117,18 @@ impl Service {
};
let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash
let s9pk_path = s9pk_dir.join(id).with_extension("s9pk");
match ctx
let Some(entry) = ctx
.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(id)
.map(|pde| pde.into_match())
{
Some(PackageDataEntryMatchModel::Installing(_)) => {
else {
return Ok(None);
};
match entry.as_state_info().as_match() {
PackageStateMatchModelRef::Installing(_) => {
if disposition == LoadDisposition::Retry {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for install: {e}");
@@ -150,14 +151,17 @@ impl Service {
.await?;
Ok(None)
}
Some(PackageDataEntryMatchModel::Updating(e)) => {
PackageStateMatchModelRef::Updating(s) => {
if disposition == LoadDisposition::Retry
&& e.as_install_progress().de()?.phases.iter().any(
|NamedProgress { name, progress }| {
&& s.as_installing_info()
.as_progress()
.de()?
.phases
.iter()
.any(|NamedProgress { name, progress }| {
name.eq_ignore_ascii_case("download")
&& progress == &Progress::Complete(true)
},
)
})
{
if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for update: {e}");
@@ -166,7 +170,7 @@ impl Service {
if let Ok(service) = Self::install(
ctx.clone(),
s9pk,
Some(e.as_installed().as_manifest().as_version().de()?),
Some(s.as_manifest().as_version().de()?),
None,
)
.await
@@ -181,24 +185,28 @@ impl Service {
let s9pk = S9pk::open(s9pk_path, Some(id)).await?;
ctx.db
.mutate({
let manifest = s9pk.as_manifest().clone();
|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&manifest.id)
.or_not_found(&manifest.id)?
.ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
static_files: e.as_static_files().de()?,
manifest,
installed: e.as_installed().de()?,
}))
.as_idx_mut(&id)
.or_not_found(&id)?
.as_state_info_mut()
.map_mutate(|s| {
if let PackageState::Updating(UpdatingState {
manifest, ..
}) = s
{
Ok(PackageState::Installed(InstalledState { manifest }))
} else {
Err(Error::new(eyre!("Race condition detected - package state changed during load"), ErrorKind::Database))
}
})
}
})
.await?;
handle_installed(s9pk, e.as_installed().clone()).await
handle_installed(s9pk, entry).await
}
Some(PackageDataEntryMatchModel::Removing(_))
| Some(PackageDataEntryMatchModel::Restoring(_)) => {
PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for removal: {e}");
tracing::debug!("{e:?}")
@@ -230,18 +238,13 @@ impl Service {
Ok(None)
}
Some(PackageDataEntryMatchModel::Installed(i)) => {
handle_installed(
S9pk::open(s9pk_path, Some(id)).await?,
i.as_installed().clone(),
)
.await
PackageStateMatchModelRef::Installed(_) => {
handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await
}
Some(PackageDataEntryMatchModel::Error(e)) => Err(Error::new(
PackageStateMatchModelRef::Error(e) => Err(Error::new(
eyre!("Failed to parse PackageDataEntry, found {e:?}"),
ErrorKind::Deserialization,
)),
None => Ok(None),
}
}
@@ -255,7 +258,6 @@ impl Service {
let manifest = s9pk.as_manifest().clone();
let developer_key = s9pk.as_archive().signer();
let icon = s9pk.icon_data_url().await?;
let static_files = StaticFiles::local(&manifest.id, &manifest.version, icon);
let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?;
service
.seed
@@ -270,32 +272,19 @@ impl Service {
}
ctx.db
.mutate(|d| {
d.as_public_mut()
let entry = d
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&manifest.id)
.or_not_found(&manifest.id)?
.ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
installed: InstalledPackageInfo {
current_dependencies: Default::default(), // TODO
current_dependents: Default::default(), // TODO
dependency_info: Default::default(), // TODO
developer_key,
status: Status {
configured: false, // TODO
main: MainStatus::Stopped, // TODO
dependency_config_errors: Default::default(), // TODO
},
interface_addresses: Default::default(), // TODO
marketplace_url: None, // TODO
manifest: manifest.clone(),
last_backup: None, // TODO
hosts: Default::default(), // TODO
store_exposed_dependents: Default::default(), // TODO
store_exposed_ui: Default::default(), // TODO
},
manifest,
static_files,
}))
.or_not_found(&manifest.id)?;
entry
.as_state_info_mut()
.ser(&PackageState::Installed(InstalledState { manifest }))?;
entry.as_developer_key_mut().ser(&developer_key)?;
entry.as_icon_mut().ser(&icon)?;
// TODO: marketplace url
// TODO: dependency info
Ok(())
})
.await?;
Ok(service)
@@ -466,11 +455,7 @@ impl Actor for ServiceActor {
seed.ctx
.db
.mutate(|d| {
if let Some(i) = d
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&id)
.and_then(|p| p.as_installed_mut())
if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id)
{
i.as_status_mut().as_main_mut().ser(&main_status)?;
}

155
core/startos/src/service/service_effect_handler.rs
View File

@@ -1,3 +1,4 @@
use std::collections::BTreeSet;
use std::ffi::OsString;
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
@@ -7,13 +8,14 @@ use std::sync::{Arc, Weak};
use clap::builder::ValueParserFactory;
use clap::Parser;
use imbl_value::{json, InternedString};
use models::{ActionId, HealthCheckId, ImageId, PackageId};
use models::{ActionId, HealthCheckId, ImageId, InvalidId, PackageId};
use patch_db::json_ptr::JsonPointer;
use rpc_toolkit::{from_fn, from_fn_async, AnyContext, Context, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::process::Command;
use ts_rs::TS;
use crate::db::model::ExposedUI;
use crate::db::model::package::{CurrentDependencies, CurrentDependencyInfo, ExposedUI};
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
@@ -131,8 +133,13 @@ pub fn service_effect_handler() -> ParentHandler {
.subcommand("clearBindings", from_fn_async(clear_bindings).no_cli())
.subcommand("bind", from_fn_async(bind).no_cli())
.subcommand("getHostInfo", from_fn_async(get_host_info).no_cli())
.subcommand(
"setDependencies",
from_fn_async(set_dependencies)
.no_display()
.with_remote_cli::<ContainerCliContext>(),
)
// TODO @DrBonez when we get the new api for 4.0
// .subcommand("setDependencies",from_fn_async(set_dependencies).no_cli())
// .subcommand("embassyGetInterface",from_fn_async(embassy_get_interface).no_cli())
// .subcommand("mount",from_fn_async(mount).no_cli())
// .subcommand("removeAction",from_fn_async(remove_action).no_cli())
@@ -459,8 +466,6 @@ async fn expose_for_dependents(
.as_package_data_mut()
.as_idx_mut(&package_id)
.or_not_found(&package_id)?
.as_installed_mut()
.or_not_found(&package_id)?
.as_store_exposed_dependents_mut()
.ser(&paths)
})
@@ -488,8 +493,6 @@ async fn expose_ui(
.as_package_data_mut()
.as_idx_mut(&package_id)
.or_not_found(&package_id)?
.as_installed_mut()
.or_not_found(&package_id)?
.as_store_exposed_ui_mut()
.ser(&paths)
})
@@ -566,8 +569,6 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error
.as_package_data()
.as_idx(&package_id)
.or_not_found(&package_id)?
.as_installed()
.or_not_found(&package_id)?
.as_status()
.as_configured()
.de()?;
@@ -583,8 +584,6 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
.as_package_data()
.as_idx(&package_id)
.or_not_found(&package_id)?
.as_installed()
.or_not_found(&package_id)?
.as_status()
.as_main()
.de()?;
@@ -600,8 +599,6 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Valu
.as_package_data()
.as_idx(&package_id)
.or_not_found(&package_id)?
.as_installed()
.or_not_found(&package_id)?
.as_status()
.as_main()
.de()?;
@@ -652,8 +649,6 @@ async fn set_configured(context: EffectContext, params: SetConfigured) -> Result
.as_package_data_mut()
.as_idx_mut(package_id)
.or_not_found(package_id)?
.as_installed_mut()
.or_not_found(package_id)?
.as_status_mut()
.as_configured_mut()
.ser(&params.configured)
@@ -733,8 +728,6 @@ async fn set_health(
.as_package_data()
.as_idx(package_id)
.or_not_found(package_id)?
.as_installed()
.or_not_found(package_id)?
.as_status()
.as_main()
.de()?;
@@ -764,8 +757,6 @@ async fn set_health(
.as_package_data_mut()
.as_idx_mut(package_id)
.or_not_found(package_id)?
.as_installed_mut()
.or_not_found(package_id)?
.as_status_mut()
.as_main_mut()
.ser(&main)
@@ -778,8 +769,6 @@ async fn set_health(
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct DestroyOverlayedImageParams {
#[ts(type = "string ")]
image_id: ImageId,
#[ts(type = "string")]
guid: InternedString,
}
@@ -787,7 +776,7 @@ pub struct DestroyOverlayedImageParams {
#[instrument(skip_all)]
pub async fn destroy_overlayed_image(
ctx: EffectContext,
DestroyOverlayedImageParams { image_id, guid }: DestroyOverlayedImageParams,
DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
if ctx
@@ -873,3 +862,125 @@ pub async fn create_overlayed_image(
))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
enum DependencyKind {
Exists,
Running,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
struct DependencyRequirement {
id: PackageId,
kind: DependencyKind,
#[serde(default)]
health_checks: BTreeSet<HealthCheckId>,
}
// filebrowser:exists,bitcoind:running:foo+bar+baz
impl FromStr for DependencyRequirement {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.split_once(":") {
Some((id, "e")) | Some((id, "exists")) => Ok(Self {
id: id.parse()?,
kind: DependencyKind::Exists,
health_checks: BTreeSet::new(),
}),
Some((id, rest)) => {
let health_checks = match rest.split_once(":") {
Some(("r", rest)) | Some(("running", rest)) => rest
.split("+")
.map(|id| id.parse().map_err(Error::from))
.collect(),
Some((kind, _)) => Err(Error::new(
eyre!("unknown dependency kind {kind}"),
ErrorKind::InvalidRequest,
)),
None => match rest {
"r" | "running" => Ok(BTreeSet::new()),
kind => Err(Error::new(
eyre!("unknown dependency kind {kind}"),
ErrorKind::InvalidRequest,
)),
},
}?;
Ok(Self {
id: id.parse()?,
kind: DependencyKind::Running,
health_checks,
})
}
None => Ok(Self {
id: s.parse()?,
kind: DependencyKind::Running,
health_checks: BTreeSet::new(),
}),
}
}
}
impl ValueParserFactory for DependencyRequirement {
type Parser = FromStrParser<Self>;
fn value_parser() -> Self::Parser {
FromStrParser::new()
}
}
#[derive(Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct SetDependenciesParams {
dependencies: Vec<DependencyRequirement>,
}
pub async fn set_dependencies(
ctx: EffectContext,
SetDependenciesParams { dependencies }: SetDependenciesParams,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
let id = &ctx.id;
ctx.ctx
.db
.mutate(|db| {
let dependencies = CurrentDependencies(
dependencies
.into_iter()
.map(
|DependencyRequirement {
id,
kind,
health_checks,
}| {
(
id,
match kind {
DependencyKind::Exists => CurrentDependencyInfo::Exists,
DependencyKind::Running => {
CurrentDependencyInfo::Running { health_checks }
}
},
)
},
)
.collect(),
);
for (dep, entry) in db.as_public_mut().as_package_data_mut().as_entries_mut()? {
if let Some(info) = dependencies.0.get(&dep) {
entry.as_current_dependents_mut().insert(id, info)?;
} else {
entry.as_current_dependents_mut().remove(id)?;
}
}
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(id)
.or_not_found(id)?
.as_current_dependencies_mut()
.ser(&dependencies)
})
.await
}

98
core/startos/src/service/service_map.rs
View File

@@ -11,9 +11,8 @@ use tokio::sync::{Mutex, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
use tracing::instrument;
use crate::context::RpcContext;
use crate::db::model::{
PackageDataEntry, PackageDataEntryInstalled, PackageDataEntryInstalling,
PackageDataEntryRestoring, PackageDataEntryUpdating, StaticFiles,
use crate::db::model::package::{
InstallingInfo, InstallingState, PackageDataEntry, PackageState, UpdatingState,
};
use crate::disk::mount::guard::GenericMountGuard;
use crate::install::PKG_ARCHIVE_DIR;
@@ -27,6 +26,7 @@ use crate::s9pk::manifest::PackageId;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::{LoadDisposition, Service};
use crate::status::{MainStatus, Status};
pub type DownloadInstallFuture = BoxFuture<'static, Result<InstallFuture, Error>>;
pub type InstallFuture = BoxFuture<'static, Result<(), Error>>;
@@ -95,9 +95,10 @@ impl ServiceMap {
mut s9pk: S9pk<S>,
recovery_source: Option<impl GenericMountGuard>,
) -> Result<DownloadInstallFuture, Error> {
let manifest = Arc::new(s9pk.as_manifest().clone());
let manifest = s9pk.as_manifest().clone();
let id = manifest.id.clone();
let icon = s9pk.icon_data_url().await?;
let developer_key = s9pk.as_archive().signer();
let mut service = self.get_mut(&id).await;
let op_name = if recovery_source.is_none() {
@@ -135,49 +136,51 @@ impl ServiceMap {
let id = id.clone();
let install_progress = progress.snapshot();
move |db| {
let pde = match db
.as_public()
.as_package_data()
.as_idx(&id)
.map(|x| x.de())
.transpose()?
{
Some(PackageDataEntry::Installed(PackageDataEntryInstalled {
installed,
static_files,
..
})) => PackageDataEntry::Updating(PackageDataEntryUpdating {
install_progress,
installed,
manifest: (*manifest).clone(),
static_files,
}),
None if restoring => {
PackageDataEntry::Restoring(PackageDataEntryRestoring {
install_progress,
static_files: StaticFiles::local(
&manifest.id,
&manifest.version,
icon,
),
manifest: (*manifest).clone(),
})
}
None => PackageDataEntry::Installing(PackageDataEntryInstalling {
install_progress,
static_files: StaticFiles::local(&manifest.id, &manifest.version, icon),
manifest: (*manifest).clone(),
}),
_ => {
return Err(Error::new(
eyre!("Cannot install over a package in a transient state"),
crate::ErrorKind::InvalidRequest,
))
}
if let Some(pde) = db.as_public_mut().as_package_data_mut().as_idx_mut(&id) {
let prev = pde.as_state_info().expect_installed()?.de()?;
pde.as_state_info_mut()
.ser(&PackageState::Updating(UpdatingState {
manifest: prev.manifest,
installing_info: InstallingInfo {
new_manifest: manifest,
progress: install_progress,
},
}))?;
} else {
let installing = InstallingState {
installing_info: InstallingInfo {
new_manifest: manifest,
progress: install_progress,
},
};
db.as_public_mut().as_package_data_mut().insert(
&id,
&PackageDataEntry {
state_info: if restoring {
PackageState::Restoring(installing)
} else {
PackageState::Installing(installing)
},
status: Status {
configured: false,
main: MainStatus::Stopped,
dependency_config_errors: Default::default(),
},
marketplace_url: None,
developer_key,
icon,
last_backup: None,
dependency_info: Default::default(),
current_dependents: Default::default(), // TODO: initialize
current_dependencies: Default::default(),
interface_addresses: Default::default(),
hosts: Default::default(),
store_exposed_ui: Default::default(),
store_exposed_dependents: Default::default(),
},
)?;
};
db.as_public_mut()
.as_package_data_mut()
.insert(&manifest.id, &pde)
Ok(())
}
}))
.await?;
@@ -200,7 +203,8 @@ impl ServiceMap {
v.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&deref_id)
.and_then(|e| e.as_install_progress_mut())
.and_then(|e| e.as_state_info_mut().as_installing_info_mut())
.map(|i| i.as_progress_mut())
},
Some(Duration::from_millis(100)),
)));