reorganize package data and write dependencies rpc (#2571)

* wip

* finish dependencies

* minor fixes

Makefile

@@ -173,7 +173,7 @@ container-runtime/node_modules: container-runtime/package.json container-runtime
 	npm --prefix container-runtime ci
 	touch container-runtime/node_modules
 
-core/startos/bindings: $(CORE_SRC) $(ENVIRONMENT_FILE) $(PLATFORM_FILE)
+core/startos/bindings: $(shell git ls-files core) $(ENVIRONMENT_FILE) $(PLATFORM_FILE)
 	(cd core/ && cargo test)
 	touch core/startos/bindings
 

container-runtime/src/Adapters/HostSystemStartOs.ts

@@ -117,10 +117,7 @@ export class HostSystemStartOs implements Effects {
       T.Effects["createOverlayedImage"]
     >
   }
-  destroyOverlayedImage(options: {
+  destroyOverlayedImage(options: { guid: string }): Promise<void> {
-    imageId: string
-    guid: string
-  }): Promise<void> {
     return this.rpcRound("destroyOverlayedImage", options) as ReturnType<
       T.Effects["destroyOverlayedImage"]
     >

core/models/src/id/health_check.rs

@@ -1,8 +1,9 @@
 use std::path::Path;
+use std::str::FromStr;
 
 use serde::{Deserialize, Deserializer, Serialize};
 
-use crate::Id;
+use crate::{Id, InvalidId};
 
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, ts_rs::TS)]
 pub struct HealthCheckId(Id);
@@ -11,6 +12,12 @@ impl std::fmt::Display for HealthCheckId {
         write!(f, "{}", &self.0)
     }
 }
+impl FromStr for HealthCheckId {
+    type Err = InvalidId;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Id::from_str(s).map(HealthCheckId)
+    }
+}
 impl AsRef<str> for HealthCheckId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()

core/models/src/id/mod.rs

@@ -1,4 +1,5 @@
 use std::borrow::Borrow;
+use std::str::FromStr;
 
 use regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -59,6 +60,12 @@ impl TryFrom<&str> for Id {
         }
     }
 }
+impl FromStr for Id {
+    type Err = InvalidId;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::try_from(s)
+    }
+}
 impl From<Id> for InternedString {
     fn from(value: Id) -> Self {
         value.0

core/startos/bindings/DependencyKind.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type DependencyKind = "exists" | "running";

core/startos/bindings/DependencyRequirement.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { DependencyKind } from "./DependencyKind";
+import type { HealthCheckId } from "./HealthCheckId";
+import type { PackageId } from "./PackageId";
+
+export interface DependencyRequirement { id: PackageId, kind: DependencyKind, healthChecks: Array<HealthCheckId>, }

core/startos/bindings/DestroyOverlayedImageParams.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface DestroyOverlayedImageParams { imageId: string , guid: string, }
+export interface DestroyOverlayedImageParams { guid: string, }

core/startos/bindings/SetDependenciesParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { DependencyRequirement } from "./DependencyRequirement";
+
+export interface SetDependenciesParams { dependencies: Array<DependencyRequirement>, }

core/startos/src/backup/backup_bulk.rs

@@ -18,7 +18,8 @@ use crate::auth::check_password_against_db;
 use crate::backup::os::OsBackup;
 use crate::backup::{BackupReport, ServerBackupReport};
 use crate::context::RpcContext;
-use crate::db::model::{BackupProgress, DatabaseModel};
+use crate::db::model::public::BackupProgress;
+use crate::db::model::DatabaseModel;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
@@ -174,7 +175,7 @@ pub async fn backup_all(
                         .as_package_data()
                         .as_entries()?
                         .into_iter()
-                        .filter(|(_, m)| m.expect_as_installed().is_ok())
+                        .filter(|(_, m)| m.as_state_info().expect_installed().is_ok())
                         .map(|(id, _)| id)
                         .collect()
                 };

core/startos/src/context/rpc.rs

@@ -19,7 +19,7 @@ use super::setup::CURRENT_SECRET;
 use crate::account::AccountInfo;
 use crate::context::config::ServerConfig;
 use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation, WebSocketHandler};
-use crate::db::model::CurrentDependents;
+use crate::db::model::package::CurrentDependents;
 use crate::db::prelude::PatchDbExt;
 use crate::dependencies::compute_dependency_config_errs;
 use crate::disk::OsPartitionInfo;
@@ -219,12 +219,7 @@ impl RpcContext {
                 for (package_id, package) in
                     f.as_public_mut().as_package_data_mut().as_entries_mut()?
                 {
-                    for (k, v) in package
+                    for (k, v) in package.clone().into_current_dependencies().into_entries()? {
-                        .as_installed_mut()
-                        .into_iter()
-                        .flat_map(|i| i.clone().into_current_dependencies().into_entries())
-                        .flatten()
-                    {
                         let mut entry: BTreeMap<_, _> =
                             current_dependents.remove(&k).unwrap_or_default();
                         entry.insert(package_id.clone(), v.de()?);
@@ -236,16 +231,7 @@ impl RpcContext {
                         .as_public_mut()
                         .as_package_data_mut()
                         .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.expect_as_installed_mut().ok())
+                        .map(|i| i.as_current_dependents_mut())
-                        .map(|i| i.as_installed_mut().as_current_dependents_mut())
-                    {
-                        deps.ser(&CurrentDependents(current_dependents))?;
-                    } else if let Some(deps) = f
-                        .as_public_mut()
-                        .as_package_data_mut()
-                        .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.expect_as_removing_mut().ok())
-                        .map(|i| i.as_removing_mut().as_current_dependents_mut())
                     {
                         deps.ser(&CurrentDependents(current_dependents))?;
                     }
@@ -261,23 +247,18 @@ impl RpcContext {
         let peek = self.db.peek().await;
         for (package_id, package) in peek.as_public().as_package_data().as_entries()?.into_iter() {
             let package = package.clone();
-            if let Some(current_dependencies) = package
+            let current_dependencies = package.as_current_dependencies().de()?;
-                .as_installed()
+            all_dependency_config_errs.insert(
-                .and_then(|x| x.as_current_dependencies().de().ok())
+                package_id.clone(),
-            {
+                compute_dependency_config_errs(
-                let manifest = package.as_manifest().de()?;
+                    self,
-                all_dependency_config_errs.insert(
+                    &peek,
-                    package_id.clone(),
+                    &package_id,
-                    compute_dependency_config_errs(
+                    &current_dependencies,
-                        self,
+                    &Default::default(),
-                        &peek,
+                )
-                        &manifest,
+                .await?,
-                        &current_dependencies,
+            );
-                        &Default::default(),
-                    )
-                    .await?,
-                );
-            }
         }
         self.db
             .mutate(|v| {
@@ -286,7 +267,6 @@ impl RpcContext {
                         .as_public_mut()
                         .as_package_data_mut()
                         .as_idx_mut(&package_id)
-                        .and_then(|pde| pde.as_installed_mut())
                         .map(|i| i.as_status_mut().as_dependency_config_errors_mut())
                     {
                         config_errors.ser(&errs)?;

core/startos/src/db/model.rs

@@ -1,627 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::net::{Ipv4Addr, Ipv6Addr};
-
-use chrono::{DateTime, Utc};
-use emver::VersionRange;
-use imbl_value::InternedString;
-use ipnet::{Ipv4Net, Ipv6Net};
-use isocountry::CountryCode;
-use itertools::Itertools;
-use models::{DataUrl, HealthCheckId, HostId, PackageId};
-use openssl::hash::MessageDigest;
-use patch_db::json_ptr::JsonPointer;
-use patch_db::{HasModel, Value};
-use reqwest::Url;
-use serde::{Deserialize, Serialize};
-use torut::onion::OnionAddressV3;
-
-use crate::account::AccountInfo;
-use crate::auth::Sessions;
-use crate::backup::target::cifs::CifsTargets;
-use crate::net::forward::AvailablePorts;
-use crate::net::host::HostInfo;
-use crate::net::keys::KeyStore;
-use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
-use crate::notifications::Notifications;
-use crate::prelude::*;
-use crate::progress::FullProgress;
-use crate::s9pk::manifest::Manifest;
-use crate::ssh::SshKeys;
-use crate::status::Status;
-use crate::util::cpupower::Governor;
-use crate::util::serde::Pem;
-use crate::util::Version;
-use crate::version::{Current, VersionT};
-use crate::{ARCH, PLATFORM};
-
-fn get_arch() -> InternedString {
-    (*ARCH).into()
-}
-
-fn get_platform() -> InternedString {
-    (&*PLATFORM).into()
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct Database {
-    pub public: Public,
-    pub private: Private,
-}
-impl Database {
-    pub fn init(account: &AccountInfo) -> Result<Self, Error> {
-        let lan_address = account.hostname.lan_address().parse().unwrap();
-        Ok(Database {
-            public: Public {
-                server_info: ServerInfo {
-                    arch: get_arch(),
-                    platform: get_platform(),
-                    id: account.server_id.clone(),
-                    version: Current::new().semver().into(),
-                    hostname: account.hostname.no_dot_host_name(),
-                    last_backup: None,
-                    last_wifi_region: None,
-                    eos_version_compat: Current::new().compat().clone(),
-                    lan_address,
-                    onion_address: account.tor_key.public().get_onion_address(),
-                    tor_address: format!(
-                        "https://{}",
-                        account.tor_key.public().get_onion_address()
-                    )
-                    .parse()
-                    .unwrap(),
-                    ip_info: BTreeMap::new(),
-                    status_info: ServerStatus {
-                        backup_progress: None,
-                        updated: false,
-                        update_progress: None,
-                        shutting_down: false,
-                        restarting: false,
-                    },
-                    wifi: WifiInfo {
-                        ssids: Vec::new(),
-                        connected: None,
-                        selected: None,
-                    },
-                    unread_notification_count: 0,
-                    connection_addresses: ConnectionAddresses {
-                        tor: Vec::new(),
-                        clearnet: Vec::new(),
-                    },
-                    password_hash: account.password.clone(),
-                    pubkey: ssh_key::PublicKey::from(&account.ssh_key)
-                        .to_openssh()
-                        .unwrap(),
-                    ca_fingerprint: account
-                        .root_ca_cert
-                        .digest(MessageDigest::sha256())
-                        .unwrap()
-                        .iter()
-                        .map(|x| format!("{x:X}"))
-                        .join(":"),
-                    ntp_synced: false,
-                    zram: true,
-                    governor: None,
-                },
-                package_data: AllPackageData::default(),
-                ui: serde_json::from_str(include_str!(concat!(
-                    env!("CARGO_MANIFEST_DIR"),
-                    "/../../web/patchdb-ui-seed.json"
-                )))
-                .unwrap(),
-            },
-            private: Private {
-                key_store: KeyStore::new(account)?,
-                password: account.password.clone(),
-                ssh_privkey: Pem(account.ssh_key.clone()),
-                ssh_pubkeys: SshKeys::new(),
-                available_ports: AvailablePorts::new(),
-                sessions: Sessions::new(),
-                notifications: Notifications::new(),
-                cifs: CifsTargets::new(),
-                package_stores: BTreeMap::new(),
-            }, // TODO
-        })
-    }
-}
-
-pub type DatabaseModel = Model<Database>;
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-// #[macro_debug]
-pub struct Public {
-    pub server_info: ServerInfo,
-    pub package_data: AllPackageData,
-    pub ui: Value,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct Private {
-    pub key_store: KeyStore,
-    pub password: String, // argon2 hash
-    pub ssh_privkey: Pem<ssh_key::PrivateKey>,
-    pub ssh_pubkeys: SshKeys,
-    pub available_ports: AvailablePorts,
-    pub sessions: Sessions,
-    pub notifications: Notifications,
-    pub cifs: CifsTargets,
-    #[serde(default)]
-    pub package_stores: BTreeMap<PackageId, Value>,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct ServerInfo {
-    #[serde(default = "get_arch")]
-    pub arch: InternedString,
-    #[serde(default = "get_platform")]
-    pub platform: InternedString,
-    pub id: String,
-    pub hostname: String,
-    pub version: Version,
-    pub last_backup: Option<DateTime<Utc>>,
-    /// Used in the wifi to determine the region to set the system to
-    pub last_wifi_region: Option<CountryCode>,
-    pub eos_version_compat: VersionRange,
-    pub lan_address: Url,
-    pub onion_address: OnionAddressV3,
-    /// for backwards compatibility
-    pub tor_address: Url,
-    pub ip_info: BTreeMap<String, IpInfo>,
-    #[serde(default)]
-    pub status_info: ServerStatus,
-    pub wifi: WifiInfo,
-    pub unread_notification_count: u64,
-    pub connection_addresses: ConnectionAddresses,
-    pub password_hash: String,
-    pub pubkey: String,
-    pub ca_fingerprint: String,
-    #[serde(default)]
-    pub ntp_synced: bool,
-    #[serde(default)]
-    pub zram: bool,
-    pub governor: Option<Governor>,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct IpInfo {
-    pub ipv4_range: Option<Ipv4Net>,
-    pub ipv4: Option<Ipv4Addr>,
-    pub ipv6_range: Option<Ipv6Net>,
-    pub ipv6: Option<Ipv6Addr>,
-}
-impl IpInfo {
-    pub async fn for_interface(iface: &str) -> Result<Self, Error> {
-        let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
-        let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
-        Ok(Self {
-            ipv4_range,
-            ipv4,
-            ipv6_range,
-            ipv6,
-        })
-    }
-}
-
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct BackupProgress {
-    pub complete: bool,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct ServerStatus {
-    pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
-    pub updated: bool,
-    pub update_progress: Option<UpdateProgress>,
-    #[serde(default)]
-    pub shutting_down: bool,
-    #[serde(default)]
-    pub restarting: bool,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct UpdateProgress {
-    pub size: Option<u64>,
-    pub downloaded: u64,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct WifiInfo {
-    pub ssids: Vec<String>,
-    pub selected: Option<String>,
-    pub connected: Option<String>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ServerSpecs {
-    pub cpu: String,
-    pub disk: String,
-    pub memory: String,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct ConnectionAddresses {
-    pub tor: Vec<String>,
-    pub clearnet: Vec<String>,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize)]
-pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
-impl Map for AllPackageData {
-    type Key = PackageId;
-    type Value = PackageDataEntry;
-    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
-        Ok(key)
-    }
-    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
-        Ok(key.clone().into())
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct StaticFiles {
-    license: String,
-    instructions: String,
-    icon: DataUrl<'static>,
-}
-impl StaticFiles {
-    pub fn local(id: &PackageId, version: &Version, icon: DataUrl<'static>) -> Self {
-        StaticFiles {
-            license: format!("/public/package-data/{}/{}/LICENSE.md", id, version),
-            instructions: format!("/public/package-data/{}/{}/INSTRUCTIONS.md", id, version),
-            icon,
-        }
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryInstalling {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub install_progress: FullProgress,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryUpdating {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub installed: InstalledPackageInfo,
-    pub install_progress: FullProgress,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryRestoring {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub install_progress: FullProgress,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryRemoving {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub removing: InstalledPackageInfo,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct PackageDataEntryInstalled {
-    pub static_files: StaticFiles,
-    pub manifest: Manifest,
-    pub installed: InstalledPackageInfo,
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(tag = "state")]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-// #[macro_debug]
-pub enum PackageDataEntry {
-    Installing(PackageDataEntryInstalling),
-    Updating(PackageDataEntryUpdating),
-    Restoring(PackageDataEntryRestoring),
-    Removing(PackageDataEntryRemoving),
-    Installed(PackageDataEntryInstalled),
-}
-impl Model<PackageDataEntry> {
-    pub fn expect_into_installed(self) -> Result<Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModel::Installed(a) = self.into_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installed(&self) -> Result<&Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModelRef::Installed(a) = self.as_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installed_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryInstalled>, Error> {
-        if let PackageDataEntryMatchModelMut::Installed(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installed state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_into_removing(self) -> Result<Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModel::Removing(a) = self.into_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_removing(&self) -> Result<&Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModelRef::Removing(a) = self.as_match() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_removing_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryRemoving>, Error> {
-        if let PackageDataEntryMatchModelMut::Removing(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in removing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn expect_as_installing_mut(
-        &mut self,
-    ) -> Result<&mut Model<PackageDataEntryInstalling>, Error> {
-        if let PackageDataEntryMatchModelMut::Installing(a) = self.as_match_mut() {
-            Ok(a)
-        } else {
-            Err(Error::new(
-                eyre!("package is not in installing state"),
-                ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    pub fn into_manifest(self) -> Model<Manifest> {
-        match self.into_match() {
-            PackageDataEntryMatchModel::Installing(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Updating(a) => a.into_installed().into_manifest(),
-            PackageDataEntryMatchModel::Restoring(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Removing(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Installed(a) => a.into_manifest(),
-            PackageDataEntryMatchModel::Error(_) => Model::from(Value::Null),
-        }
-    }
-    pub fn as_manifest(&self) -> &Model<Manifest> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Updating(a) => a.as_installed().as_manifest(),
-            PackageDataEntryMatchModelRef::Restoring(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Removing(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Installed(a) => a.as_manifest(),
-            PackageDataEntryMatchModelRef::Error(_) => (&Value::Null).into(),
-        }
-    }
-    pub fn into_installed(self) -> Option<Model<InstalledPackageInfo>> {
-        match self.into_match() {
-            PackageDataEntryMatchModel::Installing(_) => None,
-            PackageDataEntryMatchModel::Updating(a) => Some(a.into_installed()),
-            PackageDataEntryMatchModel::Restoring(_) => None,
-            PackageDataEntryMatchModel::Removing(_) => None,
-            PackageDataEntryMatchModel::Installed(a) => Some(a.into_installed()),
-            PackageDataEntryMatchModel::Error(_) => None,
-        }
-    }
-    pub fn as_installed(&self) -> Option<&Model<InstalledPackageInfo>> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(_) => None,
-            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_installed()),
-            PackageDataEntryMatchModelRef::Restoring(_) => None,
-            PackageDataEntryMatchModelRef::Removing(_) => None,
-            PackageDataEntryMatchModelRef::Installed(a) => Some(a.as_installed()),
-            PackageDataEntryMatchModelRef::Error(_) => None,
-        }
-    }
-    pub fn as_installed_mut(&mut self) -> Option<&mut Model<InstalledPackageInfo>> {
-        match self.as_match_mut() {
-            PackageDataEntryMatchModelMut::Installing(_) => None,
-            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_installed_mut()),
-            PackageDataEntryMatchModelMut::Restoring(_) => None,
-            PackageDataEntryMatchModelMut::Removing(_) => None,
-            PackageDataEntryMatchModelMut::Installed(a) => Some(a.as_installed_mut()),
-            PackageDataEntryMatchModelMut::Error(_) => None,
-        }
-    }
-    pub fn as_install_progress(&self) -> Option<&Model<FullProgress>> {
-        match self.as_match() {
-            PackageDataEntryMatchModelRef::Installing(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Restoring(a) => Some(a.as_install_progress()),
-            PackageDataEntryMatchModelRef::Removing(_) => None,
-            PackageDataEntryMatchModelRef::Installed(_) => None,
-            PackageDataEntryMatchModelRef::Error(_) => None,
-        }
-    }
-    pub fn as_install_progress_mut(&mut self) -> Option<&mut Model<FullProgress>> {
-        match self.as_match_mut() {
-            PackageDataEntryMatchModelMut::Installing(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Restoring(a) => Some(a.as_install_progress_mut()),
-            PackageDataEntryMatchModelMut::Removing(_) => None,
-            PackageDataEntryMatchModelMut::Installed(_) => None,
-            PackageDataEntryMatchModelMut::Error(_) => None,
-        }
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct InstalledPackageInfo {
-    pub status: Status,
-    pub marketplace_url: Option<Url>,
-    #[serde(default)]
-    #[serde(with = "crate::util::serde::ed25519_pubkey")]
-    pub developer_key: ed25519_dalek::VerifyingKey,
-    pub manifest: Manifest,
-    pub last_backup: Option<DateTime<Utc>>,
-    pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
-    pub current_dependents: CurrentDependents,
-    pub current_dependencies: CurrentDependencies,
-    pub interface_addresses: InterfaceAddressMap,
-    pub hosts: HostInfo,
-    pub store_exposed_ui: Vec<ExposedUI>,
-    pub store_exposed_dependents: Vec<JsonPointer>,
-}
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct ExposedDependent {
-    path: String,
-    title: String,
-    description: Option<String>,
-    masked: Option<bool>,
-    copyable: Option<bool>,
-    qr: Option<bool>,
-}
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel, ts_rs::TS)]
-#[model = "Model<Self>"]
-#[ts(export)]
-pub struct ExposedUI {
-    #[ts(type = "string")]
-    pub path: JsonPointer,
-    pub title: String,
-    pub description: Option<String>,
-    pub masked: Option<bool>,
-    pub copyable: Option<bool>,
-    pub qr: Option<bool>,
-}
-
-#[derive(Debug, Clone, Default, Deserialize, Serialize)]
-pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
-impl CurrentDependents {
-    pub fn map(
-        mut self,
-        transform: impl Fn(
-            BTreeMap<PackageId, CurrentDependencyInfo>,
-        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
-    ) -> Self {
-        self.0 = transform(self.0);
-        self
-    }
-}
-impl Map for CurrentDependents {
-    type Key = PackageId;
-    type Value = CurrentDependencyInfo;
-    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
-        Ok(key)
-    }
-    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
-        Ok(key.clone().into())
-    }
-}
-#[derive(Debug, Clone, Default, Deserialize, Serialize)]
-pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
-impl CurrentDependencies {
-    pub fn map(
-        mut self,
-        transform: impl Fn(
-            BTreeMap<PackageId, CurrentDependencyInfo>,
-        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
-    ) -> Self {
-        self.0 = transform(self.0);
-        self
-    }
-}
-impl Map for CurrentDependencies {
-    type Key = PackageId;
-    type Value = CurrentDependencyInfo;
-    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
-        Ok(key)
-    }
-    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
-        Ok(key.clone().into())
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct StaticDependencyInfo {
-    pub title: String,
-    pub icon: DataUrl<'static>,
-}
-
-#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct CurrentDependencyInfo {
-    #[serde(default)]
-    pub health_checks: BTreeSet<HealthCheckId>,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize)]
-pub struct InterfaceAddressMap(pub BTreeMap<HostId, InterfaceAddresses>);
-impl Map for InterfaceAddressMap {
-    type Key = HostId;
-    type Value = InterfaceAddresses;
-    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
-        Ok(key)
-    }
-    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
-        Ok(key.clone().into())
-    }
-}
-
-#[derive(Debug, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "kebab-case")]
-#[model = "Model<Self>"]
-pub struct InterfaceAddresses {
-    pub tor_address: Option<String>,
-    pub lan_address: Option<String>,
-}

core/startos/src/db/model/mod.rs

@@ -0,0 +1,48 @@
+use std::collections::BTreeMap;
+
+use patch_db::HasModel;
+use serde::{Deserialize, Serialize};
+
+use crate::account::AccountInfo;
+use crate::auth::Sessions;
+use crate::backup::target::cifs::CifsTargets;
+use crate::db::model::private::Private;
+use crate::db::model::public::Public;
+use crate::net::forward::AvailablePorts;
+use crate::net::keys::KeyStore;
+use crate::notifications::Notifications;
+use crate::prelude::*;
+use crate::ssh::SshKeys;
+use crate::util::serde::Pem;
+
+pub mod package;
+pub mod private;
+pub mod public;
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct Database {
+    pub public: Public,
+    pub private: Private,
+}
+impl Database {
+    pub fn init(account: &AccountInfo) -> Result<Self, Error> {
+        Ok(Self {
+            public: Public::init(account)?,
+            private: Private {
+                key_store: KeyStore::new(account)?,
+                password: account.password.clone(),
+                ssh_privkey: Pem(account.ssh_key.clone()),
+                ssh_pubkeys: SshKeys::new(),
+                available_ports: AvailablePorts::new(),
+                sessions: Sessions::new(),
+                notifications: Notifications::new(),
+                cifs: CifsTargets::new(),
+                package_stores: BTreeMap::new(),
+            }, // TODO
+        })
+    }
+}
+
+pub type DatabaseModel = Model<Database>;

core/startos/src/db/model/package.rs

@@ -0,0 +1,424 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use chrono::{DateTime, Utc};
+use imbl_value::InternedString;
+use models::{DataUrl, HealthCheckId, HostId, PackageId};
+use patch_db::json_ptr::JsonPointer;
+use patch_db::HasModel;
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::net::host::HostInfo;
+use crate::prelude::*;
+use crate::progress::FullProgress;
+use crate::s9pk::manifest::Manifest;
+use crate::status::Status;
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
+impl Map for AllPackageData {
+    type Key = PackageId;
+    type Value = PackageDataEntry;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ManifestPreference {
+    Old,
+    New,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[serde(tag = "state")]
+#[model = "Model<Self>"]
+pub enum PackageState {
+    Installing(InstallingState),
+    Restoring(InstallingState),
+    Updating(UpdatingState),
+    Installed(InstalledState),
+    Removing(InstalledState),
+}
+impl PackageState {
+    pub fn expect_installed(&self) -> Result<&InstalledState, Error> {
+        match self {
+            Self::Installed(a) => Ok(a),
+            a => Err(Error::new(
+                eyre!(
+                    "Package {} is not in installed state",
+                    self.as_manifest(ManifestPreference::Old).id
+                ),
+                ErrorKind::InvalidRequest,
+            )),
+        }
+    }
+    pub fn into_installing_info(self) -> Option<InstallingInfo> {
+        match self {
+            Self::Installing(InstallingState { installing_info })
+            | Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
+            Self::Updating(UpdatingState {
+                installing_info, ..
+            }) => Some(installing_info),
+            Self::Installed(_) | Self::Removing(_) => None,
+        }
+    }
+    pub fn as_installing_info(&self) -> Option<&InstallingInfo> {
+        match self {
+            Self::Installing(InstallingState { installing_info })
+            | Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
+            Self::Updating(UpdatingState {
+                installing_info, ..
+            }) => Some(installing_info),
+            Self::Installed(_) | Self::Removing(_) => None,
+        }
+    }
+    pub fn as_installing_info_mut(&mut self) -> Option<&mut InstallingInfo> {
+        match self {
+            Self::Installing(InstallingState { installing_info })
+            | Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
+            Self::Updating(UpdatingState {
+                installing_info, ..
+            }) => Some(installing_info),
+            Self::Installed(_) | Self::Removing(_) => None,
+        }
+    }
+    pub fn into_manifest(self, preference: ManifestPreference) -> Manifest {
+        match self {
+            Self::Installing(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            })
+            | Self::Restoring(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            }) => new_manifest,
+            Self::Updating(UpdatingState { manifest, .. })
+                if preference == ManifestPreference::Old =>
+            {
+                manifest
+            }
+            Self::Updating(UpdatingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+                ..
+            }) => new_manifest,
+            Self::Installed(InstalledState { manifest })
+            | Self::Removing(InstalledState { manifest }) => manifest,
+        }
+    }
+    pub fn as_manifest(&self, preference: ManifestPreference) -> &Manifest {
+        match self {
+            Self::Installing(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            })
+            | Self::Restoring(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            }) => new_manifest,
+            Self::Updating(UpdatingState { manifest, .. })
+                if preference == ManifestPreference::Old =>
+            {
+                manifest
+            }
+            Self::Updating(UpdatingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+                ..
+            }) => new_manifest,
+            Self::Installed(InstalledState { manifest })
+            | Self::Removing(InstalledState { manifest }) => manifest,
+        }
+    }
+    pub fn as_manifest_mut(&mut self, preference: ManifestPreference) -> &mut Manifest {
+        match self {
+            Self::Installing(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            })
+            | Self::Restoring(InstallingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+            }) => new_manifest,
+            Self::Updating(UpdatingState { manifest, .. })
+                if preference == ManifestPreference::Old =>
+            {
+                manifest
+            }
+            Self::Updating(UpdatingState {
+                installing_info: InstallingInfo { new_manifest, .. },
+                ..
+            }) => new_manifest,
+            Self::Installed(InstalledState { manifest })
+            | Self::Removing(InstalledState { manifest }) => manifest,
+        }
+    }
+}
+impl Model<PackageState> {
+    pub fn expect_installed(&self) -> Result<&Model<InstalledState>, Error> {
+        match self.as_match() {
+            PackageStateMatchModelRef::Installed(a) => Ok(a),
+            a => Err(Error::new(
+                eyre!(
+                    "Package {} is not in installed state",
+                    self.as_manifest(ManifestPreference::Old).as_id().de()?
+                ),
+                ErrorKind::InvalidRequest,
+            )),
+        }
+    }
+    pub fn into_installing_info(self) -> Option<Model<InstallingInfo>> {
+        match self.into_match() {
+            PackageStateMatchModel::Installing(s) | PackageStateMatchModel::Restoring(s) => {
+                Some(s.into_installing_info())
+            }
+            PackageStateMatchModel::Updating(s) => Some(s.into_installing_info()),
+            PackageStateMatchModel::Installed(_) | PackageStateMatchModel::Removing(_) => None,
+            PackageStateMatchModel::Error(_) => None,
+        }
+    }
+    pub fn as_installing_info(&self) -> Option<&Model<InstallingInfo>> {
+        match self.as_match() {
+            PackageStateMatchModelRef::Installing(s) | PackageStateMatchModelRef::Restoring(s) => {
+                Some(s.as_installing_info())
+            }
+            PackageStateMatchModelRef::Updating(s) => Some(s.as_installing_info()),
+            PackageStateMatchModelRef::Installed(_) | PackageStateMatchModelRef::Removing(_) => {
+                None
+            }
+            PackageStateMatchModelRef::Error(_) => None,
+        }
+    }
+    pub fn as_installing_info_mut(&mut self) -> Option<&mut Model<InstallingInfo>> {
+        match self.as_match_mut() {
+            PackageStateMatchModelMut::Installing(s) | PackageStateMatchModelMut::Restoring(s) => {
+                Some(s.as_installing_info_mut())
+            }
+            PackageStateMatchModelMut::Updating(s) => Some(s.as_installing_info_mut()),
+            PackageStateMatchModelMut::Installed(_) | PackageStateMatchModelMut::Removing(_) => {
+                None
+            }
+            PackageStateMatchModelMut::Error(_) => None,
+        }
+    }
+    pub fn into_manifest(self, preference: ManifestPreference) -> Model<Manifest> {
+        match self.into_match() {
+            PackageStateMatchModel::Installing(s) | PackageStateMatchModel::Restoring(s) => {
+                s.into_installing_info().into_new_manifest()
+            }
+            PackageStateMatchModel::Updating(s) if preference == ManifestPreference::Old => {
+                s.into_manifest()
+            }
+            PackageStateMatchModel::Updating(s) => s.into_installing_info().into_new_manifest(),
+            PackageStateMatchModel::Installed(s) | PackageStateMatchModel::Removing(s) => {
+                s.into_manifest()
+            }
+            PackageStateMatchModel::Error(_) => Value::Null.into(),
+        }
+    }
+    pub fn as_manifest(&self, preference: ManifestPreference) -> &Model<Manifest> {
+        match self.as_match() {
+            PackageStateMatchModelRef::Installing(s) | PackageStateMatchModelRef::Restoring(s) => {
+                s.as_installing_info().as_new_manifest()
+            }
+            PackageStateMatchModelRef::Updating(s) if preference == ManifestPreference::Old => {
+                s.as_manifest()
+            }
+            PackageStateMatchModelRef::Updating(s) => s.as_installing_info().as_new_manifest(),
+            PackageStateMatchModelRef::Installed(s) | PackageStateMatchModelRef::Removing(s) => {
+                s.as_manifest()
+            }
+            PackageStateMatchModelRef::Error(_) => (&Value::Null).into(),
+        }
+    }
+    pub fn as_manifest_mut(
+        &mut self,
+        preference: ManifestPreference,
+    ) -> Result<&mut Model<Manifest>, Error> {
+        Ok(match self.as_match_mut() {
+            PackageStateMatchModelMut::Installing(s) | PackageStateMatchModelMut::Restoring(s) => {
+                s.as_installing_info_mut().as_new_manifest_mut()
+            }
+            PackageStateMatchModelMut::Updating(s) if preference == ManifestPreference::Old => {
+                s.as_manifest_mut()
+            }
+            PackageStateMatchModelMut::Updating(s) => {
+                s.as_installing_info_mut().as_new_manifest_mut()
+            }
+            PackageStateMatchModelMut::Installed(s) | PackageStateMatchModelMut::Removing(s) => {
+                s.as_manifest_mut()
+            }
+            PackageStateMatchModelMut::Error(s) => {
+                return Err(Error::new(
+                    eyre!("could not determine package state to get manifest"),
+                    ErrorKind::Database,
+                ))
+            }
+        })
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InstallingState {
+    pub installing_info: InstallingInfo,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct UpdatingState {
+    pub manifest: Manifest,
+    pub installing_info: InstallingInfo,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InstalledState {
+    pub manifest: Manifest,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InstallingInfo {
+    pub new_manifest: Manifest,
+    pub progress: FullProgress,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct PackageDataEntry {
+    pub state_info: PackageState,
+    pub status: Status,
+    pub marketplace_url: Option<Url>,
+    #[serde(default)]
+    #[serde(with = "crate::util::serde::ed25519_pubkey")]
+    pub developer_key: ed25519_dalek::VerifyingKey,
+    pub icon: DataUrl<'static>,
+    pub last_backup: Option<DateTime<Utc>>,
+    pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
+    pub current_dependents: CurrentDependents,
+    pub current_dependencies: CurrentDependencies,
+    pub interface_addresses: InterfaceAddressMap,
+    pub hosts: HostInfo,
+    pub store_exposed_ui: Vec<ExposedUI>,
+    pub store_exposed_dependents: Vec<JsonPointer>,
+}
+impl AsRef<PackageDataEntry> for PackageDataEntry {
+    fn as_ref(&self) -> &PackageDataEntry {
+        self
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct ExposedDependent {
+    path: String,
+    title: String,
+    description: Option<String>,
+    masked: Option<bool>,
+    copyable: Option<bool>,
+    qr: Option<bool>,
+}
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]
+#[model = "Model<Self>"]
+#[ts(export)]
+pub struct ExposedUI {
+    #[ts(type = "string")]
+    pub path: JsonPointer,
+    pub title: String,
+    pub description: Option<String>,
+    pub masked: Option<bool>,
+    pub copyable: Option<bool>,
+    pub qr: Option<bool>,
+}
+
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependents {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependents {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
+}
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
+impl CurrentDependencies {
+    pub fn map(
+        mut self,
+        transform: impl Fn(
+            BTreeMap<PackageId, CurrentDependencyInfo>,
+        ) -> BTreeMap<PackageId, CurrentDependencyInfo>,
+    ) -> Self {
+        self.0 = transform(self.0);
+        self
+    }
+}
+impl Map for CurrentDependencies {
+    type Key = PackageId;
+    type Value = CurrentDependencyInfo;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct StaticDependencyInfo {
+    pub title: String,
+    pub icon: DataUrl<'static>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(tag = "kind")]
+pub enum CurrentDependencyInfo {
+    Exists,
+    #[serde(rename_all = "kebab-case")]
+    Running {
+        #[serde(default)]
+        health_checks: BTreeSet<HealthCheckId>,
+    },
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct InterfaceAddressMap(pub BTreeMap<HostId, InterfaceAddresses>);
+impl Map for InterfaceAddressMap {
+    type Key = HostId;
+    type Value = InterfaceAddresses;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct InterfaceAddresses {
+    pub tor_address: Option<String>,
+    pub lan_address: Option<String>,
+}

core/startos/src/db/model/private.rs

@@ -0,0 +1,30 @@
+use std::collections::BTreeMap;
+
+use models::PackageId;
+use patch_db::{HasModel, Value};
+use serde::{Deserialize, Serialize};
+
+use crate::auth::Sessions;
+use crate::backup::target::cifs::CifsTargets;
+use crate::net::forward::AvailablePorts;
+use crate::net::keys::KeyStore;
+use crate::notifications::Notifications;
+use crate::prelude::*;
+use crate::ssh::SshKeys;
+use crate::util::serde::Pem;
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct Private {
+    pub key_store: KeyStore,
+    pub password: String, // argon2 hash
+    pub ssh_privkey: Pem<ssh_key::PrivateKey>,
+    pub ssh_pubkeys: SshKeys,
+    pub available_ports: AvailablePorts,
+    pub sessions: Sessions,
+    pub notifications: Notifications,
+    pub cifs: CifsTargets,
+    #[serde(default)]
+    pub package_stores: BTreeMap<PackageId, Value>,
+}

core/startos/src/db/model/public.rs

@@ -0,0 +1,210 @@
+use std::collections::BTreeMap;
+use std::net::{Ipv4Addr, Ipv6Addr};
+
+use chrono::{DateTime, Utc};
+use emver::VersionRange;
+use imbl_value::InternedString;
+use ipnet::{Ipv4Net, Ipv6Net};
+use isocountry::CountryCode;
+use itertools::Itertools;
+use models::PackageId;
+use openssl::hash::MessageDigest;
+use patch_db::{HasModel, Value};
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use torut::onion::OnionAddressV3;
+
+use crate::account::AccountInfo;
+use crate::db::model::package::AllPackageData;
+use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
+use crate::prelude::*;
+use crate::util::cpupower::Governor;
+use crate::util::Version;
+use crate::version::{Current, VersionT};
+use crate::{ARCH, PLATFORM};
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+// #[macro_debug]
+pub struct Public {
+    pub server_info: ServerInfo,
+    pub package_data: AllPackageData,
+    pub ui: Value,
+}
+impl Public {
+    pub fn init(account: &AccountInfo) -> Result<Self, Error> {
+        let lan_address = account.hostname.lan_address().parse().unwrap();
+        Ok(Self {
+            server_info: ServerInfo {
+                arch: get_arch(),
+                platform: get_platform(),
+                id: account.server_id.clone(),
+                version: Current::new().semver().into(),
+                hostname: account.hostname.no_dot_host_name(),
+                last_backup: None,
+                last_wifi_region: None,
+                eos_version_compat: Current::new().compat().clone(),
+                lan_address,
+                onion_address: account.tor_key.public().get_onion_address(),
+                tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
+                    .parse()
+                    .unwrap(),
+                ip_info: BTreeMap::new(),
+                status_info: ServerStatus {
+                    backup_progress: None,
+                    updated: false,
+                    update_progress: None,
+                    shutting_down: false,
+                    restarting: false,
+                },
+                wifi: WifiInfo {
+                    ssids: Vec::new(),
+                    connected: None,
+                    selected: None,
+                },
+                unread_notification_count: 0,
+                connection_addresses: ConnectionAddresses {
+                    tor: Vec::new(),
+                    clearnet: Vec::new(),
+                },
+                password_hash: account.password.clone(),
+                pubkey: ssh_key::PublicKey::from(&account.ssh_key)
+                    .to_openssh()
+                    .unwrap(),
+                ca_fingerprint: account
+                    .root_ca_cert
+                    .digest(MessageDigest::sha256())
+                    .unwrap()
+                    .iter()
+                    .map(|x| format!("{x:X}"))
+                    .join(":"),
+                ntp_synced: false,
+                zram: true,
+                governor: None,
+            },
+            package_data: AllPackageData::default(),
+            ui: serde_json::from_str(include_str!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../../web/patchdb-ui-seed.json"
+            )))
+            .with_kind(ErrorKind::Deserialization)?,
+        })
+    }
+}
+
+fn get_arch() -> InternedString {
+    (*ARCH).into()
+}
+
+fn get_platform() -> InternedString {
+    (&*PLATFORM).into()
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct ServerInfo {
+    #[serde(default = "get_arch")]
+    pub arch: InternedString,
+    #[serde(default = "get_platform")]
+    pub platform: InternedString,
+    pub id: String,
+    pub hostname: String,
+    pub version: Version,
+    pub last_backup: Option<DateTime<Utc>>,
+    /// Used in the wifi to determine the region to set the system to
+    pub last_wifi_region: Option<CountryCode>,
+    pub eos_version_compat: VersionRange,
+    pub lan_address: Url,
+    pub onion_address: OnionAddressV3,
+    /// for backwards compatibility
+    pub tor_address: Url,
+    pub ip_info: BTreeMap<String, IpInfo>,
+    #[serde(default)]
+    pub status_info: ServerStatus,
+    pub wifi: WifiInfo,
+    pub unread_notification_count: u64,
+    pub connection_addresses: ConnectionAddresses,
+    pub password_hash: String,
+    pub pubkey: String,
+    pub ca_fingerprint: String,
+    #[serde(default)]
+    pub ntp_synced: bool,
+    #[serde(default)]
+    pub zram: bool,
+    pub governor: Option<Governor>,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct IpInfo {
+    pub ipv4_range: Option<Ipv4Net>,
+    pub ipv4: Option<Ipv4Addr>,
+    pub ipv6_range: Option<Ipv6Net>,
+    pub ipv6: Option<Ipv6Addr>,
+}
+impl IpInfo {
+    pub async fn for_interface(iface: &str) -> Result<Self, Error> {
+        let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
+        let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
+        Ok(Self {
+            ipv4_range,
+            ipv4,
+            ipv6_range,
+            ipv6,
+        })
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[model = "Model<Self>"]
+pub struct BackupProgress {
+    pub complete: bool,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct ServerStatus {
+    pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
+    pub updated: bool,
+    pub update_progress: Option<UpdateProgress>,
+    #[serde(default)]
+    pub shutting_down: bool,
+    #[serde(default)]
+    pub restarting: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct UpdateProgress {
+    pub size: Option<u64>,
+    pub downloaded: u64,
+}
+
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct WifiInfo {
+    pub ssids: Vec<String>,
+    pub selected: Option<String>,
+    pub connected: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ServerSpecs {
+    pub cpu: String,
+    pub disk: String,
+    pub memory: String,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct ConnectionAddresses {
+    pub tor: Vec<String>,
+    pub clearnet: Vec<String>,
+}

core/startos/src/db/prelude.rs

@@ -124,6 +124,12 @@ impl<T: Serialize + DeserializeOwned> Model<T> {
         self.ser(&orig)?;
         Ok(res)
     }
+    pub fn map_mutate(&mut self, f: impl FnOnce(T) -> Result<T, Error>) -> Result<T, Error> {
+        let mut orig = self.de()?;
+        let res = f(orig)?;
+        self.ser(&res)?;
+        Ok(res)
+    }
 }
 impl<T> Clone for Model<T> {
     fn clone(&self) -> Self {

core/startos/src/dependencies.rs

@@ -10,7 +10,8 @@ use tracing::instrument;
 
 use crate::config::{Config, ConfigSpec, ConfigureContext};
 use crate::context::RpcContext;
-use crate::db::model::{CurrentDependencies, Database};
+use crate::db::model::package::CurrentDependencies;
+use crate::db::model::Database;
 use crate::prelude::*;
 use crate::s9pk::manifest::Manifest;
 use crate::status::DependencyConfigErrors;
@@ -195,52 +196,19 @@ pub async fn configure_logic(
     todo!()
 }
 
-#[instrument(skip_all)]
-pub fn add_dependent_to_current_dependents_lists(
-    db: &mut Model<Database>,
-    dependent_id: &PackageId,
-    current_dependencies: &CurrentDependencies,
-) -> Result<(), Error> {
-    for (dependency, dep_info) in &current_dependencies.0 {
-        if let Some(dependency_dependents) = db
-            .as_public_mut()
-            .as_package_data_mut()
-            .as_idx_mut(dependency)
-            .and_then(|pde| pde.as_installed_mut())
-            .map(|i| i.as_current_dependents_mut())
-        {
-            dependency_dependents.insert(dependent_id, dep_info)?;
-        }
-    }
-    Ok(())
-}
-
 #[instrument(skip_all)]
 pub async fn compute_dependency_config_errs(
     ctx: &RpcContext,
     db: &Peeked,
-    manifest: &Manifest,
+    id: &PackageId,
     current_dependencies: &CurrentDependencies,
     dependency_config: &BTreeMap<PackageId, Config>,
 ) -> Result<DependencyConfigErrors, Error> {
     let mut dependency_config_errs = BTreeMap::new();
-    for (dependency, _dep_info) in current_dependencies
+    for (dependency, _dep_info) in current_dependencies.0.iter() {
-        .0
-        .iter()
-        .filter(|(dep_id, _)| dep_id != &&manifest.id)
-    {
         // check if config passes dependency check
-        if let Some(cfg) = &manifest
+        if let Some(error) = todo!() {
-            .dependencies
+            dependency_config_errs.insert(dependency.clone(), error);
-            .0
-            .get(dependency)
-            .or_not_found(dependency)?
-            .config
-        {
-            let error = todo!();
-            {
-                dependency_config_errs.insert(dependency.clone(), error);
-            }
         }
     }
     Ok(DependencyConfigErrors(dependency_config_errs))

core/startos/src/init.rs

@@ -11,7 +11,7 @@ use tracing::instrument;
 
 use crate::account::AccountInfo;
 use crate::context::config::ServerConfig;
-use crate::db::model::ServerStatus;
+use crate::db::model::public::ServerStatus;
 use crate::disk::mount::util::unmount;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::prelude::*;

core/startos/src/install/mod.rs

@@ -18,10 +18,7 @@ use tracing::instrument;
 
 use crate::context::{CliContext, RpcContext};
 use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
-use crate::db::model::{
+use crate::db::model::package::{ManifestPreference, PackageState, PackageStateMatchModelRef};
-    PackageDataEntry, PackageDataEntryInstalled, PackageDataEntryMatchModelRef,
-    PackageDataEntryRemoving,
-};
 use crate::prelude::*;
 use crate::progress::{FullProgress, PhasedProgressBar};
 use crate::s9pk::manifest::PackageId;
@@ -40,27 +37,27 @@ pub async fn list(ctx: RpcContext) -> Result<Value, Error> {
     Ok(ctx.db.peek().await.as_public().as_package_data().as_entries()?
         .iter()
         .filter_map(|(id, pde)| {
-            let status = match pde.as_match() {
+            let status = match pde.as_state_info().as_match() {
-                PackageDataEntryMatchModelRef::Installed(_) => {
+                PackageStateMatchModelRef::Installed(_) => {
                     "installed"
                 }
-                PackageDataEntryMatchModelRef::Installing(_) => {
+                PackageStateMatchModelRef::Installing(_) => {
                     "installing"
                 }
-                PackageDataEntryMatchModelRef::Updating(_) => {
+                PackageStateMatchModelRef::Updating(_) => {
                     "updating"
                 }
-                PackageDataEntryMatchModelRef::Restoring(_) => {
+                PackageStateMatchModelRef::Restoring(_) => {
                     "restoring"
                 }
-                PackageDataEntryMatchModelRef::Removing(_) => {
+                PackageStateMatchModelRef::Removing(_) => {
                     "removing"
                 }
-                PackageDataEntryMatchModelRef::Error(_) => {
+                PackageStateMatchModelRef::Error(_) => {
                     "error"
                 }
             };
-            serde_json::to_value(json!({ "status":status, "id": id.clone(), "version": pde.as_manifest().as_version().de().ok()?}))
+            serde_json::to_value(json!({ "status": status, "id": id.clone(), "version": pde.as_state_info().as_manifest(ManifestPreference::Old).as_version().de().ok()?}))
             .ok()
         })
         .collect())
@@ -212,7 +209,7 @@ pub async fn sideload(ctx: RpcContext) -> Result<SideloadResponse, Error> {
                                                 .as_public()
                                                 .as_package_data()
                                                 .as_idx(&id)
-                                                .and_then(|e| e.as_install_progress())
+                                                .and_then(|e| e.as_state_info().as_installing_info()).map(|i| i.as_progress())
                                             {
                                                 Ok::<_, ()>(p.de()?)
                                             } else {
@@ -407,31 +404,18 @@ pub async fn uninstall(
 ) -> Result<PackageId, Error> {
     ctx.db
         .mutate(|db| {
-            let (manifest, static_files, installed) = match db
+            let entry = db
-                .as_public()
+                .as_public_mut()
-                .as_package_data()
+                .as_package_data_mut()
-                .as_idx(&id)
+                .as_idx_mut(&id)
-                .or_not_found(&id)?
+                .or_not_found(&id)?;
-                .de()?
+            entry.as_state_info_mut().map_mutate(|s| match s {
-            {
+                PackageState::Installed(s) => Ok(PackageState::Removing(s)),
-                PackageDataEntry::Installed(PackageDataEntryInstalled {
+                _ => Err(Error::new(
-                    manifest,
+                    eyre!("Package {id} is not installed."),
-                    static_files,
+                    crate::ErrorKind::NotFound,
-                    installed,
+                )),
-                }) => (manifest, static_files, installed),
+            })
-                _ => {
-                    return Err(Error::new(
-                        eyre!("Package is not installed."),
-                        crate::ErrorKind::NotFound,
-                    ));
-                }
-            };
-            let pde = PackageDataEntry::Removing(PackageDataEntryRemoving {
-                manifest,
-                static_files,
-                removing: installed,
-            });
-            db.as_public_mut().as_package_data_mut().insert(&id, &pde)
         })
         .await?;
 

core/startos/src/net/dhcp.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::RwLock;
 
 use crate::context::{CliContext, RpcContext};
-use crate::db::model::IpInfo;
+use crate::db::model::public::IpInfo;
 use crate::net::utils::{iface_is_physical, list_interfaces};
 use crate::prelude::*;
 use crate::Error;

core/startos/src/net/net_controller.rs

@@ -205,8 +205,6 @@ impl NetService {
                     .as_package_data_mut()
                     .as_idx_mut(pkg_id)
                     .or_not_found(pkg_id)?
-                    .as_installed_mut()
-                    .or_not_found(pkg_id)?
                     .as_hosts_mut();
                 hosts.add_binding(&mut ports, kind, &id, internal_port, options)?;
                 let host = hosts

core/startos/src/properties.rs

@@ -4,9 +4,10 @@ use models::PackageId;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 
+use crate::context::RpcContext;
+use crate::db::model::package::ExposedUI;
 use crate::prelude::*;
 use crate::Error;
-use crate::{context::RpcContext, db::model::ExposedUI};
 
 pub fn display_properties(response: Value) {
     println!("{}", response);
@@ -59,8 +60,6 @@ pub async fn properties(
         .as_package_data()
         .as_idx(&id)
         .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?
         .as_store_exposed_ui()
         .de()?
         .into_properties(&data))

core/startos/src/service/mod.rs

@@ -16,9 +16,8 @@ use crate::action::ActionResult;
 use crate::config::action::ConfigRes;
 use crate::context::{CliContext, RpcContext};
 use crate::core::rpc_continuations::RequestGuid;
-use crate::db::model::{
+use crate::db::model::package::{
-    InstalledPackageInfo, PackageDataEntry, PackageDataEntryInstalled, PackageDataEntryMatchModel,
+    InstalledState, PackageDataEntry, PackageState, PackageStateMatchModelRef, UpdatingState,
-    StaticFiles,
 };
 use crate::disk::mount::guard::GenericMountGuard;
 use crate::install::PKG_ARCHIVE_DIR;
@@ -28,7 +27,7 @@ use crate::s9pk::S9pk;
 use crate::service::service_map::InstallProgressHandles;
 use crate::service::transition::TransitionKind;
 use crate::status::health_check::HealthCheckResult;
-use crate::status::{MainStatus, Status};
+use crate::status::MainStatus;
 use crate::util::actor::{Actor, BackgroundJobs, SimpleActor};
 use crate::volume::data_dir;
 
@@ -100,7 +99,7 @@ impl Service {
     ) -> Result<Option<Self>, Error> {
         let handle_installed = {
             let ctx = ctx.clone();
-            move |s9pk: S9pk, i: Model<InstalledPackageInfo>| async move {
+            move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
                 for volume_id in &s9pk.as_manifest().volumes {
                     let tmp_path =
                         data_dir(&ctx.datadir, &s9pk.as_manifest().id.clone(), volume_id);
@@ -118,16 +117,18 @@ impl Service {
         };
         let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash
         let s9pk_path = s9pk_dir.join(id).with_extension("s9pk");
-        match ctx
+        let Some(entry) = ctx
             .db
             .peek()
             .await
             .into_public()
             .into_package_data()
             .into_idx(id)
-            .map(|pde| pde.into_match())
+        else {
-        {
+            return Ok(None);
-            Some(PackageDataEntryMatchModel::Installing(_)) => {
+        };
+        match entry.as_state_info().as_match() {
+            PackageStateMatchModelRef::Installing(_) => {
                 if disposition == LoadDisposition::Retry {
                     if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
                         tracing::error!("Error opening s9pk for install: {e}");
@@ -150,14 +151,17 @@ impl Service {
                     .await?;
                 Ok(None)
             }
-            Some(PackageDataEntryMatchModel::Updating(e)) => {
+            PackageStateMatchModelRef::Updating(s) => {
                 if disposition == LoadDisposition::Retry
-                    && e.as_install_progress().de()?.phases.iter().any(
+                    && s.as_installing_info()
-                        |NamedProgress { name, progress }| {
+                        .as_progress()
+                        .de()?
+                        .phases
+                        .iter()
+                        .any(|NamedProgress { name, progress }| {
                             name.eq_ignore_ascii_case("download")
                                 && progress == &Progress::Complete(true)
-                        },
+                        })
-                    )
                 {
                     if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| {
                         tracing::error!("Error opening s9pk for update: {e}");
@@ -166,7 +170,7 @@ impl Service {
                         if let Ok(service) = Self::install(
                             ctx.clone(),
                             s9pk,
-                            Some(e.as_installed().as_manifest().as_version().de()?),
+                            Some(s.as_manifest().as_version().de()?),
                             None,
                         )
                         .await
@@ -181,24 +185,28 @@ impl Service {
                 let s9pk = S9pk::open(s9pk_path, Some(id)).await?;
                 ctx.db
                     .mutate({
-                        let manifest = s9pk.as_manifest().clone();
                         |db| {
                             db.as_public_mut()
                                 .as_package_data_mut()
-                                .as_idx_mut(&manifest.id)
+                                .as_idx_mut(&id)
-                                .or_not_found(&manifest.id)?
+                                .or_not_found(&id)?
-                                .ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
+                                .as_state_info_mut()
-                                    static_files: e.as_static_files().de()?,
+                                .map_mutate(|s| {
-                                    manifest,
+                                    if let PackageState::Updating(UpdatingState {
-                                    installed: e.as_installed().de()?,
+                                        manifest, ..
-                                }))
+                                    }) = s
+                                    {
+                                        Ok(PackageState::Installed(InstalledState { manifest }))
+                                    } else {
+                                        Err(Error::new(eyre!("Race condition detected - package state changed during load"), ErrorKind::Database))
+                                    }
+                                })
                         }
                     })
                     .await?;
-                handle_installed(s9pk, e.as_installed().clone()).await
+                handle_installed(s9pk, entry).await
             }
-            Some(PackageDataEntryMatchModel::Removing(_))
+            PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => {
-            | Some(PackageDataEntryMatchModel::Restoring(_)) => {
                 if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
                     tracing::error!("Error opening s9pk for removal: {e}");
                     tracing::debug!("{e:?}")
@@ -230,18 +238,13 @@ impl Service {
 
                 Ok(None)
             }
-            Some(PackageDataEntryMatchModel::Installed(i)) => {
+            PackageStateMatchModelRef::Installed(_) => {
-                handle_installed(
+                handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await
-                    S9pk::open(s9pk_path, Some(id)).await?,
-                    i.as_installed().clone(),
-                )
-                .await
             }
-            Some(PackageDataEntryMatchModel::Error(e)) => Err(Error::new(
+            PackageStateMatchModelRef::Error(e) => Err(Error::new(
                 eyre!("Failed to parse PackageDataEntry, found {e:?}"),
                 ErrorKind::Deserialization,
             )),
-            None => Ok(None),
         }
     }
 
@@ -255,7 +258,6 @@ impl Service {
         let manifest = s9pk.as_manifest().clone();
         let developer_key = s9pk.as_archive().signer();
         let icon = s9pk.icon_data_url().await?;
-        let static_files = StaticFiles::local(&manifest.id, &manifest.version, icon);
         let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?;
         service
             .seed
@@ -270,32 +272,19 @@ impl Service {
         }
         ctx.db
             .mutate(|d| {
-                d.as_public_mut()
+                let entry = d
+                    .as_public_mut()
                     .as_package_data_mut()
                     .as_idx_mut(&manifest.id)
-                    .or_not_found(&manifest.id)?
+                    .or_not_found(&manifest.id)?;
-                    .ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
+                entry
-                        installed: InstalledPackageInfo {
+                    .as_state_info_mut()
-                            current_dependencies: Default::default(), // TODO
+                    .ser(&PackageState::Installed(InstalledState { manifest }))?;
-                            current_dependents: Default::default(),   // TODO
+                entry.as_developer_key_mut().ser(&developer_key)?;
-                            dependency_info: Default::default(),      // TODO
+                entry.as_icon_mut().ser(&icon)?;
-                            developer_key,
+                // TODO: marketplace url
-                            status: Status {
+                // TODO: dependency info
-                                configured: false,                            // TODO
+                Ok(())
-                                main: MainStatus::Stopped,                    // TODO
-                                dependency_config_errors: Default::default(), // TODO
-                            },
-                            interface_addresses: Default::default(), // TODO
-                            marketplace_url: None,                   // TODO
-                            manifest: manifest.clone(),
-                            last_backup: None,                            // TODO
-                            hosts: Default::default(),                    // TODO
-                            store_exposed_dependents: Default::default(), // TODO
-                            store_exposed_ui: Default::default(),         // TODO
-                        },
-                        manifest,
-                        static_files,
-                    }))
             })
             .await?;
         Ok(service)
@@ -466,11 +455,7 @@ impl Actor for ServiceActor {
                     seed.ctx
                         .db
                         .mutate(|d| {
-                            if let Some(i) = d
+                            if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id)
-                                .as_public_mut()
-                                .as_package_data_mut()
-                                .as_idx_mut(&id)
-                                .and_then(|p| p.as_installed_mut())
                             {
                                 i.as_status_mut().as_main_mut().ser(&main_status)?;
                             }

core/startos/src/service/service_effect_handler.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeSet;
 use std::ffi::OsString;
 use std::os::unix::process::CommandExt;
 use std::path::{Path, PathBuf};
@@ -7,13 +8,14 @@ use std::sync::{Arc, Weak};
 use clap::builder::ValueParserFactory;
 use clap::Parser;
 use imbl_value::{json, InternedString};
-use models::{ActionId, HealthCheckId, ImageId, PackageId};
+use models::{ActionId, HealthCheckId, ImageId, InvalidId, PackageId};
 use patch_db::json_ptr::JsonPointer;
 use rpc_toolkit::{from_fn, from_fn_async, AnyContext, Context, Empty, HandlerExt, ParentHandler};
+use serde::{Deserialize, Serialize};
 use tokio::process::Command;
 use ts_rs::TS;
 
-use crate::db::model::ExposedUI;
+use crate::db::model::package::{CurrentDependencies, CurrentDependencyInfo, ExposedUI};
 use crate::disk::mount::filesystem::idmapped::IdMapped;
 use crate::disk::mount::filesystem::loop_dev::LoopDev;
 use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
@@ -131,8 +133,13 @@ pub fn service_effect_handler() -> ParentHandler {
         .subcommand("clearBindings", from_fn_async(clear_bindings).no_cli())
         .subcommand("bind", from_fn_async(bind).no_cli())
         .subcommand("getHostInfo", from_fn_async(get_host_info).no_cli())
+        .subcommand(
+            "setDependencies",
+            from_fn_async(set_dependencies)
+                .no_display()
+                .with_remote_cli::<ContainerCliContext>(),
+        )
     // TODO @DrBonez when we get the new api for 4.0
-    // .subcommand("setDependencies",from_fn_async(set_dependencies).no_cli())
     // .subcommand("embassyGetInterface",from_fn_async(embassy_get_interface).no_cli())
     // .subcommand("mount",from_fn_async(mount).no_cli())
     // .subcommand("removeAction",from_fn_async(remove_action).no_cli())
@@ -459,8 +466,6 @@ async fn expose_for_dependents(
                 .as_package_data_mut()
                 .as_idx_mut(&package_id)
                 .or_not_found(&package_id)?
-                .as_installed_mut()
-                .or_not_found(&package_id)?
                 .as_store_exposed_dependents_mut()
                 .ser(&paths)
         })
@@ -488,8 +493,6 @@ async fn expose_ui(
                 .as_package_data_mut()
                 .as_idx_mut(&package_id)
                 .or_not_found(&package_id)?
-                .as_installed_mut()
-                .or_not_found(&package_id)?
                 .as_store_exposed_ui_mut()
                 .ser(&paths)
         })
@@ -566,8 +569,6 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error
         .as_package_data()
         .as_idx(&package_id)
         .or_not_found(&package_id)?
-        .as_installed()
-        .or_not_found(&package_id)?
         .as_status()
         .as_configured()
         .de()?;
@@ -583,8 +584,6 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
         .as_package_data()
         .as_idx(&package_id)
         .or_not_found(&package_id)?
-        .as_installed()
-        .or_not_found(&package_id)?
         .as_status()
         .as_main()
         .de()?;
@@ -600,8 +599,6 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Valu
         .as_package_data()
         .as_idx(&package_id)
         .or_not_found(&package_id)?
-        .as_installed()
-        .or_not_found(&package_id)?
         .as_status()
         .as_main()
         .de()?;
@@ -652,8 +649,6 @@ async fn set_configured(context: EffectContext, params: SetConfigured) -> Result
                 .as_package_data_mut()
                 .as_idx_mut(package_id)
                 .or_not_found(package_id)?
-                .as_installed_mut()
-                .or_not_found(package_id)?
                 .as_status_mut()
                 .as_configured_mut()
                 .ser(&params.configured)
@@ -733,8 +728,6 @@ async fn set_health(
                 .as_package_data()
                 .as_idx(package_id)
                 .or_not_found(package_id)?
-                .as_installed()
-                .or_not_found(package_id)?
                 .as_status()
                 .as_main()
                 .de()?;
@@ -764,8 +757,6 @@ async fn set_health(
                 .as_package_data_mut()
                 .as_idx_mut(package_id)
                 .or_not_found(package_id)?
-                .as_installed_mut()
-                .or_not_found(package_id)?
                 .as_status_mut()
                 .as_main_mut()
                 .ser(&main)
@@ -778,8 +769,6 @@ async fn set_health(
 #[command(rename_all = "camelCase")]
 #[ts(export)]
 pub struct DestroyOverlayedImageParams {
-    #[ts(type = "string ")]
-    image_id: ImageId,
     #[ts(type = "string")]
     guid: InternedString,
 }
@@ -787,7 +776,7 @@ pub struct DestroyOverlayedImageParams {
 #[instrument(skip_all)]
 pub async fn destroy_overlayed_image(
     ctx: EffectContext,
-    DestroyOverlayedImageParams { image_id, guid }: DestroyOverlayedImageParams,
+    DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
 ) -> Result<(), Error> {
     let ctx = ctx.deref()?;
     if ctx
@@ -873,3 +862,125 @@ pub async fn create_overlayed_image(
         ))
     }
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+enum DependencyKind {
+    Exists,
+    Running,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+struct DependencyRequirement {
+    id: PackageId,
+    kind: DependencyKind,
+    #[serde(default)]
+    health_checks: BTreeSet<HealthCheckId>,
+}
+// filebrowser:exists,bitcoind:running:foo+bar+baz
+impl FromStr for DependencyRequirement {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.split_once(":") {
+            Some((id, "e")) | Some((id, "exists")) => Ok(Self {
+                id: id.parse()?,
+                kind: DependencyKind::Exists,
+                health_checks: BTreeSet::new(),
+            }),
+            Some((id, rest)) => {
+                let health_checks = match rest.split_once(":") {
+                    Some(("r", rest)) | Some(("running", rest)) => rest
+                        .split("+")
+                        .map(|id| id.parse().map_err(Error::from))
+                        .collect(),
+                    Some((kind, _)) => Err(Error::new(
+                        eyre!("unknown dependency kind {kind}"),
+                        ErrorKind::InvalidRequest,
+                    )),
+                    None => match rest {
+                        "r" | "running" => Ok(BTreeSet::new()),
+                        kind => Err(Error::new(
+                            eyre!("unknown dependency kind {kind}"),
+                            ErrorKind::InvalidRequest,
+                        )),
+                    },
+                }?;
+                Ok(Self {
+                    id: id.parse()?,
+                    kind: DependencyKind::Running,
+                    health_checks,
+                })
+            }
+            None => Ok(Self {
+                id: s.parse()?,
+                kind: DependencyKind::Running,
+                health_checks: BTreeSet::new(),
+            }),
+        }
+    }
+}
+impl ValueParserFactory for DependencyRequirement {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        FromStrParser::new()
+    }
+}
+
+#[derive(Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[command(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetDependenciesParams {
+    dependencies: Vec<DependencyRequirement>,
+}
+
+pub async fn set_dependencies(
+    ctx: EffectContext,
+    SetDependenciesParams { dependencies }: SetDependenciesParams,
+) -> Result<(), Error> {
+    let ctx = ctx.deref()?;
+    let id = &ctx.id;
+    ctx.ctx
+        .db
+        .mutate(|db| {
+            let dependencies = CurrentDependencies(
+                dependencies
+                    .into_iter()
+                    .map(
+                        |DependencyRequirement {
+                             id,
+                             kind,
+                             health_checks,
+                         }| {
+                            (
+                                id,
+                                match kind {
+                                    DependencyKind::Exists => CurrentDependencyInfo::Exists,
+                                    DependencyKind::Running => {
+                                        CurrentDependencyInfo::Running { health_checks }
+                                    }
+                                },
+                            )
+                        },
+                    )
+                    .collect(),
+            );
+            for (dep, entry) in db.as_public_mut().as_package_data_mut().as_entries_mut()? {
+                if let Some(info) = dependencies.0.get(&dep) {
+                    entry.as_current_dependents_mut().insert(id, info)?;
+                } else {
+                    entry.as_current_dependents_mut().remove(id)?;
+                }
+            }
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(id)
+                .or_not_found(id)?
+                .as_current_dependencies_mut()
+                .ser(&dependencies)
+        })
+        .await
+}

core/startos/src/service/service_map.rs

@@ -11,9 +11,8 @@ use tokio::sync::{Mutex, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
 use tracing::instrument;
 
 use crate::context::RpcContext;
-use crate::db::model::{
+use crate::db::model::package::{
-    PackageDataEntry, PackageDataEntryInstalled, PackageDataEntryInstalling,
+    InstallingInfo, InstallingState, PackageDataEntry, PackageState, UpdatingState,
-    PackageDataEntryRestoring, PackageDataEntryUpdating, StaticFiles,
 };
 use crate::disk::mount::guard::GenericMountGuard;
 use crate::install::PKG_ARCHIVE_DIR;
@@ -27,6 +26,7 @@ use crate::s9pk::manifest::PackageId;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::S9pk;
 use crate::service::{LoadDisposition, Service};
+use crate::status::{MainStatus, Status};
 
 pub type DownloadInstallFuture = BoxFuture<'static, Result<InstallFuture, Error>>;
 pub type InstallFuture = BoxFuture<'static, Result<(), Error>>;
@@ -95,9 +95,10 @@ impl ServiceMap {
         mut s9pk: S9pk<S>,
         recovery_source: Option<impl GenericMountGuard>,
     ) -> Result<DownloadInstallFuture, Error> {
-        let manifest = Arc::new(s9pk.as_manifest().clone());
+        let manifest = s9pk.as_manifest().clone();
         let id = manifest.id.clone();
         let icon = s9pk.icon_data_url().await?;
+        let developer_key = s9pk.as_archive().signer();
         let mut service = self.get_mut(&id).await;
 
         let op_name = if recovery_source.is_none() {
@@ -135,49 +136,51 @@ impl ServiceMap {
                 let id = id.clone();
                 let install_progress = progress.snapshot();
                 move |db| {
-                    let pde = match db
+                    if let Some(pde) = db.as_public_mut().as_package_data_mut().as_idx_mut(&id) {
-                        .as_public()
+                        let prev = pde.as_state_info().expect_installed()?.de()?;
-                        .as_package_data()
+                        pde.as_state_info_mut()
-                        .as_idx(&id)
+                            .ser(&PackageState::Updating(UpdatingState {
-                        .map(|x| x.de())
+                                manifest: prev.manifest,
-                        .transpose()?
+                                installing_info: InstallingInfo {
-                    {
+                                    new_manifest: manifest,
-                        Some(PackageDataEntry::Installed(PackageDataEntryInstalled {
+                                    progress: install_progress,
-                            installed,
+                                },
-                            static_files,
+                            }))?;
-                            ..
+                    } else {
-                        })) => PackageDataEntry::Updating(PackageDataEntryUpdating {
+                        let installing = InstallingState {
-                            install_progress,
+                            installing_info: InstallingInfo {
-                            installed,
+                                new_manifest: manifest,
-                            manifest: (*manifest).clone(),
+                                progress: install_progress,
-                            static_files,
+                            },
-                        }),
+                        };
-                        None if restoring => {
+                        db.as_public_mut().as_package_data_mut().insert(
-                            PackageDataEntry::Restoring(PackageDataEntryRestoring {
+                            &id,
-                                install_progress,
+                            &PackageDataEntry {
-                                static_files: StaticFiles::local(
+                                state_info: if restoring {
-                                    &manifest.id,
+                                    PackageState::Restoring(installing)
-                                    &manifest.version,
+                                } else {
-                                    icon,
+                                    PackageState::Installing(installing)
-                                ),
+                                },
-                                manifest: (*manifest).clone(),
+                                status: Status {
-                            })
+                                    configured: false,
-                        }
+                                    main: MainStatus::Stopped,
-                        None => PackageDataEntry::Installing(PackageDataEntryInstalling {
+                                    dependency_config_errors: Default::default(),
-                            install_progress,
+                                },
-                            static_files: StaticFiles::local(&manifest.id, &manifest.version, icon),
+                                marketplace_url: None,
-                            manifest: (*manifest).clone(),
+                                developer_key,
-                        }),
+                                icon,
-                        _ => {
+                                last_backup: None,
-                            return Err(Error::new(
+                                dependency_info: Default::default(),
-                                eyre!("Cannot install over a package in a transient state"),
+                                current_dependents: Default::default(), // TODO: initialize
-                                crate::ErrorKind::InvalidRequest,
+                                current_dependencies: Default::default(),
-                            ))
+                                interface_addresses: Default::default(),
-                        }
+                                hosts: Default::default(),
+                                store_exposed_ui: Default::default(),
+                                store_exposed_dependents: Default::default(),
+                            },
+                        )?;
                     };
-                    db.as_public_mut()
+                    Ok(())
-                        .as_package_data_mut()
-                        .insert(&manifest.id, &pde)
                 }
             }))
             .await?;
@@ -200,7 +203,8 @@ impl ServiceMap {
                                 v.as_public_mut()
                                     .as_package_data_mut()
                                     .as_idx_mut(&deref_id)
-                                    .and_then(|e| e.as_install_progress_mut())
+                                    .and_then(|e| e.as_state_info_mut().as_installing_info_mut())
+                                    .map(|i| i.as_progress_mut())
                             },
                             Some(Duration::from_millis(100)),
                         )));

core/startos/src/update/mod.rs

@@ -14,7 +14,7 @@ use tokio_stream::StreamExt;
 use tracing::instrument;
 
 use crate::context::RpcContext;
-use crate::db::model::UpdateProgress;
+use crate::db::model::public::UpdateProgress;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::MountGuard;

sdk/lib/config/configDependencies.ts

@@ -3,7 +3,7 @@ import { Dependency } from "../types"
 
 export type ConfigDependencies<T extends SDKManifest> = {
   exists(id: keyof T["dependencies"]): Dependency
-  running(id: keyof T["dependencies"]): Dependency
+  running(id: keyof T["dependencies"], healthChecks: string[]): Dependency
 }
 
 export const configDependenciesSet = <
@@ -16,10 +16,11 @@ export const configDependenciesSet = <
     } as Dependency
   },
 
-  running(id: keyof T["dependencies"]) {
+  running(id: keyof T["dependencies"], healthChecks: string[]) {
     return {
       id,
       kind: "running",
+      healthChecks,
     } as Dependency
   },
 })

sdk/lib/test/startosTypeValidation.test.ts

@@ -13,6 +13,7 @@ import { ExposeUiParams } from "../../../core/startos/bindings/ExposeUiParams"
 import { GetSslCertificateParams } from "../../../core/startos/bindings/GetSslCertificateParams"
 import { GetSslKeyParams } from "../../../core/startos/bindings/GetSslKeyParams"
 import { GetServiceInterfaceParams } from "../../../core/startos/bindings/GetServiceInterfaceParams"
+import { SetDependenciesParams } from "../../../core/startos/bindings/SetDependenciesParams"
 
 function typeEquality<ExpectedType>(_a: ExpectedType) {}
 describe("startosTypeValidation ", () => {
@@ -46,6 +47,7 @@ describe("startosTypeValidation ", () => {
           | "clearBindings"
           | "bind"
           | "getHostInfo"
+          | "setDependencies"
         )]: Effects[K] extends Function ? Parameters<Effects[K]>[0] : never
     }>({
       executeAction: {} as ExecuteAction,
@@ -67,6 +69,7 @@ describe("startosTypeValidation ", () => {
       getSslCertificate: {} as GetSslCertificateParams,
       getSslKey: {} as GetSslKeyParams,
       getServiceInterface: {} as GetServiceInterfaceParams,
+      setDependencies: {} as SetDependenciesParams,
     })
     typeEquality<Parameters<Effects["executeAction"]>[0]>(
       testInput as ExecuteAction,

sdk/lib/types.ts

@@ -286,10 +286,7 @@ export type Effects = {
   createOverlayedImage(options: { imageId: string }): Promise<[string, string]>
 
   /** A low level api used by destroyOverlay + makeOverlay:destroy */
-  destroyOverlayedImage(options: {
+  destroyOverlayedImage(options: { guid: string }): Promise<void>
-    imageId: string
-    guid: string
-  }): Promise<void>
 
   /** Removes all network bindings */
   clearBindings(): Promise<void>
@@ -467,7 +464,9 @@ export type Effects = {
   }): Promise<void>
 
   /** Set the dependencies of what the service needs, usually ran during the set config as a best practice */
-  setDependencies(dependencies: Dependencies): Promise<DependenciesReceipt>
+  setDependencies(options: {
+    dependencies: Dependencies
+  }): Promise<DependenciesReceipt>
   /** Exists could be useful during the runtime to know if some service exists, option dep */
   exists(options: { packageId: PackageId }): Promise<boolean>
   /** Exists could be useful during the runtime to know if some service is running, option dep */
@@ -585,7 +584,7 @@ export type KnownError =
 export type Dependency = {
   id: PackageId
   kind: DependencyKind
-}
+} & ({ kind: "exists" } | { kind: "running"; healthChecks: string[] })
 export type Dependencies = Array<Dependency>
 
 export type DeepPartial<T> = T extends {}