rename frontend to web and update contributing guide (#2509)

* rename frontend to web and update contributing guide

* rename this time

* fix build

* restructure rust code

* update documentation

* update descriptions

* Update CONTRIBUTING.md

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

core/startos/src/context/cli.rs

@@ -0,0 +1,185 @@
+use std::fs::File;
+use std::io::BufReader;
+use std::net::Ipv4Addr;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use cookie_store::{CookieStore, RawCookie};
+use josekit::jwk::Jwk;
+use reqwest::Proxy;
+use reqwest_cookie_store::CookieStoreMutex;
+use rpc_toolkit::reqwest::{Client, Url};
+use rpc_toolkit::url::Host;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tracing::instrument;
+
+use super::setup::CURRENT_SECRET;
+use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
+use crate::util::config::{load_config_from_paths, local_config_path};
+use crate::ResultExt;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct CliContextConfig {
+    pub host: Option<Url>,
+    #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
+    #[serde(default)]
+    pub proxy: Option<Url>,
+    pub cookie_path: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct CliContextSeed {
+    pub base_url: Url,
+    pub rpc_url: Url,
+    pub client: Client,
+    pub cookie_store: Arc<CookieStoreMutex>,
+    pub cookie_path: PathBuf,
+}
+impl Drop for CliContextSeed {
+    fn drop(&mut self) {
+        let tmp = format!("{}.tmp", self.cookie_path.display());
+        let parent_dir = self.cookie_path.parent().unwrap_or(Path::new("/"));
+        if !parent_dir.exists() {
+            std::fs::create_dir_all(&parent_dir).unwrap();
+        }
+        let mut writer = fd_lock_rs::FdLock::lock(
+            File::create(&tmp).unwrap(),
+            fd_lock_rs::LockType::Exclusive,
+            true,
+        )
+        .unwrap();
+        let mut store = self.cookie_store.lock().unwrap();
+        store.remove("localhost", "", "local");
+        store.save_json(&mut *writer).unwrap();
+        writer.sync_all().unwrap();
+        std::fs::rename(tmp, &self.cookie_path).unwrap();
+    }
+}
+
+const DEFAULT_HOST: Host<&'static str> = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
+const DEFAULT_PORT: u16 = 5959;
+
+#[derive(Debug, Clone)]
+pub struct CliContext(Arc<CliContextSeed>);
+impl CliContext {
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
+        let local_config_path = local_config_path();
+        let base: CliContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
+        let mut url = if let Some(host) = matches.value_of("host") {
+            host.parse()?
+        } else if let Some(host) = base.host {
+            host
+        } else {
+            "http://localhost".parse()?
+        };
+        let proxy = if let Some(proxy) = matches.value_of("proxy") {
+            Some(proxy.parse()?)
+        } else {
+            base.proxy
+        };
+
+        let cookie_path = base.cookie_path.unwrap_or_else(|| {
+            local_config_path
+                .as_deref()
+                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
+                .parent()
+                .unwrap_or(Path::new("/"))
+                .join(".cookies.json")
+        });
+        let cookie_store = Arc::new(CookieStoreMutex::new({
+            let mut store = if cookie_path.exists() {
+                CookieStore::load_json(BufReader::new(File::open(&cookie_path)?))
+                    .map_err(|e| eyre!("{}", e))
+                    .with_kind(crate::ErrorKind::Deserialization)?
+            } else {
+                CookieStore::default()
+            };
+            if let Ok(local) = std::fs::read_to_string(LOCAL_AUTH_COOKIE_PATH) {
+                store
+                    .insert_raw(
+                        &RawCookie::new("local", local),
+                        &"http://localhost".parse()?,
+                    )
+                    .with_kind(crate::ErrorKind::Network)?;
+            }
+            store
+        }));
+
+        Ok(CliContext(Arc::new(CliContextSeed {
+            base_url: url.clone(),
+            rpc_url: {
+                url.path_segments_mut()
+                    .map_err(|_| eyre!("Url cannot be base"))
+                    .with_kind(crate::ErrorKind::ParseUrl)?
+                    .push("rpc")
+                    .push("v1");
+                url
+            },
+            client: {
+                let mut builder = Client::builder().cookie_provider(cookie_store.clone());
+                if let Some(proxy) = proxy {
+                    builder =
+                        builder.proxy(Proxy::all(proxy).with_kind(crate::ErrorKind::ParseUrl)?)
+                }
+                builder.build().expect("cannot fail")
+            },
+            cookie_store,
+            cookie_path,
+        })))
+    }
+}
+impl AsRef<Jwk> for CliContext {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+impl std::ops::Deref for CliContext {
+    type Target = CliContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl Context for CliContext {
+    fn protocol(&self) -> &str {
+        self.0.base_url.scheme()
+    }
+    fn host(&self) -> Host<&str> {
+        self.0.base_url.host().unwrap_or(DEFAULT_HOST)
+    }
+    fn port(&self) -> u16 {
+        self.0.base_url.port().unwrap_or(DEFAULT_PORT)
+    }
+    fn path(&self) -> &str {
+        self.0.rpc_url.path()
+    }
+    fn url(&self) -> Url {
+        self.0.rpc_url.clone()
+    }
+    fn client(&self) -> &Client {
+        &self.0.client
+    }
+}
+/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
+#[test]
+fn test_cli_proxy_empty() {
+    serde_yaml::from_str::<CliContextConfig>(
+        "
+        bind_rpc:
+    ",
+    )
+    .unwrap();
+}

core/startos/src/context/diagnostic.rs

@@ -0,0 +1,83 @@
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tokio::sync::broadcast::Sender;
+use tracing::instrument;
+
+use crate::shutdown::Shutdown;
+use crate::util::config::load_config_from_paths;
+use crate::Error;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DiagnosticContextConfig {
+    pub datadir: Option<PathBuf>,
+}
+impl DiagnosticContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+}
+
+pub struct DiagnosticContextSeed {
+    pub datadir: PathBuf,
+    pub shutdown: Sender<Option<Shutdown>>,
+    pub error: Arc<RpcError>,
+    pub disk_guid: Option<Arc<String>>,
+}
+
+#[derive(Clone)]
+pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
+impl DiagnosticContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(
+        path: Option<P>,
+        disk_guid: Option<Arc<String>>,
+        error: Error,
+    ) -> Result<Self, Error> {
+        tracing::error!("Error: {}: Starting diagnostic UI", error);
+        tracing::debug!("{:?}", error);
+
+        let cfg = DiagnosticContextConfig::load(path).await?;
+
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+
+        Ok(Self(Arc::new(DiagnosticContextSeed {
+            datadir: cfg.datadir().to_owned(),
+            shutdown,
+            disk_guid,
+            error: Arc::new(error.into()),
+        })))
+    }
+}
+
+impl Context for DiagnosticContext {}
+impl Deref for DiagnosticContext {
+    type Target = DiagnosticContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

core/startos/src/context/install.rs

@@ -0,0 +1,58 @@
+use std::ops::Deref;
+use std::path::Path;
+use std::sync::Arc;
+
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tokio::sync::broadcast::Sender;
+use tracing::instrument;
+
+use crate::net::utils::find_eth_iface;
+use crate::util::config::load_config_from_paths;
+use crate::Error;
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct InstallContextConfig {}
+impl InstallContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+}
+
+pub struct InstallContextSeed {
+    pub ethernet_interface: String,
+    pub shutdown: Sender<()>,
+}
+
+#[derive(Clone)]
+pub struct InstallContext(Arc<InstallContextSeed>);
+impl InstallContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let _cfg = InstallContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        Ok(Self(Arc::new(InstallContextSeed {
+            ethernet_interface: find_eth_iface().await?,
+            shutdown,
+        })))
+    }
+}
+
+impl Context for InstallContext {}
+impl Deref for InstallContext {
+    type Target = InstallContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}

core/startos/src/context/mod.rs

@@ -0,0 +1,44 @@
+pub mod cli;
+pub mod diagnostic;
+pub mod install;
+pub mod rpc;
+pub mod sdk;
+pub mod setup;
+
+pub use cli::CliContext;
+pub use diagnostic::DiagnosticContext;
+pub use install::InstallContext;
+pub use rpc::RpcContext;
+pub use sdk::SdkContext;
+pub use setup::SetupContext;
+
+impl From<CliContext> for () {
+    fn from(_: CliContext) -> Self {
+        ()
+    }
+}
+impl From<DiagnosticContext> for () {
+    fn from(_: DiagnosticContext) -> Self {
+        ()
+    }
+}
+impl From<RpcContext> for () {
+    fn from(_: RpcContext) -> Self {
+        ()
+    }
+}
+impl From<SdkContext> for () {
+    fn from(_: SdkContext) -> Self {
+        ()
+    }
+}
+impl From<SetupContext> for () {
+    fn from(_: SetupContext) -> Self {
+        ()
+    }
+}
+impl From<InstallContext> for () {
+    fn from(_: InstallContext) -> Self {
+        ()
+    }
+}

core/startos/src/context/rpc.rs

@@ -0,0 +1,466 @@
+use std::collections::BTreeMap;
+use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use helpers::to_tmp_path;
+use josekit::jwk::Jwk;
+use patch_db::json_ptr::JsonPointer;
+use patch_db::PatchDb;
+use reqwest::{Client, Proxy, Url};
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
+use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
+use tokio::time::Instant;
+use tracing::instrument;
+
+use super::setup::CURRENT_SECRET;
+use crate::account::AccountInfo;
+use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
+use crate::db::model::{CurrentDependents, Database, PackageDataEntryMatchModelRef};
+use crate::db::prelude::PatchDbExt;
+use crate::dependencies::compute_dependency_config_errs;
+use crate::disk::OsPartitionInfo;
+use crate::init::init_postgres;
+use crate::install::cleanup::{cleanup_failed, uninstall};
+use crate::manager::ManagerMap;
+use crate::middleware::auth::HashSessionToken;
+use crate::net::net_controller::NetController;
+use crate::net::ssl::{root_ca_start_time, SslManager};
+use crate::net::wifi::WpaCli;
+use crate::notifications::NotificationManager;
+use crate::shutdown::Shutdown;
+use crate::status::MainStatus;
+use crate::system::get_mem_info;
+use crate::util::config::load_config_from_paths;
+use crate::util::lshw::{lshw, LshwDevice};
+use crate::{Error, ErrorKind, ResultExt};
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct RpcContextConfig {
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
+    pub bind_rpc: Option<SocketAddr>,
+    pub tor_control: Option<SocketAddr>,
+    pub tor_socks: Option<SocketAddr>,
+    pub dns_bind: Option<Vec<SocketAddr>>,
+    pub revision_cache_size: Option<usize>,
+    pub datadir: Option<PathBuf>,
+    pub log_server: Option<Url>,
+}
+impl RpcContextConfig {
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
+        let db_path = self.datadir().join("main").join("embassy.db");
+        let db = PatchDb::open(&db_path)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
+        if !db.exists(&<JsonPointer>::default()).await {
+            db.put(&<JsonPointer>::default(), &Database::init(account))
+                .await?;
+        }
+        Ok(db)
+    }
+    #[instrument(skip_all)]
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(self.datadir()).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
+        sqlx::migrate!()
+            .run(&secret_store)
+            .await
+            .with_kind(crate::ErrorKind::Database)?;
+        Ok(secret_store)
+    }
+}
+
+pub struct RpcContextSeed {
+    is_closed: AtomicBool,
+    pub os_partitions: OsPartitionInfo,
+    pub wifi_interface: Option<String>,
+    pub ethernet_interface: String,
+    pub datadir: PathBuf,
+    pub disk_guid: Arc<String>,
+    pub db: PatchDb,
+    pub secret_store: PgPool,
+    pub account: RwLock<AccountInfo>,
+    pub net_controller: Arc<NetController>,
+    pub managers: ManagerMap,
+    pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
+    pub shutdown: broadcast::Sender<Option<Shutdown>>,
+    pub tor_socks: SocketAddr,
+    pub notification_manager: NotificationManager,
+    pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
+    pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
+    pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
+    pub current_secret: Arc<Jwk>,
+    pub client: Client,
+    pub hardware: Hardware,
+    pub start_time: Instant,
+}
+
+pub struct Hardware {
+    pub devices: Vec<LshwDevice>,
+    pub ram: u64,
+}
+
+#[derive(Clone)]
+pub struct RpcContext(Arc<RpcContextSeed>);
+impl RpcContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + Sync + 'static>(
+        cfg_path: Option<P>,
+        disk_guid: Arc<String>,
+    ) -> Result<Self, Error> {
+        let base = RpcContextConfig::load(cfg_path).await?;
+        tracing::info!("Loaded Config");
+        let tor_proxy = base.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
+            Ipv4Addr::new(127, 0, 0, 1),
+            9050,
+        )));
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let secret_store = base.secret_store().await?;
+        tracing::info!("Opened Pg DB");
+        let account = AccountInfo::load(&secret_store).await?;
+        let db = base.db(&account).await?;
+        tracing::info!("Opened PatchDB");
+        let net_controller = Arc::new(
+            NetController::init(
+                base.tor_control
+                    .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
+                tor_proxy,
+                base.dns_bind
+                    .as_deref()
+                    .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
+                SslManager::new(&account, root_ca_start_time().await?)?,
+                &account.hostname,
+                &account.key,
+            )
+            .await?,
+        );
+        tracing::info!("Initialized Net Controller");
+        let managers = ManagerMap::default();
+        let metrics_cache = RwLock::<Option<crate::system::Metrics>>::new(None);
+        let notification_manager = NotificationManager::new(secret_store.clone());
+        tracing::info!("Initialized Notification Manager");
+        let tor_proxy_url = format!("socks5h://{tor_proxy}");
+        let devices = lshw().await?;
+        let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
+        let seed = Arc::new(RpcContextSeed {
+            is_closed: AtomicBool::new(false),
+            datadir: base.datadir().to_path_buf(),
+            os_partitions: base.os_partitions,
+            wifi_interface: base.wifi_interface.clone(),
+            ethernet_interface: base.ethernet_interface,
+            disk_guid,
+            db,
+            secret_store,
+            account: RwLock::new(account),
+            net_controller,
+            managers,
+            metrics_cache,
+            shutdown,
+            tor_socks: tor_proxy,
+            notification_manager,
+            open_authed_websockets: Mutex::new(BTreeMap::new()),
+            rpc_stream_continuations: Mutex::new(BTreeMap::new()),
+            wifi_manager: base
+                .wifi_interface
+                .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
+            current_secret: Arc::new(
+                Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).map_err(|e| {
+                    tracing::debug!("{:?}", e);
+                    tracing::error!("Couldn't generate ec key");
+                    Error::new(
+                        color_eyre::eyre::eyre!("Couldn't generate ec key"),
+                        crate::ErrorKind::Unknown,
+                    )
+                })?,
+            ),
+            client: Client::builder()
+                .proxy(Proxy::custom(move |url| {
+                    if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
+                        Some(tor_proxy_url.clone())
+                    } else {
+                        None
+                    }
+                }))
+                .build()
+                .with_kind(crate::ErrorKind::ParseUrl)?,
+            hardware: Hardware { devices, ram },
+            start_time: Instant::now(),
+        });
+
+        let res = Self(seed.clone());
+        res.cleanup_and_initialize().await?;
+        tracing::info!("Cleaned up transient states");
+        Ok(res)
+    }
+
+    #[instrument(skip_all)]
+    pub async fn shutdown(self) -> Result<(), Error> {
+        self.managers.empty().await?;
+        self.secret_store.close().await;
+        self.is_closed.store(true, Ordering::SeqCst);
+        tracing::info!("RPC Context is shutdown");
+        // TODO: shutdown http servers
+        Ok(())
+    }
+
+    #[instrument(skip(self))]
+    pub async fn cleanup_and_initialize(&self) -> Result<(), Error> {
+        self.db
+            .mutate(|f| {
+                let mut current_dependents = f
+                    .as_package_data()
+                    .keys()?
+                    .into_iter()
+                    .map(|k| (k.clone(), BTreeMap::new()))
+                    .collect::<BTreeMap<_, _>>();
+                for (package_id, package) in f.as_package_data_mut().as_entries_mut()? {
+                    for (k, v) in package
+                        .as_installed_mut()
+                        .into_iter()
+                        .flat_map(|i| i.clone().into_current_dependencies().into_entries())
+                        .flatten()
+                    {
+                        let mut entry: BTreeMap<_, _> =
+                            current_dependents.remove(&k).unwrap_or_default();
+                        entry.insert(package_id.clone(), v.de()?);
+                        current_dependents.insert(k, entry);
+                    }
+                }
+                for (package_id, current_dependents) in current_dependents {
+                    if let Some(deps) = f
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.expect_as_installed_mut().ok())
+                        .map(|i| i.as_installed_mut().as_current_dependents_mut())
+                    {
+                        deps.ser(&CurrentDependents(current_dependents))?;
+                    } else if let Some(deps) = f
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.expect_as_removing_mut().ok())
+                        .map(|i| i.as_removing_mut().as_current_dependents_mut())
+                    {
+                        deps.ser(&CurrentDependents(current_dependents))?;
+                    }
+                }
+                Ok(())
+            })
+            .await?;
+
+        let peek = self.db.peek().await;
+
+        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
+            let action = match package.as_match() {
+                PackageDataEntryMatchModelRef::Installing(_)
+                | PackageDataEntryMatchModelRef::Restoring(_)
+                | PackageDataEntryMatchModelRef::Updating(_) => {
+                    cleanup_failed(self, &package_id).await
+                }
+                PackageDataEntryMatchModelRef::Removing(_) => {
+                    uninstall(
+                        self,
+                        self.secret_store.acquire().await?.as_mut(),
+                        &package_id,
+                    )
+                    .await
+                }
+                PackageDataEntryMatchModelRef::Installed(m) => {
+                    let version = m.as_manifest().as_version().clone().de()?;
+                    let volumes = m.as_manifest().as_volumes().de()?;
+                    for (volume_id, volume_info) in &*volumes {
+                        let tmp_path = to_tmp_path(volume_info.path_for(
+                            &self.datadir,
+                            &package_id,
+                            &version,
+                            volume_id,
+                        ))
+                        .with_kind(ErrorKind::Filesystem)?;
+                        if tokio::fs::metadata(&tmp_path).await.is_ok() {
+                            tokio::fs::remove_dir_all(&tmp_path).await?;
+                        }
+                    }
+                    Ok(())
+                }
+                _ => continue,
+            };
+            if let Err(e) = action {
+                tracing::error!("Failed to clean up package {}: {}", package_id, e);
+                tracing::debug!("{:?}", e);
+            }
+        }
+        let peek = self
+            .db
+            .mutate(|v| {
+                for (_, pde) in v.as_package_data_mut().as_entries_mut()? {
+                    let status = pde
+                        .expect_as_installed_mut()?
+                        .as_installed_mut()
+                        .as_status_mut()
+                        .as_main_mut();
+                    let running = status.clone().de()?.running();
+                    status.ser(&if running {
+                        MainStatus::Starting
+                    } else {
+                        MainStatus::Stopped
+                    })?;
+                }
+                Ok(v.clone())
+            })
+            .await?;
+        self.managers.init(self.clone(), peek.clone()).await?;
+        tracing::info!("Initialized Package Managers");
+
+        let mut all_dependency_config_errs = BTreeMap::new();
+        for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
+            let package = package.clone();
+            if let Some(current_dependencies) = package
+                .as_installed()
+                .and_then(|x| x.as_current_dependencies().de().ok())
+            {
+                let manifest = package.as_manifest().de()?;
+                all_dependency_config_errs.insert(
+                    package_id.clone(),
+                    compute_dependency_config_errs(
+                        self,
+                        &peek,
+                        &manifest,
+                        &current_dependencies,
+                        &Default::default(),
+                    )
+                    .await?,
+                );
+            }
+        }
+        self.db
+            .mutate(|v| {
+                for (package_id, errs) in all_dependency_config_errs {
+                    if let Some(config_errors) = v
+                        .as_package_data_mut()
+                        .as_idx_mut(&package_id)
+                        .and_then(|pde| pde.as_installed_mut())
+                        .map(|i| i.as_status_mut().as_dependency_config_errors_mut())
+                    {
+                        config_errors.ser(&errs)?;
+                    }
+                }
+                Ok(())
+            })
+            .await?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn clean_continuations(&self) {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        let mut to_remove = Vec::new();
+        for (guid, cont) in &*continuations {
+            if cont.is_timed_out() {
+                to_remove.push(guid.clone());
+            }
+        }
+        for guid in to_remove {
+            continuations.remove(&guid);
+        }
+    }
+
+    #[instrument(skip_all)]
+    pub async fn add_continuation(&self, guid: RequestGuid, handler: RpcContinuation) {
+        self.clean_continuations().await;
+        self.rpc_stream_continuations
+            .lock()
+            .await
+            .insert(guid, handler);
+    }
+
+    pub async fn get_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let mut continuations = self.rpc_stream_continuations.lock().await;
+        if let Some(cont) = continuations.remove(guid) {
+            cont.into_handler().await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_ws_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
+    }
+
+    pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
+        let continuations = self.rpc_stream_continuations.lock().await;
+        if matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
+            drop(continuations);
+            self.get_continuation_handler(guid).await
+        } else {
+            None
+        }
+    }
+}
+impl AsRef<Jwk> for RpcContext {
+    fn as_ref(&self) -> &Jwk {
+        &CURRENT_SECRET
+    }
+}
+impl Context for RpcContext {}
+impl Deref for RpcContext {
+    type Target = RpcContextSeed;
+    fn deref(&self) -> &Self::Target {
+        #[cfg(feature = "unstable")]
+        if self.0.is_closed.load(Ordering::SeqCst) {
+            panic!(
+                "RpcContext used after shutdown! {}",
+                tracing_error::SpanTrace::capture()
+            );
+        }
+        &self.0
+    }
+}
+impl Drop for RpcContext {
+    fn drop(&mut self) {
+        #[cfg(feature = "unstable")]
+        if self.0.is_closed.load(Ordering::SeqCst) {
+            tracing::info!(
+                "RpcContext dropped. {} left.",
+                Arc::strong_count(&self.0) - 1
+            );
+        }
+    }
+}

core/startos/src/context/sdk.rs

@@ -0,0 +1,76 @@
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use rpc_toolkit::Context;
+use serde::Deserialize;
+use tracing::instrument;
+
+use crate::prelude::*;
+use crate::util::config::{load_config_from_paths, local_config_path};
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SdkContextConfig {
+    pub developer_key_path: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct SdkContextSeed {
+    pub developer_key_path: PathBuf,
+}
+
+#[derive(Debug, Clone)]
+pub struct SdkContext(Arc<SdkContextSeed>);
+impl SdkContext {
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
+        let local_config_path = local_config_path();
+        let base: SdkContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
+        Ok(SdkContext(Arc::new(SdkContextSeed {
+            developer_key_path: base.developer_key_path.unwrap_or_else(|| {
+                local_config_path
+                    .as_deref()
+                    .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
+                    .parent()
+                    .unwrap_or(Path::new("/"))
+                    .join("developer.key.pem")
+            }),
+        })))
+    }
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn developer_key(&self) -> Result<ed25519_dalek::SigningKey, Error> {
+        if !self.developer_key_path.exists() {
+            return Err(Error::new(eyre!("Developer Key does not exist! Please run `start-sdk init` before running this command."), crate::ErrorKind::Uninitialized));
+        }
+        let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
+            &std::fs::read_to_string(&self.developer_key_path)?,
+        )
+        .with_kind(crate::ErrorKind::Pem)?;
+        let secret = ed25519_dalek::SecretKey::try_from(&pair.secret_key[..]).map_err(|_| {
+            Error::new(
+                eyre!("pkcs8 key is of incorrect length"),
+                ErrorKind::OpenSsl,
+            )
+        })?;
+        Ok(secret.into())
+    }
+}
+impl std::ops::Deref for SdkContext {
+    type Target = SdkContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl Context for SdkContext {}

core/startos/src/context/setup.rs

@@ -0,0 +1,149 @@
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use josekit::jwk::Jwk;
+use patch_db::json_ptr::JsonPointer;
+use patch_db::PatchDb;
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::Context;
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
+use tokio::sync::broadcast::Sender;
+use tokio::sync::RwLock;
+use tracing::instrument;
+
+use crate::account::AccountInfo;
+use crate::db::model::Database;
+use crate::disk::OsPartitionInfo;
+use crate::init::init_postgres;
+use crate::setup::SetupStatus;
+use crate::util::config::load_config_from_paths;
+use crate::{Error, ResultExt};
+
+lazy_static::lazy_static! {
+    pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
+        tracing::debug!("{:?}", e);
+        tracing::error!("Couldn't generate ec key");
+        panic!("Couldn't generate ec key")
+    });
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetupResult {
+    pub tor_address: String,
+    pub lan_address: String,
+    pub root_ca: String,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetupContextConfig {
+    pub os_partitions: OsPartitionInfo,
+    pub migration_batch_rows: Option<usize>,
+    pub migration_prefetch_rows: Option<usize>,
+    pub datadir: Option<PathBuf>,
+    #[serde(default)]
+    pub disable_encryption: bool,
+}
+impl SetupContextConfig {
+    #[instrument(skip_all)]
+    pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        tokio::task::spawn_blocking(move || {
+            load_config_from_paths(
+                path.as_ref()
+                    .into_iter()
+                    .map(|p| p.as_ref())
+                    .chain(std::iter::once(Path::new(
+                        crate::util::config::DEVICE_CONFIG_PATH,
+                    )))
+                    .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+            )
+        })
+        .await
+        .unwrap()
+    }
+    pub fn datadir(&self) -> &Path {
+        self.datadir
+            .as_deref()
+            .unwrap_or_else(|| Path::new("/embassy-data"))
+    }
+}
+
+pub struct SetupContextSeed {
+    pub os_partitions: OsPartitionInfo,
+    pub config_path: Option<PathBuf>,
+    pub migration_batch_rows: usize,
+    pub migration_prefetch_rows: usize,
+    pub disable_encryption: bool,
+    pub shutdown: Sender<()>,
+    pub datadir: PathBuf,
+    pub selected_v2_drive: RwLock<Option<PathBuf>>,
+    pub cached_product_key: RwLock<Option<Arc<String>>>,
+    pub setup_status: RwLock<Option<Result<SetupStatus, RpcError>>>,
+    pub setup_result: RwLock<Option<(Arc<String>, SetupResult)>>,
+}
+
+impl AsRef<Jwk> for SetupContextSeed {
+    fn as_ref(&self) -> &Jwk {
+        &*CURRENT_SECRET
+    }
+}
+
+#[derive(Clone)]
+pub struct SetupContext(Arc<SetupContextSeed>);
+impl SetupContext {
+    #[instrument(skip_all)]
+    pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
+        let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
+        let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let datadir = cfg.datadir().to_owned();
+        Ok(Self(Arc::new(SetupContextSeed {
+            os_partitions: cfg.os_partitions,
+            config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
+            migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
+            migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
+            disable_encryption: cfg.disable_encryption,
+            shutdown,
+            datadir,
+            selected_v2_drive: RwLock::new(None),
+            cached_product_key: RwLock::new(None),
+            setup_status: RwLock::new(None),
+            setup_result: RwLock::new(None),
+        })))
+    }
+    #[instrument(skip_all)]
+    pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
+        let db_path = self.datadir.join("main").join("embassy.db");
+        let db = PatchDb::open(&db_path)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
+        if !db.exists(&<JsonPointer>::default()).await {
+            db.put(&<JsonPointer>::default(), &Database::init(account))
+                .await?;
+        }
+        Ok(db)
+    }
+    #[instrument(skip_all)]
+    pub async fn secret_store(&self) -> Result<PgPool, Error> {
+        init_postgres(&self.datadir).await?;
+        let secret_store =
+            PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
+                .await?;
+        sqlx::migrate!()
+            .run(&secret_store)
+            .await
+            .with_kind(crate::ErrorKind::Database)?;
+        Ok(secret_store)
+    }
+}
+
+impl Context for SetupContext {}
+impl Deref for SetupContext {
+    type Target = SetupContextSeed;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}