Feature/new registry (#2612)

* wip

* overhaul boot process

* wip: new registry

* wip

* wip

* wip

* wip

* wip

* wip

* os registry complete

* ui fixes

* fixes

* fixes

* more fixes

* fix merkle archive

core/startos/src/db/mod.rs

@@ -11,10 +11,11 @@ use clap::Parser;
 use futures::{FutureExt, StreamExt};
 use http::header::COOKIE;
 use http::HeaderMap;
+use itertools::Itertools;
 use patch_db::json_ptr::{JsonPointer, ROOT};
 use patch_db::{Dump, Revision};
 use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{command, from_fn_async, CallRemote, HandlerExt, ParentHandler};
+use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use tokio::sync::oneshot;
@@ -167,11 +168,11 @@ pub async fn subscribe(
     }))
 }
 
-pub fn db() -> ParentHandler {
+pub fn db<C: Context>() -> ParentHandler<C> {
     ParentHandler::new()
         .subcommand("dump", from_fn_async(cli_dump).with_display_serializable())
         .subcommand("dump", from_fn_async(dump).no_cli())
-        .subcommand("put", put())
+        .subcommand("put", put::<C>())
         .subcommand("apply", from_fn_async(cli_apply).no_display())
         .subcommand("apply", from_fn_async(apply).no_cli())
 }
@@ -195,21 +196,28 @@ pub struct CliDumpParams {
 
 #[instrument(skip_all)]
 async fn cli_dump(
-    ctx: CliContext,
-    CliDumpParams {
-        path,
-        include_private,
-    }: CliDumpParams,
+    HandlerArgs {
+        context,
+        parent_method,
+        method,
+        params: CliDumpParams {
+            include_private,
+            path,
+        },
+        ..
+    }: HandlerArgs<CliContext, CliDumpParams>,
 ) -> Result<Dump, RpcError> {
     let dump = if let Some(path) = path {
         PatchDb::open(path).await?.dump(&ROOT).await
     } else {
+        let method = parent_method.into_iter().chain(method).join(".");
         from_value::<Dump>(
-            ctx.call_remote(
-                "db.dump",
-                imbl_value::json!({ "includePrivate":include_private }),
-            )
-            .await?,
+            context
+                .call_remote::<RpcContext>(
+                    &method,
+                    imbl_value::json!({ "includePrivate":include_private }),
+                )
+                .await?,
         )?
     };
 
@@ -237,36 +245,54 @@ pub async fn dump(
     })
 }
 
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "camelCase")]
+#[command(rename_all = "kebab-case")]
+pub struct CliApplyParams {
+    expr: String,
+    path: Option<PathBuf>,
+}
+
 #[instrument(skip_all)]
 async fn cli_apply(
-    ctx: CliContext,
-    ApplyParams { expr, path }: ApplyParams,
+    HandlerArgs {
+        context,
+        parent_method,
+        method,
+        params: CliApplyParams { expr, path },
+        ..
+    }: HandlerArgs<CliContext, CliApplyParams>,
 ) -> Result<(), RpcError> {
     if let Some(path) = path {
         PatchDb::open(path)
             .await?
-            .mutate(|db| {
+            .apply_function(|db| {
                 let res = apply_expr(
-                    serde_json::to_value(patch_db::Value::from(db.clone()))
+                    serde_json::to_value(patch_db::Value::from(db))
                         .with_kind(ErrorKind::Deserialization)?
                         .into(),
                     &expr,
                 )?;
 
-                db.ser(
-                    &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(
-                        |_| {
-                            (
-                                crate::ErrorKind::Deserialization,
-                                "result does not match database model",
-                            )
-                        },
+                Ok::<_, Error>((
+                    to_value(
+                        &serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(
+                            |_| {
+                                (
+                                    crate::ErrorKind::Deserialization,
+                                    "result does not match database model",
+                                )
+                            },
+                        )?,
                     )?,
-                )
+                    (),
+                ))
             })
             .await?;
     } else {
-        ctx.call_remote("db.apply", imbl_value::json!({ "expr": expr }))
+        let method = parent_method.into_iter().chain(method).join(".");
+        context
+            .call_remote::<RpcContext>(&method, imbl_value::json!({ "expr": expr }))
             .await?;
     }
 
@@ -278,10 +304,9 @@ async fn cli_apply(
 #[command(rename_all = "kebab-case")]
 pub struct ApplyParams {
     expr: String,
-    path: Option<PathBuf>,
 }
 
-pub async fn apply(ctx: RpcContext, ApplyParams { expr, .. }: ApplyParams) -> Result<(), Error> {
+pub async fn apply(ctx: RpcContext, ApplyParams { expr }: ApplyParams) -> Result<(), Error> {
     ctx.db
         .mutate(|db| {
             let res = apply_expr(
@@ -303,12 +328,12 @@ pub async fn apply(ctx: RpcContext, ApplyParams { expr, .. }: ApplyParams) -> Re
         .await
 }
 
-pub fn put() -> ParentHandler {
+pub fn put<C: Context>() -> ParentHandler<C> {
     ParentHandler::new().subcommand(
         "ui",
         from_fn_async(ui)
             .with_display_serializable()
-            .with_remote_cli::<CliContext>(),
+            .with_call_remote::<CliContext>(),
     )
 }
 #[derive(Deserialize, Serialize, Parser, TS)]

core/startos/src/db/model/package.rs

@@ -54,7 +54,7 @@ impl PackageState {
     pub fn expect_installed(&self) -> Result<&InstalledState, Error> {
         match self {
             Self::Installed(a) => Ok(a),
-            a => Err(Error::new(
+            _ => Err(Error::new(
                 eyre!(
                     "Package {} is not in installed state",
                     self.as_manifest(ManifestPreference::Old).id
@@ -161,7 +161,7 @@ impl Model<PackageState> {
     pub fn expect_installed(&self) -> Result<&Model<InstalledState>, Error> {
         match self.as_match() {
             PackageStateMatchModelRef::Installed(a) => Ok(a),
-            a => Err(Error::new(
+            _ => Err(Error::new(
                 eyre!(
                     "Package {} is not in installed state",
                     self.as_manifest(ManifestPreference::Old).as_id().de()?
@@ -251,7 +251,7 @@ impl Model<PackageState> {
             PackageStateMatchModelMut::Installed(s) | PackageStateMatchModelMut::Removing(s) => {
                 s.as_manifest_mut()
             }
-            PackageStateMatchModelMut::Error(s) => {
+            PackageStateMatchModelMut::Error(_) => {
                 return Err(Error::new(
                     eyre!("could not determine package state to get manifest"),
                     ErrorKind::Database,
@@ -325,7 +325,7 @@ pub struct PackageDataEntry {
     pub state_info: PackageState,
     pub status: Status,
     #[ts(type = "string | null")]
-    pub marketplace_url: Option<Url>,
+    pub registry: Option<Url>,
     #[ts(type = "string")]
     pub developer_key: Pem<ed25519_dalek::VerifyingKey>,
     pub icon: DataUrl<'static>,

core/startos/src/db/model/public.rs

@@ -19,6 +19,7 @@ use crate::account::AccountInfo;
 use crate::db::model::package::AllPackageData;
 use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
 use crate::prelude::*;
+use crate::progress::FullProgress;
 use crate::util::cpupower::Governor;
 use crate::util::Version;
 use crate::version::{Current, VersionT};
@@ -175,24 +176,13 @@ pub struct BackupProgress {
 pub struct ServerStatus {
     pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
     pub updated: bool,
-    pub update_progress: Option<UpdateProgress>,
+    pub update_progress: Option<FullProgress>,
     #[serde(default)]
     pub shutting_down: bool,
     #[serde(default)]
     pub restarting: bool,
 }
 
-#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
-#[serde(rename_all = "camelCase")]
-#[model = "Model<Self>"]
-#[ts(export)]
-pub struct UpdateProgress {
-    #[ts(type = "number | null")]
-    pub size: Option<u64>,
-    #[ts(type = "number")]
-    pub downloaded: u64,
-}
-
 #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
 #[serde(rename_all = "camelCase")]
 #[model = "Model<Self>"]

core/startos/src/db/prelude.rs

@@ -1,22 +1,16 @@
 use std::collections::BTreeMap;
-use std::future::Future;
 use std::marker::PhantomData;
-use std::panic::UnwindSafe;
 use std::str::FromStr;
 
 use chrono::{DateTime, Utc};
 pub use imbl_value::Value;
-use patch_db::json_ptr::ROOT;
 use patch_db::value::InternedString;
 pub use patch_db::{HasModel, PatchDb};
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 
-use crate::db::model::DatabaseModel;
 use crate::prelude::*;
 
-pub type Peeked = Model<super::model::Database>;
-
 pub fn to_value<T>(value: &T) -> Result<Value, Error>
 where
     T: Serialize,
@@ -31,45 +25,7 @@ where
     patch_db::value::from_value(value).with_kind(ErrorKind::Deserialization)
 }
 
-pub trait PatchDbExt {
-    fn peek(&self) -> impl Future<Output = DatabaseModel> + Send;
-    fn mutate<U: UnwindSafe + Send>(
-        &self,
-        f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
-    ) -> impl Future<Output = Result<U, Error>> + Send;
-    fn map_mutate(
-        &self,
-        f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
-    ) -> impl Future<Output = Result<DatabaseModel, Error>> + Send;
-}
-impl PatchDbExt for PatchDb {
-    async fn peek(&self) -> DatabaseModel {
-        DatabaseModel::from(self.dump(&ROOT).await.value)
-    }
-    async fn mutate<U: UnwindSafe + Send>(
-        &self,
-        f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
-    ) -> Result<U, Error> {
-        Ok(self
-            .apply_function(|mut v| {
-                let model = <&mut DatabaseModel>::from(&mut v);
-                let res = f(model)?;
-                Ok::<_, Error>((v, res))
-            })
-            .await?
-            .1)
-    }
-    async fn map_mutate(
-        &self,
-        f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
-    ) -> Result<DatabaseModel, Error> {
-        Ok(DatabaseModel::from(
-            self.apply_function(|v| f(DatabaseModel::from(v)).map(|a| (a.into(), ())))
-                .await?
-                .0,
-        ))
-    }
-}
+pub type TypedPatchDb<T> = patch_db::TypedPatchDb<T, Error>;
 
 /// &mut Model<T> <=> &mut Value
 #[repr(transparent)]
@@ -125,7 +81,7 @@ impl<T: Serialize + DeserializeOwned> Model<T> {
         Ok(res)
     }
     pub fn map_mutate(&mut self, f: impl FnOnce(T) -> Result<T, Error>) -> Result<T, Error> {
-        let mut orig = self.de()?;
+        let orig = self.de()?;
         let res = f(orig)?;
         self.ser(&res)?;
         Ok(res)
@@ -262,10 +218,9 @@ where
             .into()),
         }
     }
-    pub fn upsert<F, D>(&mut self, key: &T::Key, value: F) -> Result<&mut Model<T::Value>, Error>
+    pub fn upsert<F>(&mut self, key: &T::Key, value: F) -> Result<&mut Model<T::Value>, Error>
     where
-        F: FnOnce() -> D,
-        D: AsRef<T::Value>,
+        F: FnOnce() -> T::Value,
     {
         use serde::ser::Error;
         match &mut self.value {
@@ -278,7 +233,7 @@ where
                     s.as_ref().index_or_insert(v)
                 });
                 if !exists {
-                    res.ser(value().as_ref())?;
+                    res.ser(&value())?;
                 }
                 Ok(res)
             }
@@ -375,6 +330,18 @@ where
     }
 }
 impl<T: Map> Model<T> {
+    pub fn contains_key(&self, key: &T::Key) -> Result<bool, Error> {
+        use serde::de::Error;
+        let s = T::key_str(key)?;
+        match &self.value {
+            Value::Object(o) => Ok(o.contains_key(s.as_ref())),
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Deserialization,
+            }
+            .into()),
+        }
+    }
     pub fn into_idx(self, key: &T::Key) -> Option<Model<T::Value>> {
         use patch_db::ModelExt;
         let s = T::key_str(key).ok()?;