Feature/new registry (#2612)

* wip

* overhaul boot process

* wip: new registry

* wip

* wip

* wip

* wip

* wip

* wip

* os registry complete

* ui fixes

* fixes

* fixes

* more fixes

* fix merkle archive

core/startos/src/context/cli.rs

@@ -10,7 +10,7 @@ use reqwest::Proxy;
 use reqwest_cookie_store::CookieStoreMutex;
 use rpc_toolkit::reqwest::{Client, Url};
 use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{call_remote_http, CallRemote, Context};
+use rpc_toolkit::{call_remote_http, CallRemote, Context, Empty};
 use tokio::net::TcpStream;
 use tokio::runtime::Runtime;
 use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
@@ -18,15 +18,17 @@ use tracing::instrument;
 
 use super::setup::CURRENT_SECRET;
 use crate::context::config::{local_config_path, ClientConfig};
-use crate::core::rpc_continuations::RequestGuid;
+use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext};
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
 use crate::prelude::*;
+use crate::rpc_continuations::RequestGuid;
 
 #[derive(Debug)]
 pub struct CliContextSeed {
     pub runtime: OnceCell<Runtime>,
     pub base_url: Url,
     pub rpc_url: Url,
+    pub registry_url: Option<Url>,
     pub client: Client,
     pub cookie_store: Arc<CookieStoreMutex>,
     pub cookie_path: PathBuf,
@@ -66,6 +68,8 @@ impl CliContext {
             "http://localhost".parse()?
         };
 
+        let registry = config.registry.clone();
+
         let cookie_path = config.cookie_path.unwrap_or_else(|| {
             local_config_path()
                 .as_deref()
@@ -104,6 +108,17 @@ impl CliContext {
                     .push("v1");
                 url
             },
+            registry_url: registry
+                .map(|mut registry| {
+                    registry
+                        .path_segments_mut()
+                        .map_err(|_| eyre!("Url cannot be base"))
+                        .with_kind(crate::ErrorKind::ParseUrl)?
+                        .push("rpc")
+                        .push("v0");
+                    Ok::<_, Error>(registry)
+                })
+                .transpose()?,
             client: {
                 let mut builder = Client::builder().cookie_provider(cookie_store.clone());
                 if let Some(proxy) = config.proxy {
@@ -198,6 +213,29 @@ impl CliContext {
             .await
             .with_kind(ErrorKind::Network)
     }
+
+    pub async fn call_remote<RemoteContext>(
+        &self,
+        method: &str,
+        params: Value,
+    ) -> Result<Value, RpcError>
+    where
+        Self: CallRemote<RemoteContext>,
+    {
+        <Self as CallRemote<RemoteContext, Empty>>::call_remote(&self, method, params, Empty {})
+            .await
+    }
+    pub async fn call_remote_with<RemoteContext, T>(
+        &self,
+        method: &str,
+        params: Value,
+        extra: T,
+    ) -> Result<Value, RpcError>
+    where
+        Self: CallRemote<RemoteContext, T>,
+    {
+        <Self as CallRemote<RemoteContext, T>>::call_remote(&self, method, params, extra).await
+    }
 }
 impl AsRef<Jwk> for CliContext {
     fn as_ref(&self) -> &Jwk {
@@ -223,9 +261,23 @@ impl Context for CliContext {
             .clone()
     }
 }
-#[async_trait::async_trait]
-impl CallRemote for CliContext {
-    async fn call_remote(&self, method: &str, params: Value) -> Result<Value, RpcError> {
+impl CallRemote<RpcContext> for CliContext {
+    async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
+        call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
+    }
+}
+impl CallRemote<DiagnosticContext> for CliContext {
+    async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
+        call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
+    }
+}
+impl CallRemote<SetupContext> for CliContext {
+    async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
+        call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
+    }
+}
+impl CallRemote<InstallContext> for CliContext {
+    async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
         call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
     }
 }

core/startos/src/context/config.rs

@@ -14,7 +14,7 @@ use crate::init::init_postgres;
 use crate::prelude::*;
 use crate::util::serde::IoFormat;
 
-pub const DEVICE_CONFIG_PATH: &str = "/media/embassy/config/config.yaml"; // "/media/startos/config/config.yaml";
+pub const DEVICE_CONFIG_PATH: &str = "/media/startos/config/config.yaml"; // "/media/startos/config/config.yaml";
 pub const CONFIG_PATH: &str = "/etc/startos/config.yaml";
 pub const CONFIG_PATH_LOCAL: &str = ".startos/config.yaml";
 
@@ -58,6 +58,8 @@ pub struct ClientConfig {
     pub config: Option<PathBuf>,
     #[arg(short = 'h', long = "host")]
     pub host: Option<Url>,
+    #[arg(short = 'r', long = "registry")]
+    pub registry: Option<Url>,
     #[arg(short = 'p', long = "proxy")]
     pub proxy: Option<Url>,
     #[arg(long = "cookie-path")]
@@ -71,8 +73,10 @@ impl ContextConfig for ClientConfig {
     }
     fn merge_with(&mut self, other: Self) {
         self.host = self.host.take().or(other.host);
+        self.registry = self.registry.take().or(other.registry);
         self.proxy = self.proxy.take().or(other.proxy);
         self.cookie_path = self.cookie_path.take().or(other.cookie_path);
+        self.developer_key_path = self.developer_key_path.take().or(other.developer_key_path);
     }
 }
 impl ClientConfig {

core/startos/src/context/diagnostic.rs

@@ -8,6 +8,7 @@ use tokio::sync::broadcast::Sender;
 use tracing::instrument;
 
 use crate::context::config::ServerConfig;
+use crate::rpc_continuations::RpcContinuations;
 use crate::shutdown::Shutdown;
 use crate::Error;
 
@@ -16,6 +17,7 @@ pub struct DiagnosticContextSeed {
     pub shutdown: Sender<Option<Shutdown>>,
     pub error: Arc<RpcError>,
     pub disk_guid: Option<Arc<String>>,
+    pub rpc_continuations: RpcContinuations,
 }
 
 #[derive(Clone)]
@@ -37,10 +39,15 @@ impl DiagnosticContext {
             shutdown,
             disk_guid,
             error: Arc::new(error.into()),
+            rpc_continuations: RpcContinuations::new(),
         })))
     }
 }
-
+impl AsRef<RpcContinuations> for DiagnosticContext {
+    fn as_ref(&self) -> &RpcContinuations {
+        &self.rpc_continuations
+    }
+}
 impl Context for DiagnosticContext {}
 impl Deref for DiagnosticContext {
     type Target = DiagnosticContextSeed;

core/startos/src/context/rpc.rs

@@ -6,33 +6,32 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
-use imbl_value::InternedString;
 use josekit::jwk::Jwk;
-use patch_db::PatchDb;
 use reqwest::{Client, Proxy};
-use rpc_toolkit::Context;
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::{CallRemote, Context, Empty};
 use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
 use tokio::time::Instant;
 use tracing::instrument;
 
 use super::setup::CURRENT_SECRET;
+use crate::account::AccountInfo;
 use crate::context::config::ServerConfig;
-use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation, WebSocketHandler};
-use crate::db::prelude::PatchDbExt;
+use crate::db::model::Database;
 use crate::dependencies::compute_dependency_config_errs;
 use crate::disk::OsPartitionInfo;
 use crate::init::check_time_is_synchronized;
-use crate::lxc::{LxcContainer, LxcManager};
+use crate::lxc::{ContainerId, LxcContainer, LxcManager};
 use crate::middleware::auth::HashSessionToken;
 use crate::net::net_controller::NetController;
 use crate::net::utils::{find_eth_iface, find_wifi_iface};
 use crate::net::wifi::WpaCli;
 use crate::prelude::*;
+use crate::rpc_continuations::RpcContinuations;
 use crate::service::ServiceMap;
 use crate::shutdown::Shutdown;
 use crate::system::get_mem_info;
 use crate::util::lshw::{lshw, LshwDevice};
-use crate::{account::AccountInfo, lxc::ContainerId};
 
 pub struct RpcContextSeed {
     is_closed: AtomicBool,
@@ -41,7 +40,7 @@ pub struct RpcContextSeed {
     pub ethernet_interface: String,
     pub datadir: PathBuf,
     pub disk_guid: Arc<String>,
-    pub db: PatchDb,
+    pub db: TypedPatchDb<Database>,
     pub account: RwLock<AccountInfo>,
     pub net_controller: Arc<NetController>,
     pub services: ServiceMap,
@@ -50,7 +49,7 @@ pub struct RpcContextSeed {
     pub tor_socks: SocketAddr,
     pub lxc_manager: Arc<LxcManager>,
     pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
-    pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
+    pub rpc_continuations: RpcContinuations,
     pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
     pub current_secret: Arc<Jwk>,
     pub client: Client,
@@ -80,7 +79,7 @@ impl RpcContext {
         )));
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
 
-        let db = config.db().await?;
+        let db = TypedPatchDb::<Database>::load(config.db().await?).await?;
         let peek = db.peek().await;
         let account = AccountInfo::load(&peek)?;
         tracing::info!("Opened PatchDB");
@@ -159,7 +158,7 @@ impl RpcContext {
             tor_socks: tor_proxy,
             lxc_manager: Arc::new(LxcManager::new()),
             open_authed_websockets: Mutex::new(BTreeMap::new()),
-            rpc_stream_continuations: Mutex::new(BTreeMap::new()),
+            rpc_continuations: RpcContinuations::new(),
             wifi_manager: wifi_interface
                 .clone()
                 .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
@@ -236,54 +235,27 @@ impl RpcContext {
 
         Ok(())
     }
-
-    #[instrument(skip_all)]
-    pub async fn clean_continuations(&self) {
-        let mut continuations = self.rpc_stream_continuations.lock().await;
-        let mut to_remove = Vec::new();
-        for (guid, cont) in &*continuations {
-            if cont.is_timed_out() {
-                to_remove.push(guid.clone());
-            }
-        }
-        for guid in to_remove {
-            continuations.remove(&guid);
-        }
-    }
-
-    #[instrument(skip_all)]
-    pub async fn add_continuation(&self, guid: RequestGuid, handler: RpcContinuation) {
-        self.clean_continuations().await;
-        self.rpc_stream_continuations
-            .lock()
-            .await
-            .insert(guid, handler);
-    }
-
-    pub async fn get_ws_continuation_handler(
+    pub async fn call_remote<RemoteContext>(
         &self,
-        guid: &RequestGuid,
-    ) -> Option<WebSocketHandler> {
-        let mut continuations = self.rpc_stream_continuations.lock().await;
-        if !matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
-            return None;
-        }
-        let Some(RpcContinuation::WebSocket(x)) = continuations.remove(guid) else {
-            return None;
-        };
-        x.get().await
+        method: &str,
+        params: Value,
+    ) -> Result<Value, RpcError>
+    where
+        Self: CallRemote<RemoteContext>,
+    {
+        <Self as CallRemote<RemoteContext, Empty>>::call_remote(&self, method, params, Empty {})
+            .await
     }
-
-    pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
-        let mut continuations: tokio::sync::MutexGuard<'_, BTreeMap<RequestGuid, RpcContinuation>> =
-            self.rpc_stream_continuations.lock().await;
-        if !matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
-            return None;
-        }
-        let Some(RpcContinuation::Rest(x)) = continuations.remove(guid) else {
-            return None;
-        };
-        x.get().await
+    pub async fn call_remote_with<RemoteContext, T>(
+        &self,
+        method: &str,
+        params: Value,
+        extra: T,
+    ) -> Result<Value, RpcError>
+    where
+        Self: CallRemote<RemoteContext, T>,
+    {
+        <Self as CallRemote<RemoteContext, T>>::call_remote(&self, method, params, extra).await
     }
 }
 impl AsRef<Jwk> for RpcContext {
@@ -291,6 +263,11 @@ impl AsRef<Jwk> for RpcContext {
         &CURRENT_SECRET
     }
 }
+impl AsRef<RpcContinuations> for RpcContext {
+    fn as_ref(&self) -> &RpcContinuations {
+        &self.rpc_continuations
+    }
+}
 impl Context for RpcContext {}
 impl Deref for RpcContext {
     type Target = RpcContextSeed;