Feature/new registry (#2612)

* wip

* overhaul boot process

* wip: new registry

* wip

* wip

* wip

* wip

* wip

* wip

* os registry complete

* ui fixes

* fixes

* fixes

* more fixes

* fix merkle archive

core/startos/src/service/cli.rs

@@ -5,10 +5,11 @@ use clap::Parser;
 use imbl_value::Value;
 use once_cell::sync::OnceCell;
 use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{call_remote_socket, yajrc, CallRemote, Context};
+use rpc_toolkit::{call_remote_socket, yajrc, CallRemote, Context, Empty};
 use tokio::runtime::Runtime;
 
 use crate::lxc::HOST_RPC_SERVER_SOCKET;
+use crate::service::service_effect_handler::EffectContext;
 
 #[derive(Debug, Default, Parser)]
 pub struct ContainerClientConfig {
@@ -48,9 +49,8 @@ impl Context for ContainerCliContext {
     }
 }
 
-#[async_trait::async_trait]
-impl CallRemote for ContainerCliContext {
-    async fn call_remote(&self, method: &str, params: Value) -> Result<Value, RpcError> {
+impl CallRemote<EffectContext> for ContainerCliContext {
+    async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result<Value, RpcError> {
         call_remote_socket(
             tokio::net::UnixStream::connect(&self.0.socket)
                 .await

core/startos/src/service/mod.rs

@@ -7,14 +7,13 @@ use futures::future::BoxFuture;
 use imbl::OrdMap;
 use models::{HealthCheckId, PackageId, ProcedureName};
 use persistent_container::PersistentContainer;
-use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, Handler, HandlerArgs};
+use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor};
 use serde::{Deserialize, Serialize};
 use start_stop::StartStop;
 use tokio::sync::Notify;
 use ts_rs::TS;
 
 use crate::context::{CliContext, RpcContext};
-use crate::core::rpc_continuations::RequestGuid;
 use crate::db::model::package::{
     InstalledState, PackageDataEntry, PackageState, PackageStateMatchModelRef, UpdatingState,
 };
@@ -23,6 +22,7 @@ use crate::install::PKG_ARCHIVE_DIR;
 use crate::lxc::ContainerId;
 use crate::prelude::*;
 use crate::progress::{NamedProgress, Progress};
+use crate::rpc_continuations::RequestGuid;
 use crate::s9pk::S9pk;
 use crate::service::service_map::InstallProgressHandles;
 use crate::service::transition::TransitionKind;
@@ -510,11 +510,25 @@ pub async fn connect_rpc(
 }
 
 pub async fn connect_rpc_cli(
-    handle_args: HandlerArgs<CliContext, ConnectParams>,
+    HandlerArgs {
+        context,
+        parent_method,
+        method,
+        params,
+        inherited_params,
+        raw_params,
+    }: HandlerArgs<CliContext, ConnectParams>,
 ) -> Result<(), Error> {
-    let ctx = handle_args.context.clone();
-    let guid = CallRemoteHandler::<CliContext, _>::new(from_fn_async(connect_rpc))
-        .handle_async(handle_args)
+    let ctx = context.clone();
+    let guid = CallRemoteHandler::<CliContext, _, _>::new(from_fn_async(connect_rpc))
+        .handle_async(HandlerArgs {
+            context,
+            parent_method,
+            method,
+            params: rpc_toolkit::util::Flat(params, Empty {}),
+            inherited_params,
+            raw_params,
+        })
         .await?;
 
     crate::lxc::connect_cli(&ctx, guid).await

core/startos/src/service/persistent_container.rs

@@ -10,7 +10,7 @@ use imbl_value::InternedString;
 use models::{ProcedureName, VolumeId};
 use rpc_toolkit::{Empty, Server, ShutdownHandle};
 use serde::de::DeserializeOwned;
-use tokio::fs::{create_dir_all, File};
+use tokio::fs::{ File};
 use tokio::process::Command;
 use tokio::sync::{oneshot, watch, Mutex, OnceCell};
 use tracing::instrument;

core/startos/src/service/service_effect_handler.rs

@@ -16,7 +16,7 @@ use models::{
     ActionId, DataUrl, HealthCheckId, HostId, Id, ImageId, PackageId, ServiceInterfaceId, VolumeId,
 };
 use patch_db::json_ptr::JsonPointer;
-use rpc_toolkit::{from_fn, from_fn_async, AnyContext, Context, Empty, HandlerExt, ParentHandler};
+use rpc_toolkit::{from_fn, from_fn_async, Context, Empty, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tokio::process::Command;
 use ts_rs::TS;
@@ -36,7 +36,7 @@ use crate::net::service_interface::{
     ServiceInterfaceWithHostInfo,
 };
 use crate::prelude::*;
-use crate::s9pk::merkle_archive::source::http::{HttpReader, HttpSource};
+use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::rpc::SKIP_ENV;
 use crate::s9pk::S9pk;
 use crate::service::cli::ContainerCliContext;
@@ -74,14 +74,17 @@ struct RpcData {
     method: String,
     params: Value,
 }
-pub fn service_effect_handler() -> ParentHandler {
+pub fn service_effect_handler<C: Context>() -> ParentHandler<C> {
     ParentHandler::new()
-        .subcommand("gitInfo", from_fn(crate::version::git_info))
+        .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
         .subcommand(
             "echo",
-            from_fn(echo).with_remote_cli::<ContainerCliContext>(),
+            from_fn(echo::<EffectContext>).with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "chroot",
+            from_fn(chroot::<ContainerCliContext>).no_display(),
         )
-        .subcommand("chroot", from_fn(chroot).no_display())
         .subcommand("exists", from_fn_async(exists).no_cli())
         .subcommand("executeAction", from_fn_async(execute_action).no_cli())
         .subcommand("getConfigured", from_fn_async(get_configured).no_cli())
@@ -89,35 +92,35 @@ pub fn service_effect_handler() -> ParentHandler {
             "stopped",
             from_fn_async(stopped)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "running",
             from_fn_async(running)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "restart",
             from_fn_async(restart)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "shutdown",
             from_fn_async(shutdown)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "setConfigured",
             from_fn_async(set_configured)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "setMainStatus",
-            from_fn_async(set_main_status).with_remote_cli::<ContainerCliContext>(),
+            from_fn_async(set_main_status).with_call_remote::<ContainerCliContext>(),
         )
         .subcommand("setHealth", from_fn_async(set_health).no_cli())
         .subcommand("getStore", from_fn_async(get_store).no_cli())
@@ -129,10 +132,8 @@ pub fn service_effect_handler() -> ParentHandler {
         .subcommand(
             "createOverlayedImage",
             from_fn_async(create_overlayed_image)
-                .with_custom_display_fn::<AnyContext, _>(|_, (path, _)| {
-                    Ok(println!("{}", path.display()))
-                })
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display())))
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "destroyOverlayedImage",
@@ -154,19 +155,19 @@ pub fn service_effect_handler() -> ParentHandler {
             "setDependencies",
             from_fn_async(set_dependencies)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "getDependencies",
             from_fn_async(get_dependencies)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand(
             "checkDependencies",
             from_fn_async(check_dependencies)
                 .no_display()
-                .with_remote_cli::<ContainerCliContext>(),
+                .with_call_remote::<ContainerCliContext>(),
         )
         .subcommand("getSystemSmtp", from_fn_async(get_system_smtp).no_cli())
         .subcommand("getContainerIp", from_fn_async(get_container_ip).no_cli())
@@ -493,7 +494,7 @@ struct GetHostInfoParams {
     callback: Callback,
 }
 async fn get_host_info(
-    _: AnyContext,
+    _: EffectContext,
     GetHostInfoParams { .. }: GetHostInfoParams,
 ) -> Result<Value, Error> {
     todo!()
@@ -537,7 +538,7 @@ struct GetServiceInterfaceParams {
     callback: Callback,
 }
 async fn get_service_interface(
-    _: AnyContext,
+    _: EffectContext,
     GetServiceInterfaceParams {
         callback,
         package_id,
@@ -584,8 +585,8 @@ struct ChrootParams {
     #[ts(type = "string[]")]
     args: Vec<OsString>,
 }
-fn chroot(
-    _: AnyContext,
+fn chroot<C: Context>(
+    _: C,
     ChrootParams {
         env,
         workdir,
@@ -734,7 +735,7 @@ async fn set_store(
             let model = db
                 .as_private_mut()
                 .as_package_stores_mut()
-                .upsert(&package_id, || Box::new(json!({})))?;
+                .upsert(&package_id, || json!({}))?;
             let mut model_value = model.de()?;
             if model_value.is_null() {
                 model_value = json!({});

core/startos/src/service/service_map.rs

@@ -165,7 +165,7 @@ impl ServiceMap {
                                     configured: false,
                                     main: MainStatus::Stopped,
                                 },
-                                marketplace_url: None,
+                                registry: None,
                                 developer_key: Pem::new(developer_key),
                                 icon,
                                 last_backup: None,
@@ -206,6 +206,7 @@ impl ServiceMap {
                             Some(Duration::from_millis(100)),
                         )));
 
+                    download_progress.start();
                     let mut progress_writer = ProgressTrackerWriter::new(
                         crate::util::io::create_file(&download_path).await?,
                         download_progress,
@@ -229,6 +230,7 @@ impl ServiceMap {
                 .await?;
             Ok(reload_guard
                 .handle_last(async move {
+                    finalization_progress.start();
                     let s9pk = S9pk::open(&installed_path, Some(&id), true).await?;
                     let prev = if let Some(service) = service.take() {
                         ensure_code!(
@@ -246,7 +248,6 @@ impl ServiceMap {
                         service
                             .uninstall(Some(s9pk.as_manifest().version.clone()))
                             .await?;
-                        finalization_progress.complete();
                         progress_handle.complete();
                         Some(version)
                     } else {