Feature/callbacks (#2678)

* wip

* initialize callbacks

* wip

* smtp

* list_service_interfaces

* wip

* wip

* fix domains

* fix hostname handling in NetService

* misc fixes

* getInstalledPackages

* misc fixes

* publish v6 lib

* refactor service effects

* fix import

* fix container runtime

* fix tests

* apply suggestions from review

core/startos/src/service/cli.rs

@@ -9,7 +9,7 @@ use rpc_toolkit::{call_remote_socket, yajrc, CallRemote, Context, Empty};
 use tokio::runtime::Runtime;
 
 use crate::lxc::HOST_RPC_SERVER_SOCKET;
-use crate::service::service_effect_handler::EffectContext;
+use crate::service::effects::context::EffectContext;
 
 #[derive(Debug, Default, Parser)]
 pub struct ContainerClientConfig {

core/startos/src/service/effects/action.rs

@@ -0,0 +1,101 @@
+use std::collections::BTreeMap;
+
+use models::{ActionId, PackageId};
+
+use crate::action::ActionResult;
+use crate::db::model::package::ActionMetadata;
+use crate::rpc_continuations::Guid;
+use crate::service::effects::prelude::*;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct ExportActionParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    id: ActionId,
+    metadata: ActionMetadata,
+}
+pub async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = context.seed.id.clone();
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            let model = db
+                .as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package_id)
+                .or_not_found(&package_id)?
+                .as_actions_mut();
+            let mut value = model.de()?;
+            value
+                .insert(data.id, data.metadata)
+                .map(|_| ())
+                .unwrap_or_default();
+            model.ser(&value)
+        })
+        .await?;
+    Ok(())
+}
+
+pub async fn clear_actions(context: EffectContext) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = context.seed.id.clone();
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package_id)
+                .or_not_found(&package_id)?
+                .as_actions_mut()
+                .ser(&BTreeMap::new())
+        })
+        .await?;
+    Ok(())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ExecuteAction {
+    #[serde(default)]
+    #[ts(skip)]
+    procedure_id: Guid,
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    action_id: ActionId,
+    #[ts(type = "any")]
+    input: Value,
+}
+pub async fn execute_action(
+    context: EffectContext,
+    ExecuteAction {
+        procedure_id,
+        package_id,
+        action_id,
+        input,
+    }: ExecuteAction,
+) -> Result<ActionResult, Error> {
+    let context = context.deref()?;
+
+    if let Some(package_id) = package_id {
+        context
+            .seed
+            .ctx
+            .services
+            .get(&package_id)
+            .await
+            .as_ref()
+            .or_not_found(&package_id)?
+            .action(procedure_id, action_id, input)
+            .await
+    } else {
+        context.action(procedure_id, action_id, input).await
+    }
+}

core/startos/src/service/effects/callbacks.rs

@@ -0,0 +1,311 @@
+use std::cmp::min;
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::{Arc, Mutex, Weak};
+use std::time::{Duration, SystemTime};
+
+use futures::future::join_all;
+use helpers::NonDetachingJoinHandle;
+use imbl::{vector, Vector};
+use imbl_value::InternedString;
+use models::{HostId, PackageId, ServiceInterfaceId};
+use patch_db::json_ptr::JsonPointer;
+use tracing::warn;
+
+use crate::net::ssl::FullchainCertData;
+use crate::prelude::*;
+use crate::service::effects::context::EffectContext;
+use crate::service::effects::net::ssl::Algorithm;
+use crate::service::rpc::CallbackHandle;
+use crate::service::{Service, ServiceActorSeed};
+use crate::util::collections::EqMap;
+
+#[derive(Default)]
+pub struct ServiceCallbacks(Mutex<ServiceCallbackMap>);
+
+#[derive(Default)]
+struct ServiceCallbackMap {
+    get_service_interface: BTreeMap<(PackageId, ServiceInterfaceId), Vec<CallbackHandler>>,
+    list_service_interfaces: BTreeMap<PackageId, Vec<CallbackHandler>>,
+    get_system_smtp: Vec<CallbackHandler>,
+    get_host_info: BTreeMap<(PackageId, HostId), Vec<CallbackHandler>>,
+    get_ssl_certificate: EqMap<
+        (BTreeSet<InternedString>, FullchainCertData, Algorithm),
+        (NonDetachingJoinHandle<()>, Vec<CallbackHandler>),
+    >,
+    get_store: BTreeMap<PackageId, BTreeMap<JsonPointer, Vec<CallbackHandler>>>,
+}
+
+impl ServiceCallbacks {
+    fn mutate<T>(&self, f: impl FnOnce(&mut ServiceCallbackMap) -> T) -> T {
+        let mut this = self.0.lock().unwrap();
+        f(&mut *this)
+    }
+
+    pub fn gc(&self) {
+        self.mutate(|this| {
+            this.get_service_interface.retain(|_, v| {
+                v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                !v.is_empty()
+            });
+            this.list_service_interfaces.retain(|_, v| {
+                v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                !v.is_empty()
+            });
+            this.get_system_smtp
+                .retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+            this.get_host_info.retain(|_, v| {
+                v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                !v.is_empty()
+            });
+            this.get_ssl_certificate.retain(|_, (_, v)| {
+                v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                !v.is_empty()
+            });
+            this.get_store.retain(|_, v| {
+                v.retain(|_, v| {
+                    v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                    !v.is_empty()
+                });
+                !v.is_empty()
+            });
+        })
+    }
+
+    pub(super) fn add_get_service_interface(
+        &self,
+        package_id: PackageId,
+        service_interface_id: ServiceInterfaceId,
+        handler: CallbackHandler,
+    ) {
+        self.mutate(|this| {
+            this.get_service_interface
+                .entry((package_id, service_interface_id))
+                .or_default()
+                .push(handler);
+        })
+    }
+
+    #[must_use]
+    pub fn get_service_interface(
+        &self,
+        id: &(PackageId, ServiceInterfaceId),
+    ) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            Some(CallbackHandlers(
+                this.get_service_interface.remove(id).unwrap_or_default(),
+            ))
+            .filter(|cb| !cb.0.is_empty())
+        })
+    }
+
+    pub(super) fn add_list_service_interfaces(
+        &self,
+        package_id: PackageId,
+        handler: CallbackHandler,
+    ) {
+        self.mutate(|this| {
+            this.list_service_interfaces
+                .entry(package_id)
+                .or_default()
+                .push(handler);
+        })
+    }
+
+    #[must_use]
+    pub fn list_service_interfaces(&self, id: &PackageId) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            Some(CallbackHandlers(
+                this.list_service_interfaces.remove(id).unwrap_or_default(),
+            ))
+            .filter(|cb| !cb.0.is_empty())
+        })
+    }
+
+    pub(super) fn add_get_system_smtp(&self, handler: CallbackHandler) {
+        self.mutate(|this| {
+            this.get_system_smtp.push(handler);
+        })
+    }
+
+    #[must_use]
+    pub fn get_system_smtp(&self) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            Some(CallbackHandlers(std::mem::take(&mut this.get_system_smtp)))
+                .filter(|cb| !cb.0.is_empty())
+        })
+    }
+
+    pub(super) fn add_get_host_info(
+        &self,
+        package_id: PackageId,
+        host_id: HostId,
+        handler: CallbackHandler,
+    ) {
+        self.mutate(|this| {
+            this.get_host_info
+                .entry((package_id, host_id))
+                .or_default()
+                .push(handler);
+        })
+    }
+
+    #[must_use]
+    pub fn get_host_info(&self, id: &(PackageId, HostId)) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            Some(CallbackHandlers(
+                this.get_host_info.remove(id).unwrap_or_default(),
+            ))
+            .filter(|cb| !cb.0.is_empty())
+        })
+    }
+
+    pub(super) fn add_get_ssl_certificate(
+        &self,
+        ctx: EffectContext,
+        hostnames: BTreeSet<InternedString>,
+        cert: FullchainCertData,
+        algorithm: Algorithm,
+        handler: CallbackHandler,
+    ) {
+        self.mutate(|this| {
+            this.get_ssl_certificate
+                .entry((hostnames.clone(), cert.clone(), algorithm))
+                .or_insert_with(|| {
+                    (
+                        tokio::spawn(async move {
+                            if let Err(e) = async {
+                                loop {
+                                    match cert
+                                        .expiration()
+                                        .ok()
+                                        .and_then(|e| e.duration_since(SystemTime::now()).ok())
+                                    {
+                                        Some(d) => {
+                                            tokio::time::sleep(min(Duration::from_secs(86400), d))
+                                                .await
+                                        }
+                                        _ => break,
+                                    }
+                                }
+                                let Ok(ctx) = ctx.deref() else {
+                                    return Ok(());
+                                };
+
+                                if let Some((_, callbacks)) =
+                                    ctx.seed.ctx.callbacks.mutate(|this| {
+                                        this.get_ssl_certificate
+                                            .remove(&(hostnames, cert, algorithm))
+                                    })
+                                {
+                                    CallbackHandlers(callbacks).call(vector![]).await?;
+                                }
+                                Ok::<_, Error>(())
+                            }
+                            .await
+                            {
+                                tracing::error!(
+                                    "Error in callback handler for getSslCertificate: {e}"
+                                );
+                                tracing::debug!("{e:?}");
+                            }
+                        })
+                        .into(),
+                        Vec::new(),
+                    )
+                })
+                .1
+                .push(handler);
+        })
+    }
+
+    pub(super) fn add_get_store(
+        &self,
+        package_id: PackageId,
+        path: JsonPointer,
+        handler: CallbackHandler,
+    ) {
+        self.mutate(|this| {
+            this.get_store
+                .entry(package_id)
+                .or_default()
+                .entry(path)
+                .or_default()
+                .push(handler)
+        })
+    }
+
+    #[must_use]
+    pub fn get_store(
+        &self,
+        package_id: &PackageId,
+        path: &JsonPointer,
+    ) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            if let Some(watched) = this.get_store.get_mut(package_id) {
+                let mut res = Vec::new();
+                watched.retain(|ptr, cbs| {
+                    if ptr.starts_with(path) || path.starts_with(ptr) {
+                        res.append(cbs);
+                        false
+                    } else {
+                        true
+                    }
+                });
+                Some(CallbackHandlers(res))
+            } else {
+                None
+            }
+            .filter(|cb| !cb.0.is_empty())
+        })
+    }
+}
+
+pub struct CallbackHandler {
+    handle: CallbackHandle,
+    seed: Weak<ServiceActorSeed>,
+}
+impl CallbackHandler {
+    pub fn new(service: &Service, handle: CallbackHandle) -> Self {
+        Self {
+            handle,
+            seed: Arc::downgrade(&service.seed),
+        }
+    }
+    pub async fn call(mut self, args: Vector<Value>) -> Result<(), Error> {
+        if let Some(seed) = self.seed.upgrade() {
+            seed.persistent_container
+                .callback(self.handle.take(), args)
+                .await?;
+        }
+        Ok(())
+    }
+}
+impl Drop for CallbackHandler {
+    fn drop(&mut self) {
+        if self.handle.is_active() {
+            warn!("Callback handler dropped while still active!");
+        }
+    }
+}
+
+pub struct CallbackHandlers(Vec<CallbackHandler>);
+impl CallbackHandlers {
+    pub async fn call(self, args: Vector<Value>) -> Result<(), Error> {
+        let mut err = ErrorCollection::new();
+        for res in join_all(self.0.into_iter().map(|cb| cb.call(args.clone()))).await {
+            err.handle(res);
+        }
+        err.into_result()
+    }
+}
+
+pub(super) fn clear_callbacks(context: EffectContext) -> Result<(), Error> {
+    let context = context.deref()?;
+    context
+        .seed
+        .persistent_container
+        .state
+        .send_if_modified(|s| !std::mem::take(&mut s.callbacks).is_empty());
+    context.seed.ctx.callbacks.gc();
+    Ok(())
+}

core/startos/src/service/effects/config.rs

@@ -0,0 +1,53 @@
+use models::PackageId;
+
+use crate::service::effects::prelude::*;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetConfiguredParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+}
+pub async fn get_configured(context: EffectContext) -> Result<bool, Error> {
+    let context = context.deref()?;
+    let peeked = context.seed.ctx.db.peek().await;
+    let package_id = &context.seed.id;
+    peeked
+        .as_public()
+        .as_package_data()
+        .as_idx(package_id)
+        .or_not_found(package_id)?
+        .as_status()
+        .as_configured()
+        .de()
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetConfigured {
+    configured: bool,
+}
+pub async fn set_configured(
+    context: EffectContext,
+    SetConfigured { configured }: SetConfigured,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = &context.seed.id;
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(package_id)
+                .or_not_found(package_id)?
+                .as_status_mut()
+                .as_configured_mut()
+                .ser(&configured)
+        })
+        .await?;
+    Ok(())
+}

core/startos/src/service/effects/context.rs

@@ -0,0 +1,27 @@
+use std::sync::{Arc, Weak};
+
+use rpc_toolkit::Context;
+
+use crate::prelude::*;
+use crate::service::Service;
+
+#[derive(Clone)]
+pub(in crate::service) struct EffectContext(Weak<Service>);
+impl EffectContext {
+    pub fn new(service: Weak<Service>) -> Self {
+        Self(service)
+    }
+}
+impl Context for EffectContext {}
+impl EffectContext {
+    pub(super) fn deref(&self) -> Result<Arc<Service>, Error> {
+        if let Some(seed) = Weak::upgrade(&self.0) {
+            Ok(seed)
+        } else {
+            Err(Error::new(
+                eyre!("Service has already been destroyed"),
+                ErrorKind::InvalidRequest,
+            ))
+        }
+    }
+}

core/startos/src/service/effects/control.rs

@@ -0,0 +1,66 @@
+use std::str::FromStr;
+
+use clap::builder::ValueParserFactory;
+
+use crate::service::effects::prelude::*;
+use crate::util::clap::FromStrParser;
+
+pub async fn restart(
+    context: EffectContext,
+    ProcedureId { procedure_id }: ProcedureId,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    context.restart(procedure_id).await?;
+    Ok(())
+}
+
+pub async fn shutdown(
+    context: EffectContext,
+    ProcedureId { procedure_id }: ProcedureId,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    context.stop(procedure_id).await?;
+    Ok(())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum SetMainStatusStatus {
+    Running,
+    Stopped,
+}
+impl FromStr for SetMainStatusStatus {
+    type Err = color_eyre::eyre::Report;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "running" => Ok(Self::Running),
+            "stopped" => Ok(Self::Stopped),
+            _ => Err(eyre!("unknown status {s}")),
+        }
+    }
+}
+impl ValueParserFactory for SetMainStatusStatus {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        FromStrParser::new()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetMainStatus {
+    status: SetMainStatusStatus,
+}
+pub async fn set_main_status(
+    context: EffectContext,
+    SetMainStatus { status }: SetMainStatus,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    match status {
+        SetMainStatusStatus::Running => context.seed.started(),
+        SetMainStatusStatus::Stopped => context.seed.stopped(),
+    }
+    Ok(())
+}

core/startos/src/service/effects/dependency.rs

@@ -0,0 +1,371 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use clap::builder::ValueParserFactory;
+use exver::VersionRange;
+use itertools::Itertools;
+use models::{HealthCheckId, PackageId, VolumeId};
+use patch_db::json_ptr::JsonPointer;
+
+use crate::db::model::package::{
+    CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind, ManifestPreference,
+};
+use crate::rpc_continuations::Guid;
+use crate::service::effects::prelude::*;
+use crate::status::health_check::HealthCheckResult;
+use crate::util::clap::FromStrParser;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct MountTarget {
+    package_id: PackageId,
+    volume_id: VolumeId,
+    subpath: Option<PathBuf>,
+    readonly: bool,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct MountParams {
+    location: String,
+    target: MountTarget,
+}
+pub async fn mount(
+    context: EffectContext,
+    MountParams {
+        location,
+        target:
+            MountTarget {
+                package_id,
+                volume_id,
+                subpath,
+                readonly,
+            },
+    }: MountParams,
+) -> Result<(), Error> {
+    // TODO
+    todo!()
+}
+
+pub async fn get_installed_packages(context: EffectContext) -> Result<Vec<PackageId>, Error> {
+    context
+        .deref()?
+        .seed
+        .ctx
+        .db
+        .peek()
+        .await
+        .into_public()
+        .into_package_data()
+        .keys()
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ExposeForDependentsParams {
+    #[ts(type = "string[]")]
+    paths: Vec<JsonPointer>,
+}
+pub async fn expose_for_dependents(
+    context: EffectContext,
+    ExposeForDependentsParams { paths }: ExposeForDependentsParams,
+) -> Result<(), Error> {
+    Ok(())
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum DependencyKind {
+    Exists,
+    Running,
+}
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase", tag = "kind")]
+#[serde(rename_all_fields = "camelCase")]
+#[ts(export)]
+pub enum DependencyRequirement {
+    Running {
+        id: PackageId,
+        health_checks: BTreeSet<HealthCheckId>,
+        #[ts(type = "string")]
+        version_range: VersionRange,
+    },
+    Exists {
+        id: PackageId,
+        #[ts(type = "string")]
+        version_range: VersionRange,
+    },
+}
+// filebrowser:exists,bitcoind:running:foo+bar+baz
+impl FromStr for DependencyRequirement {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.split_once(':') {
+            Some((id, "e")) | Some((id, "exists")) => Ok(Self::Exists {
+                id: id.parse()?,
+                version_range: "*".parse()?, // TODO
+            }),
+            Some((id, rest)) => {
+                let health_checks = match rest.split_once(':') {
+                    Some(("r", rest)) | Some(("running", rest)) => rest
+                        .split('+')
+                        .map(|id| id.parse().map_err(Error::from))
+                        .collect(),
+                    Some((kind, _)) => Err(Error::new(
+                        eyre!("unknown dependency kind {kind}"),
+                        ErrorKind::InvalidRequest,
+                    )),
+                    None => match rest {
+                        "r" | "running" => Ok(BTreeSet::new()),
+                        kind => Err(Error::new(
+                            eyre!("unknown dependency kind {kind}"),
+                            ErrorKind::InvalidRequest,
+                        )),
+                    },
+                }?;
+                Ok(Self::Running {
+                    id: id.parse()?,
+                    health_checks,
+                    version_range: "*".parse()?, // TODO
+                })
+            }
+            None => Ok(Self::Running {
+                id: s.parse()?,
+                health_checks: BTreeSet::new(),
+                version_range: "*".parse()?, // TODO
+            }),
+        }
+    }
+}
+impl ValueParserFactory for DependencyRequirement {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        FromStrParser::new()
+    }
+}
+#[derive(Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[command(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetDependenciesParams {
+    #[serde(default)]
+    procedure_id: Guid,
+    dependencies: Vec<DependencyRequirement>,
+}
+pub async fn set_dependencies(
+    context: EffectContext,
+    SetDependenciesParams {
+        procedure_id,
+        dependencies,
+    }: SetDependenciesParams,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    let id = &context.seed.id;
+
+    let mut deps = BTreeMap::new();
+    for dependency in dependencies {
+        let (dep_id, kind, version_range) = match dependency {
+            DependencyRequirement::Exists { id, version_range } => {
+                (id, CurrentDependencyKind::Exists, version_range)
+            }
+            DependencyRequirement::Running {
+                id,
+                health_checks,
+                version_range,
+            } => (
+                id,
+                CurrentDependencyKind::Running { health_checks },
+                version_range,
+            ),
+        };
+        let config_satisfied =
+            if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await {
+                context
+                    .dependency_config(
+                        procedure_id.clone(),
+                        dep_id.clone(),
+                        dep_service.get_config(procedure_id.clone()).await?.config,
+                    )
+                    .await?
+                    .is_none()
+            } else {
+                true
+            };
+        let info = CurrentDependencyInfo {
+            title: context
+                .seed
+                .persistent_container
+                .s9pk
+                .dependency_metadata(&dep_id)
+                .await?
+                .map(|m| m.title),
+            icon: context
+                .seed
+                .persistent_container
+                .s9pk
+                .dependency_icon_data_url(&dep_id)
+                .await?,
+            kind,
+            version_range,
+            config_satisfied,
+        };
+        deps.insert(dep_id, info);
+    }
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(id)
+                .or_not_found(id)?
+                .as_current_dependencies_mut()
+                .ser(&CurrentDependencies(deps))
+        })
+        .await
+}
+
+pub async fn get_dependencies(context: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
+    let context = context.deref()?;
+    let id = &context.seed.id;
+    let db = context.seed.ctx.db.peek().await;
+    let data = db
+        .as_public()
+        .as_package_data()
+        .as_idx(id)
+        .or_not_found(id)?
+        .as_current_dependencies()
+        .de()?;
+
+    data.0
+        .into_iter()
+        .map(|(id, current_dependency_info)| {
+            let CurrentDependencyInfo {
+                version_range,
+                kind,
+                ..
+            } = current_dependency_info;
+            Ok::<_, Error>(match kind {
+                CurrentDependencyKind::Exists => {
+                    DependencyRequirement::Exists { id, version_range }
+                }
+                CurrentDependencyKind::Running { health_checks } => {
+                    DependencyRequirement::Running {
+                        id,
+                        health_checks,
+                        version_range,
+                    }
+                }
+            })
+        })
+        .try_collect()
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct CheckDependenciesParam {
+    #[ts(optional)]
+    package_ids: Option<Vec<PackageId>>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct CheckDependenciesResult {
+    package_id: PackageId,
+    is_installed: bool,
+    is_running: bool,
+    config_satisfied: bool,
+    health_checks: BTreeMap<HealthCheckId, HealthCheckResult>,
+    #[ts(type = "string | null")]
+    version: Option<exver::ExtendedVersion>,
+}
+pub async fn check_dependencies(
+    context: EffectContext,
+    CheckDependenciesParam { package_ids }: CheckDependenciesParam,
+) -> Result<Vec<CheckDependenciesResult>, Error> {
+    let context = context.deref()?;
+    let db = context.seed.ctx.db.peek().await;
+    let current_dependencies = db
+        .as_public()
+        .as_package_data()
+        .as_idx(&context.seed.id)
+        .or_not_found(&context.seed.id)?
+        .as_current_dependencies()
+        .de()?;
+    let package_ids: Vec<_> = package_ids
+        .unwrap_or_else(|| current_dependencies.0.keys().cloned().collect())
+        .into_iter()
+        .filter_map(|x| {
+            let info = current_dependencies.0.get(&x)?;
+            Some((x, info))
+        })
+        .collect();
+    let mut results = Vec::with_capacity(package_ids.len());
+
+    for (package_id, dependency_info) in package_ids {
+        let Some(package) = db.as_public().as_package_data().as_idx(&package_id) else {
+            results.push(CheckDependenciesResult {
+                package_id,
+                is_installed: false,
+                is_running: false,
+                config_satisfied: false,
+                health_checks: Default::default(),
+                version: None,
+            });
+            continue;
+        };
+        let manifest = package.as_state_info().as_manifest(ManifestPreference::New);
+        let installed_version = manifest.as_version().de()?.into_version();
+        let satisfies = manifest.as_satisfies().de()?;
+        let version = Some(installed_version.clone());
+        if ![installed_version]
+            .into_iter()
+            .chain(satisfies.into_iter().map(|v| v.into_version()))
+            .any(|v| v.satisfies(&dependency_info.version_range))
+        {
+            results.push(CheckDependenciesResult {
+                package_id,
+                is_installed: false,
+                is_running: false,
+                config_satisfied: false,
+                health_checks: Default::default(),
+                version,
+            });
+            continue;
+        }
+        let is_installed = true;
+        let status = package.as_status().as_main().de()?;
+        let is_running = if is_installed {
+            status.running()
+        } else {
+            false
+        };
+        let health_checks =
+            if let CurrentDependencyKind::Running { health_checks } = &dependency_info.kind {
+                status
+                    .health()
+                    .cloned()
+                    .unwrap_or_default()
+                    .into_iter()
+                    .filter(|(id, _)| health_checks.contains(id))
+                    .collect()
+            } else {
+                Default::default()
+            };
+        results.push(CheckDependenciesResult {
+            package_id,
+            is_installed,
+            is_running,
+            config_satisfied: dependency_info.config_satisfied,
+            health_checks,
+            version,
+        });
+    }
+    Ok(results)
+}

core/startos/src/service/effects/health.rs

@@ -0,0 +1,46 @@
+use models::HealthCheckId;
+
+use crate::service::effects::prelude::*;
+use crate::status::health_check::HealthCheckResult;
+use crate::status::MainStatus;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetHealth {
+    id: HealthCheckId,
+    #[serde(flatten)]
+    result: HealthCheckResult,
+}
+pub async fn set_health(
+    context: EffectContext,
+    SetHealth { id, result }: SetHealth,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+
+    let package_id = &context.seed.id;
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(move |db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(package_id)
+                .or_not_found(package_id)?
+                .as_status_mut()
+                .as_main_mut()
+                .mutate(|main| {
+                    match main {
+                        &mut MainStatus::Running { ref mut health, .. }
+                        | &mut MainStatus::BackingUp { ref mut health, .. } => {
+                            health.insert(id, result);
+                        }
+                        _ => (),
+                    }
+                    Ok(())
+                })
+        })
+        .await?;
+    Ok(())
+}

core/startos/src/service/effects/image.rs

@@ -0,0 +1,163 @@
+use std::ffi::OsString;
+use std::os::unix::process::CommandExt;
+use std::path::{Path, PathBuf};
+
+use models::ImageId;
+use rpc_toolkit::Context;
+use tokio::process::Command;
+
+use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
+use crate::rpc_continuations::Guid;
+use crate::service::effects::prelude::*;
+use crate::util::Invoke;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Parser)]
+pub struct ChrootParams {
+    #[arg(short = 'e', long = "env")]
+    env: Option<PathBuf>,
+    #[arg(short = 'w', long = "workdir")]
+    workdir: Option<PathBuf>,
+    #[arg(short = 'u', long = "user")]
+    user: Option<String>,
+    path: PathBuf,
+    command: OsString,
+    args: Vec<OsString>,
+}
+pub fn chroot<C: Context>(
+    _: C,
+    ChrootParams {
+        env,
+        workdir,
+        user,
+        path,
+        command,
+        args,
+    }: ChrootParams,
+) -> Result<(), Error> {
+    let mut cmd = std::process::Command::new(command);
+    if let Some(env) = env {
+        for (k, v) in std::fs::read_to_string(env)?
+            .lines()
+            .map(|l| l.trim())
+            .filter_map(|l| l.split_once("="))
+        {
+            cmd.env(k, v);
+        }
+    }
+    nix::unistd::setsid().ok(); // https://stackoverflow.com/questions/25701333/os-setsid-operation-not-permitted
+    std::os::unix::fs::chroot(path)?;
+    if let Some(uid) = user.as_deref().and_then(|u| u.parse::<u32>().ok()) {
+        cmd.uid(uid);
+    } else if let Some(user) = user {
+        let (uid, gid) = std::fs::read_to_string("/etc/passwd")?
+            .lines()
+            .find_map(|l| {
+                let mut split = l.trim().split(":");
+                if user != split.next()? {
+                    return None;
+                }
+                split.next(); // throw away x
+                Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?))
+                // uid gid
+            })
+            .or_not_found(lazy_format!("{user} in /etc/passwd"))?;
+        cmd.uid(uid);
+        cmd.gid(gid);
+    };
+    if let Some(workdir) = workdir {
+        cmd.current_dir(workdir);
+    }
+    cmd.args(args);
+    Err(cmd.exec().into())
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct DestroyOverlayedImageParams {
+    guid: Guid,
+}
+#[instrument(skip_all)]
+pub async fn destroy_overlayed_image(
+    context: EffectContext,
+    DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    if context
+        .seed
+        .persistent_container
+        .overlays
+        .lock()
+        .await
+        .remove(&guid)
+        .is_none()
+    {
+        tracing::warn!("Could not find a guard to remove on the destroy overlayed image; assumming that it already is removed and will be skipping");
+    }
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct CreateOverlayedImageParams {
+    image_id: ImageId,
+}
+#[instrument(skip_all)]
+pub async fn create_overlayed_image(
+    context: EffectContext,
+    CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams,
+) -> Result<(PathBuf, Guid), Error> {
+    let context = context.deref()?;
+    if let Some(image) = context
+        .seed
+        .persistent_container
+        .images
+        .get(&image_id)
+        .cloned()
+    {
+        let guid = Guid::new();
+        let rootfs_dir = context
+            .seed
+            .persistent_container
+            .lxc_container
+            .get()
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("PersistentContainer has been destroyed"),
+                    ErrorKind::Incoherent,
+                )
+            })?
+            .rootfs_dir();
+        let mountpoint = rootfs_dir
+            .join("media/startos/overlays")
+            .join(guid.as_ref());
+        tokio::fs::create_dir_all(&mountpoint).await?;
+        let container_mountpoint = Path::new("/").join(
+            mountpoint
+                .strip_prefix(rootfs_dir)
+                .with_kind(ErrorKind::Incoherent)?,
+        );
+        tracing::info!("Mounting overlay {guid} for {image_id}");
+        let guard = OverlayGuard::mount(image, &mountpoint).await?;
+        Command::new("chown")
+            .arg("100000:100000")
+            .arg(&mountpoint)
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        tracing::info!("Mounted overlay {guid} for {image_id}");
+        context
+            .seed
+            .persistent_container
+            .overlays
+            .lock()
+            .await
+            .insert(guid.clone(), guard);
+        Ok((container_mountpoint, guid))
+    } else {
+        Err(Error::new(
+            eyre!("image {image_id} not found in s9pk"),
+            ErrorKind::NotFound,
+        ))
+    }
+}

core/startos/src/service/effects/mod.rs

@@ -0,0 +1,174 @@
+use rpc_toolkit::{from_fn, from_fn_async, Context, HandlerExt, ParentHandler};
+
+use crate::echo;
+use crate::prelude::*;
+use crate::service::cli::ContainerCliContext;
+use crate::service::effects::context::EffectContext;
+
+mod action;
+pub mod callbacks;
+mod config;
+pub mod context;
+mod control;
+mod dependency;
+mod health;
+mod image;
+mod net;
+mod prelude;
+mod store;
+mod system;
+
+pub fn handler<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
+        .subcommand(
+            "echo",
+            from_fn(echo::<EffectContext>).with_call_remote::<ContainerCliContext>(),
+        )
+        // action
+        .subcommand(
+            "executeAction",
+            from_fn_async(action::execute_action).no_cli(),
+        )
+        .subcommand(
+            "exportAction",
+            from_fn_async(action::export_action).no_cli(),
+        )
+        .subcommand(
+            "clearActions",
+            from_fn_async(action::clear_actions).no_cli(),
+        )
+        // callbacks
+        .subcommand(
+            "clearCallbacks",
+            from_fn(callbacks::clear_callbacks).no_cli(),
+        )
+        // config
+        .subcommand(
+            "getConfigured",
+            from_fn_async(config::get_configured).no_cli(),
+        )
+        .subcommand(
+            "setConfigured",
+            from_fn_async(config::set_configured)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        // control
+        .subcommand(
+            "restart",
+            from_fn_async(control::restart)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "shutdown",
+            from_fn_async(control::shutdown)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "setMainStatus",
+            from_fn_async(control::set_main_status)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        // dependency
+        .subcommand(
+            "setDependencies",
+            from_fn_async(dependency::set_dependencies)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "getDependencies",
+            from_fn_async(dependency::get_dependencies)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "checkDependencies",
+            from_fn_async(dependency::check_dependencies)
+                .no_display()
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand("mount", from_fn_async(dependency::mount).no_cli())
+        .subcommand(
+            "getInstalledPackages",
+            from_fn_async(dependency::get_installed_packages).no_cli(),
+        )
+        .subcommand(
+            "exposeForDependents",
+            from_fn_async(dependency::expose_for_dependents).no_cli(),
+        )
+        // health
+        .subcommand("setHealth", from_fn_async(health::set_health).no_cli())
+        // image
+        .subcommand(
+            "chroot",
+            from_fn(image::chroot::<ContainerCliContext>).no_display(),
+        )
+        .subcommand(
+            "createOverlayedImage",
+            from_fn_async(image::create_overlayed_image)
+                .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display())))
+                .with_call_remote::<ContainerCliContext>(),
+        )
+        .subcommand(
+            "destroyOverlayedImage",
+            from_fn_async(image::destroy_overlayed_image).no_cli(),
+        )
+        // net
+        .subcommand("bind", from_fn_async(net::bind::bind).no_cli())
+        .subcommand(
+            "getServicePortForward",
+            from_fn_async(net::bind::get_service_port_forward).no_cli(),
+        )
+        .subcommand(
+            "clearBindings",
+            from_fn_async(net::bind::clear_bindings).no_cli(),
+        )
+        .subcommand(
+            "getHostInfo",
+            from_fn_async(net::host::get_host_info).no_cli(),
+        )
+        .subcommand(
+            "getPrimaryUrl",
+            from_fn_async(net::host::get_primary_url).no_cli(),
+        )
+        .subcommand(
+            "getContainerIp",
+            from_fn_async(net::info::get_container_ip).no_cli(),
+        )
+        .subcommand(
+            "exportServiceInterface",
+            from_fn_async(net::interface::export_service_interface).no_cli(),
+        )
+        .subcommand(
+            "getServiceInterface",
+            from_fn_async(net::interface::get_service_interface).no_cli(),
+        )
+        .subcommand(
+            "listServiceInterfaces",
+            from_fn_async(net::interface::list_service_interfaces).no_cli(),
+        )
+        .subcommand(
+            "clearServiceInterfaces",
+            from_fn_async(net::interface::clear_service_interfaces).no_cli(),
+        )
+        .subcommand(
+            "getSslCertificate",
+            from_fn_async(net::ssl::get_ssl_certificate).no_cli(),
+        )
+        .subcommand("getSslKey", from_fn_async(net::ssl::get_ssl_key).no_cli())
+        // store
+        .subcommand("getStore", from_fn_async(store::get_store).no_cli())
+        .subcommand("setStore", from_fn_async(store::set_store).no_cli())
+        // system
+        .subcommand(
+            "getSystemSmtp",
+            from_fn_async(system::get_system_smtp).no_cli(),
+        )
+
+    // TODO Callbacks
+}

core/startos/src/service/effects/net/bind.rs

@@ -0,0 +1,56 @@
+use models::{HostId, PackageId};
+
+use crate::net::host::binding::{BindOptions, LanInfo};
+use crate::net::host::HostKind;
+use crate::service::effects::prelude::*;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct BindParams {
+    kind: HostKind,
+    id: HostId,
+    internal_port: u16,
+    #[serde(flatten)]
+    options: BindOptions,
+}
+pub async fn bind(
+    context: EffectContext,
+    BindParams {
+        kind,
+        id,
+        internal_port,
+        options,
+    }: BindParams,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    let mut svc = context.seed.persistent_container.net_service.lock().await;
+    svc.bind(kind, id, internal_port, options).await
+}
+
+pub async fn clear_bindings(context: EffectContext) -> Result<(), Error> {
+    let context = context.deref()?;
+    let mut svc = context.seed.persistent_container.net_service.lock().await;
+    svc.clear_bindings().await?;
+    Ok(())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct GetServicePortForwardParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    host_id: HostId,
+    internal_port: u32,
+}
+pub async fn get_service_port_forward(
+    context: EffectContext,
+    data: GetServicePortForwardParams,
+) -> Result<LanInfo, Error> {
+    let internal_port = data.internal_port as u16;
+
+    let context = context.deref()?;
+    let net_service = context.seed.persistent_container.net_service.lock().await;
+    net_service.get_lan_port(data.host_id, internal_port)
+}

core/startos/src/service/effects/net/host.rs

@@ -0,0 +1,73 @@
+use models::{HostId, PackageId};
+
+use crate::net::host::address::HostAddress;
+use crate::net::host::Host;
+use crate::service::effects::callbacks::CallbackHandler;
+use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct GetPrimaryUrlParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    host_id: HostId,
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_primary_url(
+    context: EffectContext,
+    GetPrimaryUrlParams {
+        package_id,
+        host_id,
+        callback,
+    }: GetPrimaryUrlParams,
+) -> Result<Option<HostAddress>, Error> {
+    let context = context.deref()?;
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
+
+    Ok(None) // TODO
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetHostInfoParams {
+    host_id: HostId,
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_host_info(
+    context: EffectContext,
+    GetHostInfoParams {
+        host_id,
+        package_id,
+        callback,
+    }: GetHostInfoParams,
+) -> Result<Option<Host>, Error> {
+    let context = context.deref()?;
+    let db = context.seed.ctx.db.peek().await;
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
+
+    let res = db
+        .as_public()
+        .as_package_data()
+        .as_idx(&package_id)
+        .and_then(|m| m.as_hosts().as_idx(&host_id))
+        .map(|m| m.de())
+        .transpose()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context.seed.ctx.callbacks.add_get_host_info(
+            package_id,
+            host_id,
+            CallbackHandler::new(&context, callback),
+        );
+    }
+
+    Ok(res)
+}

core/startos/src/service/effects/net/info.rs

@@ -0,0 +1,9 @@
+use std::net::Ipv4Addr;
+
+use crate::service::effects::prelude::*;
+
+pub async fn get_container_ip(context: EffectContext) -> Result<Ipv4Addr, Error> {
+    let context = context.deref()?;
+    let net_service = context.seed.persistent_container.net_service.lock().await;
+    Ok(net_service.get_ip())
+}

core/startos/src/service/effects/net/interface.rs

@@ -0,0 +1,188 @@
+use std::collections::BTreeMap;
+
+use imbl::vector;
+use models::{PackageId, ServiceInterfaceId};
+
+use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType};
+use crate::service::effects::callbacks::CallbackHandler;
+use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct ExportServiceInterfaceParams {
+    id: ServiceInterfaceId,
+    name: String,
+    description: String,
+    has_primary: bool,
+    disabled: bool,
+    masked: bool,
+    address_info: AddressInfo,
+    r#type: ServiceInterfaceType,
+}
+pub async fn export_service_interface(
+    context: EffectContext,
+    ExportServiceInterfaceParams {
+        id,
+        name,
+        description,
+        has_primary,
+        disabled,
+        masked,
+        address_info,
+        r#type,
+    }: ExportServiceInterfaceParams,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = context.seed.id.clone();
+
+    let service_interface = ServiceInterface {
+        id: id.clone(),
+        name,
+        description,
+        has_primary,
+        disabled,
+        masked,
+        address_info,
+        interface_type: r#type,
+    };
+
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package_id)
+                .or_not_found(&package_id)?
+                .as_service_interfaces_mut()
+                .insert(&id, &service_interface)?;
+            Ok(())
+        })
+        .await?;
+    if let Some(callbacks) = context
+        .seed
+        .ctx
+        .callbacks
+        .get_service_interface(&(package_id.clone(), id))
+    {
+        callbacks.call(vector![]).await?;
+    }
+    if let Some(callbacks) = context
+        .seed
+        .ctx
+        .callbacks
+        .list_service_interfaces(&package_id)
+    {
+        callbacks.call(vector![]).await?;
+    }
+
+    Ok(())
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetServiceInterfaceParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    service_interface_id: ServiceInterfaceId,
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_service_interface(
+    context: EffectContext,
+    GetServiceInterfaceParams {
+        package_id,
+        service_interface_id,
+        callback,
+    }: GetServiceInterfaceParams,
+) -> Result<Option<ServiceInterface>, Error> {
+    let context = context.deref()?;
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
+    let db = context.seed.ctx.db.peek().await;
+
+    let interface = db
+        .as_public()
+        .as_package_data()
+        .as_idx(&package_id)
+        .and_then(|m| m.as_service_interfaces().as_idx(&service_interface_id))
+        .map(|m| m.de())
+        .transpose()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context.seed.ctx.callbacks.add_get_service_interface(
+            package_id,
+            service_interface_id,
+            CallbackHandler::new(&context, callback),
+        );
+    }
+
+    Ok(interface)
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct ListServiceInterfacesParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn list_service_interfaces(
+    context: EffectContext,
+    ListServiceInterfacesParams {
+        package_id,
+        callback,
+    }: ListServiceInterfacesParams,
+) -> Result<BTreeMap<ServiceInterfaceId, ServiceInterface>, Error> {
+    let context = context.deref()?;
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
+
+    let res = context
+        .seed
+        .ctx
+        .db
+        .peek()
+        .await
+        .into_public()
+        .into_package_data()
+        .into_idx(&package_id)
+        .map(|m| m.into_service_interfaces().de())
+        .transpose()?
+        .unwrap_or_default();
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context
+            .seed
+            .ctx
+            .callbacks
+            .add_list_service_interfaces(package_id, CallbackHandler::new(&context, callback));
+    }
+
+    Ok(res)
+}
+
+pub async fn clear_service_interfaces(context: EffectContext) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = context.seed.id.clone();
+
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            db.as_public_mut()
+                .as_package_data_mut()
+                .as_idx_mut(&package_id)
+                .or_not_found(&package_id)?
+                .as_service_interfaces_mut()
+                .ser(&Default::default())
+        })
+        .await
+}

core/startos/src/service/effects/net/mod.rs

@@ -0,0 +1,5 @@
+pub mod bind;
+pub mod host;
+pub mod info;
+pub mod interface;
+pub mod ssl;

core/startos/src/service/effects/net/ssl.rs

@@ -0,0 +1,169 @@
+use std::collections::BTreeSet;
+
+use imbl_value::InternedString;
+use itertools::Itertools;
+use openssl::pkey::{PKey, Private};
+
+use crate::service::effects::callbacks::CallbackHandler;
+use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
+use crate::util::serde::Pem;
+
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, TS, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum Algorithm {
+    Ecdsa,
+    Ed25519,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetSslCertificateParams {
+    #[ts(type = "string[]")]
+    hostnames: BTreeSet<InternedString>,
+    #[ts(optional)]
+    algorithm: Option<Algorithm>, //"ecdsa" | "ed25519"
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_ssl_certificate(
+    ctx: EffectContext,
+    GetSslCertificateParams {
+        hostnames,
+        algorithm,
+        callback,
+    }: GetSslCertificateParams,
+) -> Result<Vec<String>, Error> {
+    let context = ctx.deref()?;
+    let algorithm = algorithm.unwrap_or(Algorithm::Ecdsa);
+
+    let cert = context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            let errfn = |h: &str| Error::new(eyre!("unknown hostname: {h}"), ErrorKind::NotFound);
+            let entries = db.as_public().as_package_data().as_entries()?;
+            let packages = entries.iter().map(|(k, _)| k).collect::<BTreeSet<_>>();
+            let allowed_hostnames = entries
+                .iter()
+                .map(|(_, m)| m.as_hosts().as_entries())
+                .flatten_ok()
+                .map_ok(|(_, m)| m.as_addresses().de())
+                .map(|a| a.and_then(|a| a))
+                .flatten_ok()
+                .map_ok(|a| InternedString::from_display(&a))
+                .try_collect::<_, BTreeSet<_>, _>()?;
+            for hostname in &hostnames {
+                if let Some(internal) = hostname
+                    .strip_suffix(".embassy")
+                    .or_else(|| hostname.strip_suffix(".startos"))
+                {
+                    if !packages.contains(internal) {
+                        return Err(errfn(&*hostname));
+                    }
+                } else {
+                    if !allowed_hostnames.contains(hostname) {
+                        return Err(errfn(&*hostname));
+                    }
+                }
+            }
+            db.as_private_mut()
+                .as_key_store_mut()
+                .as_local_certs_mut()
+                .cert_for(&hostnames)
+        })
+        .await?;
+    let fullchain = match algorithm {
+        Algorithm::Ecdsa => cert.fullchain_nistp256(),
+        Algorithm::Ed25519 => cert.fullchain_ed25519(),
+    };
+
+    let res = fullchain
+        .into_iter()
+        .map(|c| c.to_pem())
+        .map_ok(String::from_utf8)
+        .map(|a| Ok::<_, Error>(a??))
+        .try_collect()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context.seed.ctx.callbacks.add_get_ssl_certificate(
+            ctx,
+            hostnames,
+            cert,
+            algorithm,
+            CallbackHandler::new(&context, callback),
+        );
+    }
+
+    Ok(res)
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetSslKeyParams {
+    #[ts(type = "string[]")]
+    hostnames: BTreeSet<InternedString>,
+    #[ts(optional)]
+    algorithm: Option<Algorithm>, //"ecdsa" | "ed25519"
+}
+pub async fn get_ssl_key(
+    context: EffectContext,
+    GetSslKeyParams {
+        hostnames,
+        algorithm,
+    }: GetSslKeyParams,
+) -> Result<Pem<PKey<Private>>, Error> {
+    let context = context.deref()?;
+    let package_id = &context.seed.id;
+    let algorithm = algorithm.unwrap_or(Algorithm::Ecdsa);
+
+    let cert = context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            let errfn = |h: &str| Error::new(eyre!("unknown hostname: {h}"), ErrorKind::NotFound);
+            let allowed_hostnames = db
+                .as_public()
+                .as_package_data()
+                .as_idx(package_id)
+                .into_iter()
+                .map(|m| m.as_hosts().as_entries())
+                .flatten_ok()
+                .map_ok(|(_, m)| m.as_addresses().de())
+                .map(|a| a.and_then(|a| a))
+                .flatten_ok()
+                .map_ok(|a| InternedString::from_display(&a))
+                .try_collect::<_, BTreeSet<_>, _>()?;
+            for hostname in &hostnames {
+                if let Some(internal) = hostname
+                    .strip_suffix(".embassy")
+                    .or_else(|| hostname.strip_suffix(".startos"))
+                {
+                    if internal != &**package_id {
+                        return Err(errfn(&*hostname));
+                    }
+                } else {
+                    if !allowed_hostnames.contains(hostname) {
+                        return Err(errfn(&*hostname));
+                    }
+                }
+            }
+            db.as_private_mut()
+                .as_key_store_mut()
+                .as_local_certs_mut()
+                .cert_for(&hostnames)
+        })
+        .await?;
+    let key = match algorithm {
+        Algorithm::Ecdsa => cert.leaf.keys.nistp256,
+        Algorithm::Ed25519 => cert.leaf.keys.ed25519,
+    };
+
+    Ok(Pem(key))
+}

core/startos/src/service/effects/prelude.rs

@@ -0,0 +1,16 @@
+pub use clap::Parser;
+pub use serde::{Deserialize, Serialize};
+pub use ts_rs::TS;
+
+pub use crate::prelude::*;
+use crate::rpc_continuations::Guid;
+pub(super) use crate::service::effects::context::EffectContext;
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ProcedureId {
+    #[serde(default)]
+    #[arg(default_value_t, long)]
+    pub procedure_id: Guid,
+}

core/startos/src/service/effects/store.rs

@@ -0,0 +1,93 @@
+use imbl::vector;
+use imbl_value::json;
+use models::PackageId;
+use patch_db::json_ptr::JsonPointer;
+
+use crate::service::effects::callbacks::CallbackHandler;
+use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetStoreParams {
+    #[ts(optional)]
+    package_id: Option<PackageId>,
+    #[ts(type = "string")]
+    path: JsonPointer,
+    #[ts(optional)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_store(
+    context: EffectContext,
+    GetStoreParams {
+        package_id,
+        path,
+        callback,
+    }: GetStoreParams,
+) -> Result<Value, Error> {
+    let context = context.deref()?;
+    let peeked = context.seed.ctx.db.peek().await;
+    let package_id = package_id.unwrap_or(context.seed.id.clone());
+    let value = peeked
+        .as_private()
+        .as_package_stores()
+        .as_idx(&package_id)
+        .or_not_found(&package_id)?
+        .de()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context.seed.ctx.callbacks.add_get_store(
+            package_id,
+            path.clone(),
+            CallbackHandler::new(&context, callback),
+        );
+    }
+
+    Ok(path
+        .get(&value)
+        .ok_or_else(|| Error::new(eyre!("Did not find value at path"), ErrorKind::NotFound))?
+        .clone())
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SetStoreParams {
+    #[ts(type = "any")]
+    value: Value,
+    #[ts(type = "string")]
+    path: JsonPointer,
+}
+pub async fn set_store(
+    context: EffectContext,
+    SetStoreParams { value, path }: SetStoreParams,
+) -> Result<(), Error> {
+    let context = context.deref()?;
+    let package_id = &context.seed.id;
+    context
+        .seed
+        .ctx
+        .db
+        .mutate(|db| {
+            let model = db
+                .as_private_mut()
+                .as_package_stores_mut()
+                .upsert(package_id, || Ok(json!({})))?;
+            let mut model_value = model.de()?;
+            if model_value.is_null() {
+                model_value = json!({});
+            }
+            path.set(&mut model_value, value, true)
+                .with_kind(ErrorKind::ParseDbField)?;
+            model.ser(&model_value)
+        })
+        .await?;
+
+    if let Some(callbacks) = context.seed.ctx.callbacks.get_store(package_id, &path) {
+        callbacks.call(vector![]).await?;
+    }
+
+    Ok(())
+}

core/startos/src/service/effects/system.rs

@@ -0,0 +1,39 @@
+use crate::service::effects::callbacks::CallbackHandler;
+use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
+use crate::system::SmtpValue;
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct GetSystemSmtpParams {
+    #[arg(skip)]
+    callback: Option<CallbackId>,
+}
+pub async fn get_system_smtp(
+    context: EffectContext,
+    GetSystemSmtpParams { callback }: GetSystemSmtpParams,
+) -> Result<Option<SmtpValue>, Error> {
+    let context = context.deref()?;
+    let res = context
+        .seed
+        .ctx
+        .db
+        .peek()
+        .await
+        .into_public()
+        .into_server_info()
+        .into_smtp()
+        .de()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context
+            .seed
+            .ctx
+            .callbacks
+            .add_get_system_smtp(CallbackHandler::new(&context, callback));
+    }
+
+    Ok(res)
+}

core/startos/src/service/mod.rs

@@ -27,10 +27,7 @@ use crate::progress::{NamedProgress, Progress};
 use crate::rpc_continuations::Guid;
 use crate::s9pk::S9pk;
 use crate::service::service_map::InstallProgressHandles;
-use crate::service::transition::TransitionKind;
 use crate::status::health_check::HealthCheckResult;
-use crate::status::MainStatus;
-use crate::util::actor::background::BackgroundJobQueue;
 use crate::util::actor::concurrent::ConcurrentActor;
 use crate::util::actor::Actor;
 use crate::util::io::create_file;
@@ -43,11 +40,11 @@ pub mod cli;
 mod config;
 mod control;
 mod dependencies;
+pub mod effects;
 pub mod persistent_container;
 mod properties;
 mod rpc;
 mod service_actor;
-pub mod service_effect_handler;
 pub mod service_map;
 mod start_stop;
 mod transition;

core/startos/src/service/persistent_container.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 use std::sync::{Arc, Weak};
 use std::time::Duration;
@@ -6,6 +6,7 @@ use std::time::Duration;
 use futures::future::ready;
 use futures::{Future, FutureExt};
 use helpers::NonDetachingJoinHandle;
+use imbl::Vector;
 use models::{ImageId, ProcedureName, VolumeId};
 use rpc_toolkit::{Empty, Server, ShutdownHandle};
 use serde::de::DeserializeOwned;
@@ -13,8 +14,6 @@ use tokio::process::Command;
 use tokio::sync::{oneshot, watch, Mutex, OnceCell};
 use tracing::instrument;
 
-use super::service_effect_handler::{service_effect_handler, EffectContext};
-use super::transition::{TransitionKind, TransitionState};
 use crate::context::RpcContext;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::idmapped::IdMapped;
@@ -28,7 +27,11 @@ use crate::prelude::*;
 use crate::rpc_continuations::Guid;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::S9pk;
+use crate::service::effects::context::EffectContext;
+use crate::service::effects::handler;
+use crate::service::rpc::{CallbackHandle, CallbackId, CallbackParams};
 use crate::service::start_stop::StartStop;
+use crate::service::transition::{TransitionKind, TransitionState};
 use crate::service::{rpc, RunningStatus, Service};
 use crate::util::io::create_file;
 use crate::util::rpc_client::UnixRpcClient;
@@ -42,6 +45,8 @@ const RPC_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
 pub struct ServiceState {
     // This contains the start time and health check information for when the service is running. Note: Will be overwritting to the db,
     pub(super) running_status: Option<RunningStatus>,
+    // This tracks references to callbacks registered by the running service:
+    pub(super) callbacks: BTreeSet<Arc<CallbackId>>,
     /// Setting this value causes the service actor to try to bring the service to the specified state. This is done in the background job created in ServiceActor::init
     pub(super) desired_state: StartStop,
     /// Override the current desired state for the service during a transition (this is protected by a guard that sets this value to null on drop)
@@ -61,6 +66,7 @@ impl ServiceState {
     pub fn new(desired_state: StartStop) -> Self {
         Self {
             running_status: Default::default(),
+            callbacks: Default::default(),
             temp_desired_state: Default::default(),
             transition_state: Default::default(),
             desired_state,
@@ -308,10 +314,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn init(&self, seed: Weak<Service>) -> Result<(), Error> {
         let socket_server_context = EffectContext::new(seed);
-        let server = Server::new(
-            move || ready(Ok(socket_server_context.clone())),
-            service_effect_handler(),
-        );
+        let server = Server::new(move || ready(Ok(socket_server_context.clone())), handler());
         let path = self
             .lxc_container
             .get()
@@ -430,21 +433,13 @@ impl PersistentContainer {
 
     #[instrument(skip_all)]
     pub async fn start(&self) -> Result<(), Error> {
-        self.execute(
-            Guid::new(),
-            ProcedureName::StartMain,
-            Value::Null,
-            Some(Duration::from_secs(5)), // TODO
-        )
-        .await?;
+        self.rpc_client.request(rpc::Start, Empty {}).await?;
         Ok(())
     }
 
     #[instrument(skip_all)]
     pub async fn stop(&self) -> Result<(), Error> {
-        let timeout: Option<crate::util::serde::Duration> = self
-            .execute(Guid::new(), ProcedureName::StopMain, Value::Null, None)
-            .await?;
+        self.rpc_client.request(rpc::Stop, Empty {}).await?;
         Ok(())
     }
 
@@ -480,6 +475,19 @@ impl PersistentContainer {
             .and_then(from_value)
     }
 
+    #[instrument(skip_all)]
+    pub async fn callback(&self, handle: CallbackHandle, args: Vector<Value>) -> Result<(), Error> {
+        let mut params = None;
+        self.state.send_if_modified(|s| {
+            params = handle.params(&mut s.callbacks, args);
+            params.is_some()
+        });
+        if let Some(params) = params {
+            self._callback(params).await?;
+        }
+        Ok(())
+    }
+
     #[instrument(skip_all)]
     async fn _execute(
         &self,
@@ -523,6 +531,12 @@ impl PersistentContainer {
             fut.await?
         })
     }
+
+    #[instrument(skip_all)]
+    async fn _callback(&self, params: CallbackParams) -> Result<(), Error> {
+        self.rpc_client.notify(rpc::Callback, params).await?;
+        Ok(())
+    }
 }
 
 impl Drop for PersistentContainer {

core/startos/src/service/rpc.rs

@@ -1,5 +1,8 @@
+use std::collections::BTreeSet;
+use std::sync::{Arc, Weak};
 use std::time::Duration;
 
+use imbl::Vector;
 use imbl_value::Value;
 use models::ProcedureName;
 use rpc_toolkit::yajrc::RpcMethod;
@@ -8,6 +11,8 @@ use ts_rs::TS;
 
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
+use crate::service::persistent_container::PersistentContainer;
+use crate::util::Never;
 
 #[derive(Clone)]
 pub struct Init;
@@ -27,6 +32,42 @@ impl serde::Serialize for Init {
     }
 }
 
+#[derive(Clone)]
+pub struct Start;
+impl RpcMethod for Start {
+    type Params = Empty;
+    type Response = ();
+    fn as_str<'a>(&'a self) -> &'a str {
+        "start"
+    }
+}
+impl serde::Serialize for Start {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(self.as_str())
+    }
+}
+
+#[derive(Clone)]
+pub struct Stop;
+impl RpcMethod for Stop {
+    type Params = Empty;
+    type Response = ();
+    fn as_str<'a>(&'a self) -> &'a str {
+        "stop"
+    }
+}
+impl serde::Serialize for Stop {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(self.as_str())
+    }
+}
+
 #[derive(Clone)]
 pub struct Exit;
 impl RpcMethod for Exit {
@@ -104,3 +145,74 @@ impl serde::Serialize for Sandbox {
         serializer.serialize_str(self.as_str())
     }
 }
+
+#[derive(
+    Clone, Copy, Debug, serde::Deserialize, serde::Serialize, TS, PartialEq, Eq, PartialOrd, Ord,
+)]
+#[ts(type = "number")]
+pub struct CallbackId(u64);
+impl CallbackId {
+    pub fn register(self, container: &PersistentContainer) -> CallbackHandle {
+        let this = Arc::new(self);
+        let res = Arc::downgrade(&this);
+        container
+            .state
+            .send_if_modified(|s| s.callbacks.insert(this));
+        CallbackHandle(res)
+    }
+}
+
+pub struct CallbackHandle(Weak<CallbackId>);
+impl CallbackHandle {
+    pub fn is_active(&self) -> bool {
+        self.0.strong_count() > 0
+    }
+    pub fn params(
+        self,
+        registered: &mut BTreeSet<Arc<CallbackId>>,
+        args: Vector<Value>,
+    ) -> Option<CallbackParams> {
+        if let Some(id) = self.0.upgrade() {
+            if let Some(strong) = registered.get(&id) {
+                if Arc::ptr_eq(strong, &id) {
+                    registered.remove(&id);
+                    return Some(CallbackParams::new(&*id, args));
+                }
+            }
+        }
+        None
+    }
+    pub fn take(&mut self) -> Self {
+        Self(std::mem::take(&mut self.0))
+    }
+}
+
+#[derive(Clone, serde::Deserialize, serde::Serialize, TS)]
+pub struct CallbackParams {
+    id: u64,
+    #[ts(type = "any[]")]
+    args: Vector<Value>,
+}
+impl CallbackParams {
+    fn new(id: &CallbackId, args: Vector<Value>) -> Self {
+        Self { id: id.0, args }
+    }
+}
+
+#[derive(Clone)]
+pub struct Callback;
+impl RpcMethod for Callback {
+    type Params = CallbackParams;
+    type Response = Never;
+    fn as_str<'a>(&'a self) -> &'a str {
+        "callback"
+    }
+}
+impl serde::Serialize for Callback {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(self.as_str())
+    }
+}