merge 036, everything broken

core/Cargo.toml

@@ -1,10 +1,3 @@
 [workspace]
 
-members = [
-    "container-init",
-    "helpers",
-    "js-engine",
-    "models",
-    "snapshot-creator",
-    "startos",
-]
+members = ["helpers", "models", "startos"]

core/README.md

@@ -8,9 +8,6 @@
 ## Structure
 
 - `startos`: This contains the core library for StartOS that supports building `startbox`.
-- `container-init` (ignore: deprecated)
-- `js-engine`: This contains the library required to build `deno` to support running `.js` maintainer scripts for v0.3
-- `snapshot-creator`: This contains a binary used to build `v8` runtime snapshots, required for initializing `start-deno`
 - `helpers`: This contains utility functions used across both `startos` and `js-engine`
 - `models`: This contains types that are shared across `startos`, `js-engine`, and `helpers`
 
@@ -24,8 +21,6 @@ several different names for different behaviour:
   `startd` and control it similarly to the UI
 - `start-sdk`: This is a CLI tool that aids in building and packaging services
   you wish to deploy to StartOS
-- `start-deno`: This is a CLI tool invoked by startd to run `.js` maintainer scripts for v0.3
-- `avahi-alias`: This is a CLI tool invoked by startd to create aliases in `avahi` for mDNS
 
 ## Questions
 

core/build-prod.sh

@@ -18,22 +18,22 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
-alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
+if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+	RUSTFLAGS="--cfg tokio_unstable"
+fi
+
+alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl'
 
 set +e
 fail=
 echo "FEATURES=\"$FEATURES\""
 echo "RUSTFLAGS=\"$RUSTFLAGS\""
-if ! rust-gnu-builder sh -c "(cd core && cargo build --release --features avahi-alias,$FEATURES --locked --bin startbox --target=$ARCH-unknown-linux-gnu)"; then 
+if ! rust-musl-builder sh -c "(cd core && cargo build --release $(if [ -n "$FEATURES" ]; then echo "--features $FEATURES"; fi) --locked --bin startbox --target=$ARCH-unknown-linux-musl)"; then 
+	fail=true
+fi
+if ! rust-musl-builder sh -c "(cd core && cargo build --release --no-default-features --features container-runtime,$FEATURES --locked --bin containerbox --target=$ARCH-unknown-linux-musl)"; then 
 	fail=true
 fi
-for ARCH in x86_64 aarch64
-do
-	if ! rust-musl-builder sh -c "(cd core && cargo build --release --locked --bin container-init)"; then 
-		fail=true
-	fi
-done
 set -e
 cd core
 

core/build-v8-snapshot.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# Reason for this being is that we need to create a snapshot for the deno runtime. It wants to pull 3 files from build, and during the creation it gets embedded, but for some 
-# reason during the actual runtime it is looking for them. So this will create a docker in arm that creates the snaphot needed for the arm
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-
-set -e
-shopt -s expand_aliases
-
-if [ -z "$ARCH" ]; then
-	ARCH=$(uname -m)
-fi
-
-USE_TTY=
-if tty -s; then
-	USE_TTY="-it"
-fi
-
-alias 'rust-gnu-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
-
-echo "Building "
-cd ..
-rust-gnu-builder sh -c "(cd core/ && cargo build -p snapshot_creator --release --target=${ARCH}-unknown-linux-gnu)"
-cd -
-
-if [ "$ARCH" = "aarch64" ]; then
-	DOCKER_ARCH='arm64/v8'
-elif [ "$ARCH" = "x86_64" ]; then
-	DOCKER_ARCH='amd64'
-fi
-
-echo "Creating Arm v8 Snapshot"
-docker run  $USE_TTY --platform "linux/${DOCKER_ARCH}" --mount type=bind,src=$(pwd),dst=/mnt ubuntu:22.04 /bin/sh -c "cd /mnt && /mnt/target/${ARCH}-unknown-linux-gnu/release/snapshot_creator"
-sudo chown -R $USER target
-sudo chown -R $USER ~/.cargo
-sudo chown $USER JS_SNAPSHOT.bin
-sudo chmod 0644 JS_SNAPSHOT.bin
-
-sudo mv -f JS_SNAPSHOT.bin ./js-engine/src/artifacts/JS_SNAPSHOT.${ARCH}.bin

core/container-init/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-name = "container-init"
-version = "0.1.0"
-edition = "2021"
-rust = "1.66"
-
-[features]
-dev = []
-metal = []
-sound = []
-unstable = []
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[dependencies]
-async-stream = "0.3"
-# cgroups-rs = "0.2"
-color-eyre = "0.6"
-futures = "0.3"
-serde = { version = "1", features = ["derive", "rc"] }
-serde_json = "1"
-helpers = { path = "../helpers" }
-imbl = "2"
-nix = { version = "0.27", features = ["process", "signal"] }
-tokio = { version = "1", features = ["full"] }
-tokio-stream = { version = "0.1", features = ["io-util", "sync", "net"] }
-tracing = "0.1"
-tracing-error = "0.2"
-tracing-futures = "0.2"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }
-
-[target.'cfg(target_os = "linux")'.dependencies]
-procfs = "0.15"
-
-[profile.test]
-opt-level = 3
-
-[profile.dev.package.backtrace]
-opt-level = 3

core/container-init/src/lib.rs

@@ -1,214 +0,0 @@
-use nix::unistd::Pid;
-use serde::{Deserialize, Serialize, Serializer};
-use yajrc::RpcMethod;
-
-/// Know what the process is called
-#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct ProcessId(pub u32);
-impl From<ProcessId> for Pid {
-    fn from(pid: ProcessId) -> Self {
-        Pid::from_raw(pid.0 as i32)
-    }
-}
-impl From<Pid> for ProcessId {
-    fn from(pid: Pid) -> Self {
-        ProcessId(pid.as_raw() as u32)
-    }
-}
-impl From<i32> for ProcessId {
-    fn from(pid: i32) -> Self {
-        ProcessId(pid as u32)
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct ProcessGroupId(pub u32);
-
-#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[serde(rename_all = "kebab-case")]
-pub enum OutputStrategy {
-    Inherit,
-    Collect,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct RunCommand;
-impl Serialize for RunCommand {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct RunCommandParams {
-    pub gid: Option<ProcessGroupId>,
-    pub command: String,
-    pub args: Vec<String>,
-    pub output: OutputStrategy,
-}
-impl RpcMethod for RunCommand {
-    type Params = RunCommandParams;
-    type Response = ProcessId;
-    fn as_str<'a>(&'a self) -> &'a str {
-        "command"
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum LogLevel {
-    Trace(String),
-    Warn(String),
-    Error(String),
-    Info(String),
-    Debug(String),
-}
-impl LogLevel {
-    pub fn trace(&self) {
-        match self {
-            LogLevel::Trace(x) => tracing::trace!("{}", x),
-            LogLevel::Warn(x) => tracing::warn!("{}", x),
-            LogLevel::Error(x) => tracing::error!("{}", x),
-            LogLevel::Info(x) => tracing::info!("{}", x),
-            LogLevel::Debug(x) => tracing::debug!("{}", x),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Log;
-impl Serialize for Log {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct LogParams {
-    pub gid: Option<ProcessGroupId>,
-    pub level: LogLevel,
-}
-impl RpcMethod for Log {
-    type Params = LogParams;
-    type Response = ();
-    fn as_str<'a>(&'a self) -> &'a str {
-        "log"
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct ReadLineStdout;
-impl Serialize for ReadLineStdout {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ReadLineStdoutParams {
-    pub pid: ProcessId,
-}
-impl RpcMethod for ReadLineStdout {
-    type Params = ReadLineStdoutParams;
-    type Response = String;
-    fn as_str<'a>(&'a self) -> &'a str {
-        "read-line-stdout"
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct ReadLineStderr;
-impl Serialize for ReadLineStderr {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ReadLineStderrParams {
-    pub pid: ProcessId,
-}
-impl RpcMethod for ReadLineStderr {
-    type Params = ReadLineStderrParams;
-    type Response = String;
-    fn as_str<'a>(&'a self) -> &'a str {
-        "read-line-stderr"
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Output;
-impl Serialize for Output {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct OutputParams {
-    pub pid: ProcessId,
-}
-impl RpcMethod for Output {
-    type Params = OutputParams;
-    type Response = String;
-    fn as_str<'a>(&'a self) -> &'a str {
-        "output"
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SendSignal;
-impl Serialize for SendSignal {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SendSignalParams {
-    pub pid: ProcessId,
-    pub signal: u32,
-}
-impl RpcMethod for SendSignal {
-    type Params = SendSignalParams;
-    type Response = ();
-    fn as_str<'a>(&'a self) -> &'a str {
-        "signal"
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct SignalGroup;
-impl Serialize for SignalGroup {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        Serialize::serialize(Self.as_str(), serializer)
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SignalGroupParams {
-    pub gid: ProcessGroupId,
-    pub signal: u32,
-}
-impl RpcMethod for SignalGroup {
-    type Params = SignalGroupParams;
-    type Response = ();
-    fn as_str<'a>(&'a self) -> &'a str {
-        "signal-group"
-    }
-}

core/container-init/src/main.rs

@@ -1,428 +0,0 @@
-use std::collections::BTreeMap;
-use std::ops::DerefMut;
-use std::os::unix::process::ExitStatusExt;
-use std::process::Stdio;
-use std::sync::Arc;
-
-use container_init::{
-    LogParams, OutputParams, OutputStrategy, ProcessGroupId, ProcessId, RunCommandParams,
-    SendSignalParams, SignalGroupParams,
-};
-use futures::StreamExt;
-use helpers::NonDetachingJoinHandle;
-use nix::errno::Errno;
-use nix::sys::signal::Signal;
-use serde::{Deserialize, Serialize};
-use serde_json::json;
-use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
-use tokio::process::{Child, Command};
-use tokio::select;
-use tokio::sync::{watch, Mutex};
-use yajrc::{Id, RpcError};
-
-/// Outputs embedded in the JSONRpc output of the executable.
-#[derive(Debug, Clone, Serialize)]
-#[serde(untagged)]
-enum Output {
-    Command(ProcessId),
-    ReadLineStdout(String),
-    ReadLineStderr(String),
-    Output(String),
-    Log,
-    Signal,
-    SignalGroup,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "method", content = "params", rename_all = "kebab-case")]
-enum Input {
-    /// Run a new command, with the args
-    Command(RunCommandParams),
-    /// Want to log locall on the service rather than the eos
-    Log(LogParams),
-    // /// Get a line of stdout from the command
-    // ReadLineStdout(ReadLineStdoutParams),
-    // /// Get a line of stderr from the command
-    // ReadLineStderr(ReadLineStderrParams),
-    /// Get output of command
-    Output(OutputParams),
-    /// Send the sigterm to the process
-    Signal(SendSignalParams),
-    /// Signal a group of processes
-    SignalGroup(SignalGroupParams),
-}
-
-#[derive(Deserialize)]
-struct IncomingRpc {
-    id: Id,
-    #[serde(flatten)]
-    input: Input,
-}
-
-struct ChildInfo {
-    gid: Option<ProcessGroupId>,
-    child: Arc<Mutex<Option<Child>>>,
-    output: Option<InheritOutput>,
-}
-
-struct InheritOutput {
-    _thread: NonDetachingJoinHandle<()>,
-    stdout: watch::Receiver<String>,
-    stderr: watch::Receiver<String>,
-}
-
-struct HandlerMut {
-    processes: BTreeMap<ProcessId, ChildInfo>,
-    // groups: BTreeMap<ProcessGroupId, Cgroup>,
-}
-
-#[derive(Clone)]
-struct Handler {
-    children: Arc<Mutex<HandlerMut>>,
-}
-impl Handler {
-    fn new() -> Self {
-        Handler {
-            children: Arc::new(Mutex::new(HandlerMut {
-                processes: BTreeMap::new(),
-                // groups: BTreeMap::new(),
-            })),
-        }
-    }
-    async fn handle(&self, req: Input) -> Result<Output, RpcError> {
-        Ok(match req {
-            Input::Command(RunCommandParams {
-                gid,
-                command,
-                args,
-                output,
-            }) => Output::Command(self.command(gid, command, args, output).await?),
-            // Input::ReadLineStdout(ReadLineStdoutParams { pid }) => {
-            //     Output::ReadLineStdout(self.read_line_stdout(pid).await?)
-            // }
-            // Input::ReadLineStderr(ReadLineStderrParams { pid }) => {
-            //     Output::ReadLineStderr(self.read_line_stderr(pid).await?)
-            // }
-            Input::Log(LogParams { gid: _, level }) => {
-                level.trace();
-                Output::Log
-            }
-            Input::Output(OutputParams { pid }) => Output::Output(self.output(pid).await?),
-            Input::Signal(SendSignalParams { pid, signal }) => {
-                self.signal(pid, signal).await?;
-                Output::Signal
-            }
-            Input::SignalGroup(SignalGroupParams { gid, signal }) => {
-                self.signal_group(gid, signal).await?;
-                Output::SignalGroup
-            }
-        })
-    }
-
-    async fn command(
-        &self,
-        gid: Option<ProcessGroupId>,
-        command: String,
-        args: Vec<String>,
-        output: OutputStrategy,
-    ) -> Result<ProcessId, RpcError> {
-        let mut cmd = Command::new(command);
-        cmd.args(args);
-        cmd.kill_on_drop(true);
-        cmd.stdout(Stdio::piped());
-        cmd.stderr(Stdio::piped());
-        let mut child = cmd.spawn().map_err(|e| {
-            let mut err = yajrc::INTERNAL_ERROR.clone();
-            err.data = Some(json!(e.to_string()));
-            err
-        })?;
-        let pid = ProcessId(child.id().ok_or_else(|| {
-            let mut err = yajrc::INTERNAL_ERROR.clone();
-            err.data = Some(json!("Child has no pid"));
-            err
-        })?);
-        let output = match output {
-            OutputStrategy::Inherit => {
-                let (stdout_send, stdout) = watch::channel(String::new());
-                let (stderr_send, stderr) = watch::channel(String::new());
-                if let (Some(child_stdout), Some(child_stderr)) =
-                    (child.stdout.take(), child.stderr.take())
-                {
-                    Some(InheritOutput {
-                        _thread: tokio::spawn(async move {
-                            tokio::join!(
-                                async {
-                                    if let Err(e) = async {
-                                        let mut lines = BufReader::new(child_stdout).lines();
-                                        while let Some(line) = lines.next_line().await? {
-                                            tracing::info!("({}): {}", pid.0, line);
-                                            let _ = stdout_send.send(line);
-                                        }
-                                        Ok::<_, std::io::Error>(())
-                                    }
-                                    .await
-                                    {
-                                        tracing::error!(
-                                            "Error reading stdout of pid {}: {}",
-                                            pid.0,
-                                            e
-                                        );
-                                    }
-                                },
-                                async {
-                                    if let Err(e) = async {
-                                        let mut lines = BufReader::new(child_stderr).lines();
-                                        while let Some(line) = lines.next_line().await? {
-                                            tracing::warn!("({}): {}", pid.0, line);
-                                            let _ = stderr_send.send(line);
-                                        }
-                                        Ok::<_, std::io::Error>(())
-                                    }
-                                    .await
-                                    {
-                                        tracing::error!(
-                                            "Error reading stdout of pid {}: {}",
-                                            pid.0,
-                                            e
-                                        );
-                                    }
-                                }
-                            );
-                        })
-                        .into(),
-                        stdout,
-                        stderr,
-                    })
-                } else {
-                    None
-                }
-            }
-            OutputStrategy::Collect => None,
-        };
-        self.children.lock().await.processes.insert(
-            pid,
-            ChildInfo {
-                gid,
-                child: Arc::new(Mutex::new(Some(child))),
-                output,
-            },
-        );
-        Ok(pid)
-    }
-
-    async fn output(&self, pid: ProcessId) -> Result<String, RpcError> {
-        let not_found = || {
-            let mut err = yajrc::INTERNAL_ERROR.clone();
-            err.data = Some(json!(format!("Child with pid {} not found", pid.0)));
-            err
-        };
-        let mut child = {
-            self.children
-                .lock()
-                .await
-                .processes
-                .get(&pid)
-                .ok_or_else(not_found)?
-                .child
-                .clone()
-        }
-        .lock_owned()
-        .await;
-        if let Some(child) = child.take() {
-            let output = child.wait_with_output().await?;
-            if output.status.success() {
-                Ok(String::from_utf8(output.stdout).map_err(|_| yajrc::PARSE_ERROR)?)
-            } else {
-                Err(RpcError {
-                    code: output
-                        .status
-                        .code()
-                        .or_else(|| output.status.signal().map(|s| 128 + s))
-                        .unwrap_or(0),
-                    message: "Command failed".into(),
-                    data: Some(json!(String::from_utf8(if output.stderr.is_empty() {
-                        output.stdout
-                    } else {
-                        output.stderr
-                    })
-                    .map_err(|_| yajrc::PARSE_ERROR)?)),
-                })
-            }
-        } else {
-            Err(not_found())
-        }
-    }
-
-    async fn signal(&self, pid: ProcessId, signal: u32) -> Result<(), RpcError> {
-        let not_found = || {
-            let mut err = yajrc::INTERNAL_ERROR.clone();
-            err.data = Some(json!(format!("Child with pid {} not found", pid.0)));
-            err
-        };
-
-        Self::killall(pid, Signal::try_from(signal as i32)?)?;
-
-        if signal == 9 {
-            self.children
-                .lock()
-                .await
-                .processes
-                .remove(&pid)
-                .ok_or_else(not_found)?;
-        }
-        Ok(())
-    }
-
-    async fn signal_group(&self, gid: ProcessGroupId, signal: u32) -> Result<(), RpcError> {
-        let mut to_kill = Vec::new();
-        {
-            let mut children_ref = self.children.lock().await;
-            let children = std::mem::take(&mut children_ref.deref_mut().processes);
-            for (pid, child_info) in children {
-                if child_info.gid == Some(gid) {
-                    to_kill.push(pid);
-                } else {
-                    children_ref.processes.insert(pid, child_info);
-                }
-            }
-        }
-        for pid in to_kill {
-            tracing::info!("Killing pid {}", pid.0);
-            Self::killall(pid, Signal::try_from(signal as i32)?)?;
-        }
-
-        Ok(())
-    }
-
-    fn killall(pid: ProcessId, signal: Signal) -> Result<(), RpcError> {
-        for proc in procfs::process::all_processes()? {
-            let stat = proc?.stat()?;
-            if ProcessId::from(stat.ppid) == pid {
-                Self::killall(stat.pid.into(), signal)?;
-            }
-        }
-        if let Err(e) = nix::sys::signal::kill(pid.into(), Some(signal)) {
-            if e != Errno::ESRCH {
-                tracing::error!("Failed to kill pid {}: {}", pid.0, e);
-            }
-        }
-        Ok(())
-    }
-
-    async fn graceful_exit(self) {
-        let kill_all = futures::stream::iter(
-            std::mem::take(&mut self.children.lock().await.deref_mut().processes).into_iter(),
-        )
-        .for_each_concurrent(None, |(pid, child)| async move {
-            let _ = Self::killall(pid, Signal::SIGTERM);
-            if let Some(child) = child.child.lock().await.take() {
-                let _ = child.wait_with_output().await;
-            }
-        });
-        kill_all.await
-    }
-}
-
-#[tokio::main]
-async fn main() {
-    use tokio::signal::unix::{signal, SignalKind};
-    let mut sigint = signal(SignalKind::interrupt()).unwrap();
-    let mut sigterm = signal(SignalKind::terminate()).unwrap();
-    let mut sigquit = signal(SignalKind::quit()).unwrap();
-    let mut sighangup = signal(SignalKind::hangup()).unwrap();
-
-    use tracing_error::ErrorLayer;
-    use tracing_subscriber::prelude::*;
-    use tracing_subscriber::{fmt, EnvFilter};
-
-    let filter_layer = EnvFilter::new("container_init=debug");
-    let fmt_layer = fmt::layer().with_target(true);
-
-    tracing_subscriber::registry()
-        .with(filter_layer)
-        .with(fmt_layer)
-        .with(ErrorLayer::default())
-        .init();
-    color_eyre::install().unwrap();
-
-    let handler = Handler::new();
-    let handler_thread = async {
-        let listener = tokio::net::UnixListener::bind("/start9/sockets/rpc.sock")?;
-        loop {
-            let (stream, _) = listener.accept().await?;
-            let (r, w) = stream.into_split();
-            let mut lines = BufReader::new(r).lines();
-            let handler = handler.clone();
-            tokio::spawn(async move {
-                let w = Arc::new(Mutex::new(w));
-                while let Some(line) = lines.next_line().await.transpose() {
-                    let handler = handler.clone();
-                    let w = w.clone();
-                    tokio::spawn(async move {
-                        if let Err(e) = async {
-                            let req = serde_json::from_str::<IncomingRpc>(&line?)?;
-                            match handler.handle(req.input).await {
-                                Ok(output) => {
-                                    if  w.lock().await.write_all(
-                                        format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "result": output }))
-                                            .as_bytes(),
-                                    )
-                                    .await.is_err() {
-                                        tracing::error!("Error sending to {id:?}", id = req.id);
-                                    }
-                                }
-                                Err(e) =>
-                                if  w
-                                    .lock()
-                                    .await
-                                    .write_all(
-                                        format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "error": e }))
-                                            .as_bytes(),
-                                    )
-                                    .await.is_err() {
-
-                                        tracing::error!("Handle + Error sending to {id:?}", id = req.id);
-                                    },
-                            }
-                            Ok::<_, color_eyre::Report>(())
-                        }
-                        .await
-                        {
-                            tracing::error!("Error parsing RPC request: {}", e);
-                            tracing::debug!("{:?}", e);
-                        }
-                    });
-                }
-                Ok::<_, std::io::Error>(())
-            });
-        }
-        #[allow(unreachable_code)]
-        Ok::<_, std::io::Error>(())
-    };
-
-    select! {
-        res = handler_thread => {
-            match res {
-                Ok(()) => tracing::debug!("Done with inputs/outputs"),
-                Err(e) => {
-                    tracing::error!("Error reading RPC input: {}", e);
-                    tracing::debug!("{:?}", e);
-                }
-            }
-        },
-        _ = sigint.recv() => {
-            tracing::debug!("SIGINT");
-        },
-        _ = sigterm.recv() => {
-            tracing::debug!("SIGTERM");
-        },
-        _ = sigquit.recv() => {
-            tracing::debug!("SIGQUIT");
-        },
-        _ = sighangup.recv() => {
-            tracing::debug!("SIGHUP");
-        }
-    }
-    handler.graceful_exit().await;
-    ::std::process::exit(0)
-}

core/helpers/Cargo.toml

@@ -6,15 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1.64"
 color-eyre = "0.6.2"
 futures = "0.3.28"
 lazy_async_pool = "0.3.3"
 models = { path = "../models" }
 pin-project = "1.1.3"
+rpc-toolkit = "0.2.3"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0.1.14", features = ["io-util", "sync"] }
 tracing = "0.1.39"
-yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }

core/helpers/src/lib.rs

@@ -11,13 +11,9 @@ use tokio::sync::oneshot;
 use tokio::task::{JoinError, JoinHandle, LocalSet};
 
 mod byte_replacement_reader;
-mod os_api;
-mod rpc_client;
 mod rsync;
 mod script_dir;
 pub use byte_replacement_reader::*;
-pub use os_api::*;
-pub use rpc_client::{RpcClient, UnixRpcClient};
 pub use rsync::*;
 pub use script_dir::*;
 

core/helpers/src/rsync.rs

@@ -70,9 +70,6 @@ impl Rsync {
         for exclude in options.exclude {
             cmd.arg(format!("--exclude={}", exclude));
         }
-        if options.no_permissions {
-            cmd.arg("--no-perms");
-        }
         let mut command = cmd
             .arg("-actAXH")
             .arg("--info=progress2")

core/install-cli.sh

@@ -12,7 +12,4 @@ if [ -z "$PLATFORM" ]; then
   export PLATFORM=$(uname -m)
 fi
 
-cargo install --path=./startos --no-default-features --features=js_engine,sdk,cli --locked
-startbox_loc=$(which startbox)
-ln -sf $startbox_loc $(dirname $startbox_loc)/start-cli
-ln -sf $startbox_loc $(dirname $startbox_loc)/start-sdk
+cargo install --path=./startos --no-default-features --features=cli,docker --bin start-cli --locked

core/js-engine/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "js-engine"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-async-trait = "0.1.74"
-dashmap = "5.5.3"
-deno_core = "=0.222.0"
-deno_ast = { version = "=0.29.5", features = ["transpiling"] }
-container-init = { path = "../container-init" }
-reqwest = { version = "0.11.22" }
-sha2 = "0.10.8"
-itertools = "0.11.0"
-lazy_static = "1.4.0"
-models = { path = "../models" }
-helpers = { path = "../helpers" }
-serde = { version = "1.0", features = ["derive", "rc"] }
-serde_json = "1.0"
-tokio = { version = "1", features = ["full"] }
-tracing = "0.1"
-pin-project = "1"

core/js-engine/src/artifacts/loadModule.js

@@ -1,369 +0,0 @@
-import Deno from "/deno_global.js";
-import * as mainModule from "/embassy.js";
-function requireParam(param) {
-  throw new Error(`Missing required parameter ${param}`);
-}
-
-const callbackName = (() => {
-  let count = 0;
-  return () => `callback${count++}${Math.floor(Math.random() * 100000)}`;
-})();
-
-const callbackMapping = {};
-const registerCallback = (fn) => {
-  const uuid = callbackName(); // TODO
-  callbackMapping[uuid] = fn;
-  return uuid;
-};
-
-/**
- * This is using the simplified json pointer spec, using no escapes and arrays
- * @param {object} obj
- * @param {string} pointer
- * @returns
- */
-function jsonPointerValue(obj, pointer) {
-  const paths = pointer.substring(1).split("/");
-  for (const path of paths) {
-    if (obj == null) {
-      return null;
-    }
-    obj = (obj || {})[path];
-  }
-  return obj;
-}
-
-function maybeDate(value) {
-  if (!value) return value;
-  return new Date(value);
-}
-const writeFile = (
-  {
-    path = requireParam("path"),
-    volumeId = requireParam("volumeId"),
-    toWrite = requireParam("toWrite"),
-  } = requireParam("options"),
-) => Deno.core.opAsync("write_file", volumeId, path, toWrite);
-
-const readFile = (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => Deno.core.opAsync("read_file", volumeId, path);
-
-const runDaemon = (
-  { command = requireParam("command"), args = [] } = requireParam("options"),
-) => {
-  let id = Deno.core.opAsync("start_command", command, args, "inherit", null);
-  let processId = id.then((x) => x.processId);
-  let waitPromise = null;
-  return {
-    processId,
-    async wait() {
-      waitPromise = waitPromise ||
-        Deno.core.opAsync("wait_command", await processId);
-      return waitPromise;
-    },
-    async term(signal = 15) {
-      return Deno.core.opAsync("send_signal", await processId, 15);
-    },
-  };
-};
-const runCommand = async (
-  { command = requireParam("command"), args = [], timeoutMillis = 30000 } =
-    requireParam("options"),
-) => {
-  let id = Deno.core.opAsync(
-    "start_command",
-    command,
-    args,
-    "collect",
-    timeoutMillis,
-  );
-  let pid = id.then((x) => x.processId);
-  return Deno.core.opAsync("wait_command", await pid);
-};
-const bindLocal = async (
-  {
-    internalPort = requireParam("internalPort"),
-    name = requireParam("name"),
-    externalPort = requireParam("externalPort"),
-  } = requireParam("options"),
-) => {
-  return Deno.core.opAsync("bind_local", internalPort, {
-    id: name,
-    externalPort,
-  });
-};
-const bindTor = async (
-  {
-    internalPort = requireParam("internalPort"),
-    name = requireParam("name"),
-    externalPort = requireParam("externalPort"),
-  } = requireParam("options"),
-) => {
-  return Deno.core.opAsync("bind_onion", internalPort, {
-    id: name,
-    externalPort,
-  });
-};
-
-const signalGroup = async (
-  { gid = requireParam("gid"), signal = requireParam("signal") } = requireParam(
-    "gid and signal",
-  ),
-) => {
-  return Deno.core.opAsync("signal_group", gid, signal);
-};
-const sleep = (timeMs = requireParam("timeMs")) =>
-  Deno.core.opAsync("sleep", timeMs);
-
-const rename = (
-  {
-    srcVolume = requireParam("srcVolume"),
-    dstVolume = requirePapram("dstVolume"),
-    srcPath = requireParam("srcPath"),
-    dstPath = requireParam("dstPath"),
-  } = requireParam("options"),
-) => Deno.core.opAsync("rename", srcVolume, srcPath, dstVolume, dstPath);
-const metadata = async (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => {
-  const data = await Deno.core.opAsync("metadata", volumeId, path);
-  return {
-    ...data,
-    modified: maybeDate(data.modified),
-    created: maybeDate(data.created),
-    accessed: maybeDate(data.accessed),
-  };
-};
-const removeFile = (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => Deno.core.opAsync("remove_file", volumeId, path);
-const isSandboxed = () => Deno.core.ops["is_sandboxed"]();
-
-const writeJsonFile = (
-  {
-    volumeId = requireParam("volumeId"),
-    path = requireParam("path"),
-    toWrite = requireParam("toWrite"),
-  } = requireParam("options"),
-) =>
-  writeFile({
-    volumeId,
-    path,
-    toWrite: JSON.stringify(toWrite),
-  });
-
-const chown = async (
-  {
-    volumeId = requireParam("volumeId"),
-    path = requireParam("path"),
-    uid = requireParam("uid"),
-  } = requireParam("options"),
-) => {
-  return await Deno.core.opAsync("chown", volumeId, path, uid);
-};
-
-const chmod = async (
-  {
-    volumeId = requireParam("volumeId"),
-    path = requireParam("path"),
-    mode = requireParam("mode"),
-  } = requireParam("options"),
-) => {
-  return await Deno.core.opAsync("chmod", volumeId, path, mode);
-};
-const readJsonFile = async (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => JSON.parse(await readFile({ volumeId, path }));
-const createDir = (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => Deno.core.opAsync("create_dir", volumeId, path);
-
-const readDir = (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => Deno.core.opAsync("read_dir", volumeId, path);
-const removeDir = (
-  { volumeId = requireParam("volumeId"), path = requireParam("path") } =
-    requireParam("options"),
-) => Deno.core.opAsync("remove_dir", volumeId, path);
-const trace = (whatToTrace = requireParam("whatToTrace")) =>
-  Deno.core.opAsync("log_trace", whatToTrace);
-const warn = (whatToTrace = requireParam("whatToTrace")) =>
-  Deno.core.opAsync("log_warn", whatToTrace);
-const error = (whatToTrace = requireParam("whatToTrace")) =>
-  Deno.core.opAsync("log_error", whatToTrace);
-const debug = (whatToTrace = requireParam("whatToTrace")) =>
-  Deno.core.opAsync("log_debug", whatToTrace);
-const info = (whatToTrace = requireParam("whatToTrace")) =>
-  Deno.core.opAsync("log_info", whatToTrace);
-const fetch = async (url = requireParam("url"), options = null) => {
-  const { body, ...response } = await Deno.core.opAsync("fetch", url, options);
-  const textValue = Promise.resolve(body);
-  return {
-    ...response,
-    text() {
-      return textValue;
-    },
-    json() {
-      return textValue.then((x) => JSON.parse(x));
-    },
-  };
-};
-
-const runRsync = (
-  {
-    srcVolume = requireParam("srcVolume"),
-    dstVolume = requireParam("dstVolume"),
-    srcPath = requireParam("srcPath"),
-    dstPath = requireParam("dstPath"),
-    options = requireParam("options"),
-  } = requireParam("options"),
-) => {
-  let id = Deno.core.opAsync(
-    "rsync",
-    srcVolume,
-    srcPath,
-    dstVolume,
-    dstPath,
-    options,
-  );
-  let waitPromise = null;
-  return {
-    async id() {
-      return id;
-    },
-    async wait() {
-      waitPromise = waitPromise || Deno.core.opAsync("rsync_wait", await id);
-      return waitPromise;
-    },
-    async progress() {
-      return Deno.core.opAsync("rsync_progress", await id);
-    },
-  };
-};
-
-globalThis.runCallback = (uuid, args) => callbackMapping[uuid](...args);
-
-const getServiceConfig = async (
-  {
-    serviceId = requireParam("serviceId"),
-    configPath = requireParam("configPath"),
-    onChange = requireParam("onChange"),
-  } = requireParam("options"),
-) => {
-  return await Deno.core.opAsync(
-    "get_service_config",
-    serviceId,
-    configPath,
-    registerCallback(onChange),
-  );
-};
-
-const started = () => Deno.core.ops.set_started();
-const restart = () => Deno.core.opAsync("restart");
-const start = () => Deno.core.opAsync("start");
-const stop = () => Deno.core.opAsync("stop");
-
-const currentFunction = Deno.core.ops.current_function();
-const input = Deno.core.ops.get_input();
-const variable_args = Deno.core.ops.get_variable_args();
-const setState = (x) => Deno.core.ops.set_value(x);
-const effects = {
-  bindLocal,
-  bindTor,
-  chmod,
-  chown,
-  createDir,
-  debug,
-  diskUsage,
-  error,
-  fetch,
-  getServiceConfig,
-  getServiceConfig,
-  info,
-  isSandboxed,
-  metadata,
-  readDir,
-  readFile,
-  readJsonFile,
-  removeDir,
-  removeFile,
-  rename,
-  restart,
-  runCommand,
-  runDaemon,
-  runRsync,
-  signalGroup,
-  sleep,
-  start,
-  stop,
-  trace,
-  warn,
-  writeFile,
-  writeJsonFile,
-  restart,
-  start,
-  stop,
-};
-const fnSpecificArgs = {
-  main: { started },
-};
-
-const defaults = {
-  handleSignal: (effects, { gid, signal }) => {
-    return effects.signalGroup({ gid, signal });
-  },
-};
-
-function safeToString(fn, orValue = "") {
-  try {
-    return fn();
-  } catch (e) {
-    return orValue;
-  }
-}
-
-const apiVersion = mainModule?.version || defaults?.version || 0;
-const runFunction = jsonPointerValue(mainModule, currentFunction) ||
-  jsonPointerValue(defaults, currentFunction);
-const extraArgs = jsonPointerValue(fnSpecificArgs, currentFunction) || {};
-(async () => {
-  const answer = await (async () => {
-    if (typeof runFunction !== "function") {
-      error(`Expecting ${currentFunction} to be a function`);
-      throw new Error(`Expecting ${currentFunction} to be a function`);
-    }
-  })()
-    .then(() => {
-      switch (apiVersion) {
-        case 0:
-          return runFunction(effects, input, ...variable_args);
-        case 1:
-          return runFunction({
-            effects,
-            input,
-            args: variable_args,
-            ...extraArgs,
-          });
-        default:
-          return { error: `Unknown API version ${apiVersion}` };
-      }
-    })
-    .catch((e) => {
-      if ("error" in e) return e;
-      if ("error-code" in e) return e;
-      return {
-        error: safeToString(
-          () => e.toString(),
-          "Error Not able to be stringified",
-        ),
-      };
-    });
-  await setState(answer);
-})();

core/models/Cargo.toml

@@ -15,6 +15,7 @@ emver = { version = "0.1", git = "https://github.com/Start9Labs/emver-rs.git", f
   "serde",
 ] }
 ipnet = "2.8.0"
+num_enum = "0.7.1"
 openssl = { version = "0.10.57", features = ["vendored"] }
 patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
   "trace",
@@ -31,8 +32,9 @@ sqlx = { version = "0.7.2", features = [
   "postgres",
 ] }
 ssh-key = "0.6.2"
+ts-rs = "7"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
-torut = "0.2.1"
+torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies" }
 tracing = "0.1.39"
 yasi = "0.1.5"

core/models/src/errors.rs

@@ -1,14 +1,19 @@
-use std::fmt::Display;
+use std::fmt::{Debug, Display};
 
 use color_eyre::eyre::eyre;
+use num_enum::TryFromPrimitive;
 use patch_db::Revision;
 use rpc_toolkit::hyper::http::uri::InvalidUri;
 use rpc_toolkit::reqwest;
-use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::yajrc::{
+    RpcError, INVALID_PARAMS_ERROR, INVALID_REQUEST_ERROR, METHOD_NOT_FOUND_ERROR, PARSE_ERROR,
+};
+use serde::{Deserialize, Serialize};
 
 use crate::InvalidId;
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, TryFromPrimitive)]
+#[repr(i32)]
 pub enum ErrorKind {
     Unknown = 1,
     Filesystem = 2,
@@ -81,6 +86,8 @@ pub enum ErrorKind {
     CpuSettings = 69,
     Firmware = 70,
     Timeout = 71,
+    Lxc = 72,
+    Cancelled = 73,
 }
 impl ErrorKind {
     pub fn as_str(&self) -> &'static str {
@@ -157,6 +164,8 @@ impl ErrorKind {
             CpuSettings => "CPU Settings Error",
             Firmware => "Firmware Error",
             Timeout => "Timeout Error",
+            Lxc => "LXC Error",
+            Cancelled => "Cancelled",
         }
     }
 }
@@ -186,6 +195,22 @@ impl Error {
             revision: None,
         }
     }
+    pub fn clone_output(&self) -> Self {
+        Error {
+            source: ErrorData {
+                details: format!("{}", self.source),
+                debug: format!("{:?}", self.source),
+            }
+            .into(),
+            kind: self.kind,
+            revision: self.revision.clone(),
+        }
+    }
+}
+impl From<std::convert::Infallible> for Error {
+    fn from(value: std::convert::Infallible) -> Self {
+        match value {}
+    }
 }
 impl From<InvalidId> for Error {
     fn from(err: InvalidId) -> Self {
@@ -300,6 +325,53 @@ impl From<patch_db::value::Error> for Error {
     }
 }
 
+#[derive(Clone, Deserialize, Serialize)]
+pub struct ErrorData {
+    pub details: String,
+    pub debug: String,
+}
+impl Display for ErrorData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Display::fmt(&self.details, f)
+    }
+}
+impl Debug for ErrorData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Display::fmt(&self.debug, f)
+    }
+}
+impl std::error::Error for ErrorData {}
+impl From<&RpcError> for ErrorData {
+    fn from(value: &RpcError) -> Self {
+        Self {
+            details: value
+                .data
+                .as_ref()
+                .and_then(|d| {
+                    d.as_object()
+                        .and_then(|d| {
+                            d.get("details")
+                                .and_then(|d| d.as_str().map(|s| s.to_owned()))
+                        })
+                        .or_else(|| d.as_str().map(|s| s.to_owned()))
+                })
+                .unwrap_or_else(|| value.message.clone().into_owned()),
+            debug: value
+                .data
+                .as_ref()
+                .and_then(|d| {
+                    d.as_object()
+                        .and_then(|d| {
+                            d.get("debug")
+                                .and_then(|d| d.as_str().map(|s| s.to_owned()))
+                        })
+                        .or_else(|| d.as_str().map(|s| s.to_owned()))
+                })
+                .unwrap_or_else(|| value.message.clone().into_owned()),
+        }
+    }
+}
+
 impl From<Error> for RpcError {
     fn from(e: Error) -> Self {
         let mut data_object = serde_json::Map::with_capacity(3);
@@ -318,10 +390,40 @@ impl From<Error> for RpcError {
         RpcError {
             code: e.kind as i32,
             message: e.kind.as_str().into(),
-            data: Some(data_object.into()),
+            data: Some(
+                match serde_json::to_value(&ErrorData {
+                    details: format!("{}", e.source),
+                    debug: format!("{:?}", e.source),
+                }) {
+                    Ok(a) => a,
+                    Err(e) => {
+                        tracing::warn!("Error serializing revision for Error object: {}", e);
+                        serde_json::Value::Null
+                    }
+                },
+            ),
         }
     }
 }
+impl From<RpcError> for Error {
+    fn from(e: RpcError) -> Self {
+        Error::new(
+            ErrorData::from(&e),
+            if let Ok(kind) = e.code.try_into() {
+                kind
+            } else if e.code == METHOD_NOT_FOUND_ERROR.code {
+                ErrorKind::NotFound
+            } else if e.code == PARSE_ERROR.code
+                || e.code == INVALID_PARAMS_ERROR.code
+                || e.code == INVALID_REQUEST_ERROR.code
+            {
+                ErrorKind::Deserialization
+            } else {
+                ErrorKind::Unknown
+            },
+        )
+    }
+}
 
 #[derive(Debug, Default)]
 pub struct ErrorCollection(Vec<Error>);
@@ -377,10 +479,7 @@ where
     Self: Sized,
 {
     fn with_kind(self, kind: ErrorKind) -> Result<T, Error>;
-    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display + Send + Sync + 'static>(
-        self,
-        f: F,
-    ) -> Result<T, Error>;
+    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display>(self, f: F) -> Result<T, Error>;
 }
 impl<T, E> ResultExt<T, E> for Result<T, E>
 where
@@ -394,10 +493,7 @@ where
         })
     }
 
-    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display + Send + Sync + 'static>(
-        self,
-        f: F,
-    ) -> Result<T, Error> {
+    fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display>(self, f: F) -> Result<T, Error> {
         self.map_err(|e| {
             let (kind, ctx) = f(&e);
             let source = color_eyre::eyre::Error::from(e);
@@ -411,6 +507,29 @@ where
         })
     }
 }
+impl<T> ResultExt<T, Error> for Result<T, Error> {
+    fn with_kind(self, kind: ErrorKind) -> Result<T, Error> {
+        self.map_err(|e| Error {
+            source: e.source,
+            kind,
+            revision: e.revision,
+        })
+    }
+
+    fn with_ctx<F: FnOnce(&Error) -> (ErrorKind, D), D: Display>(self, f: F) -> Result<T, Error> {
+        self.map_err(|e| {
+            let (kind, ctx) = f(&e);
+            let source = e.source;
+            let ctx = format!("{}: {}", ctx, source);
+            let source = source.wrap_err(ctx);
+            Error {
+                kind,
+                source,
+                revision: e.revision,
+            }
+        })
+    }
+}
 
 pub trait OptionExt<T>
 where

core/models/src/id/action.rs

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::{Id, InvalidId};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, ts_rs::TS)]
 pub struct ActionId(Id);
 impl FromStr for ActionId {
     type Err = InvalidId;

core/models/src/id/health_check.rs

@@ -1,16 +1,23 @@
 use std::path::Path;
+use std::str::FromStr;
 
 use serde::{Deserialize, Deserializer, Serialize};
 
-use crate::Id;
+use crate::{Id, InvalidId};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, ts_rs::TS)]
 pub struct HealthCheckId(Id);
 impl std::fmt::Display for HealthCheckId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
+impl FromStr for HealthCheckId {
+    type Err = InvalidId;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Id::from_str(s).map(HealthCheckId)
+    }
+}
 impl AsRef<str> for HealthCheckId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()

core/models/src/id/host.rs

@@ -2,52 +2,63 @@ use std::path::Path;
 use std::str::FromStr;
 
 use serde::{Deserialize, Deserializer, Serialize};
+use yasi::InternedString;
 
 use crate::{Id, InvalidId};
 
 #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
-pub struct InterfaceId(Id);
-impl FromStr for InterfaceId {
+pub struct HostId(Id);
+impl FromStr for HostId {
     type Err = InvalidId;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(Self(Id::try_from(s.to_owned())?))
     }
 }
-impl From<Id> for InterfaceId {
+impl From<Id> for HostId {
     fn from(id: Id) -> Self {
         Self(id)
     }
 }
-impl std::fmt::Display for InterfaceId {
+impl From<HostId> for Id {
+    fn from(value: HostId) -> Self {
+        value.0
+    }
+}
+impl From<HostId> for InternedString {
+    fn from(value: HostId) -> Self {
+        value.0.into()
+    }
+}
+impl std::fmt::Display for HostId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
-impl std::ops::Deref for InterfaceId {
+impl std::ops::Deref for HostId {
     type Target = str;
     fn deref(&self) -> &Self::Target {
         &*self.0
     }
 }
-impl AsRef<str> for InterfaceId {
+impl AsRef<str> for HostId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<'de> Deserialize<'de> for InterfaceId {
+impl<'de> Deserialize<'de> for HostId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
-        Ok(InterfaceId(Deserialize::deserialize(deserializer)?))
+        Ok(HostId(Deserialize::deserialize(deserializer)?))
     }
 }
-impl AsRef<Path> for InterfaceId {
+impl AsRef<Path> for HostId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref().as_ref()
     }
 }
-impl<'q> sqlx::Encode<'q, sqlx::Postgres> for InterfaceId {
+impl<'q> sqlx::Encode<'q, sqlx::Postgres> for HostId {
     fn encode_by_ref(
         &self,
         buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
@@ -55,7 +66,7 @@ impl<'q> sqlx::Encode<'q, sqlx::Postgres> for InterfaceId {
         <&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
     }
 }
-impl sqlx::Type<sqlx::Postgres> for InterfaceId {
+impl sqlx::Type<sqlx::Postgres> for HostId {
     fn type_info() -> sqlx::postgres::PgTypeInfo {
         <&str as sqlx::Type<sqlx::Postgres>>::type_info()
     }

core/models/src/id/image.rs

@@ -1,12 +1,18 @@
 use std::fmt::Debug;
+use std::path::Path;
 use std::str::FromStr;
 
 use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::{Id, InvalidId, PackageId, Version};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, ts_rs::TS)]
 pub struct ImageId(Id);
+impl AsRef<Path> for ImageId {
+    fn as_ref(&self) -> &Path {
+        self.0.as_ref().as_ref()
+    }
+}
 impl std::fmt::Display for ImageId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)

core/models/src/id/mod.rs

@@ -1,4 +1,5 @@
 use std::borrow::Borrow;
+use std::str::FromStr;
 
 use regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -7,8 +8,8 @@ use yasi::InternedString;
 mod action;
 mod address;
 mod health_check;
+mod host;
 mod image;
-mod interface;
 mod invalid_id;
 mod package;
 mod volume;
@@ -16,8 +17,8 @@ mod volume;
 pub use action::ActionId;
 pub use address::AddressId;
 pub use health_check::HealthCheckId;
+pub use host::HostId;
 pub use image::ImageId;
-pub use interface::InterfaceId;
 pub use invalid_id::InvalidId;
 pub use package::{PackageId, SYSTEM_PACKAGE_ID};
 pub use volume::VolumeId;
@@ -27,12 +28,12 @@ lazy_static::lazy_static! {
     pub static ref SYSTEM_ID: Id = Id(InternedString::intern("x_system"));
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
-pub struct Id(InternedString);
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default, ts_rs::TS)]
+pub struct Id(#[ts(type = "string")] InternedString);
 impl TryFrom<InternedString> for Id {
     type Error = InvalidId;
     fn try_from(value: InternedString) -> Result<Self, Self::Error> {
-        if ID_REGEX.is_match(&*value) {
+        if ID_REGEX.is_match(&value) {
             Ok(Id(value))
         } else {
             Err(InvalidId)
@@ -52,17 +53,28 @@ impl TryFrom<String> for Id {
 impl TryFrom<&str> for Id {
     type Error = InvalidId;
     fn try_from(value: &str) -> Result<Self, Self::Error> {
-        if ID_REGEX.is_match(&value) {
+        if ID_REGEX.is_match(value) {
             Ok(Id(InternedString::intern(value)))
         } else {
             Err(InvalidId)
         }
     }
 }
+impl FromStr for Id {
+    type Err = InvalidId;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::try_from(s)
+    }
+}
+impl From<Id> for InternedString {
+    fn from(value: Id) -> Self {
+        value.0
+    }
+}
 impl std::ops::Deref for Id {
     type Target = str;
     fn deref(&self) -> &Self::Target {
-        &*self.0
+        &self.0
     }
 }
 impl std::fmt::Display for Id {
@@ -72,7 +84,7 @@ impl std::fmt::Display for Id {
 }
 impl AsRef<str> for Id {
     fn as_ref(&self) -> &str {
-        &*self.0
+        &self.0
     }
 }
 impl Borrow<str> for Id {
@@ -94,7 +106,7 @@ impl Serialize for Id {
     where
         Ser: Serializer,
     {
-        serializer.serialize_str(&*self)
+        serializer.serialize_str(self)
     }
 }
 impl<'q> sqlx::Encode<'q, sqlx::Postgres> for Id {

core/models/src/id/package.rs

@@ -3,13 +3,15 @@ use std::path::Path;
 use std::str::FromStr;
 
 use serde::{Deserialize, Serialize, Serializer};
+use ts_rs::TS;
+use yasi::InternedString;
 
 use crate::{Id, InvalidId, SYSTEM_ID};
 
 lazy_static::lazy_static! {
     pub static ref SYSTEM_PACKAGE_ID: PackageId = PackageId(SYSTEM_ID.clone());
 }
-#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, TS)]
 pub struct PackageId(Id);
 impl FromStr for PackageId {
     type Err = InvalidId;
@@ -22,10 +24,20 @@ impl From<Id> for PackageId {
         PackageId(id)
     }
 }
+impl From<PackageId> for Id {
+    fn from(value: PackageId) -> Self {
+        value.0
+    }
+}
+impl From<PackageId> for InternedString {
+    fn from(value: PackageId) -> Self {
+        value.0.into()
+    }
+}
 impl std::ops::Deref for PackageId {
     type Target = str;
     fn deref(&self) -> &Self::Target {
-        &*self.0
+        &self.0
     }
 }
 impl AsRef<PackageId> for PackageId {

core/models/src/procedure_name.rs

@@ -1,57 +1,40 @@
 use serde::{Deserialize, Serialize};
 
-use crate::{ActionId, HealthCheckId, PackageId};
+use crate::ActionId;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ProcedureName {
-    Main, // Usually just run container
-    CreateBackup,
-    RestoreBackup,
+    StartMain,
+    StopMain,
     GetConfig,
     SetConfig,
-    Migration,
-    Properties,
-    LongRunning,
-    Check(PackageId),
-    AutoConfig(PackageId),
-    Health(HealthCheckId),
-    Action(ActionId),
-    Signal,
+    CreateBackup,
+    RestoreBackup,
+    ActionMetadata,
+    RunAction(ActionId),
+    GetAction(ActionId),
+    QueryDependency(ActionId),
+    UpdateDependency(ActionId),
+    Init,
+    Uninit,
 }
 
 impl ProcedureName {
-    pub fn docker_name(&self) -> Option<String> {
+    pub fn js_function_name(&self) -> String {
         match self {
-            ProcedureName::Main => None,
-            ProcedureName::LongRunning => None,
-            ProcedureName::CreateBackup => Some("CreateBackup".to_string()),
-            ProcedureName::RestoreBackup => Some("RestoreBackup".to_string()),
-            ProcedureName::GetConfig => Some("GetConfig".to_string()),
-            ProcedureName::SetConfig => Some("SetConfig".to_string()),
-            ProcedureName::Migration => Some("Migration".to_string()),
-            ProcedureName::Properties => Some(format!("Properties-{}", rand::random::<u64>())),
-            ProcedureName::Health(id) => Some(format!("{}Health", id)),
-            ProcedureName::Action(id) => Some(format!("{}Action", id)),
-            ProcedureName::Check(_) => None,
-            ProcedureName::AutoConfig(_) => None,
-            ProcedureName::Signal => None,
-        }
-    }
-    pub fn js_function_name(&self) -> Option<String> {
-        match self {
-            ProcedureName::Main => Some("/main".to_string()),
-            ProcedureName::LongRunning => None,
-            ProcedureName::CreateBackup => Some("/createBackup".to_string()),
-            ProcedureName::RestoreBackup => Some("/restoreBackup".to_string()),
-            ProcedureName::GetConfig => Some("/getConfig".to_string()),
-            ProcedureName::SetConfig => Some("/setConfig".to_string()),
-            ProcedureName::Migration => Some("/migration".to_string()),
-            ProcedureName::Properties => Some("/properties".to_string()),
-            ProcedureName::Health(id) => Some(format!("/health/{}", id)),
-            ProcedureName::Action(id) => Some(format!("/action/{}", id)),
-            ProcedureName::Check(id) => Some(format!("/dependencies/{}/check", id)),
-            ProcedureName::AutoConfig(id) => Some(format!("/dependencies/{}/autoConfigure", id)),
-            ProcedureName::Signal => Some("/handleSignal".to_string()),
+            ProcedureName::Init => "/init".to_string(),
+            ProcedureName::Uninit => "/uninit".to_string(),
+            ProcedureName::StartMain => "/main/start".to_string(),
+            ProcedureName::StopMain => "/main/stop".to_string(),
+            ProcedureName::SetConfig => "/config/set".to_string(),
+            ProcedureName::GetConfig => "/config/get".to_string(),
+            ProcedureName::CreateBackup => "/backup/create".to_string(),
+            ProcedureName::RestoreBackup => "/backup/restore".to_string(),
+            ProcedureName::ActionMetadata => "/actions/metadata".to_string(),
+            ProcedureName::RunAction(id) => format!("/actions/{}/run", id),
+            ProcedureName::GetAction(id) => format!("/actions/{}/get", id),
+            ProcedureName::QueryDependency(id) => format!("/dependencies/{}/query", id),
+            ProcedureName::UpdateDependency(id) => format!("/dependencies/{}/update", id),
         }
     }
 }

core/snapshot-creator/Cargo.toml

@@ -1,11 +0,0 @@
-[package]
-name = "snapshot_creator"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-dashmap = "5.3.4"
-deno_core = "=0.222.0"
-deno_ast = { version = "=0.29.5", features = ["transpiling"] }

core/snapshot-creator/src/main.rs

@@ -1,11 +0,0 @@
-use deno_core::JsRuntimeForSnapshot;
-
-fn main() {
-    let runtime = JsRuntimeForSnapshot::new(Default::default());
-    let snapshot = runtime.snapshot();
-
-    let snapshot_slice: &[u8] = &*snapshot;
-    println!("Snapshot size: {}", snapshot_slice.len());
-
-    std::fs::write("JS_SNAPSHOT.bin", snapshot_slice).unwrap();
-}

core/startos/Cargo.toml

@@ -21,20 +21,26 @@ license = "MIT"
 name = "startos"
 path = "src/lib.rs"
 
+[[bin]]
+name = "containerbox"
+path = "src/main.rs"
+
+[[bin]]
+name = "start-cli"
+path = "src/main.rs"
+
 [[bin]]
 name = "startbox"
 path = "src/main.rs"
 
 [features]
-avahi = ["avahi-sys"]
-avahi-alias = ["avahi"]
 cli = []
+container-runtime = []
 daemon = []
-default = ["cli", "sdk", "daemon", "js-engine"]
+default = ["cli", "daemon"]
 dev = []
-docker = []
-sdk = []
 unstable = ["console-subscriber", "tokio/tracing"]
+docker = []
 
 [dependencies]
 aes = { version = "0.7.5", features = ["ctr"] }
@@ -45,18 +51,16 @@ async-compression = { version = "0.4.4", features = [
 ] }
 async-stream = "0.3.5"
 async-trait = "0.1.74"
-avahi-sys = { git = "https://github.com/Start9Labs/avahi-sys", version = "0.10.0", branch = "feature/dynamic-linking", features = [
-  "dynamic",
-], optional = true }
+axum = { version = "0.7.3", features = ["ws"] }
+axum-server = "0.6.0"
 base32 = "0.4.0"
 base64 = "0.21.4"
 base64ct = "1.6.0"
 basic-cookies = "0.1.4"
-bimap = { version = "0.6.2", features = ["serde"] }
 blake3 = "1.5.0"
 bytes = "1"
 chrono = { version = "0.4.31", features = ["serde"] }
-clap = "3.2.25"
+clap = "4.4.12"
 color-eyre = "0.6.2"
 console = "0.15.7"
 console-subscriber = { version = "0.2", optional = true }
@@ -73,7 +77,6 @@ ed25519-dalek = { version = "2.0.0", features = [
   "digest",
 ] }
 ed25519-dalek-v1 = { package = "ed25519-dalek", version = "1" }
-container-init = { path = "../container-init" }
 emver = { version = "0.1.7", git = "https://github.com/Start9Labs/emver-rs.git", features = [
   "serde",
 ] }
@@ -83,13 +86,11 @@ gpt = "3.1.0"
 helpers = { path = "../helpers" }
 hex = "0.4.3"
 hmac = "0.12.1"
-http = "0.2.9"
-hyper = { version = "0.14.27", features = ["full"] }
-hyper-ws-listener = "0.3.0"
-id-pool = { version = "0.2.2", features = [
-  "u16",
+http = "1.0.0"
+id-pool = { version = "0.2.2", default-features = false, features = [
   "serde",
-], default-features = false }
+  "u16",
+] }
 imbl = "2.0.2"
 imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" }
 include_dir = "0.7.3"
@@ -99,12 +100,13 @@ integer-encoding = { version = "4.0.0", features = ["tokio_async"] }
 ipnet = { version = "2.8.0", features = ["serde"] }
 iprange = { version = "0.6.7", features = ["serde"] }
 isocountry = "0.3.2"
-itertools = "0.11.0"
+itertools = "0.12.0"
 jaq-core = "0.10.1"
 jaq-std = "0.10.0"
 josekit = "0.8.4"
-js-engine = { path = '../js-engine', optional = true }
 jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
+lazy_async_pool = "0.3.3"
+lazy_format = "2.0"
 lazy_static = "1.4.0"
 libc = "0.2.149"
 log = "0.4.20"
@@ -115,6 +117,7 @@ nix = { version = "0.27.1", features = ["user", "process", "signal", "fs"] }
 nom = "7.1.3"
 num = "0.4.1"
 num_enum = "0.7.0"
+once_cell = "1.19.0"
 openssh-keys = "0.6.2"
 openssl = { version = "0.10.57", features = ["vendored"] }
 p256 = { version = "0.13.2", features = ["pem"] }
@@ -129,12 +132,12 @@ proptest = "1.3.1"
 proptest-derive = "0.4.0"
 rand = { version = "0.8.5", features = ["std"] }
 regex = "1.10.2"
-reqwest = { version = "0.11.22", features = ["stream", "json", "socks"] }
+reqwest = { version = "0.11.23", features = ["stream", "json", "socks"] }
 reqwest_cookie_store = "0.6.0"
 rpassword = "7.2.0"
-rpc-toolkit = "0.2.2"
+rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "refactor/traits" }
 rust-argon2 = "2.0.0"
-scopeguard = "1.1" # because avahi-sys fucks your shit up
+rustyline-async = "0.4.1"
 semver = { version = "1.0.20", features = ["serde"] }
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_cbor = { package = "ciborium", version = "0.2.1" }
@@ -143,6 +146,7 @@ serde_toml = { package = "toml", version = "0.8.2" }
 serde_with = { version = "3.4.0", features = ["macros", "json"] }
 serde_yaml = "0.9.25"
 sha2 = "0.10.2"
+shell-words = "1"
 simple-logging = "2.0.2"
 sqlx = { version = "0.7.2", features = [
   "chrono",
@@ -155,20 +159,23 @@ stderrlog = "0.5.4"
 tar = "0.4.40"
 thiserror = "1.0.49"
 tokio = { version = "1", features = ["full"] }
-tokio-rustls = "0.24.1"
+tokio-rustls = "0.25.0"
 tokio-socks = "0.5.1"
 tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
-tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] }
+tokio-tungstenite = { version = "0.21.0", features = ["native-tls"] }
 tokio-util = { version = "0.7.9", features = ["io"] }
-torut = "0.2.1"
+torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies", features = [
+  "serialize",
+] }
 tracing = "0.1.39"
 tracing-error = "0.2.0"
 tracing-futures = "0.2.5"
 tracing-journald = "0.3.0"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 trust-dns-server = "0.23.1"
-typed-builder = "0.17.0"
+ts-rs = "7.1.1"
+typed-builder = "0.18.0"
 url = { version = "2.4.1", features = ["serde"] }
 urlencoding = "2.1.3"
 uuid = { version = "1.4.1", features = ["v4"] }

core/startos/Effects.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface SetStoreParams { value: any, path: string, }

core/startos/bindings/AddSslOptions.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface AddSslOptions { scheme: string | null, preferredExternalPort: number, addXForwardedHeaders: boolean | null, }

core/startos/bindings/AddressInfo.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { BindOptions } from "./BindOptions";
+
+export interface AddressInfo { username: string | null, hostId: string, bindOptions: BindOptions, suffix: string, }

core/startos/bindings/Algorithm.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type Algorithm = "ecdsa" | "ed25519";

core/startos/bindings/AllowedStatuses.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type AllowedStatuses = "only-running" | "only-stopped" | "any" | "disabled";

core/startos/bindings/BindKind.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type BindKind = "static" | "single" | "multi";

core/startos/bindings/BindOptions.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { AddSslOptions } from "./AddSslOptions";
+import type { BindOptionsSecure } from "./BindOptionsSecure";
+
+export interface BindOptions { scheme: string | null, preferredExternalPort: number, addSsl: AddSslOptions | null, secure: BindOptionsSecure | null, }

core/startos/bindings/BindOptionsSecure.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface BindOptionsSecure { ssl: boolean, }

core/startos/bindings/BindParams.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { AddSslOptions } from "./AddSslOptions";
+import type { BindKind } from "./BindKind";
+import type { BindOptionsSecure } from "./BindOptionsSecure";
+
+export interface BindParams { kind: BindKind, id: string, internalPort: number, scheme: string, preferredExternalPort: number, addSsl: AddSslOptions | null, secure: BindOptionsSecure | null, }

core/startos/bindings/Callback.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type Callback = () => void;

core/startos/bindings/ChrootParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ChrootParams { env: string | null, workdir: string | null, user: string | null, path: string, command: string, args: string[], }

core/startos/bindings/CreateOverlayedImageParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface CreateOverlayedImageParams { imageId: string, }

core/startos/bindings/DependencyKind.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type DependencyKind = "exists" | "running";

core/startos/bindings/DependencyRequirement.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { DependencyKind } from "./DependencyKind";
+
+export interface DependencyRequirement { id: string, kind: DependencyKind, healthChecks: string[], }

core/startos/bindings/DestroyOverlayedImageParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface DestroyOverlayedImageParams { guid: string, }

core/startos/bindings/ExecuteAction.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ExecuteAction { serviceId: string | null, actionId: string, input: any, }

core/startos/bindings/ExportActionParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { AllowedStatuses } from "./AllowedStatuses";
+
+export interface ExportActionParams { name: string, description: string, id: string, input: {[key: string]: any}, allowedStatuses: AllowedStatuses, group: string | null, }

core/startos/bindings/ExportServiceInterfaceParams.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { AddressInfo } from "./AddressInfo";
+import type { ServiceInterfaceType } from "./ServiceInterfaceType";
+
+export interface ExportServiceInterfaceParams { id: string, name: string, description: string, hasPrimary: boolean, disabled: boolean, masked: boolean, addressInfo: AddressInfo, type: ServiceInterfaceType, }

core/startos/bindings/ExposeForDependentsParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ExposeForDependentsParams { paths: string[], }

core/startos/bindings/ExposeUiParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ExposeUiParams = { "type": "object", value: {[key: string]: ExposeUiParams}, } | { "type": "string", path: string, description: string | null, masked: boolean, copyable: boolean | null, qr: boolean | null, };

core/startos/bindings/ExposedUI.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ExposedUI = { "type": "object", value: {[key: string]: ExposedUI}, description: string | null, } | { "type": "string", path: string, description: string | null, masked: boolean, copyable: boolean | null, qr: boolean | null, };

core/startos/bindings/GetHostInfoParams.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Callback } from "./Callback";
+import type { GetHostInfoParamsKind } from "./GetHostInfoParamsKind";
+
+export interface GetHostInfoParams { kind: GetHostInfoParamsKind | null, serviceInterfaceId: string, packageId: string | null, callback: Callback, }

core/startos/bindings/GetHostInfoParamsKind.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type GetHostInfoParamsKind = "multi";

core/startos/bindings/GetPrimaryUrlParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Callback } from "./Callback";
+
+export interface GetPrimaryUrlParams { packageId: string | null, serviceInterfaceId: string, callback: Callback, }

core/startos/bindings/GetServiceInterfaceParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Callback } from "./Callback";
+
+export interface GetServiceInterfaceParams { packageId: string | null, serviceInterfaceId: string, callback: Callback, }

core/startos/bindings/GetServicePortForwardParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface GetServicePortForwardParams { packageId: string | null, internalPort: number, }

core/startos/bindings/GetSslCertificateParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Algorithm } from "./Algorithm";
+
+export interface GetSslCertificateParams { packageId: string | null, hostId: string, algorithm: Algorithm | null, }

core/startos/bindings/GetSslKeyParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Algorithm } from "./Algorithm";
+
+export interface GetSslKeyParams { packageId: string | null, hostId: string, algorithm: Algorithm | null, }

core/startos/bindings/GetStoreParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface GetStoreParams { packageId: string | null, path: string, }

core/startos/bindings/GetSystemSmtpParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Callback } from "./Callback";
+
+export interface GetSystemSmtpParams { callback: Callback, }

core/startos/bindings/HealthCheckString.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type HealthCheckString = "passing" | "disabled" | "starting" | "warning" | "failure";

core/startos/bindings/ListServiceInterfacesParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Callback } from "./Callback";
+
+export interface ListServiceInterfacesParams { packageId: string | null, callback: Callback, }

core/startos/bindings/MountParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { MountTarget } from "./MountTarget";
+
+export interface MountParams { location: string, target: MountTarget, }

core/startos/bindings/MountTarget.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface MountTarget { packageId: string, volumeId: string, subpath: string | null, readonly: boolean, }

core/startos/bindings/ParamsMaybePackageId.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ParamsMaybePackageId { packageId: string | null, }

core/startos/bindings/ParamsPackageId.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ParamsPackageId { packageId: string, }

core/startos/bindings/RemoveActionParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface RemoveActionParams { id: string, }

core/startos/bindings/RemoveAddressParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface RemoveAddressParams { id: string, }

core/startos/bindings/ReverseProxyBind.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ReverseProxyBind { ip: string | null, port: number, ssl: boolean, }

core/startos/bindings/ReverseProxyDestination.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ReverseProxyDestination { ip: string | null, port: number, ssl: boolean, }

core/startos/bindings/ReverseProxyHttp.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ReverseProxyHttp { headers: null | {[key: string]: string}, }

core/startos/bindings/ReverseProxyParams.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ReverseProxyBind } from "./ReverseProxyBind";
+import type { ReverseProxyDestination } from "./ReverseProxyDestination";
+import type { ReverseProxyHttp } from "./ReverseProxyHttp";
+
+export interface ReverseProxyParams { bind: ReverseProxyBind, dst: ReverseProxyDestination, http: ReverseProxyHttp, }

core/startos/bindings/ServiceInterfaceType.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ServiceInterfaceType = "ui" | "p2p" | "api";

core/startos/bindings/SetConfigured.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface SetConfigured { configured: boolean, }

core/startos/bindings/SetDependenciesParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { DependencyRequirement } from "./DependencyRequirement";
+
+export interface SetDependenciesParams { dependencies: Array<DependencyRequirement>, }

core/startos/bindings/SetHealth.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { HealthCheckString } from "./HealthCheckString";
+
+export interface SetHealth { name: string, status: HealthCheckString, message: string | null, }

core/startos/bindings/SetMainStatus.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Status } from "./Status";
+
+export interface SetMainStatus { status: Status, }

core/startos/bindings/SetStoreParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface SetStoreParams { value: any, path: string, }

core/startos/bindings/Status.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type Status = "running" | "stopped";

core/startos/deny.toml

@@ -14,9 +14,15 @@ allow = [
     "BSD-3-Clause",
     "LGPL-3.0",
     "OpenSSL",
+    "Unicode-DFS-2016",
+    "Zlib",
 ]
 
 clarify = [
-    { name = "webpki", expression = "ISC", license-files = [ { path = "LICENSE", hash = 0x001c7e6c } ] },
-    { name = "ring", expression = "OpenSSL", license-files = [ { path = "LICENSE", hash = 0xbd0eed23 } ] },
+    { name = "webpki", expression = "ISC", license-files = [
+        { path = "LICENSE", hash = 0x001c7e6c },
+    ] },
+    { name = "ring", expression = "OpenSSL", license-files = [
+        { path = "LICENSE", hash = 0xbd0eed23 },
+    ] },
 ]

core/startos/src/account.rs

@@ -1,15 +1,14 @@
 use std::time::SystemTime;
 
-use ed25519_dalek::SecretKey;
 use openssl::pkey::{PKey, Private};
 use openssl::x509::X509;
-use sqlx::PgExecutor;
+use torut::onion::TorSecretKeyV3;
 
+use crate::db::model::DatabaseModel;
 use crate::hostname::{generate_hostname, generate_id, Hostname};
-use crate::net::keys::Key;
 use crate::net::ssl::{generate_key, make_root_cert};
 use crate::prelude::*;
-use crate::util::crypto::ed25519_expand_key;
+use crate::util::serde::Pem;
 
 fn hash_password(password: &str) -> Result<String, Error> {
     argon2::hash_encoded(
@@ -25,103 +24,83 @@ pub struct AccountInfo {
     pub server_id: String,
     pub hostname: Hostname,
     pub password: String,
-    pub key: Key,
+    pub tor_key: TorSecretKeyV3,
     pub root_ca_key: PKey<Private>,
     pub root_ca_cert: X509,
+    pub ssh_key: ssh_key::PrivateKey,
 }
 impl AccountInfo {
     pub fn new(password: &str, start_time: SystemTime) -> Result<Self, Error> {
         let server_id = generate_id();
         let hostname = generate_hostname();
+        let tor_key = TorSecretKeyV3::generate();
         let root_ca_key = generate_key()?;
         let root_ca_cert = make_root_cert(&root_ca_key, &hostname, start_time)?;
+        let ssh_key = ssh_key::PrivateKey::from(ssh_key::private::Ed25519Keypair::random(
+            &mut rand::thread_rng(),
+        ));
         Ok(Self {
             server_id,
             hostname,
             password: hash_password(password)?,
-            key: Key::new(None),
+            tor_key,
             root_ca_key,
             root_ca_cert,
+            ssh_key,
         })
     }
 
-    pub async fn load(secrets: impl PgExecutor<'_>) -> Result<Self, Error> {
-        let r = sqlx::query!("SELECT * FROM account WHERE id = 0")
-            .fetch_one(secrets)
-            .await?;
-
-        let server_id = r.server_id.unwrap_or_else(generate_id);
-        let hostname = r.hostname.map(Hostname).unwrap_or_else(generate_hostname);
-        let password = r.password;
-        let network_key = SecretKey::try_from(r.network_key).map_err(|e| {
-            Error::new(
-                eyre!("expected vec of len 32, got len {}", e.len()),
-                ErrorKind::ParseDbField,
-            )
-        })?;
-        let tor_key = if let Some(k) = &r.tor_key {
-            <[u8; 64]>::try_from(&k[..]).map_err(|_| {
-                Error::new(
-                    eyre!("expected vec of len 64, got len {}", k.len()),
-                    ErrorKind::ParseDbField,
-                )
-            })?
-        } else {
-            ed25519_expand_key(&network_key)
-        };
-        let key = Key::from_pair(None, network_key, tor_key);
-        let root_ca_key = PKey::private_key_from_pem(r.root_ca_key_pem.as_bytes())?;
-        let root_ca_cert = X509::from_pem(r.root_ca_cert_pem.as_bytes())?;
+    pub fn load(db: &DatabaseModel) -> Result<Self, Error> {
+        let server_id = db.as_public().as_server_info().as_id().de()?;
+        let hostname = Hostname(db.as_public().as_server_info().as_hostname().de()?);
+        let password = db.as_private().as_password().de()?;
+        let key_store = db.as_private().as_key_store();
+        let tor_addr = db.as_public().as_server_info().as_onion_address().de()?;
+        let tor_key = key_store.as_onion().get_key(&tor_addr)?;
+        let cert_store = key_store.as_local_certs();
+        let root_ca_key = cert_store.as_root_key().de()?.0;
+        let root_ca_cert = cert_store.as_root_cert().de()?.0;
+        let ssh_key = db.as_private().as_ssh_privkey().de()?.0;
 
         Ok(Self {
             server_id,
             hostname,
             password,
-            key,
+            tor_key,
             root_ca_key,
             root_ca_cert,
+            ssh_key,
         })
     }
 
-    pub async fn save(&self, secrets: impl PgExecutor<'_>) -> Result<(), Error> {
-        let server_id = self.server_id.as_str();
-        let hostname = self.hostname.0.as_str();
-        let password = self.password.as_str();
-        let network_key = self.key.as_bytes();
-        let network_key = network_key.as_slice();
-        let root_ca_key = String::from_utf8(self.root_ca_key.private_key_to_pem_pkcs8()?)?;
-        let root_ca_cert = String::from_utf8(self.root_ca_cert.to_pem()?)?;
-
-        sqlx::query!(
-            r#"
-            INSERT INTO account (
-                id,
-                server_id,
-                hostname,
-                password,
-                network_key,
-                root_ca_key_pem,
-                root_ca_cert_pem
-            ) VALUES (
-                0, $1, $2, $3, $4, $5, $6
-            ) ON CONFLICT (id) DO UPDATE SET
-                server_id = EXCLUDED.server_id,
-                hostname = EXCLUDED.hostname,
-                password = EXCLUDED.password,
-                network_key = EXCLUDED.network_key,
-                root_ca_key_pem = EXCLUDED.root_ca_key_pem,
-                root_ca_cert_pem = EXCLUDED.root_ca_cert_pem
-            "#,
-            server_id,
-            hostname,
-            password,
-            network_key,
-            root_ca_key,
-            root_ca_cert,
-        )
-        .execute(secrets)
-        .await?;
-
+    pub fn save(&self, db: &mut DatabaseModel) -> Result<(), Error> {
+        let server_info = db.as_public_mut().as_server_info_mut();
+        server_info.as_id_mut().ser(&self.server_id)?;
+        server_info.as_hostname_mut().ser(&self.hostname.0)?;
+        server_info
+            .as_lan_address_mut()
+            .ser(&self.hostname.lan_address().parse()?)?;
+        server_info
+            .as_pubkey_mut()
+            .ser(&self.ssh_key.public_key().to_openssh()?)?;
+        let onion_address = self.tor_key.public().get_onion_address();
+        server_info.as_onion_address_mut().ser(&onion_address)?;
+        server_info
+            .as_tor_address_mut()
+            .ser(&format!("https://{onion_address}").parse()?)?;
+        db.as_private_mut().as_password_mut().ser(&self.password)?;
+        db.as_private_mut()
+            .as_ssh_privkey_mut()
+            .ser(Pem::new_ref(&self.ssh_key))?;
+        let key_store = db.as_private_mut().as_key_store_mut();
+        key_store.as_onion_mut().insert_key(&self.tor_key)?;
+        let cert_store = key_store.as_local_certs_mut();
+        cert_store
+            .as_root_key_mut()
+            .ser(Pem::new_ref(&self.root_ca_key))?;
+        cert_store
+            .as_root_cert_mut()
+            .ser(Pem::new_ref(&self.root_ca_cert))?;
         Ok(())
     }
 

core/startos/src/action.rs

@@ -1,26 +1,14 @@
-use std::collections::{BTreeMap, BTreeSet};
-
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
-use indexmap::IndexSet;
+use clap::Parser;
 pub use models::ActionId;
-use models::ImageId;
+use models::PackageId;
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use crate::config::{Config, ConfigSpec};
+use crate::config::Config;
 use crate::context::RpcContext;
 use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::util::Version;
-use crate::volume::Volumes;
-use crate::{Error, ResultExt};
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
-pub struct Actions(pub BTreeMap<ActionId, Action>);
+use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat};
 
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(tag = "version")]
@@ -44,72 +32,11 @@ pub enum DockerStatus {
     Stopped,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct Action {
-    pub name: String,
-    pub description: String,
-    #[serde(default)]
-    pub warning: Option<String>,
-    pub implementation: PackageProcedure,
-    pub allowed_statuses: IndexSet<DockerStatus>,
-    #[serde(default)]
-    pub input_spec: ConfigSpec,
-}
-impl Action {
-    #[instrument(skip_all)]
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.implementation
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| {
-                (
-                    crate::ErrorKind::ValidateS9pk,
-                    format!("Action {}", self.name),
-                )
-            })
+pub fn display_action_result(params: WithIoFormat<ActionParams>, result: ActionResult) {
+    if let Some(format) = params.format {
+        return display_serializable(format, result);
     }
-
-    #[instrument(skip_all)]
-    pub async fn execute(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        action_id: &ActionId,
-        volumes: &Volumes,
-        input: Option<Config>,
-    ) -> Result<ActionResult, Error> {
-        if let Some(ref input) = input {
-            self.input_spec
-                .matches(&input)
-                .with_kind(crate::ErrorKind::ConfigSpecViolation)?;
-        }
-        self.implementation
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::Action(action_id.clone()),
-                volumes,
-                input,
-                None,
-            )
-            .await?
-            .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::Action))
-    }
-}
-
-fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
-    if matches.is_present("format") {
-        return display_serializable(action_result, matches);
-    }
-    match action_result {
+    match result {
         ActionResult::V0(ar) => {
             println!(
                 "{}: {}",
@@ -120,44 +47,39 @@ fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
     }
 }
 
-#[command(about = "Executes an action", display(display_action_result))]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct ActionParams {
+    #[arg(id = "id")]
+    #[serde(rename = "id")]
+    pub package_id: PackageId,
+    #[arg(id = "action-id")]
+    #[serde(rename = "action-id")]
+    pub action_id: ActionId,
+    #[command(flatten)]
+    pub input: StdinDeserializable<Option<Config>>,
+}
+// impl C
+
+// #[command(about = "Executes an action", display(display_action_result))]
 #[instrument(skip_all)]
 pub async fn action(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "id")] pkg_id: PackageId,
-    #[arg(rename = "action-id")] action_id: ActionId,
-    #[arg(stdin, parse(parse_stdin_deserializable))] input: Option<Config>,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
+    ctx: RpcContext,
+    ActionParams {
+        package_id,
+        action_id,
+        input: StdinDeserializable(input),
+    }: ActionParams,
 ) -> Result<ActionResult, Error> {
-    let manifest = ctx
-        .db
-        .peek()
+    ctx.services
+        .get(&package_id)
+        .await
+        .as_ref()
+        .or_not_found(lazy_format!("Manager for {}", package_id))?
+        .action(
+            action_id,
+            input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(),
+        )
         .await
-        .as_package_data()
-        .as_idx(&pkg_id)
-        .or_not_found(&pkg_id)?
-        .as_installed()
-        .or_not_found(&pkg_id)?
-        .as_manifest()
-        .de()?;
-
-    if let Some(action) = manifest.actions.0.get(&action_id) {
-        action
-            .execute(
-                &ctx,
-                &manifest.id,
-                &manifest.version,
-                &action_id,
-                &manifest.volumes,
-                input,
-            )
-            .await
-    } else {
-        Err(Error::new(
-            eyre!("Action not found in manifest"),
-            crate::ErrorKind::NotFound,
-        ))
-    }
 }

core/startos/src/auth.rs

@@ -1,25 +1,43 @@
 use std::collections::BTreeMap;
-use std::marker::PhantomData;
 
 use chrono::{DateTime, Utc};
-use clap::ArgMatches;
+use clap::Parser;
 use color_eyre::eyre::eyre;
+use imbl_value::{json, InternedString};
 use josekit::jwk::Jwk;
-use rpc_toolkit::command;
-use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
 use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::{command, from_fn_async, AnyContext, CallRemote, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use sqlx::{Executor, Postgres};
 use tracing::instrument;
 
 use crate::context::{CliContext, RpcContext};
-use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
-use crate::middleware::encrypt::EncryptedWire;
+use crate::db::model::DatabaseModel;
+use crate::middleware::auth::{
+    AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken, LoginRes,
+};
 use crate::prelude::*;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::crypto::EncryptedWire;
+use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
 use crate::{ensure_code, Error, ResultExt};
+
+#[derive(Debug, Clone, Default, Deserialize, Serialize)]
+pub struct Sessions(pub BTreeMap<InternedString, Session>);
+impl Sessions {
+    pub fn new() -> Self {
+        Self(BTreeMap::new())
+    }
+}
+impl Map for Sessions {
+    type Key = InternedString;
+    type Value = Session;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone())
+    }
+}
+
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(untagged)]
 pub enum PasswordType {
@@ -61,20 +79,39 @@ impl std::str::FromStr for PasswordType {
         })
     }
 }
-
-#[command(subcommands(login, logout, session, reset_password, get_pubkey))]
-pub fn auth() -> Result<(), Error> {
-    Ok(())
-}
-
-pub fn cli_metadata() -> Value {
-    serde_json::json!({
-        "platforms": ["cli"],
-    })
-}
-
-pub fn parse_metadata(_: &str, _: &ArgMatches) -> Result<Value, Error> {
-    Ok(cli_metadata())
+pub fn auth() -> ParentHandler {
+    ParentHandler::new()
+        .subcommand(
+            "login",
+            from_fn_async(login_impl)
+                .with_metadata("login", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand("login", from_fn_async(cli_login).no_display())
+        .subcommand(
+            "logout",
+            from_fn_async(logout)
+                .with_metadata("get-session", Value::Bool(true))
+                .with_remote_cli::<CliContext>()
+                // TODO @dr-bonez
+                .no_display(),
+        )
+        .subcommand("session", session())
+        .subcommand(
+            "reset-password",
+            from_fn_async(reset_password_impl).no_cli(),
+        )
+        .subcommand(
+            "reset-password",
+            from_fn_async(cli_reset_password).no_display(),
+        )
+        .subcommand(
+            "get-pubkey",
+            from_fn_async(get_pubkey)
+                .with_metadata("authenticated", Value::Bool(false))
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
 }
 
 #[test]
@@ -89,12 +126,17 @@ fn gen_pwd() {
         .unwrap()
     )
 }
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct CliLoginParams {
+    password: Option<PasswordType>,
+}
 
 #[instrument(skip_all)]
 async fn cli_login(
     ctx: CliContext,
-    password: Option<PasswordType>,
-    metadata: Value,
+    CliLoginParams { password }: CliLoginParams,
 ) -> Result<(), RpcError> {
     let password = if let Some(password) = password {
         password.decrypt(&ctx)?
@@ -102,14 +144,16 @@ async fn cli_login(
         rpassword::prompt_password("Password: ")?
     };
 
-    rpc_toolkit::command_helpers::call_remote(
-        ctx,
+    ctx.call_remote(
         "auth.login",
-        serde_json::json!({ "password": password, "metadata": metadata }),
-        PhantomData::<()>,
+        json!({
+            "password": password,
+            "metadata": {
+                "platforms": ["cli"],
+            },
+        }),
     )
-    .await?
-    .result?;
+    .await?;
 
     Ok(())
 }
@@ -128,99 +172,110 @@ pub fn check_password(hash: &str, password: &str) -> Result<(), Error> {
     Ok(())
 }
 
-pub async fn check_password_against_db<Ex>(secrets: &mut Ex, password: &str) -> Result<(), Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let pw_hash = sqlx::query!("SELECT password FROM account")
-        .fetch_one(secrets)
-        .await?
-        .password;
+pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(), Error> {
+    let pw_hash = db.as_private().as_password().de()?;
     check_password(&pw_hash, password)?;
     Ok(())
 }
 
-#[command(
-    custom_cli(cli_login(async, context(CliContext))),
-    display(display_none),
-    metadata(authenticated = false)
-)]
-#[instrument(skip_all)]
-pub async fn login(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
-    #[response] res: &mut ResponseParts,
-    #[arg] password: Option<PasswordType>,
-    #[arg(
-        parse(parse_metadata),
-        default = "cli_metadata",
-        help = "RPC Only: This value cannot be overidden from the cli"
-    )]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct LoginParams {
+    password: Option<PasswordType>,
+    #[serde(default)]
+    user_agent: Option<String>,
+    #[serde(default)]
     metadata: Value,
-) -> Result<(), Error> {
-    let password = password.unwrap_or_default().decrypt(&ctx)?;
-    let mut handle = ctx.secret_store.acquire().await?;
-    check_password_against_db(handle.as_mut(), &password).await?;
+}
 
-    let hash_token = HashSessionToken::new();
-    let user_agent = req.headers.get("user-agent").and_then(|h| h.to_str().ok());
-    let metadata = serde_json::to_string(&metadata).with_kind(crate::ErrorKind::Database)?;
-    let hash_token_hashed = hash_token.hashed();
-    sqlx::query!(
-        "INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
-        hash_token_hashed,
+#[instrument(skip_all)]
+pub async fn login_impl(
+    ctx: RpcContext,
+    LoginParams {
+        password,
         user_agent,
         metadata,
-    )
-    .execute(handle.as_mut())
-    .await?;
-    res.headers.insert(
-        "set-cookie",
-        hash_token.header_value()?, // Should be impossible, but don't want to panic
-    );
+    }: LoginParams,
+) -> Result<LoginRes, Error> {
+    let password = password.unwrap_or_default().decrypt(&ctx)?;
 
-    Ok(())
+    ctx.db
+        .mutate(|db| {
+            check_password_against_db(db, &password)?;
+            let hash_token = HashSessionToken::new();
+            db.as_private_mut().as_sessions_mut().insert(
+                hash_token.hashed(),
+                &Session {
+                    logged_in: Utc::now(),
+                    last_active: Utc::now(),
+                    user_agent,
+                    metadata,
+                },
+            )?;
+
+            Ok(hash_token.to_login_res())
+        })
+        .await
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct LogoutParams {
+    session: InternedString,
 }
 
-#[command(display(display_none), metadata(authenticated = false))]
-#[instrument(skip_all)]
 pub async fn logout(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
+    ctx: RpcContext,
+    LogoutParams { session }: LogoutParams,
 ) -> Result<Option<HasLoggedOutSessions>, Error> {
-    let auth = match HashSessionToken::from_request_parts(req) {
-        Err(_) => return Ok(None),
-        Ok(a) => a,
-    };
-    Ok(Some(HasLoggedOutSessions::new(vec![auth], &ctx).await?))
+    Ok(Some(
+        HasLoggedOutSessions::new(vec![HashSessionToken::from_token(session)], &ctx).await?,
+    ))
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct Session {
-    logged_in: DateTime<Utc>,
-    last_active: DateTime<Utc>,
-    user_agent: Option<String>,
-    metadata: Value,
+    pub logged_in: DateTime<Utc>,
+    pub last_active: DateTime<Utc>,
+    pub user_agent: Option<String>,
+    pub metadata: Value,
 }
 
 #[derive(Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct SessionList {
-    current: String,
-    sessions: BTreeMap<String, Session>,
+    current: InternedString,
+    sessions: Sessions,
 }
 
-#[command(subcommands(list, kill))]
-pub async fn session() -> Result<(), Error> {
-    Ok(())
+pub fn session() -> ParentHandler {
+    ParentHandler::new()
+        .subcommand(
+            "list",
+            from_fn_async(list)
+                .with_metadata("get-session", Value::Bool(true))
+                .with_display_serializable()
+                .with_custom_display_fn::<AnyContext, _>(|handle, result| {
+                    Ok(display_sessions(handle.params, result))
+                })
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand(
+            "kill",
+            from_fn_async(kill)
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
 }
 
-fn display_sessions(arg: SessionList, matches: &ArgMatches) {
+fn display_sessions(params: WithIoFormat<ListParams>, arg: SessionList) {
     use prettytable::*;
 
-    if matches.is_present("format") {
-        return display_serializable(arg, matches);
+    if let Some(format) = params.format {
+        return display_serializable(format, arg);
     }
 
     let mut table = Table::new();
@@ -231,7 +286,7 @@ fn display_sessions(arg: SessionList, matches: &ArgMatches) {
         "USER AGENT",
         "METADATA",
     ]);
-    for (id, session) in arg.sessions {
+    for (id, session) in arg.sessions.0 {
         let mut row = row![
             &id,
             &format!("{}", session.logged_in),
@@ -249,67 +304,71 @@ fn display_sessions(arg: SessionList, matches: &ArgMatches) {
     table.print_tty(false).unwrap();
 }
 
-#[command(display(display_sessions))]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct ListParams {
+    #[arg(skip)]
+    session: InternedString,
+}
+
+// #[command(display(display_sessions))]
 #[instrument(skip_all)]
 pub async fn list(
-    #[context] ctx: RpcContext,
-    #[request] req: &RequestParts,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
+    ctx: RpcContext,
+    ListParams { session, .. }: ListParams,
 ) -> Result<SessionList, Error> {
     Ok(SessionList {
-        current: HashSessionToken::from_request_parts(req)?.as_hash(),
-        sessions: sqlx::query!(
-            "SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP"
-        )
-        .fetch_all(ctx.secret_store.acquire().await?.as_mut())
-        .await?
-        .into_iter()
-        .map(|row| {
-            Ok((
-                row.id,
-                Session {
-                    logged_in: DateTime::from_utc(row.logged_in, Utc),
-                    last_active: DateTime::from_utc(row.last_active, Utc),
-                    user_agent: row.user_agent,
-                    metadata: serde_json::from_str(&row.metadata)
-                        .with_kind(crate::ErrorKind::Database)?,
-                },
-            ))
-        })
-        .collect::<Result<_, Error>>()?,
+        current: HashSessionToken::from_token(session).hashed().clone(),
+        sessions: ctx.db.peek().await.into_private().into_sessions().de()?,
     })
 }
 
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<String>, RpcError> {
-    Ok(arg.split(",").map(|s| s.trim().to_owned()).collect())
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct KillSessionId(InternedString);
+
+impl KillSessionId {
+    fn new(id: String) -> Self {
+        Self(InternedString::from(id))
+    }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct KillSessionId(String);
-
 impl AsLogoutSessionId for KillSessionId {
-    fn as_logout_session_id(self) -> String {
+    fn as_logout_session_id(self) -> InternedString {
         self.0
     }
 }
 
-#[command(display(display_none))]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct KillParams {
+    ids: Vec<String>,
+}
+
 #[instrument(skip_all)]
-pub async fn kill(
-    #[context] ctx: RpcContext,
-    #[arg(parse(parse_comma_separated))] ids: Vec<String>,
-) -> Result<(), Error> {
-    HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId), &ctx).await?;
+pub async fn kill(ctx: RpcContext, KillParams { ids }: KillParams) -> Result<(), Error> {
+    HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId::new), &ctx).await?;
     Ok(())
 }
 
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct ResetPasswordParams {
+    #[arg(name = "old-password")]
+    old_password: Option<PasswordType>,
+    #[arg(name = "new-password")]
+    new_password: Option<PasswordType>,
+}
+
 #[instrument(skip_all)]
 async fn cli_reset_password(
     ctx: CliContext,
-    old_password: Option<PasswordType>,
-    new_password: Option<PasswordType>,
+    ResetPasswordParams {
+        old_password,
+        new_password,
+    }: ResetPasswordParams,
 ) -> Result<(), RpcError> {
     let old_password = if let Some(old_password) = old_password {
         old_password.decrypt(&ctx)?
@@ -331,28 +390,22 @@ async fn cli_reset_password(
         new_password
     };
 
-    rpc_toolkit::command_helpers::call_remote(
-        ctx,
+    ctx.call_remote(
         "auth.reset-password",
-        serde_json::json!({ "old-password": old_password, "new-password": new_password }),
-        PhantomData::<()>,
+        imbl_value::json!({ "old-password": old_password, "new-password": new_password }),
     )
-    .await?
-    .result?;
+    .await?;
 
     Ok(())
 }
 
-#[command(
-    rename = "reset-password",
-    custom_cli(cli_reset_password(async, context(CliContext))),
-    display(display_none)
-)]
 #[instrument(skip_all)]
-pub async fn reset_password(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "old-password")] old_password: Option<PasswordType>,
-    #[arg(rename = "new-password")] new_password: Option<PasswordType>,
+pub async fn reset_password_impl(
+    ctx: RpcContext,
+    ResetPasswordParams {
+        old_password,
+        new_password,
+    }: ResetPasswordParams,
 ) -> Result<(), Error> {
     let old_password = old_password.unwrap_or_default().decrypt(&ctx)?;
     let new_password = new_password.unwrap_or_default().decrypt(&ctx)?;
@@ -367,24 +420,23 @@ pub async fn reset_password(
         ));
     }
     account.set_password(&new_password)?;
-    account.save(&ctx.secret_store).await?;
     let account_password = &account.password;
+    let account = account.clone();
     ctx.db
         .mutate(|d| {
-            d.as_server_info_mut()
+            d.as_public_mut()
+                .as_server_info_mut()
                 .as_password_hash_mut()
-                .ser(account_password)
+                .ser(account_password)?;
+            account.save(d)?;
+
+            Ok(())
         })
         .await
 }
 
-#[command(
-    rename = "get-pubkey",
-    display(display_none),
-    metadata(authenticated = false)
-)]
 #[instrument(skip_all)]
-pub async fn get_pubkey(#[context] ctx: RpcContext) -> Result<Jwk, RpcError> {
+pub async fn get_pubkey(ctx: RpcContext) -> Result<Jwk, RpcError> {
     let secret = ctx.as_ref().clone();
     let pub_key = secret.to_public_key()?;
     Ok(pub_key)

core/startos/src/backup/backup_bulk.rs

@@ -1,17 +1,15 @@
 use std::collections::BTreeMap;
-use std::panic::UnwindSafe;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use chrono::Utc;
-use clap::ArgMatches;
+use clap::Parser;
 use color_eyre::eyre::eyre;
 use helpers::AtomicFile;
 use imbl::OrdSet;
-use models::Version;
-use rpc_toolkit::command;
+use models::PackageId;
+use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
-use tokio::sync::Mutex;
 use tracing::instrument;
 
 use super::target::BackupTargetId;
@@ -20,259 +18,264 @@ use crate::auth::check_password_against_db;
 use crate::backup::os::OsBackup;
 use crate::backup::{BackupReport, ServerBackupReport};
 use crate::context::RpcContext;
-use crate::db::model::BackupProgress;
-use crate::db::package::get_packages;
+use crate::db::model::public::BackupProgress;
+use crate::db::model::DatabaseModel;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::ReadWrite;
-use crate::disk::mount::guard::TmpMountGuard;
-use crate::manager::BackupReturn;
-use crate::notifications::NotificationLevel;
+use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
+use crate::notifications::{notify, NotificationLevel};
 use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::display_none;
 use crate::util::io::dir_copy;
 use crate::util::serde::IoFormat;
 use crate::version::VersionT;
 
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<OrdSet<PackageId>, Error> {
-    arg.split(',')
-        .map(|s| s.trim().parse::<PackageId>().map_err(Error::from))
-        .collect()
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct BackupParams {
+    target_id: BackupTargetId,
+    #[arg(long = "old-password")]
+    old_password: Option<crate::auth::PasswordType>,
+    #[arg(long = "package-ids")]
+    package_ids: Option<Vec<PackageId>>,
+    password: crate::auth::PasswordType,
+}
+
+struct BackupStatusGuard(Option<PatchDb>);
+impl BackupStatusGuard {
+    fn new(db: PatchDb) -> Self {
+        Self(Some(db))
+    }
+    async fn handle_result(
+        mut self,
+        result: Result<BTreeMap<PackageId, PackageBackupReport>, Error>,
+    ) -> Result<(), Error> {
+        if let Some(db) = self.0.as_ref() {
+            db.mutate(|v| {
+                v.as_public_mut()
+                    .as_server_info_mut()
+                    .as_status_info_mut()
+                    .as_backup_progress_mut()
+                    .ser(&None)
+            })
+            .await?;
+        }
+        if let Some(db) = self.0.take() {
+            match result {
+                Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => {
+                    db.mutate(|db| {
+                        notify(
+                            db,
+                            None,
+                            NotificationLevel::Success,
+                            "Backup Complete".to_owned(),
+                            "Your backup has completed".to_owned(),
+                            BackupReport {
+                                server: ServerBackupReport {
+                                    attempted: true,
+                                    error: None,
+                                },
+                                packages: report,
+                            },
+                        )
+                    })
+                    .await
+                }
+                Ok(report) => {
+                    db.mutate(|db| {
+                        notify(
+                            db,
+                            None,
+                            NotificationLevel::Warning,
+                            "Backup Complete".to_owned(),
+                            "Your backup has completed, but some package(s) failed to backup"
+                                .to_owned(),
+                            BackupReport {
+                                server: ServerBackupReport {
+                                    attempted: true,
+                                    error: None,
+                                },
+                                packages: report,
+                            },
+                        )
+                    })
+                    .await
+                }
+                Err(e) => {
+                    tracing::error!("Backup Failed: {}", e);
+                    tracing::debug!("{:?}", e);
+                    let err_string = e.to_string();
+                    db.mutate(|db| {
+                        notify(
+                            db,
+                            None,
+                            NotificationLevel::Error,
+                            "Backup Failed".to_owned(),
+                            "Your backup failed to complete.".to_owned(),
+                            BackupReport {
+                                server: ServerBackupReport {
+                                    attempted: true,
+                                    error: Some(err_string),
+                                },
+                                packages: BTreeMap::new(),
+                            },
+                        )
+                    })
+                    .await
+                }
+            }?;
+        }
+        Ok(())
+    }
+}
+impl Drop for BackupStatusGuard {
+    fn drop(&mut self) {
+        if let Some(db) = self.0.take() {
+            tokio::spawn(async move {
+                db.mutate(|v| {
+                    v.as_public_mut()
+                        .as_server_info_mut()
+                        .as_status_info_mut()
+                        .as_backup_progress_mut()
+                        .ser(&None)
+                })
+                .await
+                .unwrap()
+            });
+        }
+    }
 }
 
-#[command(rename = "create", display(display_none))]
 #[instrument(skip(ctx, old_password, password))]
 pub async fn backup_all(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg(rename = "old-password", long = "old-password")] old_password: Option<
-        crate::auth::PasswordType,
-    >,
-    #[arg(
-        rename = "package-ids",
-        long = "package-ids",
-        parse(parse_comma_separated)
-    )]
-    package_ids: Option<OrdSet<PackageId>>,
-    #[arg] password: crate::auth::PasswordType,
+    ctx: RpcContext,
+    BackupParams {
+        target_id,
+        old_password,
+        package_ids,
+        password,
+    }: BackupParams,
 ) -> Result<(), Error> {
-    let db = ctx.db.peek().await;
     let old_password_decrypted = old_password
         .as_ref()
         .unwrap_or(&password)
         .clone()
         .decrypt(&ctx)?;
     let password = password.decrypt(&ctx)?;
-    check_password_against_db(ctx.secret_store.acquire().await?.as_mut(), &password).await?;
-    let fs = target_id
-        .load(ctx.secret_store.acquire().await?.as_mut())
-        .await?;
+
+    let ((fs, package_ids), status_guard) = (
+        ctx.db
+            .mutate(|db| {
+                check_password_against_db(db, &password)?;
+                let fs = target_id.load(db)?;
+                let package_ids = if let Some(ids) = package_ids {
+                    ids.into_iter().collect()
+                } else {
+                    db.as_public()
+                        .as_package_data()
+                        .as_entries()?
+                        .into_iter()
+                        .filter(|(_, m)| m.as_state_info().expect_installed().is_ok())
+                        .map(|(id, _)| id)
+                        .collect()
+                };
+                assure_backing_up(db, &package_ids)?;
+                Ok((fs, package_ids))
+            })
+            .await?,
+        BackupStatusGuard::new(ctx.db.clone()),
+    );
+
     let mut backup_guard = BackupMountGuard::mount(
         TmpMountGuard::mount(&fs, ReadWrite).await?,
         &old_password_decrypted,
     )
     .await?;
-    let package_ids = if let Some(ids) = package_ids {
-        ids.into_iter()
-            .flat_map(|package_id| {
-                let version = db
-                    .as_package_data()
-                    .as_idx(&package_id)?
-                    .as_manifest()
-                    .as_version()
-                    .de()
-                    .ok()?;
-                Some((package_id, version))
-            })
-            .collect()
-    } else {
-        get_packages(db.clone())?.into_iter().collect()
-    };
     if old_password.is_some() {
         backup_guard.change_password(&password)?;
     }
-    assure_backing_up(&ctx.db, &package_ids).await?;
     tokio::task::spawn(async move {
-        let backup_res = perform_backup(&ctx, backup_guard, &package_ids).await;
-        match backup_res {
-            Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
-                .notification_manager
-                .notify(
-                    ctx.db.clone(),
-                    None,
-                    NotificationLevel::Success,
-                    "Backup Complete".to_owned(),
-                    "Your backup has completed".to_owned(),
-                    BackupReport {
-                        server: ServerBackupReport {
-                            attempted: true,
-                            error: None,
-                        },
-                        packages: report
-                            .into_iter()
-                            .map(|((package_id, _), value)| (package_id, value))
-                            .collect(),
-                    },
-                    None,
-                )
-                .await
-                .expect("failed to send notification"),
-            Ok(report) => ctx
-                .notification_manager
-                .notify(
-                    ctx.db.clone(),
-                    None,
-                    NotificationLevel::Warning,
-                    "Backup Complete".to_owned(),
-                    "Your backup has completed, but some package(s) failed to backup".to_owned(),
-                    BackupReport {
-                        server: ServerBackupReport {
-                            attempted: true,
-                            error: None,
-                        },
-                        packages: report
-                            .into_iter()
-                            .map(|((package_id, _), value)| (package_id, value))
-                            .collect(),
-                    },
-                    None,
-                )
-                .await
-                .expect("failed to send notification"),
-            Err(e) => {
-                tracing::error!("Backup Failed: {}", e);
-                tracing::debug!("{:?}", e);
-                ctx.notification_manager
-                    .notify(
-                        ctx.db.clone(),
-                        None,
-                        NotificationLevel::Error,
-                        "Backup Failed".to_owned(),
-                        "Your backup failed to complete.".to_owned(),
-                        BackupReport {
-                            server: ServerBackupReport {
-                                attempted: true,
-                                error: Some(e.to_string()),
-                            },
-                            packages: BTreeMap::new(),
-                        },
-                        None,
-                    )
-                    .await
-                    .expect("failed to send notification");
-            }
-        }
-        ctx.db
-            .mutate(|v| {
-                v.as_server_info_mut()
-                    .as_status_info_mut()
-                    .as_backup_progress_mut()
-                    .ser(&None)
-            })
-            .await?;
-        Ok::<(), Error>(())
+        status_guard
+            .handle_result(perform_backup(&ctx, backup_guard, &package_ids).await)
+            .await
+            .unwrap();
     });
     Ok(())
 }
 
 #[instrument(skip(db, packages))]
-async fn assure_backing_up(
-    db: &PatchDb,
-    packages: impl IntoIterator<Item = &(PackageId, Version)> + UnwindSafe + Send,
+fn assure_backing_up<'a>(
+    db: &mut DatabaseModel,
+    packages: impl IntoIterator<Item = &'a PackageId>,
 ) -> Result<(), Error> {
-    db.mutate(|v| {
-        let backing_up = v
-            .as_server_info_mut()
-            .as_status_info_mut()
-            .as_backup_progress_mut();
-        if backing_up
-            .clone()
-            .de()?
-            .iter()
-            .flat_map(|x| x.values())
-            .fold(false, |acc, x| {
-                if !x.complete {
-                    return true;
-                }
-                acc
-            })
-        {
-            return Err(Error::new(
-                eyre!("Server is already backing up!"),
-                ErrorKind::InvalidRequest,
-            ));
-        }
-        backing_up.ser(&Some(
-            packages
-                .into_iter()
-                .map(|(x, _)| (x.clone(), BackupProgress { complete: false }))
-                .collect(),
-        ))?;
-        Ok(())
-    })
-    .await
+    let backing_up = db
+        .as_public_mut()
+        .as_server_info_mut()
+        .as_status_info_mut()
+        .as_backup_progress_mut();
+    if backing_up
+        .clone()
+        .de()?
+        .iter()
+        .flat_map(|x| x.values())
+        .fold(false, |acc, x| {
+            if !x.complete {
+                return true;
+            }
+            acc
+        })
+    {
+        return Err(Error::new(
+            eyre!("Server is already backing up!"),
+            ErrorKind::InvalidRequest,
+        ));
+    }
+    backing_up.ser(&Some(
+        packages
+            .into_iter()
+            .map(|x| (x.clone(), BackupProgress { complete: false }))
+            .collect(),
+    ))?;
+    Ok(())
 }
 
 #[instrument(skip(ctx, backup_guard))]
 async fn perform_backup(
     ctx: &RpcContext,
     backup_guard: BackupMountGuard<TmpMountGuard>,
-    package_ids: &OrdSet<(PackageId, Version)>,
-) -> Result<BTreeMap<(PackageId, Version), PackageBackupReport>, Error> {
+    package_ids: &OrdSet<PackageId>,
+) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
     let mut backup_report = BTreeMap::new();
-    let backup_guard = Arc::new(Mutex::new(backup_guard));
+    let backup_guard = Arc::new(backup_guard);
 
-    for package_id in package_ids {
-        let (response, _report) = match ctx
-            .managers
-            .get(package_id)
-            .await
-            .ok_or_else(|| Error::new(eyre!("Manager not found"), ErrorKind::InvalidRequest))?
-            .backup(backup_guard.clone())
-            .await
-        {
-            BackupReturn::Ran { report, res } => (res, report),
-            BackupReturn::AlreadyRunning(report) => {
-                backup_report.insert(package_id.clone(), report);
-                continue;
-            }
-            BackupReturn::Error(error) => {
-                tracing::warn!("Backup thread error");
-                tracing::debug!("{error:?}");
-                backup_report.insert(
-                    package_id.clone(),
-                    PackageBackupReport {
-                        error: Some("Backup thread error".to_owned()),
-                    },
-                );
-                continue;
-            }
-        };
-        backup_report.insert(
-            package_id.clone(),
-            PackageBackupReport {
-                error: response.as_ref().err().map(|e| e.to_string()),
-            },
-        );
-
-        if let Ok(pkg_meta) = response {
-            backup_guard
-                .lock()
-                .await
-                .metadata
-                .package_backups
-                .insert(package_id.0.clone(), pkg_meta);
+    for id in package_ids {
+        if let Some(service) = &*ctx.services.get(id).await {
+            backup_report.insert(
+                id.clone(),
+                PackageBackupReport {
+                    error: service
+                        .backup(backup_guard.package_backup(id))
+                        .await
+                        .err()
+                        .map(|e| e.to_string()),
+                },
+            );
         }
     }
 
-    let ui = ctx.db.peek().await.into_ui().de()?;
+    let mut backup_guard = Arc::try_unwrap(backup_guard).map_err(|_| {
+        Error::new(
+            eyre!("leaked reference to BackupMountGuard"),
+            ErrorKind::Incoherent,
+        )
+    })?;
 
-    let mut os_backup_file = AtomicFile::new(
-        backup_guard.lock().await.as_ref().join("os-backup.cbor"),
-        None::<PathBuf>,
-    )
-    .await
-    .with_kind(ErrorKind::Filesystem)?;
+    let ui = ctx.db.peek().await.into_public().into_ui().de()?;
+
+    let mut os_backup_file =
+        AtomicFile::new(backup_guard.path().join("os-backup.cbor"), None::<PathBuf>)
+            .await
+            .with_kind(ErrorKind::Filesystem)?;
     os_backup_file
         .write_all(&IoFormat::Cbor.to_vec(&OsBackup {
             account: ctx.account.read().await.clone(),
@@ -284,11 +287,11 @@ async fn perform_backup(
         .await
         .with_kind(ErrorKind::Filesystem)?;
 
-    let luks_folder_old = backup_guard.lock().await.as_ref().join("luks.old");
+    let luks_folder_old = backup_guard.path().join("luks.old");
     if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
         tokio::fs::remove_dir_all(&luks_folder_old).await?;
     }
-    let luks_folder_bak = backup_guard.lock().await.as_ref().join("luks");
+    let luks_folder_bak = backup_guard.path().join("luks");
     if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
         tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
     }
@@ -298,14 +301,6 @@ async fn perform_backup(
     }
 
     let timestamp = Some(Utc::now());
-    let mut backup_guard = Arc::try_unwrap(backup_guard)
-        .map_err(|_err| {
-            Error::new(
-                eyre!("Backup guard could not ensure that the others where dropped"),
-                ErrorKind::Unknown,
-            )
-        })?
-        .into_inner();
 
     backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
     backup_guard.unencrypted_metadata.full = true;
@@ -315,7 +310,12 @@ async fn perform_backup(
     backup_guard.save_and_unmount().await?;
 
     ctx.db
-        .mutate(|v| v.as_server_info_mut().as_last_backup_mut().ser(&timestamp))
+        .mutate(|v| {
+            v.as_public_mut()
+                .as_server_info_mut()
+                .as_last_backup_mut()
+                .ser(&timestamp)
+        })
         .await?;
 
     Ok(backup_report)

core/startos/src/backup/mod.rs

@@ -1,33 +1,15 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::collections::BTreeMap;
 
 use chrono::{DateTime, Utc};
-use color_eyre::eyre::eyre;
-use helpers::AtomicFile;
-use models::{ImageId, OptionExt};
+use models::{HostId, PackageId};
 use reqwest::Url;
-use rpc_toolkit::command;
+use rpc_toolkit::{from_fn_async, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
-use tracing::instrument;
 
-use self::target::PackageBackupInfo;
-use crate::context::RpcContext;
-use crate::install::PKG_ARCHIVE_DIR;
-use crate::manager::manager_seed::ManagerSeed;
-use crate::net::interface::InterfaceId;
-use crate::net::keys::Key;
+use crate::context::CliContext;
+#[allow(unused_imports)]
 use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{Base32, Base64, IoFormat};
-use crate::util::Version;
-use crate::version::{Current, VersionT};
-use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
-use crate::{Error, ErrorKind, ResultExt};
+use crate::util::serde::{Base32, Base64};
 
 pub mod backup_bulk;
 pub mod os;
@@ -51,176 +33,24 @@ pub struct PackageBackupReport {
     pub error: Option<String>,
 }
 
-#[command(subcommands(backup_bulk::backup_all, target::target))]
-pub fn backup() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(rename = "backup", subcommands(restore::restore_packages_rpc))]
-pub fn package_backup() -> Result<(), Error> {
-    Ok(())
+// #[command(subcommands(backup_bulk::backup_all, target::target))]
+pub fn backup() -> ParentHandler {
+    ParentHandler::new()
+        .subcommand(
+            "create",
+            from_fn_async(backup_bulk::backup_all)
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand("target", target::target())
 }
 
 #[derive(Deserialize, Serialize)]
 struct BackupMetadata {
     pub timestamp: DateTime<Utc>,
     #[serde(default)]
-    pub network_keys: BTreeMap<InterfaceId, Base64<[u8; 32]>>,
+    pub network_keys: BTreeMap<HostId, Base64<[u8; 32]>>,
     #[serde(default)]
-    pub tor_keys: BTreeMap<InterfaceId, Base32<[u8; 64]>>, // DEPRECATED
+    pub tor_keys: BTreeMap<HostId, Base32<[u8; 64]>>, // DEPRECATED
     pub marketplace_url: Option<Url>,
 }
-
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct BackupActions {
-    pub create: PackageProcedure,
-    pub restore: PackageProcedure,
-}
-impl BackupActions {
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.create
-            .validate(eos_version, volumes, image_ids, false)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
-        self.restore
-            .validate(eos_version, volumes, image_ids, false)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
-        Ok(())
-    }
-
-    #[instrument(skip_all)]
-    pub async fn create(&self, seed: Arc<ManagerSeed>) -> Result<PackageBackupInfo, Error> {
-        let manifest = &seed.manifest;
-        let mut volumes = seed.manifest.volumes.to_readonly();
-        let ctx = &seed.ctx;
-        let pkg_id = &manifest.id;
-        let pkg_version = &manifest.version;
-        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
-        let backup_dir = backup_dir(&manifest.id);
-        if tokio::fs::metadata(&backup_dir).await.is_err() {
-            tokio::fs::create_dir_all(&backup_dir).await?
-        }
-        self.create
-            .execute::<(), NoOutput>(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::CreateBackup,
-                &volumes,
-                None,
-                None,
-            )
-            .await?
-            .map_err(|e| eyre!("{}", e.1))
-            .with_kind(crate::ErrorKind::Backup)?;
-        let (network_keys, tor_keys): (Vec<_>, Vec<_>) =
-            Key::for_package(&ctx.secret_store, pkg_id)
-                .await?
-                .into_iter()
-                .filter_map(|k| {
-                    let interface = k.interface().map(|(_, i)| i)?;
-                    Some((
-                        (interface.clone(), Base64(k.as_bytes())),
-                        (interface, Base32(k.tor_key().as_bytes())),
-                    ))
-                })
-                .unzip();
-        let marketplace_url = ctx
-            .db
-            .peek()
-            .await
-            .as_package_data()
-            .as_idx(&pkg_id)
-            .or_not_found(pkg_id)?
-            .expect_as_installed()?
-            .as_installed()
-            .as_marketplace_url()
-            .de()?;
-        let tmp_path = Path::new(BACKUP_DIR)
-            .join(pkg_id)
-            .join(format!("{}.s9pk", pkg_id));
-        let s9pk_path = ctx
-            .datadir
-            .join(PKG_ARCHIVE_DIR)
-            .join(pkg_id)
-            .join(pkg_version.as_str())
-            .join(format!("{}.s9pk", pkg_id));
-        let mut infile = File::open(&s9pk_path).await?;
-        let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
-            .await
-            .with_kind(ErrorKind::Filesystem)?;
-        tokio::io::copy(&mut infile, &mut *outfile)
-            .await
-            .with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
-                )
-            })?;
-        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
-        let timestamp = Utc::now();
-        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
-        let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
-            .await
-            .with_kind(ErrorKind::Filesystem)?;
-        let network_keys = network_keys.into_iter().collect();
-        let tor_keys = tor_keys.into_iter().collect();
-        outfile
-            .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
-                timestamp,
-                network_keys,
-                tor_keys,
-                marketplace_url,
-            })?)
-            .await?;
-        outfile.save().await.with_kind(ErrorKind::Filesystem)?;
-        Ok(PackageBackupInfo {
-            os_version: Current::new().semver().into(),
-            title: manifest.title.clone(),
-            version: pkg_version.clone(),
-            timestamp,
-        })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn restore(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-    ) -> Result<Option<Url>, Error> {
-        let mut volumes = volumes.clone();
-        volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
-        self.restore
-            .execute::<(), NoOutput>(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::RestoreBackup,
-                &volumes,
-                None,
-                None,
-            )
-            .await?
-            .map_err(|e| eyre!("{}", e.1))
-            .with_kind(crate::ErrorKind::Restore)?;
-        let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
-        let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
-            &tokio::fs::read(&metadata_path).await.with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    metadata_path.display().to_string(),
-                )
-            })?,
-        )?;
-
-        Ok(metadata.marketplace_url)
-    }
-}

core/startos/src/backup/os.rs

@@ -1,13 +1,15 @@
-use openssl::pkey::PKey;
+use openssl::pkey::{PKey, Private};
 use openssl::x509::X509;
 use patch_db::Value;
 use serde::{Deserialize, Serialize};
+use ssh_key::private::Ed25519Keypair;
+use torut::onion::TorSecretKeyV3;
 
 use crate::account::AccountInfo;
 use crate::hostname::{generate_hostname, generate_id, Hostname};
-use crate::net::keys::Key;
 use crate::prelude::*;
-use crate::util::serde::Base64;
+use crate::util::crypto::ed25519_expand_key;
+use crate::util::serde::{Base32, Base64, Pem};
 
 pub struct OsBackup {
     pub account: AccountInfo,
@@ -19,19 +21,23 @@ impl<'de> Deserialize<'de> for OsBackup {
         D: serde::Deserializer<'de>,
     {
         let tagged = OsBackupSerDe::deserialize(deserializer)?;
-        match tagged.version {
+        Ok(match tagged.version {
             0 => patch_db::value::from_value::<OsBackupV0>(tagged.rest)
                 .map_err(serde::de::Error::custom)?
                 .project()
-                .map_err(serde::de::Error::custom),
+                .map_err(serde::de::Error::custom)?,
             1 => patch_db::value::from_value::<OsBackupV1>(tagged.rest)
                 .map_err(serde::de::Error::custom)?
-                .project()
-                .map_err(serde::de::Error::custom),
-            v => Err(serde::de::Error::custom(&format!(
-                "Unknown backup version {v}"
-            ))),
-        }
+                .project(),
+            2 => patch_db::value::from_value::<OsBackupV2>(tagged.rest)
+                .map_err(serde::de::Error::custom)?
+                .project(),
+            v => {
+                return Err(serde::de::Error::custom(&format!(
+                    "Unknown backup version {v}"
+                )))
+            }
+        })
     }
 }
 impl Serialize for OsBackup {
@@ -40,11 +46,9 @@ impl Serialize for OsBackup {
         S: serde::Serializer,
     {
         OsBackupSerDe {
-            version: 1,
-            rest: patch_db::value::to_value(
-                &OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
-            )
-            .map_err(serde::ser::Error::custom)?,
+            version: 2,
+            rest: patch_db::value::to_value(&OsBackupV2::unproject(self))
+                .map_err(serde::ser::Error::custom)?,
         }
         .serialize(serializer)
     }
@@ -62,10 +66,10 @@ struct OsBackupSerDe {
 #[derive(Deserialize)]
 #[serde(rename = "kebab-case")]
 struct OsBackupV0 {
-    // tor_key: Base32<[u8; 64]>,
-    root_ca_key: String,  // PEM Encoded OpenSSL Key
-    root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
-    ui: Value,            // JSON Value
+    tor_key: Base32<[u8; 64]>, // Base32 Encoded Ed25519 Expanded Secret Key
+    root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
+    root_ca_cert: Pem<X509>,   // PEM Encoded OpenSSL X509 Certificate
+    ui: Value,                 // JSON Value
 }
 impl OsBackupV0 {
     fn project(self) -> Result<OsBackup, Error> {
@@ -74,9 +78,13 @@ impl OsBackupV0 {
                 server_id: generate_id(),
                 hostname: generate_hostname(),
                 password: Default::default(),
-                key: Key::new(None),
-                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
-                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
+                root_ca_key: self.root_ca_key.0,
+                root_ca_cert: self.root_ca_cert.0,
+                ssh_key: ssh_key::PrivateKey::random(
+                    &mut rand::thread_rng(),
+                    ssh_key::Algorithm::Ed25519,
+                )?,
+                tor_key: TorSecretKeyV3::from(self.tor_key.0),
             },
             ui: self.ui,
         })
@@ -87,36 +95,67 @@ impl OsBackupV0 {
 #[derive(Deserialize, Serialize)]
 #[serde(rename = "kebab-case")]
 struct OsBackupV1 {
-    server_id: String,         // uuidv4
-    hostname: String,          // embassy-<adjective>-<noun>
-    net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
-    root_ca_key: String,       // PEM Encoded OpenSSL Key
-    root_ca_cert: String,      // PEM Encoded OpenSSL X509 Certificate
-    ui: Value,                 // JSON Value
-                               // TODO add more
+    server_id: String,               // uuidv4
+    hostname: String,                // embassy-<adjective>-<noun>
+    net_key: Base64<[u8; 32]>,       // Ed25519 Secret Key
+    root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
+    root_ca_cert: Pem<X509>,         // PEM Encoded OpenSSL X509 Certificate
+    ui: Value,                       // JSON Value
 }
 impl OsBackupV1 {
-    fn project(self) -> Result<OsBackup, Error> {
-        Ok(OsBackup {
+    fn project(self) -> OsBackup {
+        OsBackup {
             account: AccountInfo {
                 server_id: self.server_id,
                 hostname: Hostname(self.hostname),
                 password: Default::default(),
-                key: Key::from_bytes(None, self.net_key.0),
-                root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
-                root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
+                root_ca_key: self.root_ca_key.0,
+                root_ca_cert: self.root_ca_cert.0,
+                ssh_key: ssh_key::PrivateKey::from(Ed25519Keypair::from_seed(&self.net_key.0)),
+                tor_key: TorSecretKeyV3::from(ed25519_expand_key(&self.net_key.0)),
             },
             ui: self.ui,
-        })
-    }
-    fn unproject(backup: &OsBackup) -> Result<Self, Error> {
-        Ok(Self {
-            server_id: backup.account.server_id.clone(),
-            hostname: backup.account.hostname.0.clone(),
-            net_key: Base64(backup.account.key.as_bytes()),
-            root_ca_key: String::from_utf8(backup.account.root_ca_key.private_key_to_pem_pkcs8()?)?,
-            root_ca_cert: String::from_utf8(backup.account.root_ca_cert.to_pem()?)?,
-            ui: backup.ui.clone(),
-        })
+        }
+    }
+}
+
+/// V2
+#[derive(Deserialize, Serialize)]
+#[serde(rename = "kebab-case")]
+
+struct OsBackupV2 {
+    server_id: String,                 // uuidv4
+    hostname: String,                  // <adjective>-<noun>
+    root_ca_key: Pem<PKey<Private>>,   // PEM Encoded OpenSSL Key
+    root_ca_cert: Pem<X509>,           // PEM Encoded OpenSSL X509 Certificate
+    ssh_key: Pem<ssh_key::PrivateKey>, // PEM Encoded OpenSSH Key
+    tor_key: TorSecretKeyV3,           // Base64 Encoded Ed25519 Expanded Secret Key
+    ui: Value,                         // JSON Value
+}
+impl OsBackupV2 {
+    fn project(self) -> OsBackup {
+        OsBackup {
+            account: AccountInfo {
+                server_id: self.server_id,
+                hostname: Hostname(self.hostname),
+                password: Default::default(),
+                root_ca_key: self.root_ca_key.0,
+                root_ca_cert: self.root_ca_cert.0,
+                ssh_key: self.ssh_key.0,
+                tor_key: self.tor_key,
+            },
+            ui: self.ui,
+        }
+    }
+    fn unproject(backup: &OsBackup) -> Self {
+        Self {
+            server_id: backup.account.server_id.clone(),
+            hostname: backup.account.hostname.0.clone(),
+            root_ca_key: Pem(backup.account.root_ca_key.clone()),
+            root_ca_cert: Pem(backup.account.root_ca_cert.clone()),
+            ssh_key: Pem(backup.account.ssh_key.clone()),
+            tor_key: backup.account.tor_key.clone(),
+            ui: backup.ui.clone(),
+        }
     }
 }

core/startos/src/backup/restore.rs

@@ -1,170 +1,72 @@
 use std::collections::BTreeMap;
-use std::path::Path;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
-use std::time::Duration;
 
-use clap::ArgMatches;
-use futures::future::BoxFuture;
-use futures::{stream, FutureExt, StreamExt};
+use clap::Parser;
+use futures::{stream, StreamExt};
+use models::PackageId;
 use openssl::x509::X509;
-use rpc_toolkit::command;
-use sqlx::Connection;
-use tokio::fs::File;
+use patch_db::json_ptr::ROOT;
+use serde::{Deserialize, Serialize};
 use torut::onion::OnionAddressV3;
 use tracing::instrument;
 
 use super::target::BackupTargetId;
 use crate::backup::os::OsBackup;
-use crate::backup::BackupMetadata;
-use crate::context::rpc::RpcContextConfig;
 use crate::context::{RpcContext, SetupContext};
-use crate::db::model::{PackageDataEntry, PackageDataEntryRestoring, StaticFiles};
-use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
+use crate::db::model::Database;
+use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::ReadWrite;
-use crate::disk::mount::guard::TmpMountGuard;
+use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
 use crate::hostname::Hostname;
 use crate::init::init;
-use crate::install::progress::InstallProgress;
-use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
-use crate::notifications::NotificationLevel;
 use crate::prelude::*;
-use crate::s9pk::manifest::{Manifest, PackageId};
-use crate::s9pk::reader::S9pkReader;
-use crate::setup::SetupStatus;
-use crate::util::display_none;
-use crate::util::io::dir_size;
+use crate::s9pk::S9pk;
+use crate::service::service_map::DownloadInstallFuture;
 use crate::util::serde::IoFormat;
-use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
 
-fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
-    arg.split(',')
-        .map(|s| s.trim().parse().map_err(Error::from))
-        .collect()
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct RestorePackageParams {
+    pub ids: Vec<PackageId>,
+    pub target_id: BackupTargetId,
+    pub password: String,
 }
 
-#[command(rename = "restore", display(display_none))]
+// TODO dr Why doesn't anything use this
+// #[command(rename = "restore", display(display_none))]
 #[instrument(skip(ctx, password))]
 pub async fn restore_packages_rpc(
-    #[context] ctx: RpcContext,
-    #[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg] password: String,
+    ctx: RpcContext,
+    RestorePackageParams {
+        ids,
+        target_id,
+        password,
+    }: RestorePackageParams,
 ) -> Result<(), Error> {
-    let fs = target_id
-        .load(ctx.secret_store.acquire().await?.as_mut())
-        .await?;
+    let fs = target_id.load(&ctx.db.peek().await)?;
     let backup_guard =
         BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
 
-    let (backup_guard, tasks, _) = restore_packages(&ctx, backup_guard, ids).await?;
+    let tasks = restore_packages(&ctx, backup_guard, ids).await?;
 
     tokio::spawn(async move {
-        stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
-            .for_each_concurrent(5, |(res, ctx)| async move {
-                match res.await {
-                    (Ok(_), _) => (),
-                    (Err(err), package_id) => {
-                        if let Err(err) = ctx
-                            .notification_manager
-                            .notify(
-                                ctx.db.clone(),
-                                Some(package_id.clone()),
-                                NotificationLevel::Error,
-                                "Restoration Failure".to_string(),
-                                format!("Error restoring package {}: {}", package_id, err),
-                                (),
-                                None,
-                            )
-                            .await
-                        {
-                            tracing::error!("Failed to notify: {}", err);
-                            tracing::debug!("{:?}", err);
-                        };
-                        tracing::error!("Error restoring package {}: {}", package_id, err);
+        stream::iter(tasks)
+            .for_each_concurrent(5, |(id, res)| async move {
+                match async { res.await?.await }.await {
+                    Ok(_) => (),
+                    Err(err) => {
+                        tracing::error!("Error restoring package {}: {}", id, err);
                         tracing::debug!("{:?}", err);
                     }
                 }
             })
             .await;
-        if let Err(e) = backup_guard.unmount().await {
-            tracing::error!("Error unmounting backup drive: {}", e);
-            tracing::debug!("{:?}", e);
-        }
     });
 
     Ok(())
 }
 
-async fn approximate_progress(
-    rpc_ctx: &RpcContext,
-    progress: &mut ProgressInfo,
-) -> Result<(), Error> {
-    for (id, size) in &mut progress.target_volume_size {
-        let dir = rpc_ctx.datadir.join(PKG_VOLUME_DIR).join(id).join("data");
-        if tokio::fs::metadata(&dir).await.is_err() {
-            *size = 0;
-        } else {
-            *size = dir_size(&dir, None).await?;
-        }
-    }
-    Ok(())
-}
-
-async fn approximate_progress_loop(
-    ctx: &SetupContext,
-    rpc_ctx: &RpcContext,
-    mut starting_info: ProgressInfo,
-) {
-    loop {
-        if let Err(e) = approximate_progress(rpc_ctx, &mut starting_info).await {
-            tracing::error!("Failed to approximate restore progress: {}", e);
-            tracing::debug!("{:?}", e);
-        } else {
-            *ctx.setup_status.write().await = Some(Ok(starting_info.flatten()));
-        }
-        tokio::time::sleep(Duration::from_secs(1)).await;
-    }
-}
-
-#[derive(Debug, Default)]
-struct ProgressInfo {
-    package_installs: BTreeMap<PackageId, Arc<InstallProgress>>,
-    src_volume_size: BTreeMap<PackageId, u64>,
-    target_volume_size: BTreeMap<PackageId, u64>,
-}
-impl ProgressInfo {
-    fn flatten(&self) -> SetupStatus {
-        let mut total_bytes = 0;
-        let mut bytes_transferred = 0;
-
-        for progress in self.package_installs.values() {
-            total_bytes += ((progress.size.unwrap_or(0) as f64) * 2.2) as u64;
-            bytes_transferred += progress.downloaded.load(Ordering::SeqCst);
-            bytes_transferred += ((progress.validated.load(Ordering::SeqCst) as f64) * 0.2) as u64;
-            bytes_transferred += progress.unpacked.load(Ordering::SeqCst);
-        }
-
-        for size in self.src_volume_size.values() {
-            total_bytes += *size;
-        }
-
-        for size in self.target_volume_size.values() {
-            bytes_transferred += *size;
-        }
-
-        if bytes_transferred > total_bytes {
-            bytes_transferred = total_bytes;
-        }
-
-        SetupStatus {
-            total_bytes: Some(total_bytes),
-            bytes_transferred,
-            complete: false,
-        }
-    }
-}
-
 #[instrument(skip(ctx))]
 pub async fn recover_full_embassy(
     ctx: SetupContext,
@@ -179,7 +81,7 @@ pub async fn recover_full_embassy(
     )
     .await?;
 
-    let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
+    let os_backup_path = backup_guard.path().join("os-backup.cbor");
     let mut os_backup: OsBackup = IoFormat::Cbor.from_slice(
         &tokio::fs::read(&os_backup_path)
             .await
@@ -193,17 +95,13 @@ pub async fn recover_full_embassy(
     )
     .with_kind(ErrorKind::PasswordHashGeneration)?;
 
-    let secret_store = ctx.secret_store().await?;
+    let db = ctx.db().await?;
+    db.put(&ROOT, &Database::init(&os_backup.account)?).await?;
+    drop(db);
 
-    os_backup.account.save(&secret_store).await?;
+    init(&ctx.config).await?;
 
-    secret_store.close().await;
-
-    let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
-
-    init(&cfg).await?;
-
-    let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
+    let rpc_ctx = RpcContext::init(&ctx.config, disk_guid.clone()).await?;
 
     let ids: Vec<_> = backup_guard
         .metadata
@@ -211,43 +109,25 @@ pub async fn recover_full_embassy(
         .keys()
         .cloned()
         .collect();
-    let (backup_guard, tasks, progress_info) =
-        restore_packages(&rpc_ctx, backup_guard, ids).await?;
-    let task_consumer_rpc_ctx = rpc_ctx.clone();
-    tokio::select! {
-        _ = async move {
-            stream::iter(tasks.into_iter().map(|x| (x,  task_consumer_rpc_ctx.clone())))
-                .for_each_concurrent(5, |(res, ctx)| async move {
-                    match res.await {
-                        (Ok(_), _) => (),
-                        (Err(err), package_id) => {
-                            if let Err(err) = ctx.notification_manager.notify(
-                                ctx.db.clone(),
-                                Some(package_id.clone()),
-                                NotificationLevel::Error,
-                                "Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
-                                tracing::error!("Failed to notify: {}", err);
-                                tracing::debug!("{:?}", err);
-                                };
-                            tracing::error!("Error restoring package {}: {}", package_id, err);
-                            tracing::debug!("{:?}", err);
-                        },
-                    }
-                }).await;
+    let tasks = restore_packages(&rpc_ctx, backup_guard, ids).await?;
+    stream::iter(tasks)
+        .for_each_concurrent(5, |(id, res)| async move {
+            match async { res.await?.await }.await {
+                Ok(_) => (),
+                Err(err) => {
+                    tracing::error!("Error restoring package {}: {}", id, err);
+                    tracing::debug!("{:?}", err);
+                }
+            }
+        })
+        .await;
 
-        } => {
-
-        },
-        _ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
-    }
-
-    backup_guard.unmount().await?;
     rpc_ctx.shutdown().await?;
 
     Ok((
         disk_guid,
         os_backup.account.hostname,
-        os_backup.account.key.tor_address(),
+        os_backup.account.tor_key.public().get_onion_address(),
         os_backup.account.root_ca_cert,
     ))
 }
@@ -257,205 +137,25 @@ async fn restore_packages(
     ctx: &RpcContext,
     backup_guard: BackupMountGuard<TmpMountGuard>,
     ids: Vec<PackageId>,
-) -> Result<
-    (
-        BackupMountGuard<TmpMountGuard>,
-        Vec<BoxFuture<'static, (Result<(), Error>, PackageId)>>,
-        ProgressInfo,
-    ),
-    Error,
-> {
-    let guards = assure_restoring(ctx, ids, &backup_guard).await?;
-
-    let mut progress_info = ProgressInfo::default();
-
-    let mut tasks = Vec::with_capacity(guards.len());
-    for (manifest, guard) in guards {
-        let id = manifest.id.clone();
-        let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
-        progress_info
-            .package_installs
-            .insert(id.clone(), progress.clone());
-        progress_info
-            .src_volume_size
-            .insert(id.clone(), dir_size(backup_dir(&id), None).await?);
-        progress_info.target_volume_size.insert(id.clone(), 0);
-        let package_id = id.clone();
-        tasks.push(
-            async move {
-                if let Err(e) = task.await {
-                    tracing::error!("Error restoring package {}: {}", id, e);
-                    tracing::debug!("{:?}", e);
-                    Err(e)
-                } else {
-                    Ok(())
-                }
-            }
-            .map(|x| (x, package_id))
-            .boxed(),
-        );
-    }
-
-    Ok((backup_guard, tasks, progress_info))
-}
-
-#[instrument(skip(ctx, backup_guard))]
-async fn assure_restoring(
-    ctx: &RpcContext,
-    ids: Vec<PackageId>,
-    backup_guard: &BackupMountGuard<TmpMountGuard>,
-) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
-    let mut guards = Vec::with_capacity(ids.len());
-
-    let mut insert_packages = BTreeMap::new();
-
+) -> Result<BTreeMap<PackageId, DownloadInstallFuture>, Error> {
+    let backup_guard = Arc::new(backup_guard);
+    let mut tasks = BTreeMap::new();
     for id in ids {
-        let peek = ctx.db.peek().await;
-
-        let model = peek.as_package_data().as_idx(&id);
-
-        if !model.is_none() {
-            return Err(Error::new(
-                eyre!("Can't restore over existing package: {}", id),
-                crate::ErrorKind::InvalidRequest,
-            ));
-        }
-        let guard = backup_guard.mount_package_backup(&id).await?;
-        let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
-        let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
-
-        let manifest = rdr.manifest().await?;
-        let version = manifest.version.clone();
-        let progress = Arc::new(InstallProgress::new(Some(
-            tokio::fs::metadata(&s9pk_path).await?.len(),
-        )));
-
-        let public_dir_path = ctx
-            .datadir
-            .join(PKG_PUBLIC_DIR)
-            .join(&id)
-            .join(version.as_str());
-        tokio::fs::create_dir_all(&public_dir_path).await?;
-
-        let license_path = public_dir_path.join("LICENSE.md");
-        let mut dst = File::create(&license_path).await?;
-        tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
-        dst.sync_all().await?;
-
-        let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
-        let mut dst = File::create(&instructions_path).await?;
-        tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
-        dst.sync_all().await?;
-
-        let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
-        let icon_path = public_dir_path.join(&icon_path);
-        let mut dst = File::create(&icon_path).await?;
-        tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
-        dst.sync_all().await?;
-        insert_packages.insert(
-            id.clone(),
-            PackageDataEntry::Restoring(PackageDataEntryRestoring {
-                install_progress: progress.clone(),
-                static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
-                manifest: manifest.clone(),
-            }),
-        );
-
-        guards.push((manifest, guard));
-    }
-    ctx.db
-        .mutate(|db| {
-            for (id, package) in insert_packages {
-                db.as_package_data_mut().insert(&id, &package)?;
-            }
-            Ok(())
-        })
-        .await?;
-    Ok(guards)
-}
-
-#[instrument(skip(ctx, guard))]
-async fn restore_package<'a>(
-    ctx: RpcContext,
-    manifest: Manifest,
-    guard: PackageBackupMountGuard,
-) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
-    let id = manifest.id.clone();
-    let s9pk_path = Path::new(BACKUP_DIR)
-        .join(&manifest.id)
-        .join(format!("{}.s9pk", id));
-
-    let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
-    let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
-        &tokio::fs::read(&metadata_path)
-            .await
-            .with_ctx(|_| (ErrorKind::Filesystem, metadata_path.display().to_string()))?,
-    )?;
-
-    let mut secrets = ctx.secret_store.acquire().await?;
-    let mut secrets_tx = secrets.begin().await?;
-    for (iface, key) in metadata.network_keys {
-        let k = key.0.as_slice();
-        sqlx::query!(
-            "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-            id.to_string(),
-            iface.to_string(),
-            k,
-        )
-        .execute(secrets_tx.as_mut()).await?;
-    }
-    // DEPRECATED
-    for (iface, key) in metadata.tor_keys {
-        let k = key.0.as_slice();
-        sqlx::query!(
-            "INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
-            id.to_string(),
-            iface.to_string(),
-            k,
-        )
-        .execute(secrets_tx.as_mut()).await?;
-    }
-    secrets_tx.commit().await?;
-    drop(secrets);
-
-    let len = tokio::fs::metadata(&s9pk_path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?
-        .len();
-    let file = File::open(&s9pk_path)
-        .await
-        .with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?;
-
-    let progress = InstallProgress::new(Some(len));
-    let marketplace_url = metadata.marketplace_url;
-
-    let progress = Arc::new(progress);
-
-    ctx.db
-        .mutate(|db| {
-            db.as_package_data_mut().insert(
-                &id,
-                &PackageDataEntry::Restoring(PackageDataEntryRestoring {
-                    install_progress: progress.clone(),
-                    static_files: StaticFiles::local(
-                        &id,
-                        &manifest.version,
-                        manifest.assets.icon_type(),
-                    ),
-                    manifest: manifest.clone(),
-                }),
+        let backup_dir = backup_guard.clone().package_backup(&id);
+        let task = ctx
+            .services
+            .install(
+                ctx.clone(),
+                S9pk::open(
+                    backup_dir.path().join(&id).with_extension("s9pk"),
+                    Some(&id),
+                )
+                .await?,
+                Some(backup_dir),
             )
-        })
-        .await?;
-    Ok((
-        progress.clone(),
-        async move {
-            download_install_s9pk(ctx, manifest, marketplace_url, progress, file, None).await?;
+            .await?;
+        tasks.insert(id, task);
+    }
 
-            guard.unmount().await?;
-
-            Ok(())
-        }
-        .boxed(),
-    ))
+    Ok(tasks)
 }

core/startos/src/backup/target/cifs.rs

@@ -1,21 +1,40 @@
+use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
+use clap::Parser;
 use color_eyre::eyre::eyre;
-use futures::TryStreamExt;
-use rpc_toolkit::command;
+use imbl_value::InternedString;
+use rpc_toolkit::{command, from_fn_async, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
-use sqlx::{Executor, Postgres};
 
 use super::{BackupTarget, BackupTargetId};
-use crate::context::RpcContext;
+use crate::context::{CliContext, RpcContext};
+use crate::db::model::DatabaseModel;
 use crate::disk::mount::filesystem::cifs::Cifs;
 use crate::disk::mount::filesystem::ReadOnly;
-use crate::disk::mount::guard::TmpMountGuard;
+use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
 use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
 use crate::prelude::*;
-use crate::util::display_none;
 use crate::util::serde::KeyVal;
 
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct CifsTargets(pub BTreeMap<u32, Cifs>);
+impl CifsTargets {
+    pub fn new() -> Self {
+        Self(BTreeMap::new())
+    }
+}
+impl Map for CifsTargets {
+    type Key = u32;
+    type Value = Cifs;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Self::key_string(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(InternedString::from_display(key))
+    }
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct CifsBackupTarget {
@@ -26,38 +45,70 @@ pub struct CifsBackupTarget {
     embassy_os: Option<EmbassyOsRecoveryInfo>,
 }
 
-#[command(subcommands(add, update, remove))]
-pub fn cifs() -> Result<(), Error> {
-    Ok(())
+pub fn cifs() -> ParentHandler {
+    ParentHandler::new()
+        .subcommand(
+            "add",
+            from_fn_async(add)
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand(
+            "update",
+            from_fn_async(update)
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove)
+                .no_display()
+                .with_remote_cli::<CliContext>(),
+        )
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct AddParams {
+    pub hostname: String,
+    pub path: PathBuf,
+    pub username: String,
+    pub password: Option<String>,
 }
 
-#[command(display(display_none))]
 pub async fn add(
-    #[context] ctx: RpcContext,
-    #[arg] hostname: String,
-    #[arg] path: PathBuf,
-    #[arg] username: String,
-    #[arg] password: Option<String>,
-) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
-    let cifs = Cifs {
+    ctx: RpcContext,
+    AddParams {
         hostname,
         path,
         username,
         password,
+    }: AddParams,
+) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
+    let cifs = Cifs {
+        hostname,
+        path: Path::new("/").join(path),
+        username,
+        password,
     };
     let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
-    let embassy_os = recovery_info(&guard).await?;
+    let embassy_os = recovery_info(guard.path()).await?;
     guard.unmount().await?;
-    let path_string = Path::new("/").join(&cifs.path).display().to_string();
-    let id: i32 = sqlx::query!(
-        "INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
-        cifs.hostname,
-        path_string,
-        cifs.username,
-        cifs.password,
-    )
-    .fetch_one(&ctx.secret_store)
-    .await?.id;
+    let id = ctx
+        .db
+        .mutate(|db| {
+            let id = db
+                .as_private()
+                .as_cifs()
+                .keys()?
+                .into_iter()
+                .max()
+                .map_or(0, |a| a + 1);
+            db.as_private_mut().as_cifs_mut().insert(&id, &cifs)?;
+            Ok(id)
+        })
+        .await?;
     Ok(KeyVal {
         key: BackupTargetId::Cifs { id },
         value: BackupTarget::Cifs(CifsBackupTarget {
@@ -70,14 +121,26 @@ pub async fn add(
     })
 }
 
-#[command(display(display_none))]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct UpdateParams {
+    pub id: BackupTargetId,
+    pub hostname: String,
+    pub path: PathBuf,
+    pub username: String,
+    pub password: Option<String>,
+}
+
 pub async fn update(
-    #[context] ctx: RpcContext,
-    #[arg] id: BackupTargetId,
-    #[arg] hostname: String,
-    #[arg] path: PathBuf,
-    #[arg] username: String,
-    #[arg] password: Option<String>,
+    ctx: RpcContext,
+    UpdateParams {
+        id,
+        hostname,
+        path,
+        username,
+        password,
+    }: UpdateParams,
 ) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
     let id = if let BackupTargetId::Cifs { id } = id {
         id
@@ -89,32 +152,27 @@ pub async fn update(
     };
     let cifs = Cifs {
         hostname,
-        path,
+        path: Path::new("/").join(path),
         username,
         password,
     };
     let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
-    let embassy_os = recovery_info(&guard).await?;
+    let embassy_os = recovery_info(guard.path()).await?;
     guard.unmount().await?;
-    let path_string = Path::new("/").join(&cifs.path).display().to_string();
-    if sqlx::query!(
-        "UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
-        cifs.hostname,
-        path_string,
-        cifs.username,
-        cifs.password,
-        id,
-    )
-    .execute(&ctx.secret_store)
-    .await?
-    .rows_affected()
-        == 0
-    {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
-            ErrorKind::NotFound,
-        ));
-    };
+    ctx.db
+        .mutate(|db| {
+            db.as_private_mut()
+                .as_cifs_mut()
+                .as_idx_mut(&id)
+                .ok_or_else(|| {
+                    Error::new(
+                        eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
+                        ErrorKind::NotFound,
+                    )
+                })?
+                .ser(&cifs)
+        })
+        .await?;
     Ok(KeyVal {
         key: BackupTargetId::Cifs { id },
         value: BackupTarget::Cifs(CifsBackupTarget {
@@ -127,8 +185,14 @@ pub async fn update(
     })
 }
 
-#[command(display(display_none))]
-pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Result<(), Error> {
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct RemoveParams {
+    pub id: BackupTargetId,
+}
+
+pub async fn remove(ctx: RpcContext, RemoveParams { id }: RemoveParams) -> Result<(), Error> {
     let id = if let BackupTargetId::Cifs { id } = id {
         id
     } else {
@@ -137,74 +201,46 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
             ErrorKind::NotFound,
         ));
     };
-    if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
-        .execute(&ctx.secret_store)
-        .await?
-        .rows_affected()
-        == 0
-    {
-        return Err(Error::new(
-            eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
-            ErrorKind::NotFound,
-        ));
-    };
+    ctx.db
+        .mutate(|db| db.as_private_mut().as_cifs_mut().remove(&id))
+        .await?;
     Ok(())
 }
 
-pub async fn load<Ex>(secrets: &mut Ex, id: i32) -> Result<Cifs, Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let record = sqlx::query!(
-        "SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
-        id
-    )
-    .fetch_one(secrets)
-    .await?;
-
-    Ok(Cifs {
-        hostname: record.hostname,
-        path: PathBuf::from(record.path),
-        username: record.username,
-        password: record.password,
-    })
+pub fn load(db: &DatabaseModel, id: u32) -> Result<Cifs, Error> {
+    db.as_private()
+        .as_cifs()
+        .as_idx(&id)
+        .ok_or_else(|| {
+            Error::new(
+                eyre!("Backup Target ID {} Not Found", id),
+                ErrorKind::NotFound,
+            )
+        })?
+        .de()
 }
 
-pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(i32, CifsBackupTarget)>, Error>
-where
-    for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-{
-    let mut records =
-        sqlx::query!("SELECT id, hostname, path, username, password FROM cifs_shares")
-            .fetch_many(secrets);
-
+pub async fn list(db: &DatabaseModel) -> Result<Vec<(u32, CifsBackupTarget)>, Error> {
     let mut cifs = Vec::new();
-    while let Some(query_result) = records.try_next().await? {
-        if let Some(record) = query_result.right() {
-            let mount_info = Cifs {
-                hostname: record.hostname,
-                path: PathBuf::from(record.path),
-                username: record.username,
-                password: record.password,
-            };
-            let embassy_os = async {
-                let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
-                let embassy_os = recovery_info(&guard).await?;
-                guard.unmount().await?;
-                Ok::<_, Error>(embassy_os)
-            }
-            .await;
-            cifs.push((
-                record.id,
-                CifsBackupTarget {
-                    hostname: mount_info.hostname,
-                    path: mount_info.path,
-                    username: mount_info.username,
-                    mountable: embassy_os.is_ok(),
-                    embassy_os: embassy_os.ok().and_then(|a| a),
-                },
-            ));
+    for (id, model) in db.as_private().as_cifs().as_entries()? {
+        let mount_info = model.de()?;
+        let embassy_os = async {
+            let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
+            let embassy_os = recovery_info(guard.path()).await?;
+            guard.unmount().await?;
+            Ok::<_, Error>(embassy_os)
         }
+        .await;
+        cifs.push((
+            id,
+            CifsBackupTarget {
+                hostname: mount_info.hostname,
+                path: mount_info.path,
+                username: mount_info.username,
+                mountable: embassy_os.is_ok(),
+                embassy_os: embassy_os.ok().and_then(|a| a),
+            },
+        ));
     }
 
     Ok(cifs)

core/startos/src/backup/target/mod.rs

@@ -1,31 +1,34 @@
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 
-use async_trait::async_trait;
 use chrono::{DateTime, Utc};
-use clap::ArgMatches;
+use clap::builder::ValueParserFactory;
+use clap::Parser;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
 use digest::OutputSizeUser;
-use rpc_toolkit::command;
+use models::PackageId;
+use rpc_toolkit::{command, from_fn_async, AnyContext, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
-use sqlx::{Executor, Postgres};
 use tokio::sync::Mutex;
 use tracing::instrument;
 
 use self::cifs::CifsBackupTarget;
-use crate::context::RpcContext;
+use crate::context::{CliContext, RpcContext};
+use crate::db::model::DatabaseModel;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::cifs::Cifs;
 use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
-use crate::disk::mount::guard::TmpMountGuard;
+use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
 use crate::disk::util::PartitionInfo;
 use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
-use crate::util::{display_none, Version};
+use crate::util::clap::FromStrParser;
+use crate::util::serde::{
+    deserialize_from_str, display_serializable, serialize_display, HandlerExtSerde, WithIoFormat,
+};
+use crate::util::Version;
 
 pub mod cifs;
 
@@ -46,18 +49,15 @@ pub enum BackupTarget {
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub enum BackupTargetId {
     Disk { logicalname: PathBuf },
-    Cifs { id: i32 },
+    Cifs { id: u32 },
 }
 impl BackupTargetId {
-    pub async fn load<Ex>(self, secrets: &mut Ex) -> Result<BackupTargetFS, Error>
-    where
-        for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
-    {
+    pub fn load(self, db: &DatabaseModel) -> Result<BackupTargetFS, Error> {
         Ok(match self {
             BackupTargetId::Disk { logicalname } => {
                 BackupTargetFS::Disk(BlockDev::new(logicalname))
             }
-            BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(secrets, id).await?),
+            BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(db, id)?),
         })
     }
 }
@@ -84,6 +84,12 @@ impl std::str::FromStr for BackupTargetId {
         }
     }
 }
+impl ValueParserFactory for BackupTargetId {
+    type Parser = FromStrParser<Self>;
+    fn value_parser() -> Self::Parser {
+        FromStrParser::new()
+    }
+}
 impl<'de> Deserialize<'de> for BackupTargetId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -108,9 +114,8 @@ pub enum BackupTargetFS {
     Disk(BlockDev<PathBuf>),
     Cifs(Cifs),
 }
-#[async_trait]
 impl FileSystem for BackupTargetFS {
-    async fn mount<P: AsRef<Path> + Send + Sync>(
+    async fn mount<P: AsRef<Path> + Send>(
         &self,
         mountpoint: P,
         mount_type: MountType,
@@ -130,19 +135,33 @@ impl FileSystem for BackupTargetFS {
     }
 }
 
-#[command(subcommands(cifs::cifs, list, info, mount, umount))]
-pub fn target() -> Result<(), Error> {
-    Ok(())
+// #[command(subcommands(cifs::cifs, list, info, mount, umount))]
+pub fn target() -> ParentHandler {
+    ParentHandler::new()
+        .subcommand("cifs", cifs::cifs())
+        .subcommand(
+            "list",
+            from_fn_async(list)
+                .with_display_serializable()
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand(
+            "info",
+            from_fn_async(info)
+                .with_display_serializable()
+                .with_custom_display_fn::<AnyContext, _>(|params, info| {
+                    Ok(display_backup_info(params.params, info))
+                })
+                .with_remote_cli::<CliContext>(),
+        )
 }
 
-#[command(display(display_serializable))]
-pub async fn list(
-    #[context] ctx: RpcContext,
-) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
-    let mut sql_handle = ctx.secret_store.acquire().await?;
+// #[command(display(display_serializable))]
+pub async fn list(ctx: RpcContext) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
+    let peek = ctx.db.peek().await;
     let (disks_res, cifs) = tokio::try_join!(
         crate::disk::util::list(&ctx.os_partitions),
-        cifs::list(sql_handle.as_mut()),
+        cifs::list(&peek),
     )?;
     Ok(disks_res
         .into_iter()
@@ -187,11 +206,11 @@ pub struct PackageBackupInfo {
     pub timestamp: DateTime<Utc>,
 }
 
-fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
+fn display_backup_info(params: WithIoFormat<InfoParams>, info: BackupInfo) {
     use prettytable::*;
 
-    if matches.is_present("format") {
-        return display_serializable(info, matches);
+    if let Some(format) = params.format {
+        return display_serializable(format, info);
     }
 
     let mut table = Table::new();
@@ -223,21 +242,24 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
     table.print_tty(false).unwrap();
 }
 
-#[command(display(display_backup_info))]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct InfoParams {
+    target_id: BackupTargetId,
+    password: String,
+}
+
 #[instrument(skip(ctx, password))]
 pub async fn info(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg] password: String,
+    ctx: RpcContext,
+    InfoParams {
+        target_id,
+        password,
+    }: InfoParams,
 ) -> Result<BackupInfo, Error> {
     let guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(
-            &target_id
-                .load(ctx.secret_store.acquire().await?.as_mut())
-                .await?,
-            ReadWrite,
-        )
-        .await?,
+        TmpMountGuard::mount(&target_id.load(&ctx.db.peek().await)?, ReadWrite).await?,
         &password,
     )
     .await?;
@@ -254,45 +276,51 @@ lazy_static::lazy_static! {
         Mutex::new(BTreeMap::new());
 }
 
-#[command]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct MountParams {
+    target_id: BackupTargetId,
+    password: String,
+}
+
 #[instrument(skip_all)]
 pub async fn mount(
-    #[context] ctx: RpcContext,
-    #[arg(rename = "target-id")] target_id: BackupTargetId,
-    #[arg] password: String,
+    ctx: RpcContext,
+    MountParams {
+        target_id,
+        password,
+    }: MountParams,
 ) -> Result<String, Error> {
     let mut mounts = USER_MOUNTS.lock().await;
 
     if let Some(existing) = mounts.get(&target_id) {
-        return Ok(existing.as_ref().display().to_string());
+        return Ok(existing.path().display().to_string());
     }
 
     let guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(
-            &target_id
-                .clone()
-                .load(ctx.secret_store.acquire().await?.as_mut())
-                .await?,
-            ReadWrite,
-        )
-        .await?,
+        TmpMountGuard::mount(&target_id.clone().load(&ctx.db.peek().await)?, ReadWrite).await?,
         &password,
     )
     .await?;
 
-    let res = guard.as_ref().display().to_string();
+    let res = guard.path().display().to_string();
 
     mounts.insert(target_id, guard);
 
     Ok(res)
 }
-#[command(display(display_none))]
+
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct UmountParams {
+    target_id: Option<BackupTargetId>,
+}
+
 #[instrument(skip_all)]
-pub async fn umount(
-    #[context] _ctx: RpcContext,
-    #[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
-) -> Result<(), Error> {
-    let mut mounts = USER_MOUNTS.lock().await;
+pub async fn umount(_: RpcContext, UmountParams { target_id }: UmountParams) -> Result<(), Error> {
+    let mut mounts = USER_MOUNTS.lock().await; // TODO: move to context
     if let Some(target_id) = target_id {
         if let Some(existing) = mounts.remove(&target_id) {
             existing.unmount().await?;

core/startos/src/bins/avahi_alias.rs

@@ -1,163 +0,0 @@
-use avahi_sys::{
-    self, avahi_client_errno, avahi_entry_group_add_service, avahi_entry_group_commit,
-    avahi_strerror, AvahiClient,
-};
-
-fn log_str_error(action: &str, e: i32) {
-    unsafe {
-        let e_str = avahi_strerror(e);
-        eprintln!(
-            "Could not {}: {:?}",
-            action,
-            std::ffi::CStr::from_ptr(e_str)
-        );
-    }
-}
-
-pub fn main() {
-    let aliases: Vec<_> = std::env::args().skip(1).collect();
-    unsafe {
-        let simple_poll = avahi_sys::avahi_simple_poll_new();
-        let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
-        let mut box_err = Box::pin(0 as i32);
-        let err_c: *mut i32 = box_err.as_mut().get_mut();
-        let avahi_client = avahi_sys::avahi_client_new(
-            poll,
-            avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
-            Some(client_callback),
-            std::ptr::null_mut(),
-            err_c,
-        );
-        if avahi_client == std::ptr::null_mut::<AvahiClient>() {
-            log_str_error("create Avahi client", *box_err);
-            panic!("Failed to create Avahi Client");
-        }
-        let group = avahi_sys::avahi_entry_group_new(
-            avahi_client,
-            Some(entry_group_callback),
-            std::ptr::null_mut(),
-        );
-        if group == std::ptr::null_mut() {
-            log_str_error("create Avahi entry group", avahi_client_errno(avahi_client));
-            panic!("Failed to create Avahi Entry Group");
-        }
-        let mut hostname_buf = vec![0];
-        let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
-        hostname_buf.extend_from_slice(std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul());
-        let buflen = hostname_buf.len();
-        debug_assert!(hostname_buf.ends_with(b".local\0"));
-        debug_assert!(!hostname_buf[..(buflen - 7)].contains(&b'.'));
-        // assume fixed length prefix on hostname due to local address
-        hostname_buf[0] = (buflen - 8) as u8; // set the prefix length to len - 8 (leading byte, .local, nul) for the main address
-        hostname_buf[buflen - 7] = 5; // set the prefix length to 5 for "local"
-        let mut res;
-        let http_tcp_cstr =
-            std::ffi::CString::new("_http._tcp").expect("Could not cast _http._tcp to c string");
-        res = avahi_entry_group_add_service(
-            group,
-            avahi_sys::AVAHI_IF_UNSPEC,
-            avahi_sys::AVAHI_PROTO_UNSPEC,
-            avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST,
-            hostname_raw,
-            http_tcp_cstr.as_ptr(),
-            std::ptr::null(),
-            std::ptr::null(),
-            443,
-            // below is a secret final argument that the type signature of this function does not tell you that it
-            // needs. This is because the C lib function takes a variable number of final arguments indicating the
-            // desired TXT records to add to this service entry. The way it decides when to stop taking arguments
-            // from the stack and dereferencing them is when it finds a null pointer...because fuck you, that's why.
-            // The consequence of this is that forgetting this last argument will cause segfaults or other undefined
-            // behavior. Welcome back to the stone age motherfucker.
-            std::ptr::null::<libc::c_char>(),
-        );
-        if res < avahi_sys::AVAHI_OK {
-            log_str_error("add service to Avahi entry group", res);
-            panic!("Failed to load Avahi services");
-        }
-        eprintln!("Published {:?}", std::ffi::CStr::from_ptr(hostname_raw));
-        for alias in aliases {
-            let lan_address = alias + ".local";
-            let lan_address_ptr = std::ffi::CString::new(lan_address)
-                .expect("Could not cast lan address to c string");
-            res = avahi_sys::avahi_entry_group_add_record(
-                group,
-                avahi_sys::AVAHI_IF_UNSPEC,
-                avahi_sys::AVAHI_PROTO_UNSPEC,
-                avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
-                    | avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
-                lan_address_ptr.as_ptr(),
-                avahi_sys::AVAHI_DNS_CLASS_IN as u16,
-                avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
-                avahi_sys::AVAHI_DEFAULT_TTL,
-                hostname_buf.as_ptr().cast(),
-                hostname_buf.len(),
-            );
-            if res < avahi_sys::AVAHI_OK {
-                log_str_error("add CNAME record to Avahi entry group", res);
-                panic!("Failed to load Avahi services");
-            }
-            eprintln!("Published {:?}", lan_address_ptr);
-        }
-        let commit_err = avahi_entry_group_commit(group);
-        if commit_err < avahi_sys::AVAHI_OK {
-            log_str_error("reset Avahi entry group", commit_err);
-            panic!("Failed to load Avahi services: reset");
-        }
-    }
-    std::thread::park()
-}
-
-unsafe extern "C" fn entry_group_callback(
-    _group: *mut avahi_sys::AvahiEntryGroup,
-    state: avahi_sys::AvahiEntryGroupState,
-    _userdata: *mut core::ffi::c_void,
-) {
-    match state {
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_FAILURE => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_FAILURE");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_COLLISION => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_COLLISION");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_UNCOMMITED => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_UNCOMMITED");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_ESTABLISHED => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_ESTABLISHED");
-        }
-        avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_REGISTERING => {
-            eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_REGISTERING");
-        }
-        other => {
-            eprintln!("AvahiCallback: EntryGroupState = {}", other);
-        }
-    }
-}
-
-unsafe extern "C" fn client_callback(
-    _group: *mut avahi_sys::AvahiClient,
-    state: avahi_sys::AvahiClientState,
-    _userdata: *mut core::ffi::c_void,
-) {
-    match state {
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_FAILURE => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_FAILURE");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_RUNNING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_RUNNING");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_CONNECTING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_CONNECTING");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_COLLISION => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_COLLISION");
-        }
-        avahi_sys::AvahiClientState_AVAHI_CLIENT_S_REGISTERING => {
-            eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_REGISTERING");
-        }
-        other => {
-            eprintln!("AvahiCallback: ClientState = {}", other);
-        }
-    }
-}

core/startos/src/bins/container_cli.rs

@@ -0,0 +1,38 @@
+use std::ffi::OsString;
+
+use rpc_toolkit::CliApp;
+use serde_json::Value;
+
+use crate::service::cli::{ContainerCliContext, ContainerClientConfig};
+use crate::util::logger::EmbassyLogger;
+use crate::version::{Current, VersionT};
+
+lazy_static::lazy_static! {
+    static ref VERSION_STRING: String = Current::new().semver().to_string();
+}
+
+pub fn main(args: impl IntoIterator<Item = OsString>) {
+    EmbassyLogger::init();
+    if let Err(e) = CliApp::new(
+        |cfg: ContainerClientConfig| Ok(ContainerCliContext::init(cfg)),
+        crate::service::service_effect_handler::service_effect_handler(),
+    )
+    .run(args)
+    {
+        match e.data {
+            Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
+            Some(Value::Object(o)) => {
+                if let Some(Value::String(s)) = o.get("details") {
+                    eprintln!("{}: {}", e.message, s);
+                    if let Some(Value::String(s)) = o.get("debug") {
+                        tracing::debug!("{}", s)
+                    }
+                }
+            }
+            Some(a) => eprintln!("{}: {}", e.message, a),
+            None => eprintln!("{}", e.message),
+        }
+
+        std::process::exit(e.code);
+    }
+}

core/startos/src/bins/mod.rs

@@ -1,49 +1,54 @@
+use std::collections::VecDeque;
+use std::ffi::OsString;
 use std::path::Path;
 
-#[cfg(feature = "avahi-alias")]
-pub mod avahi_alias;
+#[cfg(feature = "container-runtime")]
+pub mod container_cli;
 pub mod deprecated;
 #[cfg(feature = "cli")]
 pub mod start_cli;
-#[cfg(feature = "js-engine")]
-pub mod start_deno;
 #[cfg(feature = "daemon")]
 pub mod start_init;
-#[cfg(feature = "sdk")]
-pub mod start_sdk;
 #[cfg(feature = "daemon")]
 pub mod startd;
 
-fn select_executable(name: &str) -> Option<fn()> {
+fn select_executable(name: &str) -> Option<fn(VecDeque<OsString>)> {
     match name {
-        #[cfg(feature = "avahi-alias")]
-        "avahi-alias" => Some(avahi_alias::main),
-        #[cfg(feature = "js-engine")]
-        "start-deno" => Some(start_deno::main),
         #[cfg(feature = "cli")]
         "start-cli" => Some(start_cli::main),
-        #[cfg(feature = "sdk")]
-        "start-sdk" => Some(start_sdk::main),
+        #[cfg(feature = "container-runtime")]
+        "start-cli" => Some(container_cli::main),
         #[cfg(feature = "daemon")]
         "startd" => Some(startd::main),
-        "embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
-        "embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
-        "embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
-        "embassy-init" => Some(|| deprecated::removed("embassy-init")),
+        "embassy-cli" => Some(|_| deprecated::renamed("embassy-cli", "start-cli")),
+        "embassy-sdk" => Some(|_| deprecated::renamed("embassy-sdk", "start-sdk")),
+        "embassyd" => Some(|_| deprecated::renamed("embassyd", "startd")),
+        "embassy-init" => Some(|_| deprecated::removed("embassy-init")),
         _ => None,
     }
 }
 
 pub fn startbox() {
-    let args = std::env::args().take(2).collect::<Vec<_>>();
-    let executable = args
-        .get(0)
-        .and_then(|s| Path::new(&*s).file_name())
-        .and_then(|s| s.to_str());
-    if let Some(x) = executable.and_then(|s| select_executable(&s)) {
-        x()
-    } else {
-        eprintln!("unknown executable: {}", executable.unwrap_or("N/A"));
-        std::process::exit(1);
+    let mut args = std::env::args_os().collect::<VecDeque<_>>();
+    for _ in 0..2 {
+        if let Some(s) = args.pop_front() {
+            if let Some(x) = Path::new(&*s)
+                .file_name()
+                .and_then(|s| s.to_str())
+                .and_then(|s| select_executable(&s))
+            {
+                args.push_front(s);
+                return x(args);
+            }
+        }
     }
+    let args = std::env::args().collect::<VecDeque<_>>();
+    eprintln!(
+        "unknown executable: {}",
+        args.get(1)
+            .or_else(|| args.get(0))
+            .map(|s| s.as_str())
+            .unwrap_or("N/A")
+    );
+    std::process::exit(1);
 }

core/startos/src/bins/start_cli.rs

@@ -1,62 +1,39 @@
-use clap::Arg;
-use rpc_toolkit::run_cli;
-use rpc_toolkit::yajrc::RpcError;
+use std::ffi::OsString;
+
+use rpc_toolkit::CliApp;
 use serde_json::Value;
 
+use crate::context::config::ClientConfig;
 use crate::context::CliContext;
 use crate::util::logger::EmbassyLogger;
 use crate::version::{Current, VersionT};
-use crate::Error;
 
 lazy_static::lazy_static! {
     static ref VERSION_STRING: String = Current::new().semver().to_string();
 }
 
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: crate::main_api,
-        app: app => app
-            .name("StartOS CLI")
-            .version(&**VERSION_STRING)
-            .arg(
-                clap::Arg::with_name("config")
-                    .short('c')
-                    .long("config")
-                    .takes_value(true),
-            )
-            .arg(Arg::with_name("host").long("host").short('h').takes_value(true))
-            .arg(Arg::with_name("proxy").long("proxy").short('p').takes_value(true)),
-        context: matches => {
-            EmbassyLogger::init();
-            CliContext::init(matches)?
-        },
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
+pub fn main(args: impl IntoIterator<Item = OsString>) {
+    EmbassyLogger::init();
+    if let Err(e) = CliApp::new(
+        |cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?),
+        crate::main_api(),
+    )
+    .run(args)
+    {
+        match e.data {
+            Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
+            Some(Value::Object(o)) => {
+                if let Some(Value::String(s)) = o.get("details") {
                     eprintln!("{}: {}", e.message, s);
                     if let Some(Value::String(s)) = o.get("debug") {
                         tracing::debug!("{}", s)
                     }
                 }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
             }
-
-            std::process::exit(e.code);
+            Some(a) => eprintln!("{}: {}", e.message, a),
+            None => eprintln!("{}", e.message),
         }
-    });
-    Ok(())
-}
 
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
+        std::process::exit(e.code);
     }
 }

core/startos/src/bins/start_deno.rs

@@ -1,140 +0,0 @@
-use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{command, run_cli, Context};
-use serde_json::Value;
-
-use crate::procedure::js_scripts::ExecuteArgs;
-use crate::s9pk::manifest::PackageId;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::version::{Current, VersionT};
-use crate::Error;
-
-lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
-}
-
-struct DenoContext;
-impl Context for DenoContext {}
-
-#[command(subcommands(execute, sandbox))]
-fn deno_api() -> Result<(), Error> {
-    Ok(())
-}
-
-#[command(cli_only, display(display_serializable))]
-async fn execute(
-    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<Result<Value, (i32, String)>, Error> {
-    let ExecuteArgs {
-        procedure,
-        directory,
-        pkg_id,
-        pkg_version,
-        name,
-        volumes,
-        input,
-    } = arg;
-    PackageLogger::init(&pkg_id);
-    procedure
-        .execute_impl(&directory, &pkg_id, &pkg_version, name, &volumes, input)
-        .await
-}
-#[command(cli_only, display(display_serializable))]
-async fn sandbox(
-    #[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<Result<Value, (i32, String)>, Error> {
-    let ExecuteArgs {
-        procedure,
-        directory,
-        pkg_id,
-        pkg_version,
-        name,
-        volumes,
-        input,
-    } = arg;
-    PackageLogger::init(&pkg_id);
-    procedure
-        .sandboxed_impl(&directory, &pkg_id, &pkg_version, &volumes, input, name)
-        .await
-}
-
-use tracing::Subscriber;
-use tracing_subscriber::util::SubscriberInitExt;
-
-#[derive(Clone)]
-struct PackageLogger {}
-
-impl PackageLogger {
-    fn base_subscriber(id: &PackageId) -> impl Subscriber {
-        use tracing_error::ErrorLayer;
-        use tracing_subscriber::prelude::*;
-        use tracing_subscriber::{fmt, EnvFilter};
-
-        let filter_layer = EnvFilter::default().add_directive(
-            format!("{}=warn", std::module_path!().split("::").next().unwrap())
-                .parse()
-                .unwrap(),
-        );
-        let fmt_layer = fmt::layer().with_writer(std::io::stderr).with_target(true);
-        let journald_layer = tracing_journald::layer()
-            .unwrap()
-            .with_syslog_identifier(format!("{id}.embassy"));
-
-        let sub = tracing_subscriber::registry()
-            .with(filter_layer)
-            .with(fmt_layer)
-            .with(journald_layer)
-            .with(ErrorLayer::default());
-
-        sub
-    }
-    pub fn init(id: &PackageId) -> Self {
-        Self::base_subscriber(id).init();
-        color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times"));
-
-        Self {}
-    }
-}
-
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: deno_api,
-        app: app => app
-            .name("StartOS Deno Executor")
-            .version(&**VERSION_STRING),
-        context: _m => DenoContext,
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
-                    eprintln!("{}: {}", e.message, s);
-                    if let Some(Value::String(s)) = o.get("debug") {
-                        tracing::debug!("{}", s)
-                    }
-                }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
-            }
-
-            std::process::exit(e.code);
-        }
-    });
-    Ok(())
-}
-
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

core/startos/src/bins/start_init.rs

@@ -1,5 +1,5 @@
 use std::net::{Ipv6Addr, SocketAddr};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -7,7 +7,7 @@ use helpers::NonDetachingJoinHandle;
 use tokio::process::Command;
 use tracing::instrument;
 
-use crate::context::rpc::RpcContextConfig;
+use crate::context::config::ServerConfig;
 use crate::context::{DiagnosticContext, InstallContext, SetupContext};
 use crate::disk::fsck::{RepairStrategy, RequiresReboot};
 use crate::disk::main::DEFAULT_PASSWORD;
@@ -21,7 +21,7 @@ use crate::util::Invoke;
 use crate::{Error, ErrorKind, ResultExt, PLATFORM};
 
 #[instrument(skip_all)]
-async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+async fn setup_or_init(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
     let song = NonDetachingJoinHandle::from(tokio::spawn(async {
         loop {
             BEP.play().await.unwrap();
@@ -82,13 +82,12 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
             .invoke(crate::ErrorKind::OpenSsh)
             .await?;
 
-        let ctx = InstallContext::init(cfg_path).await?;
+        let ctx = InstallContext::init().await?;
 
         let server = WebServer::install(
             SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
             ctx.clone(),
-        )
-        .await?;
+        )?;
 
         drop(song);
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
@@ -109,26 +108,24 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
         .await
         .is_err()
     {
-        let ctx = SetupContext::init(cfg_path).await?;
+        let ctx = SetupContext::init(config)?;
 
         let server = WebServer::setup(
             SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
             ctx.clone(),
-        )
-        .await?;
+        )?;
 
         drop(song);
         tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
         CHIME.play().await?;
 
-        ctx.shutdown
-            .subscribe()
-            .recv()
-            .await
-            .expect("context dropped");
+        let mut shutdown = ctx.shutdown.subscribe();
+        shutdown.recv().await.expect("context dropped");
 
         server.shutdown().await;
 
+        drop(shutdown);
+
         tokio::task::yield_now().await;
         if let Err(e) = Command::new("killall")
             .arg("firefox-esr")
@@ -139,13 +136,12 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
             tracing::debug!("{:?}", e);
         }
     } else {
-        let cfg = RpcContextConfig::load(cfg_path).await?;
         let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
             .await?;
         let guid = guid_string.trim();
         let requires_reboot = crate::disk::main::import(
             guid,
-            cfg.datadir(),
+            config.datadir(),
             if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
                 RepairStrategy::Aggressive
             } else {
@@ -164,13 +160,13 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
                 .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
         }
         if requires_reboot.0 {
-            crate::disk::main::export(guid, cfg.datadir()).await?;
+            crate::disk::main::export(guid, config.datadir()).await?;
             Command::new("reboot")
                 .invoke(crate::ErrorKind::Unknown)
                 .await?;
         }
         tracing::info!("Loaded Disk");
-        crate::init::init(&cfg).await?;
+        crate::init::init(config).await?;
         drop(song);
     }
 
@@ -196,7 +192,7 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
 }
 
 #[instrument(skip_all)]
-async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+async fn inner_main(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
     if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
         tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
         Command::new("sync").invoke(ErrorKind::Filesystem).await?;
@@ -208,7 +204,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
 
     run_script_if_exists("/media/embassy/config/preinit.sh").await;
 
-    let res = match setup_or_init(cfg_path.clone()).await {
+    let res = match setup_or_init(config).await {
         Err(e) => {
             async move {
                 tracing::error!("{}", e.source);
@@ -216,7 +212,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
                 crate::sound::BEETHOVEN.play().await?;
 
                 let ctx = DiagnosticContext::init(
-                    cfg_path,
+                    config,
                     if tokio::fs::metadata("/media/embassy/config/disk.guid")
                         .await
                         .is_ok()
@@ -231,14 +227,12 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
                         None
                     },
                     e,
-                )
-                .await?;
+                )?;
 
                 let server = WebServer::diagnostic(
                     SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
                     ctx.clone(),
-                )
-                .await?;
+                )?;
 
                 let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
 
@@ -256,23 +250,13 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     res
 }
 
-pub fn main() {
-    let matches = clap::App::new("start-init")
-        .arg(
-            clap::Arg::with_name("config")
-                .short('c')
-                .long("config")
-                .takes_value(true),
-        )
-        .get_matches();
-
-    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
+pub fn main(config: &ServerConfig) {
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()
             .build()
             .expect("failed to initialize runtime");
-        rt.block_on(inner_main(cfg_path))
+        rt.block_on(inner_main(config))
     };
 
     match res {

core/startos/src/bins/start_sdk.rs

@@ -1,61 +0,0 @@
-use rpc_toolkit::run_cli;
-use rpc_toolkit::yajrc::RpcError;
-use serde_json::Value;
-
-use crate::context::SdkContext;
-use crate::util::logger::EmbassyLogger;
-use crate::version::{Current, VersionT};
-use crate::Error;
-
-lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
-}
-
-fn inner_main() -> Result<(), Error> {
-    run_cli!({
-        command: crate::portable_api,
-        app: app => app
-            .name("StartOS SDK")
-            .version(&**VERSION_STRING)
-            .arg(
-                clap::Arg::with_name("config")
-                    .short('c')
-                    .long("config")
-                    .takes_value(true),
-            ),
-        context: matches => {
-            if let Err(_) = std::env::var("RUST_LOG") {
-                std::env::set_var("RUST_LOG", "embassy=warn,js_engine=warn");
-            }
-            EmbassyLogger::init();
-            SdkContext::init(matches)?
-        },
-        exit: |e: RpcError| {
-            match e.data {
-                Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
-                Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
-                    eprintln!("{}: {}", e.message, s);
-                    if let Some(Value::String(s)) = o.get("debug") {
-                        tracing::debug!("{}", s)
-                    }
-                }
-                Some(a) => eprintln!("{}: {}", e.message, a),
-                None => eprintln!("{}", e.message),
-            }
-            std::process::exit(e.code);
-        }
-    });
-    Ok(())
-}
-
-pub fn main() {
-    match inner_main() {
-        Ok(_) => (),
-        Err(e) => {
-            eprintln!("{}", e.source);
-            tracing::debug!("{:?}", e.source);
-            drop(e.source);
-            std::process::exit(e.kind as i32)
-        }
-    }
-}

core/startos/src/bins/startd.rs

@@ -1,12 +1,15 @@
+use std::ffi::OsString;
 use std::net::{Ipv6Addr, SocketAddr};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 
+use clap::Parser;
 use color_eyre::eyre::eyre;
 use futures::{FutureExt, TryFutureExt};
 use tokio::signal::unix::signal;
 use tracing::instrument;
 
+use crate::context::config::ServerConfig;
 use crate::context::{DiagnosticContext, RpcContext};
 use crate::net::web_server::WebServer;
 use crate::shutdown::Shutdown;
@@ -15,10 +18,10 @@ use crate::util::logger::EmbassyLogger;
 use crate::{Error, ErrorKind, ResultExt};
 
 #[instrument(skip_all)]
-async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
+async fn inner_main(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
     let (rpc_ctx, server, shutdown) = async {
         let rpc_ctx = RpcContext::init(
-            cfg_path,
+            config,
             Arc::new(
                 tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
                     .await?
@@ -31,8 +34,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
         let server = WebServer::main(
             SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
             rpc_ctx.clone(),
-        )
-        .await?;
+        )?;
 
         let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
 
@@ -102,32 +104,23 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
     Ok(shutdown)
 }
 
-pub fn main() {
+pub fn main(args: impl IntoIterator<Item = OsString>) {
     EmbassyLogger::init();
 
+    let config = ServerConfig::parse_from(args).load().unwrap();
+
     if !Path::new("/run/embassy/initialized").exists() {
-        super::start_init::main();
+        super::start_init::main(&config);
         std::fs::write("/run/embassy/initialized", "").unwrap();
     }
 
-    let matches = clap::App::new("startd")
-        .arg(
-            clap::Arg::with_name("config")
-                .short('c')
-                .long("config")
-                .takes_value(true),
-        )
-        .get_matches();
-
-    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
-
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()
             .build()
             .expect("failed to initialize runtime");
         rt.block_on(async {
-            match inner_main(cfg_path.clone()).await {
+            match inner_main(&config).await {
                 Ok(a) => Ok(a),
                 Err(e) => {
                     async {
@@ -135,7 +128,7 @@ pub fn main() {
                         tracing::debug!("{:?}", e.source);
                         crate::sound::BEETHOVEN.play().await?;
                         let ctx = DiagnosticContext::init(
-                            cfg_path,
+                            &config,
                             if tokio::fs::metadata("/media/embassy/config/disk.guid")
                                 .await
                                 .is_ok()
@@ -150,14 +143,12 @@ pub fn main() {
                                 None
                             },
                             e,
-                        )
-                        .await?;
+                        )?;
 
                         let server = WebServer::diagnostic(
                             SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
                             ctx.clone(),
-                        )
-                        .await?;
+                        )?;
 
                         let mut shutdown = ctx.shutdown.subscribe();
 

core/startos/src/config/action.rs

@@ -1,22 +1,12 @@
 use std::collections::{BTreeMap, BTreeSet};
 
-use color_eyre::eyre::eyre;
-use models::ImageId;
-use patch_db::HasModel;
+use models::PackageId;
 use serde::{Deserialize, Serialize};
-use tracing::instrument;
 
 use super::{Config, ConfigSpec};
-use crate::context::RpcContext;
-use crate::dependencies::Dependencies;
+#[allow(unused_imports)]
 use crate::prelude::*;
-use crate::procedure::docker::DockerContainers;
-use crate::procedure::{PackageProcedure, ProcedureName};
-use crate::s9pk::manifest::PackageId;
 use crate::status::health_check::HealthCheckId;
-use crate::util::Version;
-use crate::volume::Volumes;
-use crate::{Error, ResultExt};
 
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -25,90 +15,6 @@ pub struct ConfigRes {
     pub spec: ConfigSpec,
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
-#[model = "Model<Self>"]
-pub struct ConfigActions {
-    pub get: PackageProcedure,
-    pub set: PackageProcedure,
-}
-impl ConfigActions {
-    #[instrument(skip_all)]
-    pub fn validate(
-        &self,
-        _container: &Option<DockerContainers>,
-        eos_version: &Version,
-        volumes: &Volumes,
-        image_ids: &BTreeSet<ImageId>,
-    ) -> Result<(), Error> {
-        self.get
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
-        self.set
-            .validate(eos_version, volumes, image_ids, true)
-            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
-        Ok(())
-    }
-    #[instrument(skip_all)]
-    pub async fn get(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        volumes: &Volumes,
-    ) -> Result<ConfigRes, Error> {
-        self.get
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::GetConfig,
-                volumes,
-                None::<()>,
-                None,
-            )
-            .await
-            .and_then(|res| {
-                res.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigGen))
-            })
-    }
-
-    #[instrument(skip_all)]
-    pub async fn set(
-        &self,
-        ctx: &RpcContext,
-        pkg_id: &PackageId,
-        pkg_version: &Version,
-        dependencies: &Dependencies,
-        volumes: &Volumes,
-        input: &Config,
-    ) -> Result<SetResult, Error> {
-        let res: SetResult = self
-            .set
-            .execute(
-                ctx,
-                pkg_id,
-                pkg_version,
-                ProcedureName::SetConfig,
-                volumes,
-                Some(input),
-                None,
-            )
-            .await
-            .and_then(|res| {
-                res.map_err(|e| {
-                    Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigRulesViolation)
-                })
-            })?;
-        Ok(SetResult {
-            depends_on: res
-                .depends_on
-                .into_iter()
-                .filter(|(pkg, _)| dependencies.0.contains_key(pkg))
-                .collect(),
-        })
-    }
-}
-
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct SetResult {

core/startos/src/config/mod.rs

@@ -1,34 +1,31 @@
-use std::collections::BTreeMap;
-use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Duration;
 
+use clap::Parser;
 use color_eyre::eyre::eyre;
-use indexmap::IndexSet;
+use indexmap::{IndexMap, IndexSet};
 use itertools::Itertools;
-use models::{ErrorKind, OptionExt};
+use models::{ErrorKind, OptionExt, PackageId};
 use patch_db::value::InternedString;
 use patch_db::Value;
 use regex::Regex;
-use rpc_toolkit::command;
+use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler};
+use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use crate::context::RpcContext;
+use crate::context::{CliContext, RpcContext};
 use crate::prelude::*;
-use crate::s9pk::manifest::PackageId;
-use crate::util::display_none;
-use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
-use crate::Error;
+use crate::util::serde::{HandlerExtSerde, StdinDeserializable};
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct ConfigSpec(pub IndexMap<InternedString, Value>);
 
 pub mod action;
-pub mod spec;
 pub mod util;
 
-pub use spec::{ConfigSpec, Defaultable};
 use util::NumRange;
 
 use self::action::ConfigRes;
-use self::spec::ValueSpecPointer;
 
 pub type Config = patch_db::value::InOMap<InternedString, Value>;
 pub trait TypeOf {
@@ -55,8 +52,6 @@ pub enum ConfigurationError {
     NoMatch(#[from] NoMatchWithPath),
     #[error("System Error: {0}")]
     SystemError(Error),
-    #[error("Permission Denied: {0}")]
-    PermissionDenied(ValueSpecPointer),
 }
 impl From<ConfigurationError> for Error {
     fn from(err: ConfigurationError) -> Self {
@@ -124,164 +119,102 @@ pub enum MatchError {
     PropertyMatchesUnionTag(InternedString, String),
     #[error("Name of Property {0:?} Conflicts With Map Tag Name")]
     PropertyNameMatchesMapTag(String),
-    #[error("Pointer Is Invalid: {0}")]
-    InvalidPointer(spec::ValueSpecPointer),
     #[error("Object Key Is Invalid: {0}")]
     InvalidKey(String),
     #[error("Value In List Is Not Unique")]
     ListUniquenessViolation,
 }
 
-#[command(rename = "config-spec", cli_only, blocking, display(display_none))]
-pub fn verify_spec(#[arg] path: PathBuf) -> Result<(), Error> {
-    let mut file = std::fs::File::open(&path)?;
-    let format = match path.extension().and_then(|s| s.to_str()) {
-        Some("yaml") | Some("yml") => IoFormat::Yaml,
-        Some("json") => IoFormat::Json,
-        Some("toml") => IoFormat::Toml,
-        Some("cbor") => IoFormat::Cbor,
-        _ => {
-            return Err(Error::new(
-                eyre!("Unknown file format. Expected one of yaml, json, toml, cbor."),
-                crate::ErrorKind::Deserialization,
-            ));
-        }
-    };
-    let _: ConfigSpec = format.from_reader(&mut file)?;
-
-    Ok(())
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+#[command(rename_all = "kebab-case")]
+pub struct ConfigParams {
+    pub id: PackageId,
 }
 
-#[command(subcommands(get, set))]
-pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
-    Ok(id)
+// #[command(subcommands(get, set))]
+pub fn config() -> ParentHandler<ConfigParams> {
+    ParentHandler::new()
+        .subcommand(
+            "get",
+            from_fn_async(get)
+                .with_inherited(|ConfigParams { id }, _| id)
+                .with_display_serializable()
+                .with_remote_cli::<CliContext>(),
+        )
+        .subcommand("set", set().with_inherited(|ConfigParams { id }, _| id))
 }
 
-#[command(display(display_serializable))]
 #[instrument(skip_all)]
-pub async fn get(
-    #[context] ctx: RpcContext,
-    #[parent_data] id: PackageId,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-) -> Result<ConfigRes, Error> {
-    let db = ctx.db.peek().await;
-    let manifest = db
-        .as_package_data()
-        .as_idx(&id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?
-        .as_manifest();
-    let action = manifest
-        .as_config()
-        .de()?
-        .ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
-
-    let volumes = manifest.as_volumes().de()?;
-    let version = manifest.as_version().de()?;
-    action.get(&ctx, &id, &version, &volumes).await
+pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result<ConfigRes, Error> {
+    ctx.services
+        .get(&id)
+        .await
+        .as_ref()
+        .or_not_found(lazy_format!("Manager for {id}"))?
+        .get_config()
+        .await
 }
 
-#[command(
-    subcommands(self(set_impl(async, context(RpcContext))), set_dry),
-    display(display_none),
-    metadata(sync_db = true)
-)]
+#[derive(Deserialize, Serialize, Parser)]
+#[serde(rename_all = "kebab-case")]
+pub struct SetParams {
+    #[arg(long = "timeout")]
+    pub timeout: Option<crate::util::serde::Duration>,
+    #[command(flatten)]
+    pub config: StdinDeserializable<Option<Config>>,
+}
+
+// TODO Dr Why isn't this used?
+// #[command(
+//     subcommands(self(set_impl(async, context(RpcContext))), set_dry),
+//     display(display_none),
+//     metadata(sync_db = true)
+// )]
 #[instrument(skip_all)]
-pub fn set(
-    #[parent_data] id: PackageId,
-    #[allow(unused_variables)]
-    #[arg(long = "format")]
-    format: Option<IoFormat>,
-    #[arg(long = "timeout")] timeout: Option<crate::util::serde::Duration>,
-    #[arg(stdin, parse(parse_stdin_deserializable))] config: Option<Config>,
-) -> Result<(PackageId, Option<Config>, Option<Duration>), Error> {
-    Ok((id, config, timeout.map(|d| *d)))
-}
-
-#[command(rename = "dry", display(display_serializable))]
-#[instrument(skip_all)]
-pub async fn set_dry(
-    #[context] ctx: RpcContext,
-    #[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
-) -> Result<BTreeMap<PackageId, String>, Error> {
-    let breakages = BTreeMap::new();
-    let overrides = Default::default();
-
-    let configure_context = ConfigureContext {
-        breakages,
-        timeout,
-        config,
-        dry_run: true,
-        overrides,
-    };
-    let breakages = configure(&ctx, &id, configure_context).await?;
-
-    Ok(breakages)
+pub fn set() -> ParentHandler<SetParams, PackageId> {
+    ParentHandler::new().root_handler(
+        from_fn_async(set_impl)
+            .with_metadata("sync_db", Value::Bool(true))
+            .with_inherited(|set_params, id| (id, set_params))
+            .no_display()
+            .with_remote_cli::<CliContext>(),
+    )
 }
 
+#[derive(Default)]
 pub struct ConfigureContext {
-    pub breakages: BTreeMap<PackageId, String>,
     pub timeout: Option<Duration>,
     pub config: Option<Config>,
-    pub overrides: BTreeMap<PackageId, Config>,
-    pub dry_run: bool,
 }
 
 #[instrument(skip_all)]
 pub async fn set_impl(
     ctx: RpcContext,
-    (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
+    _: Empty,
+    (
+        id,
+        SetParams {
+            timeout,
+            config: StdinDeserializable(config),
+        },
+    ): (PackageId, SetParams),
 ) -> Result<(), Error> {
-    let breakages = BTreeMap::new();
-    let overrides = Default::default();
-
     let configure_context = ConfigureContext {
-        breakages,
-        timeout,
+        timeout: timeout.map(|t| *t),
         config,
-        dry_run: false,
-        overrides,
     };
-    configure(&ctx, &id, configure_context).await?;
-    Ok(())
-}
-
-#[instrument(skip_all)]
-pub async fn configure(
-    ctx: &RpcContext,
-    id: &PackageId,
-    configure_context: ConfigureContext,
-) -> Result<BTreeMap<PackageId, String>, Error> {
-    let db = ctx.db.peek().await;
-    let package = db
-        .as_package_data()
-        .as_idx(id)
-        .or_not_found(&id)?
-        .as_installed()
-        .or_not_found(&id)?;
-    let version = package.as_manifest().as_version().de()?;
-    ctx.managers
-        .get(&(id.clone(), version.clone()))
+    ctx.services
+        .get(&id)
         .await
+        .as_ref()
         .ok_or_else(|| {
             Error::new(
-                eyre!("There is no manager running for {id:?} and {version:?}"),
+                eyre!("There is no manager running for {id}"),
                 ErrorKind::Unknown,
             )
         })?
         .configure(configure_context)
-        .await
+        .await?;
+    Ok(())
 }
-
-macro_rules! not_found {
-    ($x:expr) => {
-        crate::Error::new(
-            color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
-            crate::ErrorKind::Incoherent,
-        )
-    };
-}
-pub(crate) use not_found;

core/startos/src/context/cli.rs

@@ -1,43 +1,37 @@
 use std::fs::File;
 use std::io::BufReader;
-use std::net::Ipv4Addr;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use clap::ArgMatches;
-use color_eyre::eyre::eyre;
 use cookie_store::{CookieStore, RawCookie};
 use josekit::jwk::Jwk;
+use once_cell::sync::OnceCell;
 use reqwest::Proxy;
 use reqwest_cookie_store::CookieStoreMutex;
 use rpc_toolkit::reqwest::{Client, Url};
-use rpc_toolkit::url::Host;
-use rpc_toolkit::Context;
-use serde::Deserialize;
+use rpc_toolkit::yajrc::RpcError;
+use rpc_toolkit::{call_remote_http, CallRemote, Context};
+use tokio::net::TcpStream;
+use tokio::runtime::Runtime;
+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
 use tracing::instrument;
 
 use super::setup::CURRENT_SECRET;
+use crate::context::config::{local_config_path, ClientConfig};
+use crate::core::rpc_continuations::RequestGuid;
 use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
-use crate::util::config::{load_config_from_paths, local_config_path};
-use crate::ResultExt;
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct CliContextConfig {
-    pub host: Option<Url>,
-    #[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
-    #[serde(default)]
-    pub proxy: Option<Url>,
-    pub cookie_path: Option<PathBuf>,
-}
+use crate::prelude::*;
 
 #[derive(Debug)]
 pub struct CliContextSeed {
+    pub runtime: OnceCell<Runtime>,
     pub base_url: Url,
     pub rpc_url: Url,
     pub client: Client,
     pub cookie_store: Arc<CookieStoreMutex>,
     pub cookie_path: PathBuf,
+    pub developer_key_path: PathBuf,
+    pub developer_key: OnceCell<ed25519_dalek::SigningKey>,
 }
 impl Drop for CliContextSeed {
     fn drop(&mut self) {
@@ -60,42 +54,22 @@ impl Drop for CliContextSeed {
     }
 }
 
-const DEFAULT_HOST: Host<&'static str> = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
-const DEFAULT_PORT: u16 = 5959;
-
 #[derive(Debug, Clone)]
 pub struct CliContext(Arc<CliContextSeed>);
 impl CliContext {
     /// BLOCKING
     #[instrument(skip_all)]
-    pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let local_config_path = local_config_path();
-        let base: CliContextConfig = load_config_from_paths(
-            matches
-                .values_of("config")
-                .into_iter()
-                .flatten()
-                .map(|p| Path::new(p))
-                .chain(local_config_path.as_deref().into_iter())
-                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
-        )?;
-        let mut url = if let Some(host) = matches.value_of("host") {
-            host.parse()?
-        } else if let Some(host) = base.host {
+    pub fn init(config: ClientConfig) -> Result<Self, Error> {
+        let mut url = if let Some(host) = config.host {
             host
         } else {
             "http://localhost".parse()?
         };
-        let proxy = if let Some(proxy) = matches.value_of("proxy") {
-            Some(proxy.parse()?)
-        } else {
-            base.proxy
-        };
 
-        let cookie_path = base.cookie_path.unwrap_or_else(|| {
-            local_config_path
+        let cookie_path = config.cookie_path.unwrap_or_else(|| {
+            local_config_path()
                 .as_deref()
-                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
+                .unwrap_or_else(|| Path::new(super::config::CONFIG_PATH))
                 .parent()
                 .unwrap_or(Path::new("/"))
                 .join(".cookies.json")
@@ -120,6 +94,7 @@ impl CliContext {
         }));
 
         Ok(CliContext(Arc::new(CliContextSeed {
+            runtime: OnceCell::new(),
             base_url: url.clone(),
             rpc_url: {
                 url.path_segments_mut()
@@ -131,7 +106,7 @@ impl CliContext {
             },
             client: {
                 let mut builder = Client::builder().cookie_provider(cookie_store.clone());
-                if let Some(proxy) = proxy {
+                if let Some(proxy) = config.proxy {
                     builder =
                         builder.proxy(Proxy::all(proxy).with_kind(crate::ErrorKind::ParseUrl)?)
                 }
@@ -139,8 +114,90 @@ impl CliContext {
             },
             cookie_store,
             cookie_path,
+            developer_key_path: config.developer_key_path.unwrap_or_else(|| {
+                local_config_path()
+                    .as_deref()
+                    .unwrap_or_else(|| Path::new(super::config::CONFIG_PATH))
+                    .parent()
+                    .unwrap_or(Path::new("/"))
+                    .join("developer.key.pem")
+            }),
+            developer_key: OnceCell::new(),
         })))
     }
+
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub fn developer_key(&self) -> Result<&ed25519_dalek::SigningKey, Error> {
+        self.developer_key.get_or_try_init(|| {
+            if !self.developer_key_path.exists() {
+                return Err(Error::new(eyre!("Developer Key does not exist! Please run `start-cli init` before running this command."), crate::ErrorKind::Uninitialized));
+            }
+            let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
+                &std::fs::read_to_string(&self.developer_key_path)?,
+            )
+            .with_kind(crate::ErrorKind::Pem)?;
+            let secret = ed25519_dalek::SecretKey::try_from(&pair.secret_key[..]).map_err(|_| {
+                Error::new(
+                    eyre!("pkcs8 key is of incorrect length"),
+                    ErrorKind::OpenSsl,
+                )
+            })?;
+            Ok(secret.into())
+        })
+    }
+
+    pub async fn ws_continuation(
+        &self,
+        guid: RequestGuid,
+    ) -> Result<WebSocketStream<MaybeTlsStream<TcpStream>>, Error> {
+        let mut url = self.base_url.clone();
+        let ws_scheme = match url.scheme() {
+            "https" => "wss",
+            "http" => "ws",
+            _ => {
+                return Err(Error::new(
+                    eyre!("Cannot parse scheme from base URL"),
+                    crate::ErrorKind::ParseUrl,
+                )
+                .into())
+            }
+        };
+        url.set_scheme(ws_scheme)
+            .map_err(|_| Error::new(eyre!("Cannot set URL scheme"), crate::ErrorKind::ParseUrl))?;
+        url.path_segments_mut()
+            .map_err(|_| eyre!("Url cannot be base"))
+            .with_kind(crate::ErrorKind::ParseUrl)?
+            .push("ws")
+            .push("rpc")
+            .push(guid.as_ref());
+        let (stream, _) =
+                // base_url is "http://127.0.0.1/", with a trailing slash, so we don't put a leading slash in this path:
+                tokio_tungstenite::connect_async(url).await.with_kind(ErrorKind::Network)?;
+        Ok(stream)
+    }
+
+    pub async fn rest_continuation(
+        &self,
+        guid: RequestGuid,
+        body: reqwest::Body,
+        headers: reqwest::header::HeaderMap,
+    ) -> Result<reqwest::Response, Error> {
+        let mut url = self.base_url.clone();
+        url.path_segments_mut()
+            .map_err(|_| eyre!("Url cannot be base"))
+            .with_kind(crate::ErrorKind::ParseUrl)?
+            .push("rest")
+            .push("rpc")
+            .push(guid.as_ref());
+        self.client
+            .post(url)
+            .headers(headers)
+            .body(body)
+            .send()
+            .await
+            .with_kind(ErrorKind::Network)
+    }
 }
 impl AsRef<Jwk> for CliContext {
     fn as_ref(&self) -> &Jwk {
@@ -154,32 +211,33 @@ impl std::ops::Deref for CliContext {
     }
 }
 impl Context for CliContext {
-    fn protocol(&self) -> &str {
-        self.0.base_url.scheme()
-    }
-    fn host(&self) -> Host<&str> {
-        self.0.base_url.host().unwrap_or(DEFAULT_HOST)
-    }
-    fn port(&self) -> u16 {
-        self.0.base_url.port().unwrap_or(DEFAULT_PORT)
-    }
-    fn path(&self) -> &str {
-        self.0.rpc_url.path()
-    }
-    fn url(&self) -> Url {
-        self.0.rpc_url.clone()
-    }
-    fn client(&self) -> &Client {
-        &self.0.client
+    fn runtime(&self) -> tokio::runtime::Handle {
+        self.runtime
+            .get_or_init(|| {
+                tokio::runtime::Builder::new_multi_thread()
+                    .enable_all()
+                    .build()
+                    .unwrap()
+            })
+            .handle()
+            .clone()
     }
 }
-/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
+#[async_trait::async_trait]
+impl CallRemote for CliContext {
+    async fn call_remote(&self, method: &str, params: Value) -> Result<Value, RpcError> {
+        call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
+    }
+}
+
 #[test]
-fn test_cli_proxy_empty() {
-    serde_yaml::from_str::<CliContextConfig>(
-        "
-        bind_rpc:
-    ",
-    )
-    .unwrap();
+fn test() {
+    let ctx = CliContext::init(ClientConfig::default()).unwrap();
+    ctx.runtime().block_on(async {
+        reqwest::Client::new()
+            .get("http://example.com")
+            .send()
+            .await
+            .unwrap();
+    });
 }