Merge branch 'next' of github.com:Start9Labs/start-os into rebase/integration/refactors

libs/build-arm-v8-snapshot.sh

@@ -22,7 +22,7 @@ rust-gnu-builder sh -c "(cd libs/ && cargo build -p snapshot_creator --release -
 cd -
 
 echo "Creating Arm v8 Snapshot"
-docker run  $USE_TTY --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot_creator"
+docker run  $USE_TTY --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:22.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot_creator"
 sudo chown -R $USER target
 sudo chown -R $USER ~/.cargo
 sudo chown $USER JS_SNAPSHOT.bin

libs/embassy_container_init/Cargo.toml

@@ -20,7 +20,7 @@ serde = { version = "1", features = ["derive", "rc"] }
 serde_json = "1"
 helpers = { path = "../helpers" }
 imbl = "2"
-nix = "0.25"
+nix = { version = "0.27", features = ["process", "signal"] }
 tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["io-util", "sync", "net"] }
 tracing = "0.1"
@@ -30,7 +30,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
-procfs = "0.14"
+procfs = "0.15"
 
 [profile.test]
 opt-level = 3

libs/helpers/Cargo.toml

@@ -8,13 +8,13 @@ edition = "2021"
 [dependencies]
 async-trait = "0.1.64"
 color-eyre = "0.6.2"
-futures = "0.3.21"
+futures = "0.3.28"
 lazy_async_pool = "0.3.3"
 models = { path = "../models" }
-pin-project = "1.0.11"
+pin-project = "1.1.3"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-tokio = { version = "1.23", features = ["full"] }
-tokio-stream = { version = "0.1.9", features = ["io-util", "sync"] }
-tracing = "0.1.35"
+tokio = { version = "1", features = ["full"] }
+tokio-stream = { version = "0.1.14", features = ["io-util", "sync"] }
+tracing = "0.1.39"
 yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }

libs/helpers/src/lib.rs

@@ -72,6 +72,12 @@ pub async fn canonicalize(
 
 #[pin_project::pin_project(PinnedDrop)]
 pub struct NonDetachingJoinHandle<T>(#[pin] JoinHandle<T>);
+impl<T> NonDetachingJoinHandle<T> {
+    pub async fn wait_for_abort(self) -> Result<T, JoinError> {
+        self.abort();
+        self.await
+    }
+}
 impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
     fn from(t: JoinHandle<T>) -> Self {
         NonDetachingJoinHandle(t)

libs/js_engine/Cargo.toml

@@ -6,14 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1.56"
-dashmap = "5.3.4"
-deno_core = "=0.195.0"
-deno_ast = { version = "=0.27.2", features = ["transpiling"] }
+async-trait = "0.1.74"
+dashmap = "5.5.3"
+deno_core = "=0.222.0"
+deno_ast = { version = "=0.29.5", features = ["transpiling"] }
 embassy_container_init = { path = "../embassy_container_init" }
-reqwest = { version = "0.11.11" }
-sha2 = "0.10.2"
-itertools = "0.10.5"
+reqwest = { version = "0.11.22" }
+sha2 = "0.10.8"
+itertools = "0.11.0"
 lazy_static = "1.4.0"
 models = { path = "../models" }
 helpers = { path = "../helpers" }

libs/js_engine/src/lib.rs

@@ -13,8 +13,7 @@ use deno_core::{
     ModuleSourceFuture, ModuleSpecifier, ModuleType, OpDecl, ResolutionKind, RuntimeOptions,
     Snapshot,
 };
-use embassy_container_init::ProcessGroupId;
-use helpers::{script_dir, spawn_local, OsApi, Rsync, UnixRpcClient};
+use helpers::{script_dir, spawn_local, Rsync};
 use models::{PackageId, ProcedureName, Version, VolumeId};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -105,8 +104,6 @@ struct JsContext {
     volumes: Arc<dyn PathForVolumeId>,
     input: Value,
     variable_args: Vec<serde_json::Value>,
-    container_process_gid: ProcessGroupId,
-    container_rpc_client: Option<Arc<UnixRpcClient>>,
     rsyncs: Arc<Mutex<(usize, BTreeMap<usize, Rsync>)>>,
     callback_sender: mpsc::UnboundedSender<(Arc<String>, Vec<Value>)>,
 }
@@ -197,8 +194,6 @@ pub struct JsExecutionEnvironment {
     package_id: PackageId,
     version: Version,
     volumes: Arc<dyn PathForVolumeId>,
-    container_process_gid: ProcessGroupId,
-    container_rpc_client: Option<Arc<UnixRpcClient>>,
 }
 
 impl JsExecutionEnvironment {
@@ -208,9 +203,7 @@ impl JsExecutionEnvironment {
         package_id: &PackageId,
         version: &Version,
         volumes: Box<dyn PathForVolumeId>,
-        container_process_gid: ProcessGroupId,
-        container_rpc_client: Option<Arc<UnixRpcClient>>,
-    ) -> Result<Self, (JsError, String)> {
+    ) -> Result<JsExecutionEnvironment, (JsError, String)> {
         let data_dir = data_directory.as_ref();
         let base_directory = data_dir;
         let js_code = JsCode({
@@ -244,8 +237,6 @@ impl JsExecutionEnvironment {
             version: version.clone(),
             volumes: volumes.into(),
             sandboxed: false,
-            container_process_gid,
-            container_rpc_client,
         })
     }
     pub fn read_only_effects(mut self) -> Self {
@@ -313,12 +304,7 @@ impl JsExecutionEnvironment {
             fns::get_variable_args::decl(),
             fns::set_value::decl(),
             fns::is_sandboxed::decl(),
-            fns::start_command::decl(),
-            fns::wait_command::decl(),
             fns::sleep::decl(),
-            fns::send_signal::decl(),
-            fns::chmod::decl(),
-            fns::signal_group::decl(),
             fns::rsync::decl(),
             fns::rsync_wait::decl(),
             fns::rsync_progress::decl(),
@@ -359,9 +345,6 @@ impl JsExecutionEnvironment {
             sandboxed: self.sandboxed,
             input,
             variable_args,
-            container_process_gid: self.container_process_gid,
-            container_rpc_client: self.container_rpc_client.clone(),
-            callback_sender,
             rsyncs: Default::default(),
         };
         let ext = Extension::builder("embassy")
@@ -385,11 +368,7 @@ impl JsExecutionEnvironment {
                 .load_main_module(&"file:///loadModule.js".parse().unwrap(), None)
                 .await?;
             let evaluated = runtime.mod_evaluate(mod_id);
-            let res = RuntimeEventLoop {
-                runtime: &mut runtime,
-                callback_receiver,
-            }
-            .await;
+            let res = runtime.run_event_loop(false).await;
             res?;
             evaluated.await??;
             Ok::<_, AnyError>(())
@@ -450,23 +429,17 @@ mod fns {
     use deno_core::anyhow::{anyhow, bail};
     use deno_core::error::AnyError;
     use deno_core::*;
-    use embassy_container_init::{
-        OutputParams, OutputStrategy, ProcessGroupId, ProcessId, RunCommand, RunCommandParams,
-        SendSignal, SendSignalParams, SignalGroup, SignalGroupParams,
-    };
-    use helpers::{
-        to_tmp_path, AddressSchemaLocal, AddressSchemaOnion, AtomicFile, Callback, Rsync,
-        RsyncOptions,
-    };
+    use embassy_container_init::ProcessId;
+    use helpers::{to_tmp_path, AtomicFile, Rsync, RsyncOptions};
     use itertools::Itertools;
     use models::{PackageId, VolumeId};
     use serde::{Deserialize, Serialize};
-    use serde_json::{json, Value};
+    use serde_json::Value;
     use tokio::io::AsyncWriteExt;
     use tokio::process::Command;
 
     use super::{AnswerState, JsContext};
-    use crate::{system_time_as_unix_ms, MetadataJs, ResultType};
+    use crate::{system_time_as_unix_ms, MetadataJs};
 
     #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
     struct FetchOptions {
@@ -1206,18 +1179,6 @@ mod fns {
             let state = state.borrow();
             state.borrow::<JsContext>().clone()
         };
-        if let Some(rpc_client) = ctx.container_rpc_client {
-            return rpc_client
-                .request(
-                    embassy_container_init::Log,
-                    embassy_container_init::LogParams {
-                        gid: Some(ctx.container_process_gid),
-                        level: embassy_container_init::LogLevel::Trace(input),
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
-        }
         tracing::trace!(
             package_id = tracing::field::display(&ctx.package_id),
             run_function = tracing::field::display(&ctx.run_function),
@@ -1232,18 +1193,6 @@ mod fns {
             let state = state.borrow();
             state.borrow::<JsContext>().clone()
         };
-        if let Some(rpc_client) = ctx.container_rpc_client {
-            return rpc_client
-                .request(
-                    embassy_container_init::Log,
-                    embassy_container_init::LogParams {
-                        gid: Some(ctx.container_process_gid),
-                        level: embassy_container_init::LogLevel::Warn(input),
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
-        }
         tracing::warn!(
             package_id = tracing::field::display(&ctx.package_id),
             run_function = tracing::field::display(&ctx.run_function),
@@ -1258,18 +1207,6 @@ mod fns {
             let state = state.borrow();
             state.borrow::<JsContext>().clone()
         };
-        if let Some(rpc_client) = ctx.container_rpc_client {
-            return rpc_client
-                .request(
-                    embassy_container_init::Log,
-                    embassy_container_init::LogParams {
-                        gid: Some(ctx.container_process_gid),
-                        level: embassy_container_init::LogLevel::Error(input),
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
-        }
         tracing::error!(
             package_id = tracing::field::display(&ctx.package_id),
             run_function = tracing::field::display(&ctx.run_function),
@@ -1284,18 +1221,6 @@ mod fns {
             let state = state.borrow();
             state.borrow::<JsContext>().clone()
         };
-        if let Some(rpc_client) = ctx.container_rpc_client {
-            return rpc_client
-                .request(
-                    embassy_container_init::Log,
-                    embassy_container_init::LogParams {
-                        gid: Some(ctx.container_process_gid),
-                        level: embassy_container_init::LogLevel::Debug(input),
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
-        }
         tracing::debug!(
             package_id = tracing::field::display(&ctx.package_id),
             run_function = tracing::field::display(&ctx.run_function),
@@ -1306,28 +1231,11 @@ mod fns {
     }
     #[op]
     async fn log_info(state: Rc<RefCell<OpState>>, input: String) -> Result<(), AnyError> {
-        let (container_rpc_client, container_process_gid, package_id, run_function) = {
+        let (package_id, run_function) = {
             let state = state.borrow();
             let ctx: JsContext = state.borrow::<JsContext>().clone();
-            (
-                ctx.container_rpc_client,
-                ctx.container_process_gid,
-                ctx.package_id,
-                ctx.run_function,
-            )
+            (ctx.package_id, ctx.run_function)
         };
-        if let Some(rpc_client) = container_rpc_client {
-            return rpc_client
-                .request(
-                    embassy_container_init::Log,
-                    embassy_container_init::LogParams {
-                        gid: Some(container_process_gid),
-                        level: embassy_container_init::LogLevel::Info(input),
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
-        }
         tracing::info!(
             package_id = tracing::field::display(&package_id),
             run_function = tracing::field::display(&run_function),
@@ -1366,174 +1274,12 @@ mod fns {
         Ok(ctx.sandboxed)
     }
 
-    #[op]
-    async fn send_signal(
-        state: Rc<RefCell<OpState>>,
-        pid: u32,
-        signal: u32,
-    ) -> Result<(), AnyError> {
-        let sandboxed = {
-            let state = state.borrow();
-            let ctx: &JsContext = state.borrow();
-            ctx.sandboxed
-        };
-
-        if sandboxed {
-            bail!("Will not run sendSignal in sandboxed mode");
-        }
-
-        if let Some(rpc_client) = {
-            let state = state.borrow();
-            let ctx = state.borrow::<JsContext>();
-            ctx.container_rpc_client.clone()
-        } {
-            rpc_client
-                .request(
-                    SendSignal,
-                    SendSignalParams {
-                        pid: ProcessId(pid),
-                        signal,
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
-
-            Ok(())
-        } else {
-            Err(anyhow!("No RpcClient for command operations"))
-        }
-    }
-
-    #[op]
-    async fn signal_group(
-        state: Rc<RefCell<OpState>>,
-        gid: u32,
-        signal: u32,
-    ) -> Result<(), AnyError> {
-        let sandboxed = {
-            let state = state.borrow();
-            let ctx: &JsContext = state.borrow();
-            ctx.sandboxed
-        };
-
-        if sandboxed {
-            bail!("Will not run signalGroup in sandboxed mode");
-        }
-
-        if let Some(rpc_client) = {
-            let state = state.borrow();
-            let ctx = state.borrow::<JsContext>();
-            ctx.container_rpc_client.clone()
-        } {
-            rpc_client
-                .request(
-                    SignalGroup,
-                    SignalGroupParams {
-                        gid: ProcessGroupId(gid),
-                        signal,
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
-
-            Ok(())
-        } else {
-            Err(anyhow!("No RpcClient for command operations"))
-        }
-    }
-
     #[derive(Debug, Clone, Serialize, Deserialize)]
     #[serde(rename_all = "camelCase")]
     pub struct StartCommand {
         process_id: ProcessId,
     }
 
-    #[op]
-    async fn start_command(
-        state: Rc<RefCell<OpState>>,
-        command: String,
-        args: Vec<String>,
-        output: OutputStrategy,
-        timeout: Option<u64>,
-    ) -> Result<StartCommand, AnyError> {
-        let sandboxed = {
-            let state = state.borrow();
-            let ctx: &JsContext = state.borrow();
-            ctx.sandboxed
-        };
-
-        if sandboxed {
-            bail!("Will not run command in sandboxed mode");
-        }
-
-        if let (gid, Some(rpc_client)) = {
-            let state = state.borrow();
-            let ctx = state.borrow::<JsContext>();
-            (ctx.container_process_gid, ctx.container_rpc_client.clone())
-        } {
-            let pid = rpc_client
-                .request(
-                    RunCommand,
-                    RunCommandParams {
-                        gid: Some(gid),
-                        command,
-                        args,
-                        output,
-                    },
-                )
-                .await
-                .map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
-
-            if let Some(timeout) = timeout {
-                tokio::spawn(async move {
-                    tokio::time::sleep(Duration::from_millis(timeout)).await;
-                    if let Err(err) = rpc_client
-                        .request(SendSignal, SendSignalParams { pid, signal: 9 })
-                        .await
-                        .map_err(|e| anyhow!("{}: {:?}", e.message, e.data))
-                    {
-                        tracing::warn!("Could not kill process {pid:?}");
-                        tracing::debug!("{err:?}");
-                    }
-                });
-            }
-
-            Ok(StartCommand { process_id: pid })
-        } else {
-            Err(anyhow!("No RpcClient for command operations"))
-        }
-    }
-
-    #[op]
-    async fn wait_command(
-        state: Rc<RefCell<OpState>>,
-        pid: ProcessId,
-    ) -> Result<ResultType, AnyError> {
-        if let Some(rpc_client) = {
-            let state = state.borrow();
-            let ctx = state.borrow::<JsContext>();
-            ctx.container_rpc_client.clone()
-        } {
-            Ok(
-                match rpc_client
-                    .request(embassy_container_init::Output, OutputParams { pid })
-                    .await
-                {
-                    Ok(a) => ResultType::Result(json!(a)),
-                    Err(e) => ResultType::ErrorCode(
-                        e.code,
-                        match e.data {
-                            Some(Value::String(s)) => s,
-                            e => format!("{:?}", e),
-                        },
-                    ),
-                },
-            )
-        } else {
-            Err(anyhow!("No RpcClient for command operations"))
-        }
-    }
-
     #[op]
     async fn sleep(time_ms: u64) -> Result<(), AnyError> {
         tokio::time::sleep(Duration::from_millis(time_ms)).await;

libs/models/Cargo.toml

@@ -6,34 +6,33 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-base64 = "0.21.0"
-color-eyre = "0.6.1"
-ed25519-dalek = { version = "1.0.1", features = ["serde"] }
+base64 = "0.21.4"
+color-eyre = "0.6.2"
+ed25519-dalek = { version = "2.0.0", features = ["serde"] }
 lazy_static = "1.4"
-mbrman = "0.5.0"
+mbrman = "0.5.2"
 emver = { version = "0.1", git = "https://github.com/Start9Labs/emver-rs.git", features = [
   "serde",
 ] }
-ipnet = "2.7.1"
-openssl = { version = "0.10.41", features = ["vendored"] }
+ipnet = "2.8.0"
+openssl = { version = "0.10.57", features = ["vendored"] }
 patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
   "trace",
 ] }
-rand = "0.8"
-regex = "1.7.1"
-reqwest = "0.11.14"
-rpc-toolkit = "0.2.1"
+rand = "0.8.5"
+regex = "1.10.2"
+reqwest = "0.11.22"
+rpc-toolkit = "0.2.2"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-sqlx = { version = "0.6.0", features = [
+sqlx = { version = "0.7.2", features = [
   "chrono",
-  "offline",
   "runtime-tokio-rustls",
   "postgres",
 ] }
-ssh-key = "0.5.1"
+ssh-key = "0.6.2"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 torut = "0.2.1"
-tracing = "0.1.35"
-yasi = "0.1.2"
+tracing = "0.1.39"
+yasi = "0.1.5"

libs/models/src/errors.rs

@@ -78,6 +78,9 @@ pub enum ErrorKind {
     OpenSsh = 66,
     Zram = 67,
     Lshw = 68,
+    CpuSettings = 69,
+    Firmware = 70,
+    Timeout = 71,
 }
 impl ErrorKind {
     pub fn as_str(&self) -> &'static str {
@@ -151,6 +154,9 @@ impl ErrorKind {
             OpenSsh => "OpenSSH Error",
             Zram => "Zram Error",
             Lshw => "LSHW Error",
+            CpuSettings => "CPU Settings Error",
+            Firmware => "Firmware Error",
+            Timeout => "Timeout Error",
         }
     }
 }

libs/models/src/procedure_name.rs

@@ -1,6 +1,8 @@
+use serde::{Deserialize, Serialize};
+
 use crate::{ActionId, HealthCheckId, PackageId};
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ProcedureName {
     Main, // Usually just run container
     CreateBackup,

libs/snapshot_creator/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2021"
 
 [dependencies]
 dashmap = "5.3.4"
-deno_core = "0.195.0"
-deno_ast = { version = "0.27.2", features = ["transpiling"] }
+deno_core = "=0.222.0"
+deno_ast = { version = "=0.29.5", features = ["transpiling"] }

libs/snapshot_creator/src/main.rs

@@ -1,7 +1,7 @@
 use deno_core::JsRuntimeForSnapshot;
 
 fn main() {
-    let runtime = JsRuntimeForSnapshot::new(Default::default(), Default::default());
+    let runtime = JsRuntimeForSnapshot::new(Default::default());
     let snapshot = runtime.snapshot();
 
     let snapshot_slice: &[u8] = &*snapshot;