Merge branch 'master' of github.com:Start9Labs/start-os into rebase/integration/refactors

libs/build-arm-v8-snapshot.sh

@@ -2,6 +2,7 @@
 # Reason for this being is that we need to create a snapshot for the deno runtime. It wants to pull 3 files from build, and during the creation it gets embedded, but for some 
 # reason during the actual runtime it is looking for them. So this will create a docker in arm that creates the snaphot needed for the arm
 set -e
+shopt -s expand_aliases
 
 if [ "$0" != "./build-arm-v8-snapshot.sh" ]; then
 	>&2 echo "Must be run from backend/workspace directory"
@@ -13,9 +14,11 @@ if tty -s; then
 	USE_TTY="-it"
 fi
 
+alias 'rust-gnu-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/home/rust/src -w /home/rust/src -P start9/rust-arm-cross:aarch64'
+
 echo "Building "
 cd ..
-docker run --rm $USE_TTY -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64 sh -c "(cd libs/ && cargo build -p snapshot_creator --release )"
+rust-gnu-builder sh -c "(cd libs/ && cargo build -p snapshot_creator --release --target=aarch64-unknown-linux-gnu)"
 cd -
 
 echo "Creating Arm v8 Snapshot"

libs/embassy_container_init/src/main.rs

@@ -371,7 +371,7 @@ async fn main() {
                                         tracing::error!("Error sending to {id:?}", id = req.id);
                                     }
                                 }
-                                Err(e) => 
+                                Err(e) =>
                                 if let Err(err) = w
                                     .lock()
                                     .await

libs/js_engine/Cargo.toml

@@ -8,35 +8,13 @@ edition = "2021"
 [dependencies]
 async-trait = "0.1.56"
 dashmap = "5.3.4"
-deno_core = "=0.136.0"
-deno_ast = { version = "=0.15.0", features = ["transpiling"] }
-dprint-swc-ext = "=0.1.1"
+deno_core = "0.195.0"
+deno_ast = { version = "0.27.2", features = ["transpiling"] }
 embassy_container_init = { path = "../embassy_container_init" }
 reqwest = { version = "0.11.11" }
-swc_atoms = "=0.2.11"
-swc_common = "=0.18.7"
-swc_config = "=0.1.1"
-swc_config_macro = "=0.1.0"
-swc_ecma_ast = "=0.78.1"
-swc_ecma_codegen = "=0.108.6"
-swc_ecma_codegen_macros = "=0.7.0"
-swc_ecma_parser = "=0.104.2"
-swc_ecma_transforms = "=0.154.0"
-swc_ecma_transforms_base = "=0.85.4"
-swc_ecma_transforms_classes = "=0.73.0"
-swc_ecma_transforms_macros = "=0.3.0"
-swc_ecma_transforms_proposal = "=0.107.0"
-swc_ecma_transforms_react = "=0.114.1"
-swc_ecma_transforms_typescript = "=0.117.2"
-swc_ecma_utils = "=0.85.1"
-swc_ecma_visit = "=0.64.0"
-swc_ecmascript = "=0.157.0"
-swc_eq_ignore_macros = "=0.1.0"
-swc_macros_common = "=0.3.5"
-swc_visit = "=0.3.0"
-swc_visit_macros = "=0.3.1"
 sha2 = "0.10.2"
 itertools = "0.10.5"
+lazy_static = "1.4.0"
 models = { path = "../models" }
 helpers = { path = "../helpers" }
 serde = { version = "1.0", features = ["derive", "rc"] }

libs/js_engine/src/artifacts/loadModule.js

@@ -141,7 +141,7 @@ const removeFile = (
   { volumeId = requireParam("volumeId"), path = requireParam("path") } =
     requireParam("options"),
 ) => Deno.core.opAsync("remove_file", volumeId, path);
-const isSandboxed = () => Deno.core.opSync("is_sandboxed");
+const isSandboxed = () => Deno.core.ops["is_sandboxed"]();
 
 const writeJsonFile = (
   {
@@ -265,15 +265,15 @@ const getServiceConfig = async (
   );
 };
 
-const started = () => Deno.core.opSync("set_started");
+const started = () => Deno.core.ops.set_started();
 const restart = () => Deno.core.opAsync("restart");
 const start = () => Deno.core.opAsync("start");
 const stop = () => Deno.core.opAsync("stop");
 
-const currentFunction = Deno.core.opSync("current_function");
-const input = Deno.core.opSync("get_input");
-const variable_args = Deno.core.opSync("get_variable_args");
-const setState = (x) => Deno.core.opAsync("set_value", x);
+const currentFunction = Deno.core.ops.current_function();
+const input = Deno.core.ops.get_input();
+const variable_args = Deno.core.ops.get_variable_args();
+const setState = (x) => Deno.core.ops.set_value(x);
 const effects = {
   bindLocal,
   bindTor,

libs/js_engine/src/lib.rs

@@ -9,8 +9,9 @@ use std::time::SystemTime;
 use deno_core::anyhow::{anyhow, bail};
 use deno_core::error::AnyError;
 use deno_core::{
-    resolve_import, Extension, JsRuntime, ModuleLoader, ModuleSource, ModuleSourceFuture,
-    ModuleSpecifier, ModuleType, OpDecl, RuntimeOptions, Snapshot,
+    resolve_import, Extension, FastString, JsRuntime, ModuleLoader, ModuleSource,
+    ModuleSourceFuture, ModuleSpecifier, ModuleType, OpDecl, ResolutionKind, RuntimeOptions,
+    Snapshot,
 };
 use embassy_container_init::ProcessGroupId;
 use helpers::{script_dir, spawn_local, OsApi, Rsync, UnixRpcClient};
@@ -21,6 +22,12 @@ use tokio::io::AsyncReadExt;
 use tokio::sync::{mpsc, Mutex};
 use tracing::instrument;
 
+lazy_static::lazy_static! {
+    static ref DENO_GLOBAL_JS: ModuleSpecifier = "file:///deno_global.js".parse().unwrap();
+    static ref LOAD_MODULE_JS: ModuleSpecifier = "file:///loadModule.js".parse().unwrap();
+    static ref EMBASSY_JS: ModuleSpecifier = "file:///embassy.js".parse().unwrap();
+}
+
 pub trait PathForVolumeId: Send + Sync {
     fn path_for(
         &self,
@@ -32,8 +39,8 @@ pub trait PathForVolumeId: Send + Sync {
     fn readonly(&self, volume_id: &VolumeId) -> bool;
 }
 
-#[derive(Serialize, Deserialize, Debug, Default, Clone)]
-pub struct JsCode(String);
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct JsCode(Arc<str>);
 
 #[derive(Debug, Clone, Copy)]
 pub enum JsError {
@@ -131,7 +138,7 @@ impl ModuleLoader for ModsLoader {
         &self,
         specifier: &str,
         referrer: &str,
-        _is_main: bool,
+        _is_main: ResolutionKind,
     ) -> Result<ModuleSpecifier, AnyError> {
         if referrer.contains("embassy") {
             bail!("Embassy.js cannot import anything else");
@@ -143,49 +150,42 @@ impl ModuleLoader for ModsLoader {
     fn load(
         &self,
         module_specifier: &ModuleSpecifier,
-        maybe_referrer: Option<ModuleSpecifier>,
+        maybe_referrer: Option<&ModuleSpecifier>,
         is_dyn_import: bool,
     ) -> Pin<Box<ModuleSourceFuture>> {
         let module_specifier = module_specifier.as_str().to_owned();
         let module = match &*module_specifier {
-            "file:///deno_global.js" => Ok(ModuleSource {
-                module_url_specified: "file:///deno_global.js".to_string(),
-                module_url_found: "file:///deno_global.js".to_string(),
-                code: "const old_deno = Deno; Deno = null; export default old_deno"
-                    .as_bytes()
-                    .to_vec()
-                    .into_boxed_slice(),
-                module_type: ModuleType::JavaScript,
-            }),
-            "file:///loadModule.js" => Ok(ModuleSource {
-                module_url_specified: "file:///loadModule.js".to_string(),
-                module_url_found: "file:///loadModule.js".to_string(),
-                code: include_str!("./artifacts/loadModule.js")
-                    .as_bytes()
-                    .to_vec()
-                    .into_boxed_slice(),
-                module_type: ModuleType::JavaScript,
-            }),
-            "file:///embassy.js" => Ok(ModuleSource {
-                module_url_specified: "file:///embassy.js".to_string(),
-                module_url_found: "file:///embassy.js".to_string(),
-                code: self.code.0.as_bytes().to_vec().into_boxed_slice(),
-                module_type: ModuleType::JavaScript,
-            }),
+            "file:///deno_global.js" => Ok(ModuleSource::new(
+                ModuleType::JavaScript,
+                FastString::Static("const old_deno = Deno; Deno = null; export default old_deno"),
+                &*DENO_GLOBAL_JS,
+            )),
+            "file:///loadModule.js" => Ok(ModuleSource::new(
+                ModuleType::JavaScript,
+                FastString::Static(include_str!("./artifacts/loadModule.js")),
+                &*LOAD_MODULE_JS,
+            )),
+            "file:///embassy.js" => Ok(ModuleSource::new(
+                ModuleType::JavaScript,
+                self.code.0.clone().into(),
+                &*EMBASSY_JS,
+            )),
+
             x => Err(anyhow!("Not allowed to import: {}", x)),
         };
-        Box::pin(async move {
+        let module = module.and_then(|m| {
             if is_dyn_import {
                 bail!("Will not import dynamic");
             }
             match &maybe_referrer {
                 Some(x) if x.as_str() == "file:///embassy.js" => {
-                    bail!("Embassy is not allowed to import")
+                    bail!("StartJS is not allowed to import")
                 }
                 _ => (),
             }
-            module
-        })
+            Ok(m)
+        });
+        Box::pin(async move { module })
     }
 }
 
@@ -234,7 +234,7 @@ impl JsExecutionEnvironment {
                     format!("The file reading created error: {}", err),
                 ));
             };
-            buffer
+            buffer.into()
         });
         Ok(JsExecutionEnvironment {
             os,
@@ -364,12 +364,11 @@ impl JsExecutionEnvironment {
             callback_sender,
             rsyncs: Default::default(),
         };
-        let ext = Extension::builder()
+        let ext = Extension::builder("embassy")
             .ops(Self::declarations())
             .state(move |state| {
                 state.put(ext_answer_state.clone());
                 state.put(js_ctx.clone());
-                Ok(())
             })
             .build();
         let loader = std::rc::Rc::new(self.module_loader.clone());

libs/snapshot_creator/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2021"
 
 [dependencies]
 dashmap = "5.3.4"
-deno_core = "=0.136.0"
-deno_ast = { version = "=0.15.0", features = ["transpiling"] }
+deno_core = "0.195.0"
+deno_ast = { version = "0.27.2", features = ["transpiling"] }

libs/snapshot_creator/src/main.rs

@@ -1,10 +1,7 @@
-use deno_core::{JsRuntime, RuntimeOptions};
+use deno_core::JsRuntimeForSnapshot;
 
 fn main() {
-    let mut runtime = JsRuntime::new(RuntimeOptions {
-        will_snapshot: true,
-        ..Default::default()
-    });
+    let runtime = JsRuntimeForSnapshot::new(Default::default(), Default::default());
     let snapshot = runtime.snapshot();
 
     let snapshot_slice: &[u8] = &*snapshot;