Fix/making js work (#1456)

* Feat: js action

wip: Getting async js

feat: Have execute get action config

feat: Read + Write

chore: Add typing for globals

chore: Change the default path, include error on missing function, and add json File Read Write

chore: Change the default path, include error on missing function, and add json File Read Write

wip: Fix the unit test

wip: Fix the unit test

feat: module loading

* fix: Change the source + add input

* fix: Change the source + add input

wip: Fix missing js files during running

fix: Change the source + add input

wip: Fix missing js files during running

* fix: other paths

* feat: Build the arm js snapshot

* fix: test with more

* chore: Make the is_subset a result

backend/src/bin/build-js-snapshots.rs

@@ -0,0 +1,12 @@
+fn main() {
+    let mut runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
+        will_snapshot: true,
+        ..Default::default()
+    });
+    let snapshot = runtime.snapshot();
+
+    let snapshot_slice: &[u8] = &*snapshot;
+    println!("Snapshot size: {}", snapshot_slice.len());
+
+    std::fs::write("JS_SNAPSHOT.bin", snapshot_slice).unwrap();
+}

backend/src/config/mod.rs

@@ -187,7 +187,6 @@ impl ConfigGetReceipts {
         locks: &mut Vec<LockTargetId>,
         id: &PackageId,
     ) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
-
         let manifest_version = crate::db::DatabaseModel::new()
             .package_data()
             .idx_model(id)
@@ -680,6 +679,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
                         dependent,
                         &manifest.version,
                         &manifest.volumes,
+                        dependent,
                         &config,
                     )
                     .await?

backend/src/dependencies.rs

@@ -14,7 +14,6 @@ use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use crate::config::action::{ConfigActions, ConfigRes};
 use crate::config::spec::PackagePointerSpec;
 use crate::config::{not_found, Config, ConfigReceipts, ConfigSpec};
 use crate::context::RpcContext;
@@ -27,6 +26,10 @@ use crate::util::serde::display_serializable;
 use crate::util::{display_none, Version};
 use crate::volume::Volumes;
 use crate::Error;
+use crate::{
+    config::action::{ConfigActions, ConfigRes},
+    procedure::ProcedureName,
+};
 
 #[command(subcommands(configure))]
 pub fn dependency() -> Result<(), Error> {
@@ -257,6 +260,7 @@ impl DependencyError {
                                 id,
                                 &dependent_manifest.version,
                                 &dependent_manifest.volumes,
+                                dependency,
                                 &dependency_config,
                             )
                             .await?
@@ -496,6 +500,7 @@ impl DependencyConfig {
         dependent_id: &PackageId,
         dependent_version: &Version,
         dependent_volumes: &Volumes,
+        dependency_id: &PackageId,
         dependency_config: &Config,
     ) -> Result<Result<NoOutput, String>, Error> {
         Ok(self
@@ -507,6 +512,7 @@ impl DependencyConfig {
                 dependent_volumes,
                 Some(dependency_config),
                 None,
+                ProcedureName::Check(dependency_id.clone()),
             )
             .await?
             .map_err(|(_, e)| e))
@@ -527,6 +533,7 @@ impl DependencyConfig {
                 dependent_volumes,
                 Some(old),
                 None,
+                ProcedureName::AutoConfig(dependent_id.clone()),
             )
             .await?
             .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))
@@ -730,6 +737,7 @@ pub async fn configure_logic(
             &pkg_volumes,
             Some(&old_config),
             None,
+            ProcedureName::AutoConfig(dependency_id.clone()),
         )
         .await?
         .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;

backend/src/procedure/js_scripts.rs

@@ -30,7 +30,7 @@ pub enum JsError {
 pub struct JsProcedure {}
 
 impl JsProcedure {
-    pub fn validate(&self, volumes: &Volumes) -> Result<(), color_eyre::eyre::Report> {
+    pub fn validate(&self, _volumes: &Volumes) -> Result<(), color_eyre::eyre::Report> {
         Ok(())
     }
 
@@ -53,7 +53,6 @@ impl JsProcedure {
                 volumes.clone(),
             )
             .await?
-            .with_effects()
             .run_action(name, input);
             let output: O = match timeout {
                 Some(timeout_duration) => tokio::time::timeout(timeout_duration, running_action)
@@ -76,6 +75,7 @@ impl JsProcedure {
         volumes: &Volumes,
         input: Option<I>,
         timeout: Option<Duration>,
+        name: ProcedureName,
     ) -> Result<Result<O, (i32, String)>, Error> {
         Ok(async move {
             let running_action = JsExecutionEnvironment::load_from_package(
@@ -86,7 +86,7 @@ impl JsProcedure {
             )
             .await?
             .read_only_effects()
-            .run_action(ProcedureName::GetConfig, input);
+            .run_action(name, input);
             let output: O = match timeout {
                 Some(timeout_duration) => tokio::time::timeout(timeout_duration, running_action)
                     .await
@@ -111,6 +111,7 @@ mod js_runtime {
     use deno_core::ModuleSpecifier;
     use deno_core::ModuleType;
     use deno_core::RuntimeOptions;
+    use deno_core::Snapshot;
     use deno_core::{Extension, OpDecl};
     use serde::{Deserialize, Serialize};
     use serde_json::Value;
@@ -120,11 +121,16 @@ mod js_runtime {
 
     use crate::s9pk::manifest::PackageId;
     use crate::util::Version;
-    use crate::volume::{script_dir, Volumes};
+    use crate::volume::script_dir;
+    use crate::volume::Volumes;
 
     use super::super::ProcedureName;
     use super::{JsCode, JsError};
+    #[cfg(target_arch = "x86_64")]
+    const SNAPSHOT_BYTES: &[u8] = include_bytes!("./js_scripts/JS_SNAPSHOT.bin");
 
+    #[cfg(target_arch = "aarch64")]
+    const SNAPSHOT_BYTES: &[u8] = include_bytes!("./js_scripts/ARM_JS_SNAPSHOT.bin");
     #[derive(Clone, Deserialize, Serialize)]
     struct JsContext {
         sandboxed: bool,
@@ -212,7 +218,6 @@ mod js_runtime {
         module_loader: ModsLoader,
         package_id: PackageId,
         version: Version,
-        operations: Vec<OpDecl>,
         volumes: Arc<Volumes>,
     }
 
@@ -230,6 +235,7 @@ mod js_runtime {
                 let mut file = match tokio::fs::File::open(file_path.clone()).await {
                     Ok(x) => x,
                     Err(e) => {
+                        tracing::debug!("path: {:?}", file_path);
                         tracing::debug!("{:?}", e);
                         return Err((
                             JsError::FileSystem,
@@ -250,7 +256,6 @@ mod js_runtime {
             Ok(Self {
                 base_directory: base_directory.to_owned(),
                 module_loader: ModsLoader { code: js_code },
-                operations: Default::default(),
                 package_id: package_id.clone(),
                 version: version.clone(),
                 volumes: Arc::new(volumes),
@@ -259,29 +264,9 @@ mod js_runtime {
         }
         pub fn read_only_effects(mut self) -> Self {
             self.sandboxed = true;
-            self.with_effects()
-        }
-
-        pub fn with_effects(mut self) -> Self {
-            self.operations = vec![
-                fns::read_file::decl(),
-                fns::write_file::decl(),
-                fns::create_dir::decl(),
-                fns::remove_dir::decl(),
-                fns::get_context::decl(),
-                fns::log_trace::decl(),
-                fns::log_warn::decl(),
-                fns::log_error::decl(),
-                fns::log_debug::decl(),
-                fns::log_info::decl(),
-                fns::current_function::decl(),
-                fns::set_value::decl(),
-                fns::remove_file::decl(),
-                fns::is_sandboxed::decl(),
-                fns::get_input::decl(),
-            ];
             self
         }
+
         pub async fn run_action<I: Serialize, O: for<'de> Deserialize<'de>>(
             self,
             procedure_name: ProcedureName,
@@ -318,6 +303,25 @@ mod js_runtime {
                 }
             }
         }
+        fn declarations() -> Vec<OpDecl> {
+            vec![
+                fns::read_file::decl(),
+                fns::write_file::decl(),
+                fns::remove_file::decl(),
+                fns::create_dir::decl(),
+                fns::remove_dir::decl(),
+                fns::current_function::decl(),
+                fns::log_trace::decl(),
+                fns::log_warn::decl(),
+                fns::log_error::decl(),
+                fns::log_debug::decl(),
+                fns::log_info::decl(),
+                fns::get_context::decl(),
+                fns::get_input::decl(),
+                fns::set_value::decl(),
+                fns::is_sandboxed::decl(),
+            ]
+        }
 
         fn execute(
             &self,
@@ -337,7 +341,7 @@ mod js_runtime {
                 input,
             };
             let ext = Extension::builder()
-                .ops(self.operations.clone())
+                .ops(Self::declarations())
                 .state(move |state| {
                     state.put(ext_answer_state.clone());
                     state.put(js_ctx.clone());
@@ -346,11 +350,13 @@ mod js_runtime {
                 .build();
 
             let loader = std::rc::Rc::new(self.module_loader.clone());
-            let mut runtime = JsRuntime::new(RuntimeOptions {
+            let runtime_options = RuntimeOptions {
                 module_loader: Some(loader),
                 extensions: vec![ext],
+                startup_snapshot: Some(Snapshot::Static(SNAPSHOT_BYTES)),
                 ..Default::default()
-            });
+            };
+            let mut runtime = JsRuntime::new(runtime_options);
 
             let future = async move {
                 let mod_id = runtime
@@ -358,8 +364,8 @@ mod js_runtime {
                     .await?;
                 let evaluated = runtime.mod_evaluate(mod_id);
                 let res = runtime.run_event_loop(false).await;
-                evaluated.await??;
                 res?;
+                evaluated.await??;
                 Ok::<_, AnyError>(())
             };
 
@@ -367,7 +373,7 @@ mod js_runtime {
                 .block_on(future)
                 .map_err(|e| {
                     tracing::debug!("{:?}", e);
-                    (JsError::Javascript, format!("Execution error: {}", e))
+                    (JsError::Javascript, format!("{}", e))
                 })?;
 
             let answer = answer_state.0.lock().clone();
@@ -377,10 +383,17 @@ mod js_runtime {
 
     /// Note: Make sure that we have the assumption that all these methods are callable at any time, and all call restrictions should be in rust
     mod fns {
-        use deno_core::{anyhow::bail, error::AnyError, *};
+        use deno_core::{
+            anyhow::{anyhow, bail},
+            error::AnyError,
+            *,
+        };
         use serde_json::Value;
 
-        use std::{convert::TryFrom, path::PathBuf};
+        use std::{
+            convert::TryFrom,
+            path::{Path, PathBuf},
+        };
 
         use crate::volume::VolumeId;
 
@@ -402,8 +415,12 @@ mod js_runtime {
                 volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
             //get_path_for in volume.rs
             let new_file = volume_path.join(path_in);
-            if !new_file.starts_with(volume_path) {
-                bail!("Path has broken away from parent");
+            if !is_subset(&volume_path, &new_file).await? {
+                bail!(
+                    "Path '{}' has broken away from parent '{}'",
+                    new_file.to_string_lossy(),
+                    volume_path.to_string_lossy(),
+                );
             }
             let answer = tokio::fs::read_to_string(new_file).await?;
             Ok(answer)
@@ -429,10 +446,18 @@ mod js_runtime {
             }
             let volume_path =
                 volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
+
             let new_file = volume_path.join(path_in);
+            let parent_new_file = new_file
+                .parent()
+                .ok_or_else(|| anyhow!("Expecting that file is not root"))?;
             // With the volume check
-            if !new_file.starts_with(volume_path) {
-                bail!("Path has broken away from parent");
+            if !is_subset(&volume_path, &parent_new_file).await? {
+                bail!(
+                    "Path '{}' has broken away from parent '{}'",
+                    new_file.to_string_lossy(),
+                    volume_path.to_string_lossy(),
+                );
             }
             tokio::fs::write(new_file, write).await?;
             Ok(())
@@ -457,10 +482,14 @@ mod js_runtime {
             }
             let volume_path =
                 volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
-            let new_file = volume_path.clone().join(path_in);
+            let new_file = volume_path.join(path_in);
             // With the volume check
-            if !new_file.starts_with(volume_path) {
-                bail!("Path has broken away from parent");
+            if !is_subset(&volume_path, &new_file).await? {
+                bail!(
+                    "Path '{}' has broken away from parent '{}'",
+                    new_file.to_string_lossy(),
+                    volume_path.to_string_lossy(),
+                );
             }
             tokio::fs::remove_file(new_file).await?;
             Ok(())
@@ -485,10 +514,14 @@ mod js_runtime {
             }
             let volume_path =
                 volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
-            let new_file = volume_path.clone().join(path_in);
+            let new_file = volume_path.join(path_in);
             // With the volume check
-            if !new_file.starts_with(volume_path) {
-                bail!("Path has broken away from parent");
+            if !is_subset(&volume_path, &new_file).await? {
+                bail!(
+                    "Path '{}' has broken away from parent '{}'",
+                    new_file.to_string_lossy(),
+                    volume_path.to_string_lossy(),
+                );
             }
             tokio::fs::remove_dir_all(new_file).await?;
             Ok(())
@@ -513,10 +546,17 @@ mod js_runtime {
             }
             let volume_path =
                 volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
-            let new_file = volume_path.clone().join(path_in);
+            let new_file = volume_path.join(path_in);
+            let parent_new_file = new_file
+                .parent()
+                .ok_or_else(|| anyhow!("Expecting that file is not root"))?;
             // With the volume check
-            if !new_file.starts_with(volume_path) {
-                bail!("Path has broken away from parent");
+            if !is_subset(&volume_path, &parent_new_file).await? {
+                bail!(
+                    "Path '{}' has broken away from parent '{}'",
+                    new_file.to_string_lossy(),
+                    volume_path.to_string_lossy(),
+                );
             }
             tokio::fs::create_dir_all(new_file).await?;
             Ok(())
@@ -605,12 +645,24 @@ mod js_runtime {
             let ctx = state.borrow::<JsContext>();
             Ok(ctx.sandboxed)
         }
+
+        /// We need to make sure that during the file accessing, we don't reach beyond our scope of control
+        async fn is_subset(parent: impl AsRef<Path>, child: impl AsRef<Path>) -> Result<bool, AnyError> {
+            let child = tokio::fs::canonicalize(child).await?;
+            let parent = tokio::fs::canonicalize(parent).await?;
+            Ok(child.starts_with(parent))
+        }
     }
 }
+
 #[tokio::test]
 async fn js_action_execute() {
     let js_action = JsProcedure {};
-    let path: PathBuf = "test/js_action_execute/".parse().unwrap();
+    let path: PathBuf = "test/js_action_execute/"
+        .parse::<PathBuf>()
+        .unwrap()
+        .canonicalize()
+        .unwrap();
     let package_id = "test-package".parse().unwrap();
     let package_version: Version = "0.3.0.3".parse().unwrap();
     let name = ProcedureName::GetConfig;
@@ -631,7 +683,7 @@ async fn js_action_execute() {
     }))
     .unwrap();
     let input: Option<serde_json::Value> = Some(serde_json::json!({"test":123}));
-    let timeout = None;
+    let timeout = Some(Duration::from_secs(10));
     let _output: crate::config::action::ConfigRes = js_action
         .execute(
             &path,

backend/src/procedure/js_scripts/loadModule.js

@@ -1,6 +1,6 @@
 //@ts-check
 // @ts-ignore
-import  Deno from "/deno_global.js";
+import Deno from "/deno_global.js";
 // @ts-ignore
 import * as mainModule from "/embassy.js";
 /**
@@ -23,36 +23,28 @@ function jsonPointerValue(obj, pointer) {
 // @ts-ignore
 const context = Deno.core.opSync("get_context");
 // @ts-ignore
-const writeFile = ({ path, volumeId, toWrite }) =>
-  Deno.core.opAsync("write_file", context, volumeId, path, toWrite);
+const writeFile = ({ path, volumeId, toWrite }) => Deno.core.opAsync("write_file", context, volumeId, path, toWrite);
 
 // @ts-ignore
-const readFile = ({ volumeId, path }) =>
-  Deno.core.opAsync("read_file", context, volumeId, path);
+const readFile = ({ volumeId, path }) => Deno.core.opAsync("read_file", context, volumeId, path);
 // @ts-ignore
-const removeFile = ({ volumeId, path }) =>
-  Deno.core.opAsync("remove_file", context, volumeId, path);
+const removeFile = ({ volumeId, path }) => Deno.core.opAsync("remove_file", context, volumeId, path);
 // @ts-ignore
 const isSandboxed = () => Deno.core.opSync("is_sandboxed");
 
 // @ts-ignore
 const writeJsonFile = ({ volumeId, path, toWrite }) =>
-  Deno.core.opAsync(
-    "write_file",
-    context,
+  writeFile({
     volumeId,
     path,
-    JSON.stringify(toWrite)
-  );
+    toWrite: JSON.stringify(toWrite),
+  });
 // @ts-ignore
-const readJsonFile = ({ volumeId, path }) =>
-  JSON.parse(Deno.core.opAsync("read_file", context, volumeId, path));
+const readJsonFile = async ({ volumeId, path }) => JSON.parse(await readFile({ volumeId, path }));
 // @ts-ignore
-const createDir = ({ volumeId, path }) =>
-  Deno.core.opAsync("create_dir", context, volumeId, path);
+const createDir = ({ volumeId, path }) => Deno.core.opAsync("create_dir", context, volumeId, path);
 // @ts-ignore
-const removeDir = ({ volumeId, path }) =>
-  Deno.core.opAsync("remove_dir", context, volumeId, path);
+const removeDir = ({ volumeId, path }) => Deno.core.opAsync("remove_dir", context, volumeId, path);
 // @ts-ignore
 const trace = (x) => Deno.core.opSync("log_trace", x);
 // @ts-ignore
@@ -86,14 +78,11 @@ const effects = {
   removeDir,
 };
 
-const runFunction = jsonPointerValue(
-  mainModule,
-  currentFunction
-);
+const runFunction = jsonPointerValue(mainModule, currentFunction);
 (async () => {
   if (typeof runFunction !== "function") {
-    error(`Expecting ${{ currentFunction }} to be a function`);
-    throw new Error(`Expecting ${{ currentFunction }} to be a function`);
+    error(`Expecting ${ currentFunction } to be a function`);
+    throw new Error(`Expecting ${ currentFunction } to be a function`);
   }
   const answer = await runFunction(effects, input);
   setState(answer);

backend/src/procedure/mod.rs

@@ -28,6 +28,8 @@ pub enum ProcedureName {
     SetConfig,
     Migration,
     Properties,
+    Check(PackageId),
+    AutoConfig(PackageId),
     Health(HealthCheckId),
     Action(ActionId),
 }
@@ -44,11 +46,13 @@ impl ProcedureName {
             ProcedureName::Properties => Some(format!("Properties-{}", rand::random::<u64>())),
             ProcedureName::Health(id) => Some(format!("{}Health", id)),
             ProcedureName::Action(id) => Some(format!("{}Action", id)),
+            ProcedureName::Check(_) => None,
+            ProcedureName::AutoConfig(_) => None,
         }
     }
     fn js_function_name(&self) -> String {
         match self {
-            ProcedureName::Main => todo!(),
+            ProcedureName::Main => "/main".to_string(),
             ProcedureName::CreateBackup => "/createBackup".to_string(),
             ProcedureName::RestoreBackup => "/restoreBackup".to_string(),
             ProcedureName::GetConfig => "/getConfig".to_string(),
@@ -57,6 +61,8 @@ impl ProcedureName {
             ProcedureName::Properties => "/properties".to_string(),
             ProcedureName::Health(id) => format!("/health/{}", id),
             ProcedureName::Action(id) => format!("/action/{}", id),
+            ProcedureName::Check(id) => format!("/dependencies/{}/check", id),
+            ProcedureName::AutoConfig(id) => format!("/dependencies/{}/autoConfigure", id),
         }
     }
 }
@@ -136,6 +142,7 @@ impl PackageProcedure {
         volumes: &Volumes,
         input: Option<I>,
         timeout: Option<Duration>,
+        name: ProcedureName,
     ) -> Result<Result<O, (i32, String)>, Error> {
         match self {
             PackageProcedure::Docker(procedure) => {
@@ -145,7 +152,7 @@ impl PackageProcedure {
             }
             PackageProcedure::Script(procedure) => {
                 procedure
-                    .sandboxed(ctx, pkg_id, pkg_version, volumes, input, timeout)
+                    .sandboxed(ctx, pkg_id, pkg_version, volumes, input, timeout, name)
                     .await
             }
         }