diff --git a/core/Cargo.lock b/core/Cargo.lock index 8a6c1dd5c..3c3b693a2 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2560,27 +2560,6 @@ dependencies = [ "time", ] -[[package]] -name = "js-engine" -version = "0.1.0" -dependencies = [ - "async-trait", - "container-init", - "dashmap", - "deno_ast", - "deno_core", - "helpers", - "itertools 0.11.0", - "lazy_static", - "models", - "reqwest", - "serde", - "serde_json", - "sha2 0.10.8", - "tokio", - "tracing", -] - [[package]] name = "js-sys" version = "0.3.65" @@ -4995,7 +4974,6 @@ dependencies = [ "jaq-core", "jaq-std", "josekit", - "js-engine", "jsonpath_lib", "lazy_static", "libc", diff --git a/core/Cargo.toml b/core/Cargo.toml index 894362522..143a830fc 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,10 +1,3 @@ [workspace] -members = [ - "container-init", - "helpers", - "js-engine", - "models", - "snapshot-creator", - "startos", -] +members = ["container-init", "helpers", "models", "snapshot-creator", "startos"] diff --git a/core/js-engine/Cargo.toml b/core/js-engine/Cargo.toml deleted file mode 100644 index 14205109b..000000000 --- a/core/js-engine/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "js-engine" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -async-trait = "0.1.74" -dashmap = "5.5.3" -deno_core = "=0.222.0" -deno_ast = { version = "=0.29.5", features = ["transpiling"] } -container-init = { path = "../container-init" } -reqwest = { version = "0.11.22" } -sha2 = "0.10.8" -itertools = "0.11.0" -lazy_static = "1.4.0" -models = { path = "../models" } -helpers = { path = "../helpers" } -serde = { version = "1.0", features = ["derive", "rc"] } -serde_json = "1.0" -tokio = { version = "1", features = ["full"] } -tracing = "0.1" diff --git a/core/js-engine/src/artifacts/JS_SNAPSHOT.aarch64.bin b/core/js-engine/src/artifacts/JS_SNAPSHOT.aarch64.bin deleted file mode 100644 index 305aa2d4c..000000000 Binary files a/core/js-engine/src/artifacts/JS_SNAPSHOT.aarch64.bin and /dev/null differ diff --git a/core/js-engine/src/artifacts/JS_SNAPSHOT.x86_64.bin b/core/js-engine/src/artifacts/JS_SNAPSHOT.x86_64.bin deleted file mode 100644 index 7f7d10689..000000000 Binary files a/core/js-engine/src/artifacts/JS_SNAPSHOT.x86_64.bin and /dev/null differ diff --git a/core/js-engine/src/artifacts/loadModule.js b/core/js-engine/src/artifacts/loadModule.js deleted file mode 100644 index de30eac89..000000000 --- a/core/js-engine/src/artifacts/loadModule.js +++ /dev/null @@ -1,242 +0,0 @@ -import Deno from "/deno_global.js"; -import * as mainModule from "/embassy.js"; - -function requireParam(param) { - throw new Error(`Missing required parameter ${param}`); -} - -/** - * This is using the simplified json pointer spec, using no escapes and arrays - * @param {object} obj - * @param {string} pointer - * @returns - */ -function jsonPointerValue(obj, pointer) { - const paths = pointer.substring(1).split("/"); - for (const path of paths) { - if (obj == null) { - return null; - } - obj = (obj || {})[path]; - } - return obj; -} - -function maybeDate(value) { - if (!value) return value; - return new Date(value); -} -const writeFile = ( - { - path = requireParam("path"), - volumeId = requireParam("volumeId"), - toWrite = requireParam("toWrite"), - } = requireParam("options"), -) => Deno.core.opAsync("write_file", volumeId, path, toWrite); - -const readFile = ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => Deno.core.opAsync("read_file", volumeId, path); - - - -const runDaemon = ( - { command = requireParam("command"), args = [] } = requireParam("options"), -) => { - let id = Deno.core.opAsync("start_command", command, args, "inherit", null); - let processId = id.then(x => x.processId) - let waitPromise = null; - return { - processId, - async wait() { - waitPromise = waitPromise || Deno.core.opAsync("wait_command", await processId) - return waitPromise - }, - async term(signal = 15) { - return Deno.core.opAsync("send_signal", await processId, 15) - } - } -}; -const runCommand = async ( - { command = requireParam("command"), args = [], timeoutMillis = 30000 } = requireParam("options"), -) => { - let id = Deno.core.opAsync("start_command", command, args, "collect", timeoutMillis); - let pid = id.then(x => x.processId) - return Deno.core.opAsync("wait_command", await pid) -}; -const signalGroup = async ( - { gid = requireParam("gid"), signal = requireParam("signal") } = requireParam("gid and signal") -) => { - return Deno.core.opAsync("signal_group", gid, signal); -}; -const sleep = (timeMs = requireParam("timeMs"), -) => Deno.core.opAsync("sleep", timeMs); - -const rename = ( - { - srcVolume = requireParam("srcVolume"), - dstVolume = requirePapram("dstVolume"), - srcPath = requireParam("srcPath"), - dstPath = requireParam("dstPath"), - } = requireParam("options"), -) => Deno.core.opAsync("rename", srcVolume, srcPath, dstVolume, dstPath); -const metadata = async ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => { - const data = await Deno.core.opAsync("metadata", volumeId, path); - return { - ...data, - modified: maybeDate(data.modified), - created: maybeDate(data.created), - accessed: maybeDate(data.accessed), - }; -}; -const removeFile = ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => Deno.core.opAsync("remove_file", volumeId, path); -const isSandboxed = () => Deno.core.ops["is_sandboxed"](); - -const writeJsonFile = ( - { - volumeId = requireParam("volumeId"), - path = requireParam("path"), - toWrite = requireParam("toWrite"), - } = requireParam("options"), -) => - writeFile({ - volumeId, - path, - toWrite: JSON.stringify(toWrite), - }); - -const chown = async ( - { - volumeId = requireParam("volumeId"), - path = requireParam("path"), - uid = requireParam("uid"), - } = requireParam("options"), -) => { - return await Deno.core.opAsync("chown", volumeId, path, uid); -}; - -const chmod = async ( - { - volumeId = requireParam("volumeId"), - path = requireParam("path"), - mode = requireParam("mode"), - } = requireParam("options"), -) => { - return await Deno.core.opAsync("chmod", volumeId, path, mode); -}; -const readJsonFile = async ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => JSON.parse(await readFile({ volumeId, path })); -const createDir = ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => Deno.core.opAsync("create_dir", volumeId, path); - -const readDir = ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => Deno.core.opAsync("read_dir", volumeId, path); -const removeDir = ( - { volumeId = requireParam("volumeId"), path = requireParam("path") } = requireParam("options"), -) => Deno.core.opAsync("remove_dir", volumeId, path); -const trace = (whatToTrace = requireParam('whatToTrace')) => Deno.core.opAsync("log_trace", whatToTrace); -const warn = (whatToTrace = requireParam('whatToTrace')) => Deno.core.opAsync("log_warn", whatToTrace); -const error = (whatToTrace = requireParam('whatToTrace')) => Deno.core.opAsync("log_error", whatToTrace); -const debug = (whatToTrace = requireParam('whatToTrace')) => Deno.core.opAsync("log_debug", whatToTrace); -const info = (whatToTrace = requireParam('whatToTrace')) => Deno.core.opAsync("log_info", whatToTrace); -const fetch = async (url = requireParam ('url'), options = null) => { - const { body, ...response } = await Deno.core.opAsync("fetch", url, options); - const textValue = Promise.resolve(body); - return { - ...response, - text() { - return textValue; - }, - json() { - return textValue.then((x) => JSON.parse(x)); - }, - }; -}; - -const runRsync = ( - { - srcVolume = requireParam("srcVolume"), - dstVolume = requireParam("dstVolume"), - srcPath = requireParam("srcPath"), - dstPath = requireParam("dstPath"), - options = requireParam("options"), - } = requireParam("options"), -) => { - let id = Deno.core.opAsync("rsync", srcVolume, srcPath, dstVolume, dstPath, options); - let waitPromise = null; - return { - async id() { - return id - }, - async wait() { - waitPromise = waitPromise || Deno.core.opAsync("rsync_wait", await id) - return waitPromise - }, - async progress() { - return Deno.core.opAsync("rsync_progress", await id) - } - } -}; - -const diskUsage = async ({ - volumeId = requireParam("volumeId"), - path = requireParam("path"), -} = { volumeId: null, path: null }) => { - const [used, total] = await Deno.core.opAsync("disk_usage", volumeId, path); - return { used, total } -} - -const currentFunction = Deno.core.ops.current_function(); -const input = Deno.core.ops.get_input(); -const variable_args = Deno.core.ops.get_variable_args(); -const setState = (x) => Deno.core.ops.set_value(x); -const effects = { - chmod, - chown, - writeFile, - readFile, - writeJsonFile, - readJsonFile, - error, - warn, - debug, - trace, - info, - isSandboxed, - fetch, - removeFile, - createDir, - removeDir, - metadata, - rename, - runCommand, - sleep, - runDaemon, - signalGroup, - runRsync, - readDir, - diskUsage, -}; - -const defaults = { - "handleSignal": (effects, { gid, signal }) => { - return effects.signalGroup({ gid, signal }) - } -} - -const runFunction = jsonPointerValue(mainModule, currentFunction) || jsonPointerValue(defaults, currentFunction); -(async () => { - if (typeof runFunction !== "function") { - error(`Expecting ${currentFunction} to be a function`); - throw new Error(`Expecting ${currentFunction} to be a function`); - } - const answer = await runFunction(effects, input, ...variable_args); - setState(answer); -})(); diff --git a/core/js-engine/src/lib.rs b/core/js-engine/src/lib.rs deleted file mode 100644 index b0b9bea37..000000000 --- a/core/js-engine/src/lib.rs +++ /dev/null @@ -1,1219 +0,0 @@ -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::sync::Arc; -use std::time::SystemTime; - -use deno_core::anyhow::{anyhow, bail}; -use deno_core::error::AnyError; -use deno_core::{ - resolve_import, Extension, FastString, JsRuntime, ModuleLoader, ModuleSource, - ModuleSourceFuture, ModuleSpecifier, ModuleType, OpDecl, ResolutionKind, RuntimeOptions, - Snapshot, -}; -use helpers::{script_dir, spawn_local, Rsync}; -use models::{PackageId, ProcedureName, Version, VolumeId}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tokio::io::AsyncReadExt; -use tokio::sync::Mutex; - -lazy_static::lazy_static! { - static ref DENO_GLOBAL_JS: ModuleSpecifier = "file:///deno_global.js".parse().unwrap(); - static ref LOAD_MODULE_JS: ModuleSpecifier = "file:///loadModule.js".parse().unwrap(); - static ref EMBASSY_JS: ModuleSpecifier = "file:///embassy.js".parse().unwrap(); -} - -pub trait PathForVolumeId: Send + Sync { - fn path_for( - &self, - data_dir: &Path, - package_id: &PackageId, - version: &Version, - volume_id: &VolumeId, - ) -> Option; - fn readonly(&self, volume_id: &VolumeId) -> bool; -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct JsCode(Arc); - -#[derive(Debug, Clone, Copy)] -pub enum JsError { - Unknown, - Javascript, - Engine, - BoundryLayerSerDe, - Tokio, - FileSystem, - Code(i32), - Timeout, - NotValidProcedureName, -} - -impl JsError { - pub fn as_code_num(&self) -> i32 { - match self { - JsError::Unknown => 1, - JsError::Javascript => 2, - JsError::Engine => 3, - JsError::BoundryLayerSerDe => 4, - JsError::Tokio => 5, - JsError::FileSystem => 6, - JsError::NotValidProcedureName => 7, - JsError::Code(code) => *code, - JsError::Timeout => 143, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MetadataJs { - file_type: String, - is_dir: bool, - is_file: bool, - is_symlink: bool, - len: u64, - modified: Option, - accessed: Option, - created: Option, - readonly: bool, - gid: u32, - mode: u32, - uid: u32, -} - -#[cfg(target_arch = "x86_64")] -const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/JS_SNAPSHOT.x86_64.bin"); - -#[cfg(target_arch = "aarch64")] -const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/JS_SNAPSHOT.aarch64.bin"); - -#[derive(Clone)] -struct JsContext { - sandboxed: bool, - datadir: PathBuf, - run_function: String, - version: Version, - package_id: PackageId, - volumes: Arc, - input: Value, - variable_args: Vec, - rsyncs: Arc)>>, -} -#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "kebab-case")] -enum ResultType { - Error(String), - ErrorCode(i32, String), - Result(serde_json::Value), -} -#[derive(Clone, Default)] -struct AnswerState(std::sync::Arc>); - -#[derive(Clone, Debug)] -struct ModsLoader { - code: JsCode, -} - -impl ModuleLoader for ModsLoader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - _is_main: ResolutionKind, - ) -> Result { - if referrer.contains("embassy") { - bail!("Embassy.js cannot import anything else"); - } - let s = resolve_import(specifier, referrer).unwrap(); - Ok(s) - } - - fn load( - &self, - module_specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - is_dyn_import: bool, - ) -> Pin> { - let module_specifier = module_specifier.as_str().to_owned(); - let module = match &*module_specifier { - "file:///deno_global.js" => Ok(ModuleSource::new( - ModuleType::JavaScript, - FastString::Static("const old_deno = Deno; Deno = null; export default old_deno"), - &DENO_GLOBAL_JS, - )), - "file:///loadModule.js" => Ok(ModuleSource::new( - ModuleType::JavaScript, - FastString::Static(include_str!("./artifacts/loadModule.js")), - &LOAD_MODULE_JS, - )), - "file:///embassy.js" => Ok(ModuleSource::new( - ModuleType::JavaScript, - self.code.0.clone().into(), - &EMBASSY_JS, - )), - - x => Err(anyhow!("Not allowed to import: {}", x)), - }; - let module = module.and_then(|m| { - if is_dyn_import { - bail!("Will not import dynamic"); - } - match &maybe_referrer { - Some(x) if x.as_str() == "file:///embassy.js" => { - bail!("StartJS is not allowed to import") - } - _ => (), - } - Ok(m) - }); - Box::pin(async move { module }) - } -} - -pub struct JsExecutionEnvironment { - sandboxed: bool, - base_directory: PathBuf, - module_loader: ModsLoader, - package_id: PackageId, - version: Version, - volumes: Arc, -} - -impl JsExecutionEnvironment { - pub async fn load_from_package( - data_directory: impl AsRef, - package_id: &PackageId, - version: &Version, - volumes: Box, - ) -> Result { - let data_dir = data_directory.as_ref(); - let base_directory = data_dir; - let js_code = JsCode({ - let file_path = script_dir(data_dir, package_id, version).join("embassy.js"); - let mut file = match tokio::fs::File::open(file_path.clone()).await { - Ok(x) => x, - Err(e) => { - tracing::debug!("path: {:?}", file_path); - tracing::debug!("{:?}", e); - return Err(( - JsError::FileSystem, - format!("The file opening '{:?}' created error: {}", file_path, e), - )); - } - }; - let mut buffer = Default::default(); - if let Err(err) = file.read_to_string(&mut buffer).await { - tracing::debug!("{:?}", err); - return Err(( - JsError::FileSystem, - format!("The file reading created error: {}", err), - )); - }; - buffer.into() - }); - Ok(JsExecutionEnvironment { - base_directory: base_directory.to_owned(), - module_loader: ModsLoader { code: js_code }, - package_id: package_id.clone(), - version: version.clone(), - volumes: volumes.into(), - sandboxed: false, - }) - } - pub fn read_only_effects(mut self) -> Self { - self.sandboxed = true; - self - } - - pub async fn run_action Deserialize<'de>>( - self, - procedure_name: ProcedureName, - input: Option, - variable_args: Vec, - ) -> Result { - let input = match serde_json::to_value(input) { - Ok(a) => a, - Err(err) => { - tracing::error!("{}", err); - tracing::debug!("{:?}", err); - return Err(( - JsError::BoundryLayerSerDe, - "Couldn't convert input".to_string(), - )); - } - }; - let safer_handle = spawn_local(|| self.execute(procedure_name, input, variable_args)).await; - let output = safer_handle.await.unwrap()?; - match serde_json::from_value(output.clone()) { - Ok(x) => Ok(x), - Err(err) => { - tracing::error!("{}", err); - tracing::debug!("{:?}", err); - Err(( - JsError::BoundryLayerSerDe, - format!( - "Couldn't convert output = {:#?} to the correct type", - serde_json::to_string_pretty(&output).unwrap_or_default() - ), - )) - } - } - } - fn declarations() -> Vec { - vec![ - fns::chown::decl(), - fns::chmod::decl(), - fns::fetch::decl(), - fns::read_file::decl(), - fns::metadata::decl(), - fns::write_file::decl(), - fns::rename::decl(), - fns::remove_file::decl(), - fns::create_dir::decl(), - fns::remove_dir::decl(), - fns::read_dir::decl(), - fns::disk_usage::decl(), - fns::current_function::decl(), - fns::log_trace::decl(), - fns::log_warn::decl(), - fns::log_error::decl(), - fns::log_debug::decl(), - fns::log_info::decl(), - fns::get_input::decl(), - fns::get_variable_args::decl(), - fns::set_value::decl(), - fns::is_sandboxed::decl(), - fns::sleep::decl(), - fns::rsync::decl(), - fns::rsync_wait::decl(), - fns::rsync_progress::decl(), - ] - } - - async fn execute( - self, - procedure_name: ProcedureName, - input: Value, - variable_args: Vec, - ) -> Result { - let base_directory = self.base_directory.clone(); - let answer_state = AnswerState::default(); - let ext_answer_state = answer_state.clone(); - let js_ctx = JsContext { - datadir: base_directory, - run_function: procedure_name - .js_function_name() - .map(Ok) - .unwrap_or_else(|| { - Err(( - JsError::NotValidProcedureName, - format!("procedure is not value: {:?}", procedure_name), - )) - })?, - package_id: self.package_id.clone(), - volumes: self.volumes.clone(), - version: self.version.clone(), - sandboxed: self.sandboxed, - input, - variable_args, - rsyncs: Default::default(), - }; - let ext = Extension::builder("embassy") - .ops(Self::declarations()) - .state(move |state| { - state.put(ext_answer_state.clone()); - state.put(js_ctx); - }) - .build(); - - let loader = std::rc::Rc::new(self.module_loader.clone()); - let runtime_options = RuntimeOptions { - module_loader: Some(loader), - extensions: vec![ext], - startup_snapshot: Some(Snapshot::Static(SNAPSHOT_BYTES)), - ..Default::default() - }; - let mut runtime = JsRuntime::new(runtime_options); - - let future = async move { - let mod_id = runtime - .load_main_module(&"file:///loadModule.js".parse().unwrap(), None) - .await?; - let evaluated = runtime.mod_evaluate(mod_id); - let res = runtime.run_event_loop(false).await; - res?; - evaluated.await??; - Ok::<_, AnyError>(()) - }; - - future.await.map_err(|e| { - tracing::debug!("{:?}", e); - (JsError::Javascript, format!("{}", e)) - })?; - - let answer = answer_state.0.lock().clone(); - Ok(answer) - } -} - -/// Note: Make sure that we have the assumption that all these methods are callable at any time, and all call restrictions should be in rust -mod fns { - use std::cell::RefCell; - use std::collections::BTreeMap; - use std::convert::TryFrom; - use std::fs::Permissions; - use std::os::unix::fs::MetadataExt; - use std::os::unix::prelude::PermissionsExt; - use std::path::{Path, PathBuf}; - use std::rc::Rc; - use std::time::Duration; - - use container_init::ProcessId; - use deno_core::anyhow::{anyhow, bail}; - use deno_core::error::AnyError; - use deno_core::*; - use helpers::{to_tmp_path, AtomicFile, Rsync, RsyncOptions}; - use itertools::Itertools; - use models::VolumeId; - use serde::{Deserialize, Serialize}; - use serde_json::Value; - use tokio::io::AsyncWriteExt; - use tokio::process::Command; - - use super::{AnswerState, JsContext}; - use crate::{system_time_as_unix_ms, MetadataJs}; - - #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)] - struct FetchOptions { - method: Option, - headers: Option>, - body: Option, - } - #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)] - struct FetchResponse { - method: String, - ok: bool, - status: u32, - headers: BTreeMap, - body: Option, - } - #[op] - async fn fetch( - state: Rc>, - url: url::Url, - options: Option, - ) -> Result { - let sandboxed = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.sandboxed - }; - - if sandboxed { - bail!("Will not run fetch in sandboxed mode"); - } - - let client = reqwest::Client::new(); - let options = options.unwrap_or_default(); - let method = options - .method - .unwrap_or_else(|| "GET".to_string()) - .to_uppercase(); - let mut request_builder = match &*method { - "GET" => client.get(url), - "POST" => client.post(url), - "PUT" => client.put(url), - "DELETE" => client.delete(url), - "HEAD" => client.head(url), - "PATCH" => client.patch(url), - x => bail!("Unsupported method: {}", x), - }; - if let Some(headers) = options.headers { - for (key, value) in headers { - request_builder = request_builder.header(key, value); - } - } - if let Some(body) = options.body { - request_builder = request_builder.body(body); - } - let response = request_builder.send().await?; - - let fetch_response = FetchResponse { - method, - ok: response.status().is_success(), - status: response.status().as_u16() as u32, - headers: response - .headers() - .iter() - .filter_map(|(head, value)| { - Some((format!("{}", head), value.to_str().ok()?.to_string())) - }) - .collect(), - body: response.text().await.ok(), - }; - - Ok(fetch_response) - } - - #[op] - async fn read_file( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result { - let volume_path = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))? - }; - //get_path_for in volume.rs - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - let answer = tokio::fs::read_to_string(new_file).await?; - Ok(answer) - } - #[op] - async fn metadata( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result { - let volume_path = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))? - }; - //get_path_for in volume.rs - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - let answer = tokio::fs::metadata(new_file).await?; - let metadata_js = MetadataJs { - file_type: format!("{:?}", answer.file_type()), - is_dir: answer.is_dir(), - is_file: answer.is_file(), - is_symlink: answer.is_symlink(), - len: answer.len(), - modified: answer - .modified() - .ok() - .as_ref() - .and_then(system_time_as_unix_ms), - accessed: answer - .accessed() - .ok() - .as_ref() - .and_then(system_time_as_unix_ms), - created: answer - .created() - .ok() - .as_ref() - .and_then(system_time_as_unix_ms), - readonly: answer.permissions().readonly(), - gid: answer.gid(), - mode: answer.mode(), - uid: answer.uid(), - }; - - Ok(metadata_js) - } - #[op] - async fn write_file( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - write: String, - ) -> Result<(), AnyError> { - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - let parent_new_file = new_file - .parent() - .ok_or_else(|| anyhow!("Expecting that file is not root"))?; - // With the volume check - if !is_subset(&volume_path, &parent_new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - let new_volume_tmp = to_tmp_path(&volume_path).map_err(|e| anyhow!("{}", e))?; - let hashed_name = { - use std::os::unix::ffi::OsStrExt; - - use sha2::{Digest, Sha256}; - let mut hasher = Sha256::new(); - - hasher.update(path_in.as_os_str().as_bytes()); - let result = hasher.finalize(); - format!("{:X}", result) - }; - let temp_file = new_volume_tmp.join(&hashed_name); - let mut file = AtomicFile::new(&new_file, Some(&temp_file)) - .await - .map_err(|e| anyhow!("{}", e))?; - file.write_all(write.as_bytes()).await?; - file.save().await.map_err(|e| anyhow!("{}", e))?; - Ok(()) - } - #[op] - async fn rename( - state: Rc>, - src_volume: VolumeId, - src_path: PathBuf, - dst_volume: VolumeId, - dst_path: PathBuf, - ) -> Result<(), AnyError> { - let (volumes, volume_path, volume_path_out) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &src_volume) - .ok_or_else(|| anyhow!("There is no {} in volumes", src_volume))?; - let volume_path_out = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &dst_volume) - .ok_or_else(|| anyhow!("There is no {} in volumes", dst_volume))?; - (ctx.volumes.clone(), volume_path, volume_path_out) - }; - if volumes.readonly(&dst_volume) { - bail!("Volume {} is readonly", dst_volume); - } - - let src_path = src_path.strip_prefix("/").unwrap_or(&src_path); - let old_file = volume_path.join(src_path); - let parent_old_file = old_file - .parent() - .ok_or_else(|| anyhow!("Expecting that file is not root"))?; - // With the volume check - if !is_subset(&volume_path, &parent_old_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - old_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - - let dst_path = dst_path.strip_prefix("/").unwrap_or(&dst_path); - let new_file = volume_path_out.join(dst_path); - let parent_new_file = new_file - .parent() - .ok_or_else(|| anyhow!("Expecting that file is not root"))?; - // With the volume check - if !is_subset(&volume_path_out, &parent_new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path_out.to_string_lossy(), - ); - } - tokio::fs::rename(old_file, new_file).await?; - Ok(()) - } - - #[op] - async fn rsync( - state: Rc>, - src_volume: VolumeId, - src_path: PathBuf, - dst_volume: VolumeId, - dst_path: PathBuf, - options: RsyncOptions, - ) -> Result { - let (volumes, volume_path, volume_path_out, rsyncs) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &src_volume) - .ok_or_else(|| anyhow!("There is no {} in volumes", src_volume))?; - let volume_path_out = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &dst_volume) - .ok_or_else(|| anyhow!("There is no {} in volumes", dst_volume))?; - ( - ctx.volumes.clone(), - volume_path, - volume_path_out, - ctx.rsyncs.clone(), - ) - }; - if volumes.readonly(&dst_volume) { - bail!("Volume {} is readonly", dst_volume); - } - - let src_path = src_path.strip_prefix("/").unwrap_or(&src_path); - let src = volume_path.join(src_path); - // With the volume check - if !is_subset(&volume_path, &src).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - src.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - if tokio::fs::metadata(&src).await.is_err() { - bail!("Source at {} does not exists", src.to_string_lossy()); - } - - let dst_path = src_path.strip_prefix("/").unwrap_or(&dst_path); - let dst = volume_path_out.join(dst_path); - // With the volume check - if !is_subset(&volume_path_out, &dst).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - dst.to_string_lossy(), - volume_path_out.to_string_lossy(), - ); - } - - let running_rsync = Rsync::new(src, dst, options) - .await - .map_err(|e| anyhow::anyhow!("{:?}", e.source))?; - let insert_id = { - let mut rsyncs = rsyncs.lock().await; - let next = rsyncs.0 + 1; - rsyncs.0 = next; - rsyncs.1.insert(next, running_rsync); - next - }; - Ok(insert_id) - } - - #[op] - async fn rsync_wait(state: Rc>, id: usize) -> Result<(), AnyError> { - let rsyncs = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.rsyncs.clone() - }; - let running_rsync = match rsyncs.lock().await.1.remove(&id) { - Some(a) => a, - None => bail!("Couldn't find rsync at id {id}"), - }; - running_rsync - .wait() - .await - .map_err(|x| anyhow::anyhow!("{}", x.source))?; - Ok(()) - } - #[op] - async fn rsync_progress(state: Rc>, id: usize) -> Result { - use futures::StreamExt; - let rsyncs = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.rsyncs.clone() - }; - let mut running_rsync = match rsyncs.lock().await.1.remove(&id) { - Some(a) => a, - None => bail!("Couldn't find rsync at id {id}"), - }; - let progress = running_rsync.progress.next().await.unwrap_or_default(); - rsyncs.lock().await.1.insert(id, running_rsync); - Ok(progress) - } - #[op] - async fn remove_file( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result<(), AnyError> { - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - tokio::fs::remove_file(new_file).await?; - Ok(()) - } - #[op] - async fn remove_dir( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result<(), AnyError> { - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - tokio::fs::remove_dir_all(new_file).await?; - Ok(()) - } - #[op] - async fn create_dir( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result<(), AnyError> { - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - tokio::fs::create_dir_all(new_file).await?; - Ok(()) - } - #[op] - async fn read_dir( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ) -> Result, AnyError> { - let volume_path = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))? - }; - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - let mut reader = tokio::fs::read_dir(&new_file).await?; - let mut paths: Vec = Vec::new(); - let origin_path = format!("{}/", new_file.to_str().unwrap_or_default()); - let remove_new_file = |other_path: String| other_path.replacen(&origin_path, "", 1); - let has_origin_path = |other_path: &String| other_path.starts_with(&origin_path); - while let Some(entry) = reader.next_entry().await? { - entry - .path() - .to_str() - .into_iter() - .map(ToString::to_string) - .filter(&has_origin_path) - .map(&remove_new_file) - .for_each(|x| paths.push(x)); - } - paths.sort(); - Ok(paths) - } - - #[op] - async fn disk_usage( - state: Rc>, - volume_id: Option, - path_in: Option, - ) -> Result<(u64, u64), AnyError> { - let (base_path, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = if let Some(volume_id) = volume_id { - Some( - ctx.volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?, - ) - } else { - None - }; - (ctx.datadir.join("package-data"), volume_path) - }; - let path = if let (Some(volume_path), Some(path_in)) = (volume_path, path_in) { - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - Some(volume_path.join(path_in)) - } else { - None - }; - - if let Some(path) = path { - let size = String::from_utf8( - Command::new("df") - .arg("--output=size") - .arg("--block-size=1") - .arg(&base_path) - .stdout(std::process::Stdio::piped()) - .output() - .await? - .stdout, - )? - .lines() - .nth(1) - .unwrap_or_default() - .parse()?; - let used = String::from_utf8( - Command::new("du") - .arg("-s") - .arg("--block-size=1") - .arg(path) - .stdout(std::process::Stdio::piped()) - .output() - .await? - .stdout, - )? - .split_ascii_whitespace() - .next() - .unwrap_or_default() - .parse()?; - Ok((used, size)) - } else { - String::from_utf8( - Command::new("df") - .arg("--output=used,size") - .arg("--block-size=1") - .arg(&base_path) - .stdout(std::process::Stdio::piped()) - .output() - .await? - .stdout, - )? - .lines() - .nth(1) - .unwrap_or_default() - .split_ascii_whitespace() - .next_tuple() - .and_then(|(used, size)| Some((used.parse().ok()?, size.parse().ok()?))) - .ok_or_else(|| anyhow!("invalid output from df")) - } - } - - #[op] - fn current_function(state: &mut OpState) -> Result { - let ctx = state.borrow::(); - Ok(ctx.run_function.clone()) - } - - #[op] - async fn log_trace(state: Rc>, input: String) -> Result<(), AnyError> { - let ctx = { - let state = state.borrow(); - state.borrow::().clone() - }; - tracing::trace!( - package_id = tracing::field::display(&ctx.package_id), - run_function = tracing::field::display(&ctx.run_function), - "{}", - input - ); - Ok(()) - } - #[op] - async fn log_warn(state: Rc>, input: String) -> Result<(), AnyError> { - let ctx = { - let state = state.borrow(); - state.borrow::().clone() - }; - tracing::warn!( - package_id = tracing::field::display(&ctx.package_id), - run_function = tracing::field::display(&ctx.run_function), - "{}", - input - ); - Ok(()) - } - #[op] - async fn log_error(state: Rc>, input: String) -> Result<(), AnyError> { - let ctx = { - let state = state.borrow(); - state.borrow::().clone() - }; - tracing::error!( - package_id = tracing::field::display(&ctx.package_id), - run_function = tracing::field::display(&ctx.run_function), - "{}", - input - ); - Ok(()) - } - #[op] - async fn log_debug(state: Rc>, input: String) -> Result<(), AnyError> { - let ctx = { - let state = state.borrow(); - state.borrow::().clone() - }; - tracing::debug!( - package_id = tracing::field::display(&ctx.package_id), - run_function = tracing::field::display(&ctx.run_function), - "{}", - input - ); - Ok(()) - } - #[op] - async fn log_info(state: Rc>, input: String) -> Result<(), AnyError> { - let (package_id, run_function) = { - let state = state.borrow(); - let ctx: JsContext = state.borrow::().clone(); - (ctx.package_id, ctx.run_function) - }; - tracing::info!( - package_id = tracing::field::display(&package_id), - run_function = tracing::field::display(&run_function), - "{}", - input - ); - Ok(()) - } - - #[op] - fn get_input(state: &mut OpState) -> Result { - let ctx = state.borrow::(); - Ok(ctx.input.clone()) - } - #[op] - fn get_variable_args(state: &mut OpState) -> Result, AnyError> { - let ctx = state.borrow::(); - Ok(ctx.variable_args.clone()) - } - #[op] - fn set_value(state: &mut OpState, value: Value) -> Result<(), AnyError> { - let mut answer = state.borrow::().0.lock(); - *answer = value; - Ok(()) - } - #[op] - fn is_sandboxed(state: &mut OpState) -> Result { - let ctx = state.borrow::(); - Ok(ctx.sandboxed) - } - - #[derive(Debug, Clone, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct StartCommand { - process_id: ProcessId, - } - - #[op] - async fn sleep(time_ms: u64) -> Result<(), AnyError> { - tokio::time::sleep(Duration::from_millis(time_ms)).await; - - Ok(()) - } - - #[op] - async fn chown( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - ownership: u32, - ) -> Result<(), AnyError> { - let sandboxed = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.sandboxed - }; - - if sandboxed { - bail!("Will not run chown in sandboxed mode"); - } - - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - let output = tokio::process::Command::new("chown") - .arg("--recursive") - .arg(format!("{ownership}")) - .arg(new_file.as_os_str()) - .output() - .await?; - if !output.status.success() { - return Err(anyhow!("Chown Error")); - } - Ok(()) - } - #[op] - async fn chmod( - state: Rc>, - volume_id: VolumeId, - path_in: PathBuf, - mode: u32, - ) -> Result<(), AnyError> { - let sandboxed = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - ctx.sandboxed - }; - - if sandboxed { - bail!("Will not run chmod in sandboxed mode"); - } - - let (volumes, volume_path) = { - let state = state.borrow(); - let ctx: &JsContext = state.borrow(); - let volume_path = ctx - .volumes - .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) - .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; - (ctx.volumes.clone(), volume_path) - }; - if volumes.readonly(&volume_id) { - bail!("Volume {} is readonly", volume_id); - } - let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); - let new_file = volume_path.join(path_in); - // With the volume check - if !is_subset(&volume_path, &new_file).await? { - bail!( - "Path '{}' has broken away from parent '{}'", - new_file.to_string_lossy(), - volume_path.to_string_lossy(), - ); - } - tokio::fs::set_permissions(new_file, Permissions::from_mode(mode)).await?; - Ok(()) - } - /// We need to make sure that during the file accessing, we don't reach beyond our scope of control - async fn is_subset( - parent: impl AsRef, - child: impl AsRef, - ) -> Result { - let child = { - let mut child_count = 0; - let mut child = child.as_ref(); - loop { - if child.ends_with("..") { - child_count += 1; - } else if child_count > 0 { - child_count -= 1; - } else { - let meta = tokio::fs::metadata(child).await; - if meta.is_ok() { - break; - } - } - child = match child.parent() { - Some(child) => child, - None => { - return Ok(false); - } - }; - } - tokio::fs::canonicalize(child).await? - }; - let parent = tokio::fs::canonicalize(parent).await?; - Ok(child.starts_with(parent)) - } - - #[tokio::test] - async fn test_is_subset() { - let home = std::env::var("HOME").unwrap(); - let home = Path::new(&home); - assert!(!is_subset(home, &home.join("code/fakedir/../../..")) - .await - .unwrap()) - } -} - -fn system_time_as_unix_ms(system_time: &SystemTime) -> Option { - system_time - .duration_since(SystemTime::UNIX_EPOCH) - .ok()? - .as_millis() - .try_into() - .ok() -} diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index bd1beba64..bad982996 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -30,7 +30,7 @@ avahi = ["avahi-sys"] avahi-alias = ["avahi"] cli = [] daemon = [] -default = ["cli", "sdk", "daemon", "js-engine"] +default = ["cli", "sdk", "daemon"] dev = [] docker = [] sdk = [] @@ -98,7 +98,6 @@ itertools = "0.11.0" jaq-core = "0.10.1" jaq-std = "0.10.0" josekit = "0.8.4" -js-engine = { path = '../js-engine', optional = true } jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" } lazy_static = "1.4.0" libc = "0.2.149" diff --git a/core/startos/src/bins/mod.rs b/core/startos/src/bins/mod.rs index c391338fe..76329e094 100644 --- a/core/startos/src/bins/mod.rs +++ b/core/startos/src/bins/mod.rs @@ -5,8 +5,6 @@ pub mod avahi_alias; pub mod deprecated; #[cfg(feature = "cli")] pub mod start_cli; -#[cfg(feature = "js-engine")] -pub mod start_deno; #[cfg(feature = "daemon")] pub mod start_init; #[cfg(feature = "sdk")] @@ -18,8 +16,6 @@ fn select_executable(name: &str) -> Option { match name { #[cfg(feature = "avahi-alias")] "avahi-alias" => Some(avahi_alias::main), - #[cfg(feature = "js_engine")] - "start-deno" => Some(start_deno::main), #[cfg(feature = "cli")] "start-cli" => Some(start_cli::main), #[cfg(feature = "sdk")] diff --git a/core/startos/src/procedure/js_scripts.rs b/core/startos/src/procedure/js_scripts.rs index 88f240e4f..131ceef84 100644 --- a/core/startos/src/procedure/js_scripts.rs +++ b/core/startos/src/procedure/js_scripts.rs @@ -4,8 +4,6 @@ use std::time::Duration; use container_init::ProcessGroupId; use helpers::UnixRpcClient; -pub use js_engine::JsError; -use js_engine::{JsExecutionEnvironment, PathForVolumeId}; use models::VolumeId; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -28,23 +26,6 @@ enum ErrorValue { Result(serde_json::Value), } -impl PathForVolumeId for Volumes { - fn path_for( - &self, - data_dir: &Path, - package_id: &PackageId, - version: &Version, - volume_id: &VolumeId, - ) -> Option { - let volume = self.get(volume_id)?; - Some(volume.path_for(data_dir, package_id, version, volume_id)) - } - - fn readonly(&self, volume_id: &VolumeId) -> bool { - self.get(volume_id).map(|x| x.readonly()).unwrap_or(false) - } -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct ExecuteArgs { pub procedure: JsProcedure, @@ -68,27 +49,3 @@ impl JsProcedure { Ok(()) } } - -fn unwrap_known_error( - error_value: Option, -) -> Result { - let error_value = error_value.unwrap_or_else(|| ErrorValue::Result(serde_json::Value::Null)); - match error_value { - ErrorValue::Error(error) => Err((JsError::Javascript, error)), - ErrorValue::ErrorCode((code, message)) => Err((JsError::Code(code), message)), - ErrorValue::Result(ref value) => match serde_json::from_value(value.clone()) { - Ok(a) => Ok(a), - Err(err) => { - tracing::error!("{}", err); - tracing::debug!("{:?}", err); - Err(( - JsError::BoundryLayerSerDe, - format!( - "Couldn't convert output = {:#?} to the correct type", - serde_json::to_string_pretty(&error_value).unwrap_or_default() - ), - )) - } - }, - } -} diff --git a/core/startos/src/procedure/mod.rs b/core/startos/src/procedure/mod.rs index be074c2b5..aa3d4092d 100644 --- a/core/startos/src/procedure/mod.rs +++ b/core/startos/src/procedure/mod.rs @@ -17,7 +17,6 @@ use crate::volume::Volumes; use crate::{Error, ErrorKind}; pub mod docker; -#[cfg(feature = "js-engine")] pub mod js_scripts; pub use models::ProcedureName; @@ -27,15 +26,12 @@ pub use models::ProcedureName; #[model = "Model"] pub enum PackageProcedure { Docker(DockerProcedure), - - #[cfg(feature = "js-engine")] Script(js_scripts::JsProcedure), } impl PackageProcedure { pub fn is_script(&self) -> bool { match self { - #[cfg(feature = "js-engine")] Self::Script(_) => true, _ => false, } @@ -52,7 +48,6 @@ impl PackageProcedure { PackageProcedure::Docker(action) => { action.validate(eos_version, volumes, image_ids, expected_io) } - #[cfg(feature = "js-engine")] PackageProcedure::Script(action) => action.validate(volumes), } } @@ -116,7 +111,6 @@ impl std::fmt::Display for PackageProcedure { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { PackageProcedure::Docker(_) => write!(f, "Docker")?, - #[cfg(feature = "js-engine")] PackageProcedure::Script(_) => write!(f, "JS")?, } Ok(()) diff --git a/core/startos/src/s9pk/builder.rs b/core/startos/src/s9pk/builder.rs deleted file mode 100644 index 199742439..000000000 --- a/core/startos/src/s9pk/builder.rs +++ /dev/null @@ -1,145 +0,0 @@ -use sha2::{Digest, Sha512}; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}; -use tracing::instrument; -use typed_builder::TypedBuilder; - -use super::header::{FileSection, Header}; -use super::manifest::Manifest; -use super::SIG_CONTEXT; -use crate::util::io::to_cbor_async_writer; -use crate::util::HashWriter; -use crate::{Error, ResultExt}; - -#[derive(TypedBuilder)] -pub struct S9pkPacker< - 'a, - W: AsyncWriteExt + AsyncSeekExt, - RLicense: AsyncReadExt + Unpin, - RInstructions: AsyncReadExt + Unpin, - RIcon: AsyncReadExt + Unpin, - RDockerImages: AsyncReadExt + Unpin, - RAssets: AsyncReadExt + Unpin, - RScripts: AsyncReadExt + Unpin, -> { - writer: W, - manifest: &'a Manifest, - license: RLicense, - instructions: RInstructions, - icon: RIcon, - docker_images: RDockerImages, - assets: RAssets, - scripts: Option, -} -impl< - 'a, - W: AsyncWriteExt + AsyncSeekExt + Unpin, - RLicense: AsyncReadExt + Unpin, - RInstructions: AsyncReadExt + Unpin, - RIcon: AsyncReadExt + Unpin, - RDockerImages: AsyncReadExt + Unpin, - RAssets: AsyncReadExt + Unpin, - RScripts: AsyncReadExt + Unpin, - > S9pkPacker<'a, W, RLicense, RInstructions, RIcon, RDockerImages, RAssets, RScripts> -{ - /// BLOCKING - #[instrument(skip_all)] - pub async fn pack(mut self, key: &ed25519_dalek::SigningKey) -> Result<(), Error> { - let header_pos = self.writer.stream_position().await?; - if header_pos != 0 { - tracing::warn!("Appending to non-empty file."); - } - let mut header = Header::placeholder(); - header.serialize(&mut self.writer).await.with_ctx(|_| { - ( - crate::ErrorKind::Serialization, - "Writing Placeholder Header", - ) - })?; - let mut position = self.writer.stream_position().await?; - - let mut writer = HashWriter::new(Sha512::new(), &mut self.writer); - // manifest - to_cbor_async_writer(&mut writer, self.manifest).await?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.manifest = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // license - tokio::io::copy(&mut self.license, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying License"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.license = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // instructions - tokio::io::copy(&mut self.instructions, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Instructions"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.instructions = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // icon - tokio::io::copy(&mut self.icon, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Icon"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.icon = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // docker_images - tokio::io::copy(&mut self.docker_images, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Docker Images"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.docker_images = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // assets - tokio::io::copy(&mut self.assets, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Assets"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.assets = FileSection { - position, - length: new_pos - position, - }; - position = new_pos; - // scripts - if let Some(mut scripts) = self.scripts { - tokio::io::copy(&mut scripts, &mut writer) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Scripts"))?; - let new_pos = writer.inner_mut().stream_position().await?; - header.table_of_contents.scripts = Some(FileSection { - position, - length: new_pos - position, - }); - position = new_pos; - } - - // header - let (hash, _) = writer.finish(); - self.writer.seek(SeekFrom::Start(header_pos)).await?; - header.pubkey = key.into(); - header.signature = key.sign_prehashed(hash, Some(SIG_CONTEXT))?; - header - .serialize(&mut self.writer) - .await - .with_ctx(|_| (crate::ErrorKind::Serialization, "Writing Header"))?; - self.writer.seek(SeekFrom::Start(position)).await?; - - Ok(()) - } -} diff --git a/core/startos/src/s9pk/docker.rs b/core/startos/src/s9pk/docker.rs deleted file mode 100644 index be93905fb..000000000 --- a/core/startos/src/s9pk/docker.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::borrow::Cow; -use std::collections::BTreeSet; -use std::io::SeekFrom; -use std::path::Path; - -use color_eyre::eyre::eyre; -use futures::{FutureExt, TryStreamExt}; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt}; -use tokio_tar::{Archive, Entry}; - -use crate::util::io::from_cbor_async_reader; -use crate::{Error, ErrorKind, ARCH}; - -#[derive(Default, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct DockerMultiArch { - pub default: String, - pub available: BTreeSet, -} - -#[pin_project::pin_project(project = DockerReaderProject)] -#[derive(Debug)] -pub enum DockerReader { - SingleArch(#[pin] R), - MultiArch(#[pin] Entry>), -} -impl DockerReader { - pub async fn new(mut rdr: R) -> Result { - let arch = if let Some(multiarch) = tokio_tar::Archive::new(&mut rdr) - .entries()? - .try_filter_map(|e| { - async move { - Ok(if &*e.path()? == Path::new("multiarch.cbor") { - Some(e) - } else { - None - }) - } - .boxed() - }) - .try_next() - .await? - { - let multiarch: DockerMultiArch = from_cbor_async_reader(multiarch).await?; - Some(if multiarch.available.contains(&**ARCH) { - Cow::Borrowed(&**ARCH) - } else { - Cow::Owned(multiarch.default) - }) - } else { - None - }; - rdr.seek(SeekFrom::Start(0)).await?; - if let Some(arch) = arch { - if let Some(image) = tokio_tar::Archive::new(rdr) - .entries()? - .try_filter_map(|e| { - let arch = arch.clone(); - async move { - Ok(if &*e.path()? == Path::new(&format!("{}.tar", arch)) { - Some(e) - } else { - None - }) - } - .boxed() - }) - .try_next() - .await? - { - Ok(Self::MultiArch(image)) - } else { - Err(Error::new( - eyre!("Docker image section does not contain tarball for architecture"), - ErrorKind::ParseS9pk, - )) - } - } else { - Ok(Self::SingleArch(rdr)) - } - } -} -impl AsyncRead for DockerReader { - fn poll_read( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - match self.project() { - DockerReaderProject::SingleArch(r) => r.poll_read(cx, buf), - DockerReaderProject::MultiArch(r) => r.poll_read(cx, buf), - } - } -} diff --git a/core/startos/src/s9pk/git_hash.rs b/core/startos/src/s9pk/git_hash.rs deleted file mode 100644 index b2990a111..000000000 --- a/core/startos/src/s9pk/git_hash.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::path::Path; - -use crate::Error; - -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct GitHash(String); - -impl GitHash { - pub async fn from_path(path: impl AsRef) -> Result { - let hash = tokio::process::Command::new("git") - .args(["describe", "--always", "--abbrev=40", "--dirty=-modified"]) - .current_dir(path) - .output() - .await?; - if !hash.status.success() { - return Err(Error::new( - color_eyre::eyre::eyre!("Could not get hash: {}", String::from_utf8(hash.stderr)?), - crate::ErrorKind::Filesystem, - )); - } - Ok(GitHash(String::from_utf8(hash.stdout)?)) - } -} - -impl AsRef for GitHash { - fn as_ref(&self) -> &str { - &self.0 - } -} - -// #[tokio::test] -// async fn test_githash_for_current() { -// let answer: GitHash = GitHash::from_path(std::env::current_dir().unwrap()) -// .await -// .unwrap(); -// let answer_str: &str = answer.as_ref(); -// assert!( -// !answer_str.is_empty(), -// "Should have a hash for this current working" -// ); -// } diff --git a/core/startos/src/s9pk/header.rs b/core/startos/src/s9pk/header.rs deleted file mode 100644 index 4f77ad855..000000000 --- a/core/startos/src/s9pk/header.rs +++ /dev/null @@ -1,187 +0,0 @@ -use std::collections::BTreeMap; - -use color_eyre::eyre::eyre; -use ed25519_dalek::{Signature, VerifyingKey}; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; - -use crate::Error; - -pub const MAGIC: [u8; 2] = [59, 59]; -pub const VERSION: u8 = 1; - -#[derive(Debug)] -pub struct Header { - pub pubkey: VerifyingKey, - pub signature: Signature, - pub table_of_contents: TableOfContents, -} -impl Header { - pub fn placeholder() -> Self { - Header { - pubkey: VerifyingKey::default(), - signature: Signature::from_bytes(&[0; 64]), - table_of_contents: Default::default(), - } - } - // MUST BE SAME SIZE REGARDLESS OF DATA - pub async fn serialize(&self, mut writer: W) -> std::io::Result<()> { - writer.write_all(&MAGIC).await?; - writer.write_all(&[VERSION]).await?; - writer.write_all(self.pubkey.as_bytes()).await?; - writer.write_all(&self.signature.to_bytes()).await?; - self.table_of_contents.serialize(writer).await?; - Ok(()) - } - pub async fn deserialize(mut reader: R) -> Result { - let mut magic = [0; 2]; - reader.read_exact(&mut magic).await?; - if magic != MAGIC { - return Err(Error::new( - eyre!("Incorrect Magic: {:?}", magic), - crate::ErrorKind::ParseS9pk, - )); - } - let mut version = [0]; - reader.read_exact(&mut version).await?; - if version[0] != VERSION { - return Err(Error::new( - eyre!("Unknown Version: {}", version[0]), - crate::ErrorKind::ParseS9pk, - )); - } - let mut pubkey_bytes = [0; 32]; - reader.read_exact(&mut pubkey_bytes).await?; - let pubkey = VerifyingKey::from_bytes(&pubkey_bytes) - .map_err(|e| Error::new(e, crate::ErrorKind::ParseS9pk))?; - let mut sig_bytes = [0; 64]; - reader.read_exact(&mut sig_bytes).await?; - let signature = Signature::from_bytes(&sig_bytes); - let table_of_contents = TableOfContents::deserialize(reader).await?; - - Ok(Header { - pubkey, - signature, - table_of_contents, - }) - } -} - -#[derive(Debug, Default)] -pub struct TableOfContents { - pub manifest: FileSection, - pub license: FileSection, - pub instructions: FileSection, - pub icon: FileSection, - pub docker_images: FileSection, - pub assets: FileSection, - pub scripts: Option, -} -impl TableOfContents { - pub async fn serialize(&self, mut writer: W) -> std::io::Result<()> { - let len: u32 = ((1 + "manifest".len() + 16) - + (1 + "license".len() + 16) - + (1 + "instructions".len() + 16) - + (1 + "icon".len() + 16) - + (1 + "docker_images".len() + 16) - + (1 + "assets".len() + 16) - + (1 + "scripts".len() + 16)) as u32; - writer.write_all(&u32::to_be_bytes(len)).await?; - self.manifest - .serialize_entry("manifest", &mut writer) - .await?; - self.license.serialize_entry("license", &mut writer).await?; - self.instructions - .serialize_entry("instructions", &mut writer) - .await?; - self.icon.serialize_entry("icon", &mut writer).await?; - self.docker_images - .serialize_entry("docker_images", &mut writer) - .await?; - self.assets.serialize_entry("assets", &mut writer).await?; - self.scripts - .unwrap_or_default() - .serialize_entry("scripts", &mut writer) - .await?; - Ok(()) - } - pub async fn deserialize(mut reader: R) -> std::io::Result { - let mut toc_len = [0; 4]; - reader.read_exact(&mut toc_len).await?; - let toc_len = u32::from_be_bytes(toc_len); - let mut reader = reader.take(toc_len as u64); - let mut table = BTreeMap::new(); - while let Some((label, section)) = FileSection::deserialize_entry(&mut reader).await? { - table.insert(label, section); - } - fn from_table( - table: &BTreeMap, FileSection>, - label: &str, - ) -> std::io::Result { - table.get(label.as_bytes()).copied().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - format!("Missing Required Label: {}", label), - ) - }) - } - #[allow(dead_code)] - fn as_opt(fs: FileSection) -> Option { - if fs.position | fs.length == 0 { - // 0/0 is not a valid file section - None - } else { - Some(fs) - } - } - Ok(TableOfContents { - manifest: from_table(&table, "manifest")?, - license: from_table(&table, "license")?, - instructions: from_table(&table, "instructions")?, - icon: from_table(&table, "icon")?, - docker_images: from_table(&table, "docker_images")?, - assets: from_table(&table, "assets")?, - scripts: table.get("scripts".as_bytes()).cloned(), - }) - } -} - -#[derive(Clone, Copy, Debug, Default)] -pub struct FileSection { - pub position: u64, - pub length: u64, -} -impl FileSection { - pub async fn serialize_entry( - self, - label: &str, - mut writer: W, - ) -> std::io::Result<()> { - writer.write_all(&[label.len() as u8]).await?; - writer.write_all(label.as_bytes()).await?; - writer.write_all(&u64::to_be_bytes(self.position)).await?; - writer.write_all(&u64::to_be_bytes(self.length)).await?; - Ok(()) - } - pub async fn deserialize_entry( - mut reader: R, - ) -> std::io::Result, Self)>> { - let mut label_len = [0]; - let read = reader.read(&mut label_len).await?; - if read == 0 { - return Ok(None); - } - let mut label = vec![0; label_len[0] as usize]; - reader.read_exact(&mut label).await?; - let mut pos = [0; 8]; - reader.read_exact(&mut pos).await?; - let mut len = [0; 8]; - reader.read_exact(&mut len).await?; - Ok(Some(( - label, - FileSection { - position: u64::from_be_bytes(pos), - length: u64::from_be_bytes(len), - }, - ))) - } -} diff --git a/core/startos/src/s9pk/manifest.rs b/core/startos/src/s9pk/manifest.rs deleted file mode 100644 index 3eee540ed..000000000 --- a/core/startos/src/s9pk/manifest.rs +++ /dev/null @@ -1,211 +0,0 @@ -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use color_eyre::eyre::eyre; -pub use models::PackageId; -use serde::{Deserialize, Serialize}; -use url::Url; - -use super::git_hash::GitHash; -use crate::action::Actions; -use crate::backup::BackupActions; -use crate::config::action::ConfigActions; -use crate::dependencies::Dependencies; -use crate::migration::Migrations; -use crate::net::interface::Interfaces; -use crate::prelude::*; -use crate::procedure::docker::DockerContainers; -use crate::procedure::PackageProcedure; -use crate::status::health_check::HealthChecks; -use crate::util::serde::Regex; -use crate::util::Version; -use crate::version::{Current, VersionT}; -use crate::volume::Volumes; -use crate::Error; - -fn current_version() -> Version { - Current::new().semver().into() -} - -#[derive(Clone, Debug, Deserialize, Serialize, HasModel)] -#[serde(rename_all = "kebab-case")] -#[model = "Model"] -pub struct Manifest { - #[serde(default = "current_version")] - pub eos_version: Version, - pub id: PackageId, - #[serde(default)] - pub git_hash: Option, - pub title: String, - pub version: Version, - pub description: Description, - #[serde(default)] - pub assets: Assets, - #[serde(default)] - pub build: Option>, - pub release_notes: String, - pub license: String, // type of license - pub wrapper_repo: Url, - pub upstream_repo: Url, - pub support_site: Option, - pub marketing_site: Option, - pub donation_url: Option, - #[serde(default)] - pub alerts: Alerts, - pub main: PackageProcedure, - pub health_checks: HealthChecks, - pub config: Option, - pub properties: Option, - pub volumes: Volumes, - // #[serde(default)] - pub interfaces: Interfaces, - // #[serde(default)] - pub backup: BackupActions, - #[serde(default)] - pub migrations: Migrations, - #[serde(default)] - pub actions: Actions, - // #[serde(default)] - // pub permissions: Permissions, - #[serde(default)] - pub dependencies: Dependencies, - pub containers: Option, - - #[serde(default)] - pub replaces: Vec, - - #[serde(default)] - pub hardware_requirements: HardwareRequirements, -} - -impl Manifest { - pub fn package_procedures(&self) -> impl Iterator { - use std::iter::once; - let main = once(&self.main); - let cfg_get = self.config.as_ref().map(|a| &a.get).into_iter(); - let cfg_set = self.config.as_ref().map(|a| &a.set).into_iter(); - let props = self.properties.iter(); - let backups = vec![&self.backup.create, &self.backup.restore].into_iter(); - let migrations = self - .migrations - .to - .values() - .chain(self.migrations.from.values()); - let actions = self.actions.0.values().map(|a| &a.implementation); - main.chain(cfg_get) - .chain(cfg_set) - .chain(props) - .chain(backups) - .chain(migrations) - .chain(actions) - } - - pub fn with_git_hash(mut self, git_hash: GitHash) -> Self { - self.git_hash = Some(git_hash); - self - } -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct HardwareRequirements { - #[serde(default)] - device: BTreeMap, - ram: Option, - pub arch: Option>, -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct Assets { - #[serde(default)] - pub license: Option, - #[serde(default)] - pub instructions: Option, - #[serde(default)] - pub icon: Option, - #[serde(default)] - pub docker_images: Option, - #[serde(default)] - pub assets: Option, - #[serde(default)] - pub scripts: Option, -} -impl Assets { - pub fn license_path(&self) -> &Path { - self.license - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("LICENSE.md")) - } - pub fn instructions_path(&self) -> &Path { - self.instructions - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("INSTRUCTIONS.md")) - } - pub fn icon_path(&self) -> &Path { - self.icon - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("icon.png")) - } - pub fn icon_type(&self) -> &str { - self.icon - .as_ref() - .and_then(|icon| icon.extension()) - .and_then(|ext| ext.to_str()) - .unwrap_or("png") - } - pub fn docker_images_path(&self) -> &Path { - self.docker_images - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("docker-images")) - } - pub fn assets_path(&self) -> &Path { - self.assets - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("assets")) - } - pub fn scripts_path(&self) -> &Path { - self.scripts - .as_ref() - .map(|a| a.as_path()) - .unwrap_or(Path::new("scripts")) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Description { - pub short: String, - pub long: String, -} -impl Description { - pub fn validate(&self) -> Result<(), Error> { - if self.short.chars().skip(160).next().is_some() { - return Err(Error::new( - eyre!("Short description must be 160 characters or less."), - crate::ErrorKind::ValidateS9pk, - )); - } - if self.long.chars().skip(5000).next().is_some() { - return Err(Error::new( - eyre!("Long description must be 5000 characters or less."), - crate::ErrorKind::ValidateS9pk, - )); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct Alerts { - pub install: Option, - pub uninstall: Option, - pub restore: Option, - pub start: Option, - pub stop: Option, -} diff --git a/core/startos/src/s9pk/reader.rs b/core/startos/src/s9pk/reader.rs deleted file mode 100644 index 61b5e46a8..000000000 --- a/core/startos/src/s9pk/reader.rs +++ /dev/null @@ -1,406 +0,0 @@ -use std::collections::BTreeSet; -use std::io::SeekFrom; -use std::ops::Range; -use std::path::Path; -use std::pin::Pin; -use std::str::FromStr; -use std::task::{Context, Poll}; - -use color_eyre::eyre::eyre; -use digest::Output; -use ed25519_dalek::VerifyingKey; -use futures::TryStreamExt; -use models::ImageId; -use sha2::{Digest, Sha512}; -use tokio::fs::File; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf}; -use tracing::instrument; - -use super::header::{FileSection, Header, TableOfContents}; -use super::manifest::{Manifest, PackageId}; -use super::SIG_CONTEXT; -use crate::install::progress::InstallProgressTracker; -use crate::s9pk::docker::DockerReader; -use crate::util::Version; -use crate::{Error, ResultExt}; - -const MAX_REPLACES: usize = 10; -const MAX_TITLE_LEN: usize = 30; - -#[pin_project::pin_project] -#[derive(Debug)] -pub struct ReadHandle<'a, R = File> { - pos: &'a mut u64, - range: Range, - #[pin] - rdr: &'a mut R, -} -impl<'a, R: AsyncRead + Unpin> ReadHandle<'a, R> { - pub async fn to_vec(mut self) -> std::io::Result> { - let mut buf = vec![0; (self.range.end - self.range.start) as usize]; - self.read_exact(&mut buf).await?; - Ok(buf) - } -} -impl<'a, R: AsyncRead + Unpin> AsyncRead for ReadHandle<'a, R> { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - let this = self.project(); - let start = buf.filled().len(); - let mut take_buf = buf.take(this.range.end.saturating_sub(**this.pos) as usize); - let res = AsyncRead::poll_read(this.rdr, cx, &mut take_buf); - let n = take_buf.filled().len(); - unsafe { buf.assume_init(start + n) }; - buf.advance(n); - **this.pos += n as u64; - res - } -} -impl<'a, R: AsyncSeek + Unpin> AsyncSeek for ReadHandle<'a, R> { - fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> { - let this = self.project(); - AsyncSeek::start_seek( - this.rdr, - match position { - SeekFrom::Current(n) => SeekFrom::Current(n), - SeekFrom::End(n) => SeekFrom::Start((this.range.end as i64 + n) as u64), - SeekFrom::Start(n) => SeekFrom::Start(this.range.start + n), - }, - ) - } - fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); - match AsyncSeek::poll_complete(this.rdr, cx) { - Poll::Ready(Ok(n)) => { - let res = n.saturating_sub(this.range.start); - **this.pos = this.range.start + res; - Poll::Ready(Ok(res)) - } - a => a, - } - } -} - -#[derive(Debug)] -pub struct ImageTag { - pub package_id: PackageId, - pub image_id: ImageId, - pub version: Version, -} -impl ImageTag { - #[instrument(skip_all)] - pub fn validate(&self, id: &PackageId, version: &Version) -> Result<(), Error> { - if id != &self.package_id { - return Err(Error::new( - eyre!( - "Contains image for incorrect package: id {}", - self.package_id, - ), - crate::ErrorKind::ValidateS9pk, - )); - } - if version != &self.version { - return Err(Error::new( - eyre!( - "Contains image with incorrect version: expected {} received {}", - version, - self.version, - ), - crate::ErrorKind::ValidateS9pk, - )); - } - Ok(()) - } -} -impl FromStr for ImageTag { - type Err = Error; - fn from_str(s: &str) -> Result { - let rest = s.strip_prefix("start9/").ok_or_else(|| { - Error::new( - eyre!("Invalid image tag prefix: expected start9/"), - crate::ErrorKind::ValidateS9pk, - ) - })?; - let (package, rest) = rest.split_once("/").ok_or_else(|| { - Error::new( - eyre!("Image tag missing image id"), - crate::ErrorKind::ValidateS9pk, - ) - })?; - let (image, version) = rest.split_once(":").ok_or_else(|| { - Error::new( - eyre!("Image tag missing version"), - crate::ErrorKind::ValidateS9pk, - ) - })?; - Ok(ImageTag { - package_id: package.parse()?, - image_id: image.parse()?, - version: version.parse()?, - }) - } -} - -pub struct S9pkReader { - hash: Option>, - hash_string: Option, - developer_key: VerifyingKey, - toc: TableOfContents, - pos: u64, - rdr: R, -} -impl S9pkReader { - pub async fn open>(path: P, check_sig: bool) -> Result { - let p = path.as_ref(); - let rdr = File::open(p) - .await - .with_ctx(|_| (crate::error::ErrorKind::Filesystem, p.display().to_string()))?; - - Self::from_reader(rdr, check_sig).await - } -} -impl S9pkReader> { - pub fn validated(&mut self) { - self.rdr.validated() - } -} -impl S9pkReader { - #[instrument(skip_all)] - pub async fn validate(&mut self) -> Result<(), Error> { - if self.toc.icon.length > 102_400 { - // 100 KiB - return Err(Error::new( - eyre!("icon must be less than 100KiB"), - crate::ErrorKind::ValidateS9pk, - )); - } - let image_tags = self.image_tags().await?; - let man = self.manifest().await?; - let containers = &man.containers; - let validated_image_ids = image_tags - .into_iter() - .map(|i| i.validate(&man.id, &man.version).map(|_| i.image_id)) - .collect::, _>>()?; - man.description.validate()?; - man.actions.0.iter().try_for_each(|(_, action)| { - action.validate( - containers, - &man.eos_version, - &man.volumes, - &validated_image_ids, - ) - })?; - man.backup.validate( - containers, - &man.eos_version, - &man.volumes, - &validated_image_ids, - )?; - if let Some(cfg) = &man.config { - cfg.validate( - containers, - &man.eos_version, - &man.volumes, - &validated_image_ids, - )?; - } - man.health_checks - .validate(&man.eos_version, &man.volumes, &validated_image_ids)?; - man.interfaces.validate()?; - man.main - .validate(&man.eos_version, &man.volumes, &validated_image_ids, false) - .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Main"))?; - man.migrations.validate( - containers, - &man.eos_version, - &man.volumes, - &validated_image_ids, - )?; - - #[cfg(feature = "js-engine")] - if man.containers.is_some() - || matches!(man.main, crate::procedure::PackageProcedure::Script(_)) - { - return Err(Error::new( - eyre!("Right now we don't support the containers and the long running main"), - crate::ErrorKind::ValidateS9pk, - )); - } - - if man.replaces.len() >= MAX_REPLACES { - return Err(Error::new( - eyre!("Cannot have more than {MAX_REPLACES} replaces"), - crate::ErrorKind::ValidateS9pk, - )); - } - if let Some(too_big) = man.replaces.iter().find(|x| x.len() >= MAX_REPLACES) { - return Err(Error::new( - eyre!("We have found a replaces of ({too_big}) that exceeds the max length of {MAX_TITLE_LEN} "), - crate::ErrorKind::ValidateS9pk, - )); - } - if man.title.len() >= MAX_TITLE_LEN { - return Err(Error::new( - eyre!("Cannot have more than a length of {MAX_TITLE_LEN} for title"), - crate::ErrorKind::ValidateS9pk, - )); - } - - if man.containers.is_some() - && matches!(man.main, crate::procedure::PackageProcedure::Docker(_)) - { - return Err(Error::new( - eyre!("Cannot have a main docker and a main in containers"), - crate::ErrorKind::ValidateS9pk, - )); - } - if let Some(props) = &man.properties { - props - .validate(&man.eos_version, &man.volumes, &validated_image_ids, true) - .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Properties"))?; - } - man.volumes.validate(&man.interfaces)?; - - Ok(()) - } - #[instrument(skip_all)] - pub async fn image_tags(&mut self) -> Result, Error> { - let mut tar = tokio_tar::Archive::new(self.docker_images().await?); - let mut entries = tar.entries()?; - while let Some(mut entry) = entries.try_next().await? { - if &*entry.path()? != Path::new("manifest.json") { - continue; - } - let mut buf = Vec::with_capacity(entry.header().size()? as usize); - entry.read_to_end(&mut buf).await?; - #[derive(serde::Deserialize)] - struct ManEntry { - #[serde(rename = "RepoTags")] - tags: Vec, - } - let man_entries = serde_json::from_slice::>(&buf) - .with_ctx(|_| (crate::ErrorKind::Deserialization, "manifest.json"))?; - return man_entries - .iter() - .flat_map(|e| &e.tags) - .map(|t| t.parse()) - .collect(); - } - Err(Error::new( - eyre!("image.tar missing manifest.json"), - crate::ErrorKind::ParseS9pk, - )) - } - #[instrument(skip_all)] - pub async fn from_reader(mut rdr: R, check_sig: bool) -> Result { - let header = Header::deserialize(&mut rdr).await?; - - let (hash, hash_string) = if check_sig { - let mut hasher = Sha512::new(); - let mut buf = [0; 1024]; - let mut read; - while { - read = rdr.read(&mut buf).await?; - read != 0 - } { - hasher.update(&buf[0..read]); - } - let hash = hasher.clone().finalize(); - header - .pubkey - .verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?; - ( - Some(hash), - Some(base32::encode( - base32::Alphabet::RFC4648 { padding: false }, - hash.as_slice(), - )), - ) - } else { - (None, None) - }; - - let pos = rdr.stream_position().await?; - - Ok(S9pkReader { - hash_string, - hash, - developer_key: header.pubkey, - toc: header.table_of_contents, - pos, - rdr, - }) - } - - pub fn hash(&self) -> Option<&Output> { - self.hash.as_ref() - } - - pub fn hash_str(&self) -> Option<&str> { - self.hash_string.as_ref().map(|s| s.as_str()) - } - - pub fn developer_key(&self) -> &VerifyingKey { - &self.developer_key - } - - pub async fn reset(&mut self) -> Result<(), Error> { - self.rdr.seek(SeekFrom::Start(0)).await?; - Ok(()) - } - - async fn read_handle<'a>( - &'a mut self, - section: FileSection, - ) -> Result, Error> { - if self.pos != section.position { - self.rdr.seek(SeekFrom::Start(section.position)).await?; - self.pos = section.position; - } - Ok(ReadHandle { - range: self.pos..(self.pos + section.length), - pos: &mut self.pos, - rdr: &mut self.rdr, - }) - } - - pub async fn manifest_raw(&mut self) -> Result, Error> { - self.read_handle(self.toc.manifest).await - } - - pub async fn manifest(&mut self) -> Result { - let slice = self.manifest_raw().await?.to_vec().await?; - serde_cbor::de::from_reader(slice.as_slice()) - .with_ctx(|_| (crate::ErrorKind::ParseS9pk, "Deserializing Manifest (CBOR)")) - } - - pub async fn license(&mut self) -> Result, Error> { - self.read_handle(self.toc.license).await - } - - pub async fn instructions(&mut self) -> Result, Error> { - self.read_handle(self.toc.instructions).await - } - - pub async fn icon(&mut self) -> Result, Error> { - self.read_handle(self.toc.icon).await - } - - pub async fn docker_images(&mut self) -> Result>, Error> { - DockerReader::new(self.read_handle(self.toc.docker_images).await?).await - } - - pub async fn assets(&mut self) -> Result, Error> { - self.read_handle(self.toc.assets).await - } - - pub async fn scripts(&mut self) -> Result>, Error> { - Ok(match self.toc.scripts { - None => None, - Some(a) => Some(self.read_handle(a).await?), - }) - } -} diff --git a/core/startos/src/s9pk/specv2.md b/core/startos/src/s9pk/specv2.md deleted file mode 100644 index 9bf993463..000000000 --- a/core/startos/src/s9pk/specv2.md +++ /dev/null @@ -1,28 +0,0 @@ -## Header - -### Magic - -2B: `0x3b3b` - -### Version - -varint: `0x02` - -### Pubkey - -32B: ed25519 pubkey - -### TOC - -- number of sections (varint) -- FOREACH section - - sig (32B: ed25519 signature of BLAKE-3 of rest of section) - - name (varstring) - - TYPE (varint) - - TYPE=FILE (`0x01`) - - mime (varstring) - - pos (32B: u64 BE) - - len (32B: u64 BE) - - hash (32B: BLAKE-3 of file contents) - - TYPE=TOC (`0x02`) - - recursively defined diff --git a/core/startos/src/s9pk/v1/reader.rs b/core/startos/src/s9pk/v1/reader.rs index 61b5e46a8..e901b1a14 100644 --- a/core/startos/src/s9pk/v1/reader.rs +++ b/core/startos/src/s9pk/v1/reader.rs @@ -220,16 +220,6 @@ impl S9pkReader { &validated_image_ids, )?; - #[cfg(feature = "js-engine")] - if man.containers.is_some() - || matches!(man.main, crate::procedure::PackageProcedure::Script(_)) - { - return Err(Error::new( - eyre!("Right now we don't support the containers and the long running main"), - crate::ErrorKind::ValidateS9pk, - )); - } - if man.replaces.len() >= MAX_REPLACES { return Err(Error::new( eyre!("Cannot have more than {MAX_REPLACES} replaces"),