Chore/version 0 3 1 0 (#1475)

* feat: move over to workspaces

* chore: Move to libs

* chore:fix(build): Compat

* chore: fixing pr
This commit is contained in:
J M
2022-06-01 10:22:00 -06:00
committed by GitHub
parent 37344f99a7
commit b8751e7add
49 changed files with 4586 additions and 1837 deletions

10
libs/.gitignore vendored Normal file
View File

@@ -0,0 +1,10 @@
/target
**/*.rs.bk
.DS_Store
.vscode
secrets.db
*.s9pk
*.sqlite3
.env
.editorconfig
proptest-regressions/*

2287
libs/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

8
libs/Cargo.toml Normal file
View File

@@ -0,0 +1,8 @@
[workspace]
members = [
"snapshot-creator",
"models",
"js_engine",
"helpers"
]

21
libs/build-arm-v8-snapshot.sh Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/bash
# Reason for this being is that we need to create a snapshot for the deno runtime. It wants to pull 3 files from build, and during the creation it gets embedded, but for some
# reason during the actual runtime it is looking for them. So this will create a docker in arm that creates the snaphot needed for the arm
set -e
if [ "$0" != "./build-arm-v8-snapshot.sh" ]; then
>&2 echo "Must be run from backend/workspace directory"
exit 1
fi
echo "Building "
cd ..
docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64 sh -c "(cd libs/ && cargo build -p snapshot-creator --release )"
cd -
echo "Creating Arm v8 Snapshot"
docker run --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot-creator"
sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin
sudo chmod 0644 JS_SNAPSHOT.bin
sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin

16
libs/build-v8-snapshot.sh Executable file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Reason for this being is that we need to create a snapshot for the deno runtime. It wants to pull 3 files from build, and during the creation it gets embedded, but for some
# reason during the actual runtime it is looking for them. So this will create a docker in arm that creates the snaphot needed for the arm
set -e
if [ "$0" != "./build-v8-snapshot.sh" ]; then
>&2 echo "Must be run from backend/workspace directory"
exit 1
fi
echo "Creating v8 Snapshot"
cargo run -p snapshot-creator --release
sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin
sudo chmod 0644 JS_SNAPSHOT.bin
sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/JS_SNAPSHOT.bin

11
libs/helpers/Cargo.toml Normal file
View File

@@ -0,0 +1,11 @@
[package]
name = "helpers"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
pin-project = "1.0.8"
tokio = { version = "1.15.*", features = ["full"] }
models = {path = "../models"}

31
libs/helpers/src/lib.rs Normal file
View File

@@ -0,0 +1,31 @@
use std::future::Future;
use tokio::task::{JoinError, JoinHandle};
mod script_dir;
pub use script_dir::*;
#[pin_project::pin_project(PinnedDrop)]
pub struct NonDetachingJoinHandle<T>(#[pin] JoinHandle<T>);
impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
fn from(t: JoinHandle<T>) -> Self {
NonDetachingJoinHandle(t)
}
}
#[pin_project::pinned_drop]
impl<T> PinnedDrop for NonDetachingJoinHandle<T> {
fn drop(self: std::pin::Pin<&mut Self>) {
let this = self.project();
this.0.into_ref().get_ref().abort()
}
}
impl<T> Future for NonDetachingJoinHandle<T> {
type Output = Result<T, JoinError>;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = self.project();
this.0.poll(cx)
}
}

View File

@@ -0,0 +1,13 @@
use std::path::{Path, PathBuf};
use models::{PackageId, Version};
pub const PKG_SCRIPT_DIR: &str = "package-data/scripts";
pub fn script_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, version: &Version) -> PathBuf {
datadir
.as_ref()
.join(&*PKG_SCRIPT_DIR)
.join(pkg_id)
.join(version.as_str())
}

17
libs/js_engine/Cargo.toml Normal file
View File

@@ -0,0 +1,17 @@
[package]
name = "js_engine"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dashmap = "5.1.0"
deno_core = "0.136.0"
deno_ast = {version="0.15.0", features = ["transpiling"]}
models = {path = "../models"}
helpers = {path = "../helpers"}
serde = { version = "1.0.*", features = ["derive", "rc"] }
serde_json = "1.0.*"
tokio = { version = "1.*.*", features = ["full"] }
tracing = "0.1"

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,87 @@
//@ts-check
// @ts-ignore
import Deno from "/deno_global.js";
// @ts-ignore
import * as mainModule from "/embassy.js";
/**
* This is using the simplified json pointer spec, using no escapes and arrays
* @param {object} obj
* @param {string} pointer
* @returns
*/
function jsonPointerValue(obj, pointer) {
const paths = pointer.substring(1).split("/");
for (const path of paths) {
if (obj == null) {
return null;
}
obj = (obj || {})[path];
}
return obj;
}
// @ts-ignore
const writeFile = ({ path, volumeId, toWrite }) => Deno.core.opAsync("write_file", volumeId, path, toWrite);
// @ts-ignore
const readFile = ({ volumeId, path }) => Deno.core.opAsync("read_file", volumeId, path);
// @ts-ignore
const removeFile = ({ volumeId, path }) => Deno.core.opAsync("remove_file", volumeId, path);
// @ts-ignore
const isSandboxed = () => Deno.core.opSync("is_sandboxed");
// @ts-ignore
const writeJsonFile = ({ volumeId, path, toWrite }) =>
writeFile({
volumeId,
path,
toWrite: JSON.stringify(toWrite),
});
// @ts-ignore
const readJsonFile = async ({ volumeId, path }) => JSON.parse(await readFile({ volumeId, path }));
// @ts-ignore
const createDir = ({ volumeId, path }) => Deno.core.opAsync("create_dir", volumeId, path);
// @ts-ignore
const removeDir = ({ volumeId, path }) => Deno.core.opAsync("remove_dir", volumeId, path);
// @ts-ignore
const trace = (x) => Deno.core.opSync("log_trace", x);
// @ts-ignore
const warn = (x) => Deno.core.opSync("log_warn", x);
// @ts-ignore
const error = (x) => Deno.core.opSync("log_error", x);
// @ts-ignore
const debug = (x) => Deno.core.opSync("log_debug", x);
// @ts-ignore
const info = (x) => Deno.core.opSync("log_info", x);
// @ts-ignore
const currentFunction = Deno.core.opSync("current_function");
//@ts-ignore
const input = Deno.core.opSync("get_input");
// @ts-ignore
const setState = (x) => Deno.core.opSync("set_value", x);
const effects = {
writeFile,
readFile,
writeJsonFile,
readJsonFile,
error,
warn,
debug,
trace,
info,
isSandboxed,
removeFile,
createDir,
removeDir,
};
const runFunction = jsonPointerValue(mainModule, currentFunction);
(async () => {
if (typeof runFunction !== "function") {
error(`Expecting ${ currentFunction } to be a function`);
throw new Error(`Expecting ${ currentFunction } to be a function`);
}
const answer = await runFunction(effects, input);
setState(answer);
})();

550
libs/js_engine/src/lib.rs Normal file
View File

@@ -0,0 +1,550 @@
use deno_core::anyhow::{anyhow, bail};
use deno_core::error::AnyError;
use deno_core::resolve_import;
use deno_core::JsRuntime;
use deno_core::ModuleLoader;
use deno_core::ModuleSource;
use deno_core::ModuleSourceFuture;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::RuntimeOptions;
use deno_core::Snapshot;
use deno_core::{Extension, OpDecl};
use helpers::script_dir;
use helpers::NonDetachingJoinHandle;
use models::{PackageId, ProcedureName, Version, VolumeId};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{path::Path, sync::Arc};
use std::{path::PathBuf, pin::Pin};
use tokio::io::AsyncReadExt;
pub trait PathForVolumeId: Send + Sync {
fn path_for(
&self,
data_dir: &Path,
package_id: &PackageId,
version: &Version,
volume_id: &VolumeId,
) -> Option<PathBuf>;
fn readonly(&self, volume_id: &VolumeId) -> bool;
}
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
pub struct JsCode(String);
#[derive(Debug, Clone, Copy)]
pub enum JsError {
Unknown = 1,
Javascript = 2,
Engine = 3,
BoundryLayerSerDe = 4,
Tokio = 5,
FileSystem = 6,
Timeout = 143,
}
#[cfg(target_arch = "x86_64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/JS_SNAPSHOT.bin");
#[cfg(target_arch = "aarch64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/ARM_JS_SNAPSHOT.bin");
#[derive(Clone)]
struct JsContext {
sandboxed: bool,
datadir: PathBuf,
run_function: String,
version: Version,
package_id: PackageId,
volumes: Arc<dyn PathForVolumeId>,
input: Value,
}
#[derive(Clone, Default)]
struct AnswerState(std::sync::Arc<deno_core::parking_lot::Mutex<Value>>);
#[derive(Clone, Debug)]
struct ModsLoader {
code: JsCode,
}
impl ModuleLoader for ModsLoader {
fn resolve(
&self,
specifier: &str,
referrer: &str,
_is_main: bool,
) -> Result<ModuleSpecifier, AnyError> {
if referrer.contains("embassy") {
bail!("Embassy.js cannot import anything else");
}
let s = resolve_import(specifier, referrer).unwrap();
Ok(s)
}
fn load(
&self,
module_specifier: &ModuleSpecifier,
maybe_referrer: Option<ModuleSpecifier>,
is_dyn_import: bool,
) -> Pin<Box<ModuleSourceFuture>> {
let module_specifier = module_specifier.as_str().to_owned();
let module = match &*module_specifier {
"file:///deno_global.js" => Ok(ModuleSource {
module_url_specified: "file:///deno_global.js".to_string(),
module_url_found: "file:///deno_global.js".to_string(),
code: "const old_deno = Deno; Deno = null; export default old_deno"
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///loadModule.js" => Ok(ModuleSource {
module_url_specified: "file:///loadModule.js".to_string(),
module_url_found: "file:///loadModule.js".to_string(),
code: include_str!("./artifacts/loadModule.js")
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///embassy.js" => Ok(ModuleSource {
module_url_specified: "file:///embassy.js".to_string(),
module_url_found: "file:///embassy.js".to_string(),
code: self.code.0.as_bytes().to_vec().into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
x => Err(anyhow!("Not allowed to import: {}", x)),
};
Box::pin(async move {
if is_dyn_import {
bail!("Will not import dynamic");
}
match &maybe_referrer {
Some(x) if x.as_str() == "file:///embassy.js" => {
bail!("Embassy is not allowed to import")
}
_ => (),
}
module
})
}
}
pub struct JsExecutionEnvironment {
sandboxed: bool,
base_directory: PathBuf,
module_loader: ModsLoader,
package_id: PackageId,
version: Version,
volumes: Arc<dyn PathForVolumeId>,
}
impl JsExecutionEnvironment {
pub async fn load_from_package(
data_directory: impl AsRef<std::path::Path>,
package_id: &PackageId,
version: &Version,
volumes: Box<dyn PathForVolumeId>,
) -> Result<Self, (JsError, String)> {
let data_dir = data_directory.as_ref();
let base_directory = data_dir;
let js_code = JsCode({
let file_path = script_dir(data_dir, package_id, version).join("embassy.js");
let mut file = match tokio::fs::File::open(file_path.clone()).await {
Ok(x) => x,
Err(e) => {
tracing::debug!("path: {:?}", file_path);
tracing::debug!("{:?}", e);
return Err((
JsError::FileSystem,
format!("The file opening '{:?}' created error: {}", file_path, e),
));
}
};
let mut buffer = Default::default();
if let Err(err) = file.read_to_string(&mut buffer).await {
tracing::debug!("{:?}", err);
return Err((
JsError::FileSystem,
format!("The file reading created error: {}", err),
));
};
buffer
});
Ok(Self {
base_directory: base_directory.to_owned(),
module_loader: ModsLoader { code: js_code },
package_id: package_id.clone(),
version: version.clone(),
volumes: volumes.into(),
sandboxed: false,
})
}
pub fn read_only_effects(mut self) -> Self {
self.sandboxed = true;
self
}
pub async fn run_action<I: Serialize, O: for<'de> Deserialize<'de>>(
self,
procedure_name: ProcedureName,
input: Option<I>,
) -> Result<O, (JsError, String)> {
let input = match serde_json::to_value(input) {
Ok(a) => a,
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
"Couldn't convert input".to_string(),
));
}
};
let safer_handle: NonDetachingJoinHandle<_> =
tokio::task::spawn_blocking(move || self.execute(procedure_name, input)).into();
let output = safer_handle
.await
.map_err(|err| (JsError::Tokio, format!("Tokio gave us the error: {}", err)))??;
match serde_json::from_value(output.clone()) {
Ok(x) => Ok(x),
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
format!(
"Couldn't convert output = {:#?} to the correct type",
serde_json::to_string_pretty(&output).unwrap_or_default()
),
));
}
}
}
fn declarations() -> Vec<OpDecl> {
vec![
fns::read_file::decl(),
fns::write_file::decl(),
fns::remove_file::decl(),
fns::create_dir::decl(),
fns::remove_dir::decl(),
fns::current_function::decl(),
fns::log_trace::decl(),
fns::log_warn::decl(),
fns::log_error::decl(),
fns::log_debug::decl(),
fns::log_info::decl(),
fns::get_input::decl(),
fns::set_value::decl(),
fns::is_sandboxed::decl(),
]
}
fn execute(
&self,
procedure_name: ProcedureName,
input: Value,
) -> Result<Value, (JsError, String)> {
let base_directory = self.base_directory.clone();
let answer_state = AnswerState::default();
let ext_answer_state = answer_state.clone();
let js_ctx = JsContext {
datadir: base_directory,
run_function: procedure_name.js_function_name(),
package_id: self.package_id.clone(),
volumes: self.volumes.clone(),
version: self.version.clone(),
sandboxed: self.sandboxed,
input,
};
let ext = Extension::builder()
.ops(Self::declarations())
.state(move |state| {
state.put(ext_answer_state.clone());
state.put(js_ctx.clone());
Ok(())
})
.build();
let loader = std::rc::Rc::new(self.module_loader.clone());
let runtime_options = RuntimeOptions {
module_loader: Some(loader),
extensions: vec![ext],
startup_snapshot: Some(Snapshot::Static(SNAPSHOT_BYTES)),
..Default::default()
};
let mut runtime = JsRuntime::new(runtime_options);
let future = async move {
let mod_id = runtime
.load_main_module(&"file:///loadModule.js".parse().unwrap(), None)
.await?;
let evaluated = runtime.mod_evaluate(mod_id);
let res = runtime.run_event_loop(false).await;
res?;
evaluated.await??;
Ok::<_, AnyError>(())
};
tokio::runtime::Handle::current()
.block_on(future)
.map_err(|e| {
tracing::debug!("{:?}", e);
(JsError::Javascript, format!("{}", e))
})?;
let answer = answer_state.0.lock().clone();
Ok(answer)
}
}
/// Note: Make sure that we have the assumption that all these methods are callable at any time, and all call restrictions should be in rust
mod fns {
use deno_core::{
anyhow::{anyhow, bail},
error::AnyError,
*,
};
use serde_json::Value;
use std::{
cell::RefCell,
convert::TryFrom,
path::{Path, PathBuf},
rc::Rc,
};
use models::VolumeId;
use super::{AnswerState, JsContext};
#[op]
async fn read_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<String, AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
//get_path_for in volume.rs
let new_file = volume_path.join(path_in);
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
let answer = tokio::fs::read_to_string(new_file).await?;
Ok(answer)
}
#[op]
async fn write_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
write: String,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::write(new_file, write).await?;
Ok(())
}
#[op]
async fn remove_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_file(new_file).await?;
Ok(())
}
#[op]
async fn remove_dir(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_dir_all(new_file).await?;
Ok(())
}
#[op]
async fn create_dir(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::create_dir_all(new_file).await?;
Ok(())
}
#[op]
fn current_function(state: &mut OpState) -> Result<String, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.run_function.clone())
}
#[op]
fn log_trace(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::trace!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_warn(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::warn!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_error(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::error!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_debug(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::debug!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_info(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::info!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn get_input(state: &mut OpState) -> Result<Value, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.input.clone())
}
#[op]
fn set_value(state: &mut OpState, value: Value) -> Result<(), AnyError> {
let mut answer = state.borrow::<AnswerState>().0.lock();
*answer = value;
Ok(())
}
#[op]
fn is_sandboxed(state: &mut OpState) -> Result<bool, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.sandboxed)
}
/// We need to make sure that during the file accessing, we don't reach beyond our scope of control
async fn is_subset(
parent: impl AsRef<Path>,
child: impl AsRef<Path>,
) -> Result<bool, AnyError> {
let child = tokio::fs::canonicalize(child).await?;
let parent = tokio::fs::canonicalize(parent).await?;
Ok(child.starts_with(parent))
}
}

15
libs/models/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "models"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
"trace",
] }
serde = { version = "1.0.*", features = ["derive", "rc"] }
thiserror = "1.0.*"
emver = { version = "0.1.6", features = ["serde"] }
rand = "0.7.*"

View File

@@ -0,0 +1,52 @@
use std::{str::FromStr, path::Path};
use serde::{Serialize, Deserialize};
use crate::{Id, InvalidId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct ActionId<S: AsRef<str> = String>(Id<S>);
impl FromStr for ActionId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(ActionId(Id::try_from(s.to_owned())?))
}
}
impl From<ActionId> for String {
fn from(value: ActionId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> AsRef<ActionId<S>> for ActionId<S> {
fn as_ref(&self) -> &ActionId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for ActionId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for ActionId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for ActionId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for ActionId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(ActionId(serde::Deserialize::deserialize(deserializer)?))
}
}

View File

@@ -0,0 +1,35 @@
use std::path::Path;
use serde::{Serialize, Deserialize, Deserializer};
use crate::Id;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct HealthCheckId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> std::fmt::Display for HealthCheckId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for HealthCheckId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for HealthCheckId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(HealthCheckId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for HealthCheckId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}

75
libs/models/src/id.rs Normal file
View File

@@ -0,0 +1,75 @@
use std::borrow::Borrow;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use crate::{invalid_id::InvalidId, id_unchecked::IdUnchecked};
pub const SYSTEM_ID: Id<&'static str> = Id("x_system");
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Id<S: AsRef<str> = String>(S);
impl<S: AsRef<str>> Id<S> {
pub fn try_from(value: S) -> Result<Self, InvalidId> {
if value
.as_ref()
.chars()
.all(|c| c.is_ascii_lowercase() || c == '-')
{
Ok(Id(value))
} else {
Err(InvalidId)
}
}
}
impl<'a> Id<&'a str> {
pub fn owned(&self) -> Id {
Id(self.0.to_owned())
}
}
impl From<Id> for String {
fn from(value: Id) -> Self {
value.0
}
}
impl<S: AsRef<str>> std::ops::Deref for Id<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<S: AsRef<str>> std::fmt::Display for Id<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_ref())
}
}
impl<S: AsRef<str>> AsRef<str> for Id<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for Id<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for Id<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Id::try_from(unchecked.0).map_err(serde::de::Error::custom)
}
}
impl<S: AsRef<str>> Serialize for Id<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
serializer.serialize_str(self.as_ref())
}
}

View File

@@ -0,0 +1,56 @@
use std::borrow::Cow;
use serde::{Deserialize, Deserializer};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct IdUnchecked<S: AsRef<str>>(pub S);
impl<'de> Deserialize<'de> for IdUnchecked<Cow<'de, str>> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = IdUnchecked<Cow<'de, str>>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a valid ID")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v.to_owned())))
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v)))
}
fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Borrowed(v)))
}
}
deserializer.deserialize_any(Visitor)
}
}
impl<'de> Deserialize<'de> for IdUnchecked<String> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(String::deserialize(deserializer)?))
}
}
impl<'de> Deserialize<'de> for IdUnchecked<&'de str> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(<&'de str>::deserialize(deserializer)?))
}
}

View File

@@ -0,0 +1,47 @@
use std::path::Path;
use serde::{Serialize, Deserialize, Deserializer};
use crate::Id;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct InterfaceId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> From<Id<S>> for InterfaceId<S> {
fn from(id: Id<S>) -> Self {
Self(id)
}
}
impl<S: AsRef<str>> std::fmt::Display for InterfaceId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> std::ops::Deref for InterfaceId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<str> for InterfaceId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for InterfaceId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(InterfaceId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for InterfaceId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}

View File

@@ -0,0 +1,4 @@
#[derive(Debug, thiserror::Error)]
#[error("Invalid ID")]
pub struct InvalidId;

21
libs/models/src/lib.rs Normal file
View File

@@ -0,0 +1,21 @@
mod id;
mod invalid_id;
mod id_unchecked;
mod version;
mod volume_id;
mod interface_id;
mod package_id;
mod procedure_name;
mod health_check_id;
mod action_id;
pub use id::*;
pub use invalid_id::*;
pub use id_unchecked::*;
pub use version::*;
pub use volume_id::*;
pub use interface_id::*;
pub use package_id::*;
pub use procedure_name::*;
pub use health_check_id::*;
pub use action_id::*;

View File

@@ -0,0 +1,85 @@
use std::{str::FromStr, borrow::Borrow, path::Path};
use serde::{Deserialize, Serialize, Serializer};
use crate::{Id, InvalidId, SYSTEM_ID};
pub const SYSTEM_PACKAGE_ID: PackageId<&'static str> = PackageId(SYSTEM_ID);
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct PackageId<S: AsRef<str> = String>(Id<S>);
impl<'a> PackageId<&'a str> {
pub fn owned(&self) -> PackageId {
PackageId(self.0.owned())
}
}
impl FromStr for PackageId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(PackageId(Id::try_from(s.to_owned())?))
}
}
impl From<PackageId> for String {
fn from(value: PackageId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> From<Id<S>> for PackageId<S> {
fn from(id: Id<S>) -> Self {
PackageId(id)
}
}
impl<S: AsRef<str>> std::ops::Deref for PackageId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<PackageId<S>> for PackageId<S> {
fn as_ref(&self) -> &PackageId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for PackageId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for PackageId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for PackageId<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for PackageId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for PackageId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(PackageId(Deserialize::deserialize(deserializer)?))
}
}
impl<S> Serialize for PackageId<S>
where
S: AsRef<str>,
{
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
Serialize::serialize(&self.0, serializer)
}
}

View File

@@ -0,0 +1,50 @@
use crate::{PackageId, HealthCheckId, ActionId};
#[derive(Debug, Clone)]
pub enum ProcedureName {
Main, // Usually just run container
CreateBackup,
RestoreBackup,
GetConfig,
SetConfig,
Migration,
Properties,
Check(PackageId),
AutoConfig(PackageId),
Health(HealthCheckId),
Action(ActionId),
}
impl ProcedureName {
pub fn docker_name(&self) -> Option<String> {
match self {
ProcedureName::Main => None,
ProcedureName::CreateBackup => Some("CreateBackup".to_string()),
ProcedureName::RestoreBackup => Some("RestoreBackup".to_string()),
ProcedureName::GetConfig => Some("GetConfig".to_string()),
ProcedureName::SetConfig => Some("SetConfig".to_string()),
ProcedureName::Migration => Some("Migration".to_string()),
ProcedureName::Properties => Some(format!("Properties-{}", rand::random::<u64>())),
ProcedureName::Health(id) => Some(format!("{}Health", id)),
ProcedureName::Action(id) => Some(format!("{}Action", id)),
ProcedureName::Check(_) => None,
ProcedureName::AutoConfig(_) => None,
}
}
pub fn js_function_name(&self) -> String {
match self {
ProcedureName::Main => "/main".to_string(),
ProcedureName::CreateBackup => "/createBackup".to_string(),
ProcedureName::RestoreBackup => "/restoreBackup".to_string(),
ProcedureName::GetConfig => "/getConfig".to_string(),
ProcedureName::SetConfig => "/setConfig".to_string(),
ProcedureName::Migration => "/migration".to_string(),
ProcedureName::Properties => "/properties".to_string(),
ProcedureName::Health(id) => format!("/health/{}", id),
ProcedureName::Action(id) => format!("/action/{}", id),
ProcedureName::Check(id) => format!("/dependencies/{}/check", id),
ProcedureName::AutoConfig(id) => format!("/dependencies/{}/autoConfigure", id),
}
}
}

110
libs/models/src/version.rs Normal file
View File

@@ -0,0 +1,110 @@
use std::{str::FromStr, ops::Deref};
use std::hash::{Hash, Hasher};
use patch_db::{HasModel, Model};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Debug, Clone)]
pub struct Version {
version: emver::Version,
string: String,
}
impl Version {
pub fn as_str(&self) -> &str {
self.string.as_str()
}
pub fn into_version(self) -> emver::Version {
self.version
}
}
impl std::fmt::Display for Version {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.string)
}
}
impl std::str::FromStr for Version {
type Err = <emver::Version as FromStr>::Err;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Version {
string: s.to_owned(),
version: s.parse()?,
})
}
}
impl From<emver::Version> for Version {
fn from(v: emver::Version) -> Self {
Version {
string: v.to_string(),
version: v,
}
}
}
impl From<Version> for emver::Version {
fn from(v: Version) -> Self {
v.version
}
}
impl Default for Version {
fn default() -> Self {
Self::from(emver::Version::default())
}
}
impl Deref for Version {
type Target = emver::Version;
fn deref(&self) -> &Self::Target {
&self.version
}
}
impl AsRef<emver::Version> for Version {
fn as_ref(&self) -> &emver::Version {
&self.version
}
}
impl AsRef<str> for Version {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl PartialEq for Version {
fn eq(&self, other: &Version) -> bool {
self.version.eq(&other.version)
}
}
impl Eq for Version {}
impl PartialOrd for Version {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.version.partial_cmp(&other.version)
}
}
impl Ord for Version {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.version.cmp(&other.version)
}
}
impl Hash for Version {
fn hash<H: Hasher>(&self, state: &mut H) {
self.version.hash(state)
}
}
impl<'de> Deserialize<'de> for Version {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let string = String::deserialize(deserializer)?;
let version = emver::Version::from_str(&string).map_err(::serde::de::Error::custom)?;
Ok(Self { string, version })
}
}
impl Serialize for Version {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
self.string.serialize(serializer)
}
}
impl HasModel for Version {
type Model = Model<Version>;
}

View File

@@ -0,0 +1,62 @@
use std::{borrow::Borrow, path::Path};
use serde::{Deserialize, Serialize, Deserializer};
use crate::{Id, IdUnchecked};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum VolumeId<S: AsRef<str> = String> {
Backup,
Custom(Id<S>),
}
impl<S: AsRef<str>> std::fmt::Display for VolumeId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VolumeId::Backup => write!(f, "BACKUP"),
VolumeId::Custom(id) => write!(f, "{}", id),
}
}
}
impl<S: AsRef<str>> AsRef<str> for VolumeId<S> {
fn as_ref(&self) -> &str {
match self {
VolumeId::Backup => "BACKUP",
VolumeId::Custom(id) => id.as_ref(),
}
}
}
impl<S: AsRef<str>> Borrow<str> for VolumeId<S> {
fn borrow(&self) -> &str {
self.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for VolumeId<S> {
fn as_ref(&self) -> &Path {
AsRef::<str>::as_ref(self).as_ref()
}
}
impl<'de, S> Deserialize<'de> for VolumeId<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Ok(match unchecked.0.as_ref() {
"BACKUP" => VolumeId::Backup,
_ => VolumeId::Custom(Id::try_from(unchecked.0).map_err(serde::de::Error::custom)?),
})
}
}
impl<S: AsRef<str>> Serialize for VolumeId<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: serde::Serializer,
{
serializer.serialize_str(self.as_ref())
}
}

View File

@@ -0,0 +1,11 @@
[package]
name = "snapshot-creator"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dashmap = "5.1.0"
deno_core = "0.136.0"
deno_ast = {version="0.15.0", features = ["transpiling"]}

View File

@@ -0,0 +1,14 @@
use deno_core::{JsRuntime, RuntimeOptions};
fn main() {
let mut runtime = JsRuntime::new(RuntimeOptions {
will_snapshot: true,
..Default::default()
});
let snapshot = runtime.snapshot();
let snapshot_slice: &[u8] = &*snapshot;
println!("Snapshot size: {}", snapshot_slice.len());
std::fs::write("JS_SNAPSHOT.bin", snapshot_slice).unwrap();
}