Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-04-01 21:13:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

rebased and compiling again

Browse Source
This commit is contained in:
Matt Hill
2023-11-09 15:35:47 -07:00
parent c491dfdd3a d9cc21f761
commit 7a55c91349
333 changed files with 15121 additions and 18641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1738
libs/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

2
libs/build-arm-v8-snapshot.sh
View File

@@ -22,7 +22,7 @@ rust-gnu-builder sh -c "(cd libs/ && cargo build -p snapshot_creator --release -
cd -
echo "Creating Arm v8 Snapshot"
docker run $USE_TTY --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot_creator"
docker run $USE_TTY --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:22.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot_creator"
sudo chown -R $USER target
sudo chown -R $USER ~/.cargo
sudo chown $USER JS_SNAPSHOT.bin

4
libs/embassy_container_init/Cargo.toml
View File

@@ -20,7 +20,7 @@ serde = { version = "1", features = ["derive", "rc"] }
serde_json = "1"
helpers = { path = "../helpers" }
imbl = "2"
nix = "0.25"
nix = { version = "0.27", features = ["process", "signal"] }
tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1", features = ["io-util", "sync", "net"] }
tracing = "0.1"
@@ -30,7 +30,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = "0.14"
procfs = "0.15"
[profile.test]
opt-level = 3

16
libs/embassy_container_init/src/main.rs
View File

@@ -5,8 +5,8 @@ use std::process::Stdio;
use std::sync::Arc;
use embassy_container_init::{
LogParams, OutputParams, OutputStrategy, ProcessGroupId, ProcessId, ReadLineStderrParams,
ReadLineStdoutParams, RunCommandParams, SendSignalParams, SignalGroupParams,
LogParams, OutputParams, OutputStrategy, ProcessGroupId, ProcessId, RunCommandParams,
SendSignalParams, SignalGroupParams,
};
use futures::StreamExt;
use helpers::NonDetachingJoinHandle;
@@ -15,7 +15,7 @@ use nix::sys::signal::Signal;
use serde::{Deserialize, Serialize};
use serde_json::json;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, ChildStderr, ChildStdout, Command};
use tokio::process::{Child, Command};
use tokio::select;
use tokio::sync::{watch, Mutex};
use yajrc::{Id, RpcError};
@@ -103,7 +103,7 @@ impl Handler {
// Input::ReadLineStderr(ReadLineStderrParams { pid }) => {
// Output::ReadLineStderr(self.read_line_stderr(pid).await?)
// }
Input::Log(LogParams { gid, level }) => {
Input::Log(LogParams { gid: _, level }) => {
level.trace();
Output::Log
}
@@ -363,23 +363,23 @@ async fn main() {
let req = serde_json::from_str::<IncomingRpc>(&line?)?;
match handler.handle(req.input).await {
Ok(output) => {
if let Err(err) = w.lock().await.write_all(
if w.lock().await.write_all(
format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "result": output }))
.as_bytes(),
)
.await {
.await.is_err() {
tracing::error!("Error sending to {id:?}", id = req.id);
}
}
Err(e) =>
if let Err(err) = w
if w
.lock()
.await
.write_all(
format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "error": e }))
.as_bytes(),
)
.await {
.await.is_err() {
tracing::error!("Handle + Error sending to {id:?}", id = req.id);
},

10
libs/helpers/Cargo.toml
View File

@@ -8,13 +8,13 @@ edition = "2021"
[dependencies]
async-trait = "0.1.64"
color-eyre = "0.6.2"
futures = "0.3.21"
futures = "0.3.28"
lazy_async_pool = "0.3.3"
models = { path = "../models" }
pin-project = "1.0.11"
pin-project = "1.1.3"
serde = { version = "1.0", features = ["derive", "rc"] }
serde_json = "1.0"
tokio = { version = "1.23", features = ["full"] }
tokio-stream = { version = "0.1.9", features = ["io-util", "sync"] }
tracing = "0.1.35"
tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1.14", features = ["io-util", "sync"] }
tracing = "0.1.39"
yajrc = { version = "*", git = "https://github.com/dr-bonez/yajrc.git", branch = "develop" }

29
libs/helpers/src/lib.rs
View File

@@ -72,11 +72,29 @@ pub async fn canonicalize(
#[pin_project::pin_project(PinnedDrop)]
pub struct NonDetachingJoinHandle<T>(#[pin] JoinHandle<T>);
impl<T> NonDetachingJoinHandle<T> {
pub async fn wait_for_abort(self) -> Result<T, JoinError> {
self.abort();
self.await
}
}
impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
fn from(t: JoinHandle<T>) -> Self {
NonDetachingJoinHandle(t)
}
}
impl<T> Deref for NonDetachingJoinHandle<T> {
type Target = JoinHandle<T>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> DerefMut for NonDetachingJoinHandle<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[pin_project::pinned_drop]
impl<T> PinnedDrop for NonDetachingJoinHandle<T> {
fn drop(self: std::pin::Pin<&mut Self>) {
@@ -94,17 +112,6 @@ impl<T> Future for NonDetachingJoinHandle<T> {
this.0.poll(cx)
}
}
impl<T> Deref for NonDetachingJoinHandle<T> {
type Target = JoinHandle<T>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> DerefMut for NonDetachingJoinHandle<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
pub struct AtomicFile {
tmp_path: PathBuf,

2
libs/helpers/src/rsync.rs
View File

@@ -74,7 +74,7 @@ impl Rsync {
cmd.arg("--no-perms");
}
let mut command = cmd
.arg("-acAXH")
.arg("-actAXH")
.arg("--info=progress2")
.arg("--no-inc-recursive")
.arg(src.as_ref())

14
libs/js_engine/Cargo.toml
View File

@@ -6,14 +6,14 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait = "0.1.56"
dashmap = "5.3.4"
deno_core = "0.195.0"
deno_ast = { version = "0.27.2", features = ["transpiling"] }
async-trait = "0.1.74"
dashmap = "5.5.3"
deno_core = "=0.222.0"
deno_ast = { version = "=0.29.5", features = ["transpiling"] }
embassy_container_init = { path = "../embassy_container_init" }
reqwest = { version = "0.11.11" }
sha2 = "0.10.2"
itertools = "0.10.5"
reqwest = { version = "0.11.22" }
sha2 = "0.10.8"
itertools = "0.11.0"
lazy_static = "1.4.0"
models = { path = "../models" }
helpers = { path = "../helpers" }

BIN
libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
View File

Binary file not shown.

BIN
libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
View File

Binary file not shown.

296
libs/js_engine/src/lib.rs
View File

@@ -13,8 +13,7 @@ use deno_core::{
ModuleSourceFuture, ModuleSpecifier, ModuleType, OpDecl, ResolutionKind, RuntimeOptions,
Snapshot,
};
use embassy_container_init::ProcessGroupId;
use helpers::{script_dir, spawn_local, OsApi, Rsync, UnixRpcClient};
use helpers::{script_dir, spawn_local, Rsync};
use models::{PackageId, ProcedureName, Version, VolumeId};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@@ -105,8 +104,6 @@ struct JsContext {
volumes: Arc<dyn PathForVolumeId>,
input: Value,
variable_args: Vec<serde_json::Value>,
container_process_gid: ProcessGroupId,
container_rpc_client: Option<Arc<UnixRpcClient>>,
rsyncs: Arc<Mutex<(usize, BTreeMap<usize, Rsync>)>>,
callback_sender: mpsc::UnboundedSender<(Arc<String>, Vec<Value>)>,
}
@@ -158,17 +155,17 @@ impl ModuleLoader for ModsLoader {
"file:///deno_global.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
FastString::Static("const old_deno = Deno; Deno = null; export default old_deno"),
&*DENO_GLOBAL_JS,
&DENO_GLOBAL_JS,
)),
"file:///loadModule.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
FastString::Static(include_str!("./artifacts/loadModule.js")),
&*LOAD_MODULE_JS,
&LOAD_MODULE_JS,
)),
"file:///embassy.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
self.code.0.clone().into(),
&*EMBASSY_JS,
&EMBASSY_JS,
)),
x => Err(anyhow!("Not allowed to import: {}", x)),
@@ -197,8 +194,6 @@ pub struct JsExecutionEnvironment {
package_id: PackageId,
version: Version,
volumes: Arc<dyn PathForVolumeId>,
container_process_gid: ProcessGroupId,
container_rpc_client: Option<Arc<UnixRpcClient>>,
}
impl JsExecutionEnvironment {
@@ -208,9 +203,7 @@ impl JsExecutionEnvironment {
package_id: &PackageId,
version: &Version,
volumes: Box<dyn PathForVolumeId>,
container_process_gid: ProcessGroupId,
container_rpc_client: Option<Arc<UnixRpcClient>>,
) -> Result<Self, (JsError, String)> {
) -> Result<JsExecutionEnvironment, (JsError, String)> {
let data_dir = data_directory.as_ref();
let base_directory = data_dir;
let js_code = JsCode({
@@ -244,8 +237,6 @@ impl JsExecutionEnvironment {
version: version.clone(),
volumes: volumes.into(),
sandboxed: false,
container_process_gid,
container_rpc_client,
})
}
pub fn read_only_effects(mut self) -> Self {
@@ -313,12 +304,7 @@ impl JsExecutionEnvironment {
fns::get_variable_args::decl(),
fns::set_value::decl(),
fns::is_sandboxed::decl(),
fns::start_command::decl(),
fns::wait_command::decl(),
fns::sleep::decl(),
fns::send_signal::decl(),
fns::chmod::decl(),
fns::signal_group::decl(),
fns::rsync::decl(),
fns::rsync_wait::decl(),
fns::rsync_progress::decl(),
@@ -359,16 +345,13 @@ impl JsExecutionEnvironment {
sandboxed: self.sandboxed,
input,
variable_args,
container_process_gid: self.container_process_gid,
container_rpc_client: self.container_rpc_client.clone(),
callback_sender,
rsyncs: Default::default(),
};
let ext = Extension::builder("embassy")
.ops(Self::declarations())
.state(move |state| {
state.put(ext_answer_state.clone());
state.put(js_ctx.clone());
state.put(js_ctx);
})
.build();
let loader = std::rc::Rc::new(self.module_loader.clone());
@@ -385,11 +368,7 @@ impl JsExecutionEnvironment {
.load_main_module(&"file:///loadModule.js".parse().unwrap(), None)
.await?;
let evaluated = runtime.mod_evaluate(mod_id);
let res = RuntimeEventLoop {
runtime: &mut runtime,
callback_receiver,
}
.await;
let res = runtime.run_event_loop(false).await;
res?;
evaluated.await??;
Ok::<_, AnyError>(())
@@ -425,7 +404,7 @@ impl<'a> Future for RuntimeEventLoop<'a> {
if let Poll::Ready(Some((uuid, args))) = this.callback_receiver.poll_recv(cx) {
match this.runtime.execute_script(
"callback",
&format!("globalThis.runCallback(\"{uuid}\", {})", Value::Array(args)),
format!("globalThis.runCallback(\"{uuid}\", {})", Value::Array(args)).into(),
) {
Ok(_) => (),
Err(e) => return Poll::Ready(Err(e)),
@@ -450,23 +429,17 @@ mod fns {
use deno_core::anyhow::{anyhow, bail};
use deno_core::error::AnyError;
use deno_core::*;
use embassy_container_init::{
OutputParams, OutputStrategy, ProcessGroupId, ProcessId, RunCommand, RunCommandParams,
SendSignal, SendSignalParams, SignalGroup, SignalGroupParams,
};
use helpers::{
to_tmp_path, AddressSchemaLocal, AddressSchemaOnion, AtomicFile, Callback, Rsync,
RsyncOptions,
};
use embassy_container_init::ProcessId;
use helpers::{to_tmp_path, AtomicFile, Rsync, RsyncOptions};
use itertools::Itertools;
use models::{PackageId, VolumeId};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use serde_json::Value;
use tokio::io::AsyncWriteExt;
use tokio::process::Command;
use super::{AnswerState, JsContext};
use crate::{system_time_as_unix_ms, MetadataJs, ResultType};
use crate::{system_time_as_unix_ms, MetadataJs};
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
struct FetchOptions {
@@ -650,7 +623,7 @@ mod fns {
}
let path_in = path_in.strip_prefix("/").unwrap_or(&path_in);
let new_file = volume_path.join(&path_in);
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
@@ -1082,11 +1055,9 @@ mod fns {
let volume_path = {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
ctx.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
volume_path
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?
};
let path_in = path_in.strip_prefix("/").unwrap_or(&path_in);
let new_file = volume_path.join(path_in);
@@ -1157,8 +1128,7 @@ mod fns {
.stdout,
)?
.lines()
.skip(1)
.next()
.nth(1)
.unwrap_or_default()
.parse()?;
let used = String::from_utf8(
@@ -1188,8 +1158,7 @@ mod fns {
.stdout,
)?
.lines()
.skip(1)
.next()
.nth(1)
.unwrap_or_default()
.split_ascii_whitespace()
.next_tuple()
@@ -1210,18 +1179,6 @@ mod fns {
let state = state.borrow();
state.borrow::<JsContext>().clone()
};
if let Some(rpc_client) = ctx.container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
level: embassy_container_init::LogLevel::Trace(input),
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::trace!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
@@ -1236,18 +1193,6 @@ mod fns {
let state = state.borrow();
state.borrow::<JsContext>().clone()
};
if let Some(rpc_client) = ctx.container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
level: embassy_container_init::LogLevel::Warn(input),
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::warn!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
@@ -1262,18 +1207,6 @@ mod fns {
let state = state.borrow();
state.borrow::<JsContext>().clone()
};
if let Some(rpc_client) = ctx.container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
level: embassy_container_init::LogLevel::Error(input),
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::error!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
@@ -1288,18 +1221,6 @@ mod fns {
let state = state.borrow();
state.borrow::<JsContext>().clone()
};
if let Some(rpc_client) = ctx.container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
level: embassy_container_init::LogLevel::Debug(input),
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::debug!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
@@ -1310,25 +1231,14 @@ mod fns {
}
#[op]
async fn log_info(state: Rc<RefCell<OpState>>, input: String) -> Result<(), AnyError> {
let ctx = {
let (package_id, run_function) = {
let state = state.borrow();
state.borrow::<JsContext>().clone()
let ctx: JsContext = state.borrow::<JsContext>().clone();
(ctx.package_id, ctx.run_function)
};
if let Some(rpc_client) = ctx.container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
level: embassy_container_init::LogLevel::Info(input),
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::info!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
package_id = tracing::field::display(&package_id),
run_function = tracing::field::display(&run_function),
"{}",
input
);
@@ -1364,174 +1274,12 @@ mod fns {
Ok(ctx.sandboxed)
}
#[op]
async fn send_signal(
state: Rc<RefCell<OpState>>,
pid: u32,
signal: u32,
) -> Result<(), AnyError> {
let sandboxed = {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
ctx.sandboxed
};
if sandboxed {
bail!("Will not run sendSignal in sandboxed mode");
}
if let Some(rpc_client) = {
let state = state.borrow();
let ctx = state.borrow::<JsContext>();
ctx.container_rpc_client.clone()
} {
rpc_client
.request(
SendSignal,
SendSignalParams {
pid: ProcessId(pid),
signal,
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
Ok(())
} else {
Err(anyhow!("No RpcClient for command operations"))
}
}
#[op]
async fn signal_group(
state: Rc<RefCell<OpState>>,
gid: u32,
signal: u32,
) -> Result<(), AnyError> {
let sandboxed = {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
ctx.sandboxed
};
if sandboxed {
bail!("Will not run signalGroup in sandboxed mode");
}
if let Some(rpc_client) = {
let state = state.borrow();
let ctx = state.borrow::<JsContext>();
ctx.container_rpc_client.clone()
} {
rpc_client
.request(
SignalGroup,
SignalGroupParams {
gid: ProcessGroupId(gid),
signal,
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
Ok(())
} else {
Err(anyhow!("No RpcClient for command operations"))
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StartCommand {
process_id: ProcessId,
}
#[op]
async fn start_command(
state: Rc<RefCell<OpState>>,
command: String,
args: Vec<String>,
output: OutputStrategy,
timeout: Option<u64>,
) -> Result<StartCommand, AnyError> {
let sandboxed = {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
ctx.sandboxed
};
if sandboxed {
bail!("Will not run command in sandboxed mode");
}
if let (gid, Some(rpc_client)) = {
let state = state.borrow();
let ctx = state.borrow::<JsContext>();
(ctx.container_process_gid, ctx.container_rpc_client.clone())
} {
let pid = rpc_client
.request(
RunCommand,
RunCommandParams {
gid: Some(gid),
command,
args,
output,
},
)
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data))?;
if let Some(timeout) = timeout {
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(timeout)).await;
if let Err(err) = rpc_client
.request(SendSignal, SendSignalParams { pid, signal: 9 })
.await
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data))
{
tracing::warn!("Could not kill process {pid:?}");
tracing::debug!("{err:?}");
}
});
}
Ok(StartCommand { process_id: pid })
} else {
Err(anyhow!("No RpcClient for command operations"))
}
}
#[op]
async fn wait_command(
state: Rc<RefCell<OpState>>,
pid: ProcessId,
) -> Result<ResultType, AnyError> {
if let Some(rpc_client) = {
let state = state.borrow();
let ctx = state.borrow::<JsContext>();
ctx.container_rpc_client.clone()
} {
Ok(
match rpc_client
.request(embassy_container_init::Output, OutputParams { pid })
.await
{
Ok(a) => ResultType::Result(json!(a)),
Err(e) => ResultType::ErrorCode(
e.code,
match e.data {
Some(Value::String(s)) => s,
e => format!("{:?}", e),
},
),
},
)
} else {
Err(anyhow!("No RpcClient for command operations"))
}
}
#[op]
async fn sleep(time_ms: u64) -> Result<(), AnyError> {
tokio::time::sleep(Duration::from_millis(time_ms)).await;

28
libs/models/Cargo.toml
View File

@@ -6,33 +6,33 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bollard = "0.13.0"
color-eyre = "0.6.1"
ed25519-dalek = { version = "1.0.1", features = ["serde"] }
base64 = "0.21.4"
color-eyre = "0.6.2"
ed25519-dalek = { version = "2.0.0", features = ["serde"] }
lazy_static = "1.4"
mbrman = "0.5.0"
mbrman = "0.5.2"
emver = { version = "0.1", git = "https://github.com/Start9Labs/emver-rs.git", features = [
"serde",
] }
internment = { version = "0.7.0", features = ["arc", "serde"] }
ipnet = "2.7.1"
openssl = { version = "0.10.41", features = ["vendored"] }
ipnet = "2.8.0"
openssl = { version = "0.10.57", features = ["vendored"] }
patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
"trace",
] }
rand = "0.8"
regex = "1.7.1"
rpc-toolkit = "0.2.1"
rand = "0.8.5"
regex = "1.10.2"
reqwest = "0.11.22"
rpc-toolkit = "0.2.2"
serde = { version = "1.0", features = ["derive", "rc"] }
serde_json = "1.0"
sqlx = { version = "0.6.0", features = [
sqlx = { version = "0.7.2", features = [
"chrono",
"offline",
"runtime-tokio-rustls",
"postgres",
] }
ssh-key = "0.5.1"
ssh-key = "0.6.2"
thiserror = "1.0"
tokio = { version = "1", features = ["full"] }
torut = "0.2.1"
tracing = "0.1.35"
tracing = "0.1.39"
yasi = "0.1.5"

171
libs/models/src/data_url.rs Normal file
View File

@@ -0,0 +1,171 @@
use std::borrow::Cow;
use std::path::Path;
use base64::Engine;
use color_eyre::eyre::eyre;
use reqwest::header::CONTENT_TYPE;
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncRead, AsyncReadExt};
use yasi::InternedString;
use crate::{mime, Error, ErrorKind, ResultExt};
#[derive(Clone)]
pub struct DataUrl<'a> {
mime: InternedString,
data: Cow<'a, [u8]>,
}
impl<'a> DataUrl<'a> {
pub const DEFAULT_MIME: &'static str = "application/octet-stream";
pub const MAX_SIZE: u64 = 100 * 1024;
// data:{mime};base64,{data}
pub fn to_string(&self) -> String {
use std::fmt::Write;
let mut res = String::with_capacity(self.data_url_len_without_mime() + self.mime.len());
let _ = write!(res, "data:{};base64,", self.mime);
base64::engine::general_purpose::STANDARD.encode_string(&self.data, &mut res);
res
}
fn data_url_len_without_mime(&self) -> usize {
5 + 8 + (4 * self.data.len() / 3) + 3
}
pub fn data_url_len(&self) -> usize {
self.data_url_len_without_mime() + self.mime.len()
}
pub fn from_slice(mime: &str, data: &'a [u8]) -> Self {
Self {
mime: InternedString::intern(mime),
data: Cow::Borrowed(data),
}
}
}
impl DataUrl<'static> {
pub async fn from_reader(
mime: &str,
rdr: impl AsyncRead + Unpin,
size_est: Option<u64>,
) -> Result<Self, Error> {
let check_size = |s| {
if s > Self::MAX_SIZE {
Err(Error::new(
eyre!("Data URLs must be smaller than 100KiB"),
ErrorKind::Filesystem,
))
} else {
Ok(s)
}
};
let mut buf = size_est
.map(check_size)
.transpose()?
.map(|s| Vec::with_capacity(s as usize))
.unwrap_or_default();
rdr.take(Self::MAX_SIZE + 1).read_to_end(&mut buf).await?;
check_size(buf.len() as u64)?;
Ok(Self {
mime: InternedString::intern(mime),
data: Cow::Owned(buf),
})
}
pub async fn from_path(path: impl AsRef<Path>) -> Result<Self, Error> {
let path = path.as_ref();
let f = tokio::fs::File::open(path).await?;
let m = f.metadata().await?;
let mime = path
.extension()
.and_then(|s| s.to_str())
.and_then(mime)
.unwrap_or(Self::DEFAULT_MIME);
Self::from_reader(mime, f, Some(m.len())).await
}
pub async fn from_response(res: reqwest::Response) -> Result<Self, Error> {
let mime = InternedString::intern(
res.headers()
.get(CONTENT_TYPE)
.and_then(|h| h.to_str().ok())
.unwrap_or(Self::DEFAULT_MIME),
);
let data = res.bytes().await.with_kind(ErrorKind::Network)?.to_vec();
Ok(Self {
mime,
data: Cow::Owned(data),
})
}
pub fn from_vec(mime: &str, data: Vec<u8>) -> Self {
Self {
mime: InternedString::intern(mime),
data: Cow::Owned(data),
}
}
}
impl<'a> std::fmt::Debug for DataUrl<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.to_string())
}
}
impl<'de> Deserialize<'de> for DataUrl<'static> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = DataUrl<'static>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a valid base64 data url")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
v.strip_prefix("data:")
.and_then(|v| v.split_once(";base64,"))
.and_then(|(mime, data)| {
Some(DataUrl {
mime: InternedString::intern(mime),
data: Cow::Owned(
base64::engine::general_purpose::STANDARD
.decode(data)
.ok()?,
),
})
})
.ok_or_else(|| {
E::invalid_value(serde::de::Unexpected::Str(v), &"a valid base64 data url")
})
}
}
deserializer.deserialize_any(Visitor)
}
}
impl<'a> Serialize for DataUrl<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
#[test]
fn doesnt_reallocate() {
let random: [u8; 10] = rand::random();
for i in 0..10 {
let icon = DataUrl {
mime: InternedString::intern("png"),
data: Cow::Borrowed(&random[..i]),
};
assert_eq!(dbg!(icon.to_string()).capacity(), icon.data_url_len());
}
}

54
libs/models/src/errors.rs
View File

@@ -3,6 +3,7 @@ use std::fmt::Display;
use color_eyre::eyre::eyre;
use patch_db::Revision;
use rpc_toolkit::hyper::http::uri::InvalidUri;
use rpc_toolkit::reqwest;
use rpc_toolkit::yajrc::RpcError;
use crate::InvalidId;
@@ -77,6 +78,9 @@ pub enum ErrorKind {
OpenSsh = 66,
Zram = 67,
Lshw = 68,
CpuSettings = 69,
Firmware = 70,
Timeout = 71,
}
impl ErrorKind {
pub fn as_str(&self) -> &'static str {
@@ -150,6 +154,9 @@ impl ErrorKind {
OpenSsh => "OpenSSH Error",
Zram => "Zram Error",
Lshw => "LSHW Error",
CpuSettings => "CPU Settings Error",
Firmware => "Firmware Error",
Timeout => "Timeout Error",
}
}
}
@@ -235,19 +242,14 @@ impl From<ed25519_dalek::SignatureError> for Error {
Error::new(e, ErrorKind::InvalidSignature)
}
}
impl From<bollard::errors::Error> for Error {
fn from(e: bollard::errors::Error) -> Self {
Error::new(e, ErrorKind::Docker)
impl From<std::net::AddrParseError> for Error {
fn from(e: std::net::AddrParseError) -> Self {
Error::new(e, ErrorKind::ParseNetAddress)
}
}
impl From<torut::control::ConnError> for Error {
fn from(e: torut::control::ConnError) -> Self {
Error::new(eyre!("{:?}", e), ErrorKind::Tor)
}
}
impl From<std::net::AddrParseError> for Error {
fn from(e: std::net::AddrParseError) -> Self {
Error::new(e, ErrorKind::ParseNetAddress)
Error::new(e, ErrorKind::Tor)
}
}
impl From<ipnet::AddrParseError> for Error {
@@ -275,6 +277,28 @@ impl From<ssh_key::Error> for Error {
Error::new(e, ErrorKind::OpenSsh)
}
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self {
let kind = match e {
_ if e.is_builder() => ErrorKind::ParseUrl,
_ if e.is_decode() => ErrorKind::Deserialization,
_ => ErrorKind::Network,
};
Error::new(e, kind)
}
}
impl From<patch_db::value::Error> for Error {
fn from(value: patch_db::value::Error) -> Self {
match value.kind {
patch_db::value::ErrorKind::Serialization => {
Error::new(value.source, ErrorKind::Serialization)
}
patch_db::value::ErrorKind::Deserialization => {
Error::new(value.source, ErrorKind::Deserialization)
}
}
}
}
impl From<Error> for RpcError {
fn from(e: Error) -> Self {
@@ -388,6 +412,18 @@ where
}
}
pub trait OptionExt<T>
where
Self: Sized,
{
fn or_not_found(self, message: impl std::fmt::Display) -> Result<T, Error>;
}
impl<T> OptionExt<T> for Option<T> {
fn or_not_found(self, message: impl std::fmt::Display) -> Result<T, Error> {
self.ok_or_else(|| Error::new(eyre!("{}", message), ErrorKind::NotFound))
}
}
#[macro_export]
macro_rules! ensure_code {
($x:expr, $c:expr, $fmt:expr $(, $arg:expr)*) => {

0
libs/models/src/action_id.rs → libs/models/src/id/action.rs
View File

59
libs/models/src/id/address.rs Normal file
View File

@@ -0,0 +1,59 @@
use std::path::Path;
use serde::{Deserialize, Deserializer, Serialize};
use crate::Id;
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct AddressId(Id);
impl From<Id> for AddressId {
fn from(id: Id) -> Self {
Self(id)
}
}
impl std::fmt::Display for AddressId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl std::ops::Deref for AddressId {
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl AsRef<str> for AddressId {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de> Deserialize<'de> for AddressId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(AddressId(Deserialize::deserialize(deserializer)?))
}
}
impl AsRef<Path> for AddressId {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for AddressId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for AddressId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

0
libs/models/src/health_check_id.rs → libs/models/src/id/health_check.rs
View File

0
libs/models/src/image_id.rs → libs/models/src/id/image.rs
View File

19
libs/models/src/interface_id.rs → libs/models/src/id/interface.rs
View File

@@ -17,7 +17,7 @@ impl std::fmt::Display for InterfaceId {
}
}
impl std::ops::Deref for InterfaceId {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -40,3 +40,20 @@ impl AsRef<Path> for InterfaceId {
self.0.as_ref().as_ref()
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for InterfaceId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for InterfaceId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

0
libs/models/src/invalid_id.rs → libs/models/src/id/invalid_id.rs
View File

55
libs/models/src/id.rs → libs/models/src/id/mod.rs
View File

@@ -1,21 +1,37 @@
use std::borrow::Borrow;
use internment::ArcIntern;
use regex::Regex;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use yasi::InternedString;
use crate::invalid_id::InvalidId;
mod action;
mod address;
mod health_check;
mod image;
mod interface;
mod invalid_id;
mod package;
mod volume;
pub use action::ActionId;
pub use address::AddressId;
pub use health_check::HealthCheckId;
pub use image::ImageId;
pub use interface::InterfaceId;
pub use invalid_id::InvalidId;
pub use package::{PackageId, SYSTEM_PACKAGE_ID};
pub use volume::VolumeId;
lazy_static::lazy_static! {
static ref ID_REGEX: Regex = Regex::new("^[a-z]+(-[a-z]+)*$").unwrap();
pub static ref SYSTEM_ID: Id = Id(ArcIntern::from_ref("x_system"));
pub static ref SYSTEM_ID: Id = Id(InternedString::intern("x_system"));
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub struct Id(ArcIntern<String>);
impl TryFrom<ArcIntern<String>> for Id {
pub struct Id(InternedString);
impl TryFrom<InternedString> for Id {
type Error = InvalidId;
fn try_from(value: ArcIntern<String>) -> Result<Self, Self::Error> {
fn try_from(value: InternedString) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&*value) {
Ok(Id(value))
} else {
@@ -27,7 +43,7 @@ impl TryFrom<String> for Id {
type Error = InvalidId;
fn try_from(value: String) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&value) {
Ok(Id(ArcIntern::new(value)))
Ok(Id(InternedString::intern(value)))
} else {
Err(InvalidId)
}
@@ -37,14 +53,14 @@ impl TryFrom<&str> for Id {
type Error = InvalidId;
fn try_from(value: &str) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&value) {
Ok(Id(ArcIntern::from_ref(value)))
Ok(Id(InternedString::intern(value)))
} else {
Err(InvalidId)
}
}
}
impl std::ops::Deref for Id {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -69,7 +85,7 @@ impl<'de> Deserialize<'de> for Id {
where
D: Deserializer<'de>,
{
let unchecked: String = Deserialize::deserialize(deserializer)?;
let unchecked: InternedString = Deserialize::deserialize(deserializer)?;
Id::try_from(unchecked).map_err(serde::de::Error::custom)
}
}
@@ -78,6 +94,23 @@ impl Serialize for Id {
where
Ser: Serializer,
{
serializer.serialize_str(self.as_ref())
serializer.serialize_str(&*self)
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for Id {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for Id {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

19
libs/models/src/package_id.rs → libs/models/src/id/package.rs
View File

@@ -23,7 +23,7 @@ impl From<Id> for PackageId {
}
}
impl std::ops::Deref for PackageId {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -69,3 +69,20 @@ impl Serialize for PackageId {
Serialize::serialize(&self.0, serializer)
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for PackageId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for PackageId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

0
libs/models/src/volume_id.rs → libs/models/src/id/volume.rs
View File

18
libs/models/src/lib.rs
View File

@@ -1,23 +1,13 @@
mod action_id;
mod data_url;
mod errors;
mod health_check_id;
mod id;
mod image_id;
mod interface_id;
mod invalid_id;
mod package_id;
mod mime;
mod procedure_name;
mod version;
mod volume_id;
pub use action_id::*;
pub use data_url::*;
pub use errors::*;
pub use health_check_id::*;
pub use id::*;
pub use image_id::*;
pub use interface_id::*;
pub use invalid_id::*;
pub use package_id::*;
pub use mime::*;
pub use procedure_name::*;
pub use version::*;
pub use volume_id::*;

47
libs/models/src/mime.rs Normal file
View File

@@ -0,0 +1,47 @@
pub fn mime(extension: &str) -> Option<&'static str> {
match extension {
"apng" => Some("image/apng"),
"avif" => Some("image/avif"),
"flif" => Some("image/flif"),
"gif" => Some("image/gif"),
"jpg" | "jpeg" | "jfif" | "pjpeg" | "pjp" => Some("image/jpeg"),
"jxl" => Some("image/jxl"),
"png" => Some("image/png"),
"svg" => Some("image/svg+xml"),
"webp" => Some("image/webp"),
"mng" | "x-mng" => Some("image/x-mng"),
"css" => Some("text/css"),
"csv" => Some("text/csv"),
"html" => Some("text/html"),
"php" => Some("text/php"),
"plain" | "md" | "txt" => Some("text/plain"),
"xml" => Some("text/xml"),
"js" => Some("text/javascript"),
"wasm" => Some("application/wasm"),
_ => None,
}
}
pub fn unmime(mime: &str) -> Option<&'static str> {
match mime {
"image/apng" => Some("apng"),
"image/avif" => Some("avif"),
"image/flif" => Some("flif"),
"image/gif" => Some("gif"),
"jpg" | "jpeg" | "jfif" | "pjpeg" | "image/jpeg" => Some("pjp"),
"image/jxl" => Some("jxl"),
"image/png" => Some("png"),
"image/svg+xml" => Some("svg"),
"image/webp" => Some("webp"),
"mng" | "image/x-mng" => Some("x-mng"),
"text/css" => Some("css"),
"text/csv" => Some("csv"),
"text/html" => Some("html"),
"text/php" => Some("php"),
"plain" | "md" | "text/plain" => Some("txt"),
"text/xml" => Some("xml"),
"text/javascript" => Some("js"),
"application/wasm" => Some("wasm"),
_ => None,
}
}

4
libs/models/src/procedure_name.rs
View File

@@ -1,6 +1,8 @@
use serde::{Deserialize, Serialize};
use crate::{ActionId, HealthCheckId, PackageId};
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ProcedureName {
Main, // Usually just run container
CreateBackup,

4
libs/models/src/version.rs
View File

@@ -2,7 +2,6 @@ use std::hash::{Hash, Hasher};
use std::ops::Deref;
use std::str::FromStr;
use patch_db::{HasModel, Model};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Debug, Clone)]
@@ -105,6 +104,3 @@ impl Serialize for Version {
self.string.serialize(serializer)
}
}
impl HasModel for Version {
type Model = Model<Version>;
}

4
libs/snapshot_creator/Cargo.toml
View File

@@ -7,5 +7,5 @@ edition = "2021"
[dependencies]
dashmap = "5.3.4"
deno_core = "0.195.0"
deno_ast = { version = "0.27.2", features = ["transpiling"] }
deno_core = "=0.222.0"
deno_ast = { version = "=0.29.5", features = ["transpiling"] }

2
libs/snapshot_creator/src/main.rs
View File

@@ -1,7 +1,7 @@
use deno_core::JsRuntimeForSnapshot;
fn main() {
let runtime = JsRuntimeForSnapshot::new(Default::default(), Default::default());
let runtime = JsRuntimeForSnapshot::new(Default::default());
let snapshot = runtime.snapshot();
let snapshot_slice: &[u8] = &*snapshot;