Merge branch 'next' of github.com:Start9Labs/start-os into rebase/integration/refactors

This commit is contained in:
Aiden McClelland
2023-09-28 13:27:41 -06:00
167 changed files with 20082 additions and 7978 deletions

304
libs/Cargo.lock generated
View File

@@ -45,6 +45,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [
"cfg-if",
"getrandom 0.2.10",
"once_cell",
"version_check",
]
@@ -243,15 +244,6 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "bitmaps"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
dependencies = [
"typenum",
]
[[package]]
name = "bitmaps"
version = "3.2.0"
@@ -295,44 +287,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
[[package]]
name = "bollard"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5"
dependencies = [
"base64 0.13.1",
"bollard-stubs",
"bytes",
"futures-core",
"futures-util",
"hex",
"http",
"hyper",
"hyperlocal",
"log",
"pin-project-lite",
"serde",
"serde_derive",
"serde_json",
"serde_urlencoded",
"thiserror",
"tokio",
"tokio-util",
"url",
"winapi",
]
[[package]]
name = "bollard-stubs"
version = "1.42.0-rc.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864"
dependencies = [
"serde",
"serde_with",
]
[[package]]
name = "bumpalo"
version = "3.13.0"
@@ -411,7 +365,7 @@ dependencies = [
"indenter",
"once_cell",
"owo-colors",
"tracing-error 0.2.0",
"tracing-error",
]
[[package]]
@@ -423,7 +377,7 @@ dependencies = [
"once_cell",
"owo-colors",
"tracing-core",
"tracing-error 0.2.0",
"tracing-error",
]
[[package]]
@@ -560,41 +514,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "darling"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]]
name = "dashmap"
version = "5.5.0"
@@ -898,7 +817,7 @@ dependencies = [
"color-eyre",
"futures",
"helpers",
"imbl 2.0.0",
"imbl",
"nix 0.25.1",
"procfs",
"serde",
@@ -906,9 +825,9 @@ dependencies = [
"tokio",
"tokio-stream",
"tracing",
"tracing-error 0.2.0",
"tracing-error",
"tracing-futures",
"tracing-subscriber 0.3.17",
"tracing-subscriber",
"yajrc 0.1.0 (git+https://github.com/dr-bonez/yajrc.git?branch=develop)",
]
@@ -1259,8 +1178,14 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [
"ahash 0.7.6",
"ahash 0.8.3",
]
[[package]]
@@ -1282,15 +1207,6 @@ dependencies = [
"hashbrown 0.14.0",
]
[[package]]
name = "heck"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "heck"
version = "0.4.1"
@@ -1438,19 +1354,6 @@ dependencies = [
"tokio-native-tls",
]
[[package]]
name = "hyperlocal"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c"
dependencies = [
"futures-util",
"hex",
"hyper",
"pin-project",
"tokio",
]
[[package]]
name = "iana-time-zone"
version = "0.1.57"
@@ -1474,12 +1377,6 @@ dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.4.0"
@@ -1496,30 +1393,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
[[package]]
name = "imbl"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "543682c9082b25e63d03b5acbd65ad111fd49dd93e70843e5175db4ff81d606b"
dependencies = [
"bitmaps 2.1.0",
"rand_core 0.6.4",
"rand_xoshiro",
"sized-chunks",
"typenum",
"version_check",
]
[[package]]
name = "imbl"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2806b69cd9f4664844027b64465eacb444c67c1db9c778e341adff0c25cdb0d"
dependencies = [
"bitmaps 3.2.0",
"bitmaps",
"imbl-sized-chunks",
"rand_core 0.6.4",
"rand_xoshiro",
"serde",
"version_check",
]
@@ -1529,7 +1413,19 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6957ea0b2541c5ca561d3ef4538044af79f8a05a1eb3a3b148936aaceaa1076"
dependencies = [
"bitmaps 3.2.0",
"bitmaps",
]
[[package]]
name = "imbl-value"
version = "0.1.0"
source = "git+https://github.com/Start9Labs/imbl-value.git#929395141c3a882ac366c12ac9402d0ebaa2201b"
dependencies = [
"imbl",
"serde",
"serde_json",
"treediff",
"yasi",
]
[[package]]
@@ -1567,20 +1463,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "internment"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "161079c3ad892faa215fcfcf3fd7a6a3c9288df2b06a2c2bad7fbfad4f01d69d"
dependencies = [
"ahash 0.7.6",
"dashmap",
"hashbrown 0.12.3",
"once_cell",
"parking_lot 0.12.1",
"serde",
]
[[package]]
name = "io-lifetimes"
version = "1.0.11"
@@ -1670,9 +1552,9 @@ dependencies = [
name = "json-patch"
version = "0.2.7-alpha.0"
dependencies = [
"imbl-value",
"json-ptr",
"serde",
"serde_json",
"treediff",
]
@@ -1680,8 +1562,9 @@ dependencies = [
name = "json-ptr"
version = "0.1.0"
dependencies = [
"imbl",
"imbl-value",
"serde",
"serde_json",
"thiserror",
]
@@ -1895,6 +1778,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -1931,11 +1823,10 @@ dependencies = [
name = "models"
version = "0.1.0"
dependencies = [
"bollard",
"base64 0.21.2",
"color-eyre",
"ed25519-dalek",
"emver",
"internment",
"ipnet",
"lazy_static",
"mbrman",
@@ -1943,6 +1834,7 @@ dependencies = [
"patch-db",
"rand 0.8.5",
"regex",
"reqwest",
"rpc-toolkit",
"serde",
"serde_json",
@@ -1952,6 +1844,7 @@ dependencies = [
"tokio",
"torut",
"tracing",
"yasi",
]
[[package]]
@@ -1978,19 +1871,6 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
[[package]]
name = "nix"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
dependencies = [
"bitflags 1.3.2",
"cc",
"cfg-if",
"libc",
"memoffset",
]
[[package]]
name = "nix"
version = "0.24.3"
@@ -2000,7 +1880,7 @@ dependencies = [
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset",
"memoffset 0.6.5",
]
[[package]]
@@ -2013,7 +1893,20 @@ dependencies = [
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset",
"memoffset 0.6.5",
"pin-utils",
]
[[package]]
name = "nix"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
dependencies = [
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset 0.7.1",
"pin-utils",
]
@@ -2284,19 +2177,19 @@ dependencies = [
"async-trait",
"fd-lock-rs",
"futures",
"imbl 1.0.1",
"imbl",
"imbl-value",
"json-patch",
"json-ptr",
"lazy_static",
"nix 0.23.2",
"nix 0.26.4",
"patch-db-macro",
"serde",
"serde_cbor 0.11.1",
"serde_json",
"thiserror",
"tokio",
"tracing",
"tracing-error 0.1.2",
"tracing-error",
]
[[package]]
@@ -2312,7 +2205,7 @@ dependencies = [
name = "patch-db-macro-internals"
version = "0.1.0"
dependencies = [
"heck 0.3.3",
"heck",
"proc-macro2",
"quote",
"syn 1.0.109",
@@ -3087,28 +2980,6 @@ dependencies = [
"v8",
]
[[package]]
name = "serde_with"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff"
dependencies = [
"serde",
"serde_with_macros",
]
[[package]]
name = "serde_with_macros"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "sha-1"
version = "0.10.0"
@@ -3201,16 +3072,6 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]]
name = "sized-chunks"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
dependencies = [
"bitmaps 2.1.0",
"typenum",
]
[[package]]
name = "slab"
version = "0.4.8"
@@ -3370,7 +3231,7 @@ checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9"
dependencies = [
"dotenvy",
"either",
"heck 0.4.1",
"heck",
"hex",
"once_cell",
"proc-macro2",
@@ -3519,7 +3380,7 @@ version = "0.24.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
dependencies = [
"heck 0.4.1",
"heck",
"proc-macro2",
"quote",
"rustversion",
@@ -4138,16 +3999,6 @@ dependencies = [
"valuable",
]
[[package]]
name = "tracing-error"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24"
dependencies = [
"tracing",
"tracing-subscriber 0.2.25",
]
[[package]]
name = "tracing-error"
version = "0.2.0"
@@ -4155,7 +4006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
dependencies = [
"tracing",
"tracing-subscriber 0.3.17",
"tracing-subscriber",
]
[[package]]
@@ -4179,17 +4030,6 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71"
dependencies = [
"sharded-slab",
"thread_local",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.17"
@@ -4697,6 +4537,18 @@ dependencies = [
"thiserror",
]
[[package]]
name = "yasi"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f355ab62ebe30b758c1f4ab096a306722c4b7dbfb9d8c07d18c70d71a945588"
dependencies = [
"ahash 0.8.3",
"hashbrown 0.13.2",
"lazy_static",
"serde",
]
[[package]]
name = "zeroize"
version = "1.6.0"

View File

@@ -5,8 +5,8 @@ use std::process::Stdio;
use std::sync::Arc;
use embassy_container_init::{
LogParams, OutputParams, OutputStrategy, ProcessGroupId, ProcessId, ReadLineStderrParams,
ReadLineStdoutParams, RunCommandParams, SendSignalParams, SignalGroupParams,
LogParams, OutputParams, OutputStrategy, ProcessGroupId, ProcessId, RunCommandParams,
SendSignalParams, SignalGroupParams,
};
use futures::StreamExt;
use helpers::NonDetachingJoinHandle;
@@ -15,7 +15,7 @@ use nix::sys::signal::Signal;
use serde::{Deserialize, Serialize};
use serde_json::json;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, ChildStderr, ChildStdout, Command};
use tokio::process::{Child, Command};
use tokio::select;
use tokio::sync::{watch, Mutex};
use yajrc::{Id, RpcError};
@@ -103,7 +103,7 @@ impl Handler {
// Input::ReadLineStderr(ReadLineStderrParams { pid }) => {
// Output::ReadLineStderr(self.read_line_stderr(pid).await?)
// }
Input::Log(LogParams { gid, level }) => {
Input::Log(LogParams { gid: _, level }) => {
level.trace();
Output::Log
}
@@ -363,23 +363,23 @@ async fn main() {
let req = serde_json::from_str::<IncomingRpc>(&line?)?;
match handler.handle(req.input).await {
Ok(output) => {
if let Err(err) = w.lock().await.write_all(
if w.lock().await.write_all(
format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "result": output }))
.as_bytes(),
)
.await {
.await.is_err() {
tracing::error!("Error sending to {id:?}", id = req.id);
}
}
Err(e) =>
if let Err(err) = w
if w
.lock()
.await
.write_all(
format!("{}\n", json!({ "id": req.id, "jsonrpc": "2.0", "error": e }))
.as_bytes(),
)
.await {
.await.is_err() {
tracing::error!("Handle + Error sending to {id:?}", id = req.id);
},

View File

@@ -77,6 +77,18 @@ impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
NonDetachingJoinHandle(t)
}
}
impl<T> Deref for NonDetachingJoinHandle<T> {
type Target = JoinHandle<T>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> DerefMut for NonDetachingJoinHandle<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[pin_project::pinned_drop]
impl<T> PinnedDrop for NonDetachingJoinHandle<T> {
fn drop(self: std::pin::Pin<&mut Self>) {
@@ -94,17 +106,6 @@ impl<T> Future for NonDetachingJoinHandle<T> {
this.0.poll(cx)
}
}
impl<T> Deref for NonDetachingJoinHandle<T> {
type Target = JoinHandle<T>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> DerefMut for NonDetachingJoinHandle<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
pub struct AtomicFile {
tmp_path: PathBuf,

View File

@@ -8,8 +8,8 @@ edition = "2021"
[dependencies]
async-trait = "0.1.56"
dashmap = "5.3.4"
deno_core = "0.195.0"
deno_ast = { version = "0.27.2", features = ["transpiling"] }
deno_core = "=0.195.0"
deno_ast = { version = "=0.27.2", features = ["transpiling"] }
embassy_container_init = { path = "../embassy_container_init" }
reqwest = { version = "0.11.11" }
sha2 = "0.10.2"

View File

@@ -158,17 +158,17 @@ impl ModuleLoader for ModsLoader {
"file:///deno_global.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
FastString::Static("const old_deno = Deno; Deno = null; export default old_deno"),
&*DENO_GLOBAL_JS,
&DENO_GLOBAL_JS,
)),
"file:///loadModule.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
FastString::Static(include_str!("./artifacts/loadModule.js")),
&*LOAD_MODULE_JS,
&LOAD_MODULE_JS,
)),
"file:///embassy.js" => Ok(ModuleSource::new(
ModuleType::JavaScript,
self.code.0.clone().into(),
&*EMBASSY_JS,
&EMBASSY_JS,
)),
x => Err(anyhow!("Not allowed to import: {}", x)),
@@ -368,7 +368,7 @@ impl JsExecutionEnvironment {
.ops(Self::declarations())
.state(move |state| {
state.put(ext_answer_state.clone());
state.put(js_ctx.clone());
state.put(js_ctx);
})
.build();
let loader = std::rc::Rc::new(self.module_loader.clone());
@@ -650,7 +650,7 @@ mod fns {
}
let path_in = path_in.strip_prefix("/").unwrap_or(&path_in);
let new_file = volume_path.join(&path_in);
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
@@ -1082,11 +1082,9 @@ mod fns {
let volume_path = {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
ctx.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
volume_path
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?
};
let path_in = path_in.strip_prefix("/").unwrap_or(&path_in);
let new_file = volume_path.join(path_in);
@@ -1157,8 +1155,7 @@ mod fns {
.stdout,
)?
.lines()
.skip(1)
.next()
.nth(1)
.unwrap_or_default()
.parse()?;
let used = String::from_utf8(
@@ -1188,8 +1185,7 @@ mod fns {
.stdout,
)?
.lines()
.skip(1)
.next()
.nth(1)
.unwrap_or_default()
.split_ascii_whitespace()
.next_tuple()
@@ -1310,16 +1306,22 @@ mod fns {
}
#[op]
async fn log_info(state: Rc<RefCell<OpState>>, input: String) -> Result<(), AnyError> {
let ctx = {
let (container_rpc_client, container_process_gid, package_id, run_function) = {
let state = state.borrow();
state.borrow::<JsContext>().clone()
let ctx: JsContext = state.borrow::<JsContext>().clone();
(
ctx.container_rpc_client,
ctx.container_process_gid,
ctx.package_id,
ctx.run_function,
)
};
if let Some(rpc_client) = ctx.container_rpc_client {
if let Some(rpc_client) = container_rpc_client {
return rpc_client
.request(
embassy_container_init::Log,
embassy_container_init::LogParams {
gid: Some(ctx.container_process_gid),
gid: Some(container_process_gid),
level: embassy_container_init::LogLevel::Info(input),
},
)
@@ -1327,8 +1329,8 @@ mod fns {
.map_err(|e| anyhow!("{}: {:?}", e.message, e.data));
}
tracing::info!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
package_id = tracing::field::display(&package_id),
run_function = tracing::field::display(&run_function),
"{}",
input
);

View File

@@ -6,7 +6,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bollard = "0.13.0"
base64 = "0.21.0"
color-eyre = "0.6.1"
ed25519-dalek = { version = "1.0.1", features = ["serde"] }
lazy_static = "1.4"
@@ -14,7 +14,6 @@ mbrman = "0.5.0"
emver = { version = "0.1", git = "https://github.com/Start9Labs/emver-rs.git", features = [
"serde",
] }
internment = { version = "0.7.0", features = ["arc", "serde"] }
ipnet = "2.7.1"
openssl = { version = "0.10.41", features = ["vendored"] }
patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
@@ -22,6 +21,7 @@ patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
] }
rand = "0.8"
regex = "1.7.1"
reqwest = "0.11.14"
rpc-toolkit = "0.2.1"
serde = { version = "1.0", features = ["derive", "rc"] }
serde_json = "1.0"
@@ -36,3 +36,4 @@ thiserror = "1.0"
tokio = { version = "1", features = ["full"] }
torut = "0.2.1"
tracing = "0.1.35"
yasi = "0.1.2"

171
libs/models/src/data_url.rs Normal file
View File

@@ -0,0 +1,171 @@
use std::borrow::Cow;
use std::path::Path;
use base64::Engine;
use color_eyre::eyre::eyre;
use reqwest::header::CONTENT_TYPE;
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncRead, AsyncReadExt};
use yasi::InternedString;
use crate::{mime, Error, ErrorKind, ResultExt};
#[derive(Clone)]
pub struct DataUrl<'a> {
mime: InternedString,
data: Cow<'a, [u8]>,
}
impl<'a> DataUrl<'a> {
pub const DEFAULT_MIME: &'static str = "application/octet-stream";
pub const MAX_SIZE: u64 = 100 * 1024;
// data:{mime};base64,{data}
pub fn to_string(&self) -> String {
use std::fmt::Write;
let mut res = String::with_capacity(self.data_url_len_without_mime() + self.mime.len());
let _ = write!(res, "data:{};base64,", self.mime);
base64::engine::general_purpose::STANDARD.encode_string(&self.data, &mut res);
res
}
fn data_url_len_without_mime(&self) -> usize {
5 + 8 + (4 * self.data.len() / 3) + 3
}
pub fn data_url_len(&self) -> usize {
self.data_url_len_without_mime() + self.mime.len()
}
pub fn from_slice(mime: &str, data: &'a [u8]) -> Self {
Self {
mime: InternedString::intern(mime),
data: Cow::Borrowed(data),
}
}
}
impl DataUrl<'static> {
pub async fn from_reader(
mime: &str,
rdr: impl AsyncRead + Unpin,
size_est: Option<u64>,
) -> Result<Self, Error> {
let check_size = |s| {
if s > Self::MAX_SIZE {
Err(Error::new(
eyre!("Data URLs must be smaller than 100KiB"),
ErrorKind::Filesystem,
))
} else {
Ok(s)
}
};
let mut buf = size_est
.map(check_size)
.transpose()?
.map(|s| Vec::with_capacity(s as usize))
.unwrap_or_default();
rdr.take(Self::MAX_SIZE + 1).read_to_end(&mut buf).await?;
check_size(buf.len() as u64)?;
Ok(Self {
mime: InternedString::intern(mime),
data: Cow::Owned(buf),
})
}
pub async fn from_path(path: impl AsRef<Path>) -> Result<Self, Error> {
let path = path.as_ref();
let f = tokio::fs::File::open(path).await?;
let m = f.metadata().await?;
let mime = path
.extension()
.and_then(|s| s.to_str())
.and_then(mime)
.unwrap_or(Self::DEFAULT_MIME);
Self::from_reader(mime, f, Some(m.len())).await
}
pub async fn from_response(res: reqwest::Response) -> Result<Self, Error> {
let mime = InternedString::intern(
res.headers()
.get(CONTENT_TYPE)
.and_then(|h| h.to_str().ok())
.unwrap_or(Self::DEFAULT_MIME),
);
let data = res.bytes().await.with_kind(ErrorKind::Network)?.to_vec();
Ok(Self {
mime,
data: Cow::Owned(data),
})
}
pub fn from_vec(mime: &str, data: Vec<u8>) -> Self {
Self {
mime: InternedString::intern(mime),
data: Cow::Owned(data),
}
}
}
impl<'a> std::fmt::Debug for DataUrl<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.to_string())
}
}
impl<'de> Deserialize<'de> for DataUrl<'static> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = DataUrl<'static>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a valid base64 data url")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
v.strip_prefix("data:")
.and_then(|v| v.split_once(";base64,"))
.and_then(|(mime, data)| {
Some(DataUrl {
mime: InternedString::intern(mime),
data: Cow::Owned(
base64::engine::general_purpose::STANDARD
.decode(data)
.ok()?,
),
})
})
.ok_or_else(|| {
E::invalid_value(serde::de::Unexpected::Str(v), &"a valid base64 data url")
})
}
}
deserializer.deserialize_any(Visitor)
}
}
impl<'a> Serialize for DataUrl<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
#[test]
fn doesnt_reallocate() {
let random: [u8; 10] = rand::random();
for i in 0..10 {
let icon = DataUrl {
mime: InternedString::intern("png"),
data: Cow::Borrowed(&random[..i]),
};
assert_eq!(dbg!(icon.to_string()).capacity(), icon.data_url_len());
}
}

View File

@@ -3,6 +3,7 @@ use std::fmt::Display;
use color_eyre::eyre::eyre;
use patch_db::Revision;
use rpc_toolkit::hyper::http::uri::InvalidUri;
use rpc_toolkit::reqwest;
use rpc_toolkit::yajrc::RpcError;
use crate::InvalidId;
@@ -235,19 +236,14 @@ impl From<ed25519_dalek::SignatureError> for Error {
Error::new(e, ErrorKind::InvalidSignature)
}
}
impl From<bollard::errors::Error> for Error {
fn from(e: bollard::errors::Error) -> Self {
Error::new(e, ErrorKind::Docker)
impl From<std::net::AddrParseError> for Error {
fn from(e: std::net::AddrParseError) -> Self {
Error::new(e, ErrorKind::ParseNetAddress)
}
}
impl From<torut::control::ConnError> for Error {
fn from(e: torut::control::ConnError) -> Self {
Error::new(eyre!("{:?}", e), ErrorKind::Tor)
}
}
impl From<std::net::AddrParseError> for Error {
fn from(e: std::net::AddrParseError) -> Self {
Error::new(e, ErrorKind::ParseNetAddress)
Error::new(e, ErrorKind::Tor)
}
}
impl From<ipnet::AddrParseError> for Error {
@@ -275,6 +271,28 @@ impl From<ssh_key::Error> for Error {
Error::new(e, ErrorKind::OpenSsh)
}
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self {
let kind = match e {
_ if e.is_builder() => ErrorKind::ParseUrl,
_ if e.is_decode() => ErrorKind::Deserialization,
_ => ErrorKind::Network,
};
Error::new(e, kind)
}
}
impl From<patch_db::value::Error> for Error {
fn from(value: patch_db::value::Error) -> Self {
match value.kind {
patch_db::value::ErrorKind::Serialization => {
Error::new(value.source, ErrorKind::Serialization)
}
patch_db::value::ErrorKind::Deserialization => {
Error::new(value.source, ErrorKind::Deserialization)
}
}
}
}
impl From<Error> for RpcError {
fn from(e: Error) -> Self {
@@ -388,6 +406,18 @@ where
}
}
pub trait OptionExt<T>
where
Self: Sized,
{
fn or_not_found(self, message: impl std::fmt::Display) -> Result<T, Error>;
}
impl<T> OptionExt<T> for Option<T> {
fn or_not_found(self, message: impl std::fmt::Display) -> Result<T, Error> {
self.ok_or_else(|| Error::new(eyre!("{}", message), ErrorKind::NotFound))
}
}
#[macro_export]
macro_rules! ensure_code {
($x:expr, $c:expr, $fmt:expr $(, $arg:expr)*) => {

View File

@@ -0,0 +1,59 @@
use std::path::Path;
use serde::{Deserialize, Deserializer, Serialize};
use crate::Id;
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct AddressId(Id);
impl From<Id> for AddressId {
fn from(id: Id) -> Self {
Self(id)
}
}
impl std::fmt::Display for AddressId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl std::ops::Deref for AddressId {
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl AsRef<str> for AddressId {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de> Deserialize<'de> for AddressId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(AddressId(Deserialize::deserialize(deserializer)?))
}
}
impl AsRef<Path> for AddressId {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for AddressId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for AddressId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

View File

@@ -17,7 +17,7 @@ impl std::fmt::Display for InterfaceId {
}
}
impl std::ops::Deref for InterfaceId {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -40,3 +40,20 @@ impl AsRef<Path> for InterfaceId {
self.0.as_ref().as_ref()
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for InterfaceId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for InterfaceId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

View File

@@ -1,21 +1,37 @@
use std::borrow::Borrow;
use internment::ArcIntern;
use regex::Regex;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use yasi::InternedString;
use crate::invalid_id::InvalidId;
mod action;
mod address;
mod health_check;
mod image;
mod interface;
mod invalid_id;
mod package;
mod volume;
pub use action::ActionId;
pub use address::AddressId;
pub use health_check::HealthCheckId;
pub use image::ImageId;
pub use interface::InterfaceId;
pub use invalid_id::InvalidId;
pub use package::{PackageId, SYSTEM_PACKAGE_ID};
pub use volume::VolumeId;
lazy_static::lazy_static! {
static ref ID_REGEX: Regex = Regex::new("^[a-z]+(-[a-z]+)*$").unwrap();
pub static ref SYSTEM_ID: Id = Id(ArcIntern::from_ref("x_system"));
pub static ref SYSTEM_ID: Id = Id(InternedString::intern("x_system"));
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub struct Id(ArcIntern<String>);
impl TryFrom<ArcIntern<String>> for Id {
pub struct Id(InternedString);
impl TryFrom<InternedString> for Id {
type Error = InvalidId;
fn try_from(value: ArcIntern<String>) -> Result<Self, Self::Error> {
fn try_from(value: InternedString) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&*value) {
Ok(Id(value))
} else {
@@ -27,7 +43,7 @@ impl TryFrom<String> for Id {
type Error = InvalidId;
fn try_from(value: String) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&value) {
Ok(Id(ArcIntern::new(value)))
Ok(Id(InternedString::intern(value)))
} else {
Err(InvalidId)
}
@@ -37,14 +53,14 @@ impl TryFrom<&str> for Id {
type Error = InvalidId;
fn try_from(value: &str) -> Result<Self, Self::Error> {
if ID_REGEX.is_match(&value) {
Ok(Id(ArcIntern::from_ref(value)))
Ok(Id(InternedString::intern(value)))
} else {
Err(InvalidId)
}
}
}
impl std::ops::Deref for Id {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -69,7 +85,7 @@ impl<'de> Deserialize<'de> for Id {
where
D: Deserializer<'de>,
{
let unchecked: String = Deserialize::deserialize(deserializer)?;
let unchecked: InternedString = Deserialize::deserialize(deserializer)?;
Id::try_from(unchecked).map_err(serde::de::Error::custom)
}
}
@@ -78,6 +94,23 @@ impl Serialize for Id {
where
Ser: Serializer,
{
serializer.serialize_str(self.as_ref())
serializer.serialize_str(&*self)
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for Id {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for Id {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

View File

@@ -23,7 +23,7 @@ impl From<Id> for PackageId {
}
}
impl std::ops::Deref for PackageId {
type Target = String;
type Target = str;
fn deref(&self) -> &Self::Target {
&*self.0
}
@@ -69,3 +69,20 @@ impl Serialize for PackageId {
Serialize::serialize(&self.0, serializer)
}
}
impl<'q> sqlx::Encode<'q, sqlx::Postgres> for PackageId {
fn encode_by_ref(
&self,
buf: &mut <sqlx::Postgres as sqlx::database::HasArguments<'q>>::ArgumentBuffer,
) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<'q, sqlx::Postgres>>::encode_by_ref(&&**self, buf)
}
}
impl sqlx::Type<sqlx::Postgres> for PackageId {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}

View File

@@ -1,23 +1,13 @@
mod action_id;
mod data_url;
mod errors;
mod health_check_id;
mod id;
mod image_id;
mod interface_id;
mod invalid_id;
mod package_id;
mod mime;
mod procedure_name;
mod version;
mod volume_id;
pub use action_id::*;
pub use data_url::*;
pub use errors::*;
pub use health_check_id::*;
pub use id::*;
pub use image_id::*;
pub use interface_id::*;
pub use invalid_id::*;
pub use package_id::*;
pub use mime::*;
pub use procedure_name::*;
pub use version::*;
pub use volume_id::*;

47
libs/models/src/mime.rs Normal file
View File

@@ -0,0 +1,47 @@
pub fn mime(extension: &str) -> Option<&'static str> {
match extension {
"apng" => Some("image/apng"),
"avif" => Some("image/avif"),
"flif" => Some("image/flif"),
"gif" => Some("image/gif"),
"jpg" | "jpeg" | "jfif" | "pjpeg" | "pjp" => Some("image/jpeg"),
"jxl" => Some("image/jxl"),
"png" => Some("image/png"),
"svg" => Some("image/svg+xml"),
"webp" => Some("image/webp"),
"mng" | "x-mng" => Some("image/x-mng"),
"css" => Some("text/css"),
"csv" => Some("text/csv"),
"html" => Some("text/html"),
"php" => Some("text/php"),
"plain" | "md" | "txt" => Some("text/plain"),
"xml" => Some("text/xml"),
"js" => Some("text/javascript"),
"wasm" => Some("application/wasm"),
_ => None,
}
}
pub fn unmime(mime: &str) -> Option<&'static str> {
match mime {
"image/apng" => Some("apng"),
"image/avif" => Some("avif"),
"image/flif" => Some("flif"),
"image/gif" => Some("gif"),
"jpg" | "jpeg" | "jfif" | "pjpeg" | "image/jpeg" => Some("pjp"),
"image/jxl" => Some("jxl"),
"image/png" => Some("png"),
"image/svg+xml" => Some("svg"),
"image/webp" => Some("webp"),
"mng" | "image/x-mng" => Some("x-mng"),
"text/css" => Some("css"),
"text/csv" => Some("csv"),
"text/html" => Some("html"),
"text/php" => Some("php"),
"plain" | "md" | "text/plain" => Some("txt"),
"text/xml" => Some("xml"),
"text/javascript" => Some("js"),
"application/wasm" => Some("wasm"),
_ => None,
}
}

View File

@@ -2,7 +2,6 @@ use std::hash::{Hash, Hasher};
use std::ops::Deref;
use std::str::FromStr;
use patch_db::{HasModel, Model};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Debug, Clone)]
@@ -105,6 +104,3 @@ impl Serialize for Version {
self.string.serialize(serializer)
}
}
impl HasModel for Version {
type Model = Model<Version>;
}