From 71a15cf2222410a18f2ad876d251d9d48ae71a37 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Fri, 9 Jun 2023 21:46:44 +0000 Subject: [PATCH] add diskUsage effect (#2297) --- backend/Cargo.lock | 1 + build/lib/depends | 2 + libs/Cargo.lock | 1 + libs/js_engine/Cargo.toml | 1 + libs/js_engine/src/artifacts/loadModule.js | 11 ++- libs/js_engine/src/lib.rs | 98 +++++++++++++++++++++- 6 files changed, 112 insertions(+), 2 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 78ec8ec5c..2cabc9371 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2439,6 +2439,7 @@ dependencies = [ "dprint-swc-ext", "embassy_container_init", "helpers", + "itertools 0.10.5", "models", "reqwest", "serde", diff --git a/build/lib/depends b/build/lib/depends index 4ca4ee3f6..bc02a173d 100644 --- a/build/lib/depends +++ b/build/lib/depends @@ -28,6 +28,7 @@ magic-wormhole ncdu net-tools network-manager +nvme-cli nyx openssh-server pgloader @@ -35,6 +36,7 @@ postgresql psmisc rsync samba-common-bin +smartmontools sqlite3 squashfs-tools systemd diff --git a/libs/Cargo.lock b/libs/Cargo.lock index 0b6c47c9f..e70d1b5f8 100644 --- a/libs/Cargo.lock +++ b/libs/Cargo.lock @@ -1647,6 +1647,7 @@ dependencies = [ "dprint-swc-ext", "embassy_container_init", "helpers", + "itertools 0.10.5", "models", "reqwest", "serde", diff --git a/libs/js_engine/Cargo.toml b/libs/js_engine/Cargo.toml index 582f3a258..d2f25bbc5 100644 --- a/libs/js_engine/Cargo.toml +++ b/libs/js_engine/Cargo.toml @@ -36,6 +36,7 @@ swc_macros_common = "=0.3.5" swc_visit = "=0.3.0" swc_visit_macros = "=0.3.1" sha2 = "0.10.2" +itertools = "0.10.5" models = { path = "../models" } helpers = { path = "../helpers" } serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/libs/js_engine/src/artifacts/loadModule.js b/libs/js_engine/src/artifacts/loadModule.js index 140d610c2..7dfd4a03e 100644 --- a/libs/js_engine/src/artifacts/loadModule.js +++ b/libs/js_engine/src/artifacts/loadModule.js @@ -185,6 +185,14 @@ const runRsync = ( } }; +const diskUsage = async ({ + volumeId = requireParam("volumeId"), + path = requireParam("path"), +} = { volumeId: null, path: null }) => { + const [used, total] = await Deno.core.opAsync("disk_usage", volumeId, path); + return { used, total } +} + const currentFunction = Deno.core.opSync("current_function"); const input = Deno.core.opSync("get_input"); const variable_args = Deno.core.opSync("get_variable_args"); @@ -213,7 +221,8 @@ const effects = { runDaemon, signalGroup, runRsync, - readDir + readDir, + diskUsage, }; const defaults = { diff --git a/libs/js_engine/src/lib.rs b/libs/js_engine/src/lib.rs index a4e290b52..b00f4c5eb 100644 --- a/libs/js_engine/src/lib.rs +++ b/libs/js_engine/src/lib.rs @@ -284,6 +284,7 @@ impl JsExecutionEnvironment { fns::create_dir::decl(), fns::remove_dir::decl(), fns::read_dir::decl(), + fns::disk_usage::decl(), fns::current_function::decl(), fns::log_trace::decl(), fns::log_warn::decl(), @@ -396,10 +397,12 @@ mod fns { SendSignal, SendSignalParams, SignalGroup, SignalGroupParams, }; use helpers::{to_tmp_path, AtomicFile, Rsync, RsyncOptions}; - use models::VolumeId; + use itertools::Itertools; + use models::{ErrorKind, VolumeId}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use tokio::io::AsyncWriteExt; + use tokio::process::Command; use super::{AnswerState, JsContext}; use crate::{system_time_as_unix_ms, MetadataJs, ResultType}; @@ -490,6 +493,7 @@ mod fns { .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))? }; //get_path_for in volume.rs + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); if !is_subset(&volume_path, &new_file).await? { bail!( @@ -515,6 +519,7 @@ mod fns { .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))? }; //get_path_for in volume.rs + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); if !is_subset(&volume_path, &new_file).await? { bail!( @@ -573,6 +578,7 @@ mod fns { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(&path_in); let parent_new_file = new_file .parent() @@ -629,6 +635,7 @@ mod fns { bail!("Volume {} is readonly", dst_volume); } + let src_path = src_path.strip_prefix("/").unwrap_or(&src_path); let old_file = volume_path.join(src_path); let parent_old_file = old_file .parent() @@ -642,6 +649,7 @@ mod fns { ); } + let dst_path = dst_path.strip_prefix("/").unwrap_or(&dst_path); let new_file = volume_path_out.join(dst_path); let parent_new_file = new_file .parent() @@ -689,6 +697,7 @@ mod fns { bail!("Volume {} is readonly", dst_volume); } + let src_path = src_path.strip_prefix("/").unwrap_or(&src_path); let src = volume_path.join(src_path); // With the volume check if !is_subset(&volume_path, &src).await? { @@ -702,6 +711,7 @@ mod fns { bail!("Source at {} does not exists", src.to_string_lossy()); } + let dst_path = src_path.strip_prefix("/").unwrap_or(&dst_path); let dst = volume_path_out.join(dst_path); // With the volume check if !is_subset(&volume_path_out, &dst).await? { @@ -776,6 +786,7 @@ mod fns { if volumes.readonly(&volume_id) { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check if !is_subset(&volume_path, &new_file).await? { @@ -806,6 +817,7 @@ mod fns { if volumes.readonly(&volume_id) { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check if !is_subset(&volume_path, &new_file).await? { @@ -836,6 +848,7 @@ mod fns { if volumes.readonly(&volume_id) { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check @@ -864,6 +877,7 @@ mod fns { .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?; volume_path }; + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check @@ -893,6 +907,86 @@ mod fns { Ok(paths) } + #[op] + async fn disk_usage( + state: Rc>, + volume_id: Option, + path_in: Option, + ) -> Result<(u64, u64), AnyError> { + let (base_path, volume_path) = { + let state = state.borrow(); + let ctx: &JsContext = state.borrow(); + let volume_path = if let Some(volume_id) = volume_id { + Some( + ctx.volumes + .path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id) + .ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?, + ) + } else { + None + }; + (ctx.datadir.join("package-data"), volume_path) + }; + let path = if let (Some(volume_path), Some(path_in)) = (volume_path, path_in) { + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); + Some(volume_path.join(path_in)) + } else { + None + }; + + if let Some(path) = path { + let size = String::from_utf8( + Command::new("df") + .arg("--output=size") + .arg("--block-size=1") + .arg(&base_path) + .stdout(std::process::Stdio::piped()) + .output() + .await? + .stdout, + )? + .lines() + .skip(1) + .next() + .unwrap_or_default() + .parse()?; + let used = String::from_utf8( + Command::new("du") + .arg("-s") + .arg("--block-size=1") + .arg(path) + .stdout(std::process::Stdio::piped()) + .output() + .await? + .stdout, + )? + .split_ascii_whitespace() + .next() + .unwrap_or_default() + .parse()?; + Ok((used, size)) + } else { + String::from_utf8( + Command::new("df") + .arg("--output=used,size") + .arg("--block-size=1") + .arg(&base_path) + .stdout(std::process::Stdio::piped()) + .output() + .await? + .stdout, + )? + .lines() + .skip(1) + .next() + .unwrap_or_default() + .split_ascii_whitespace() + .next_tuple() + .and_then(|(used, size)| Some((used.parse().ok()?, size.parse().ok()?))) + .ok_or_else(|| anyhow!("invalid output from df")) + } + } + #[op] fn current_function(state: &mut OpState) -> Result { let ctx = state.borrow::(); @@ -1216,6 +1310,7 @@ mod fns { if volumes.readonly(&volume_id) { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check if !is_subset(&volume_path, &new_file).await? { @@ -1265,6 +1360,7 @@ mod fns { if volumes.readonly(&volume_id) { bail!("Volume {} is readonly", volume_id); } + let path_in = path_in.strip_prefix("/").unwrap_or(&path_in); let new_file = volume_path.join(path_in); // With the volume check if !is_subset(&volume_path, &new_file).await? {