From 9e03ac084e32ee440bf8732d61d9c47f0d91b49b Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Tue, 25 Jul 2023 16:22:58 -0600 Subject: [PATCH] add cli & rpc to edit db with jq syntax (#2372) * add cli & rpc to edit db with jq syntax * build fixes * fix build * fix build * update cargo.lock --- .github/workflows/startos-iso.yaml | 5 +- Makefile | 5 +- backend/Cargo.lock | 61 +++++++++++++++++++++ backend/Cargo.toml | 2 + backend/src/db/mod.rs | 85 +++++++++++++++++++++++++++++- system-images/compat/Cargo.lock | 61 +++++++++++++++++++++ 6 files changed, 211 insertions(+), 8 deletions(-) diff --git a/.github/workflows/startos-iso.yaml b/.github/workflows/startos-iso.yaml index 891c83fed..1ec009dfb 100644 --- a/.github/workflows/startos-iso.yaml +++ b/.github/workflows/startos-iso.yaml @@ -67,8 +67,8 @@ jobs: fromJson('{ "x86_64": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"], "x86_64-nonfree": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"], - "aarch64": ["buildjet-8vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"], - "aarch64-nonfree": ["buildjet-8vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"], + "aarch64": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"], + "aarch64-nonfree": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"], "raspberrypi": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"], }')[matrix.platform][github.event.inputs.platform == matrix.platform] ) @@ -77,6 +77,7 @@ jobs: steps: - name: Free space run: df -h && rm -rf /opt/hostedtoolcache* && df -h + if: ${{ github.event.inputs.runner != 'fast' }} - run: | sudo mount -t tmpfs tmpfs . diff --git a/Makefile b/Makefile index a7785eb87..a6d73e819 100644 --- a/Makefile +++ b/Makefile @@ -185,8 +185,5 @@ ui: frontend/dist/raw/ui # used by github actions backend: $(EMBASSY_BINS) -cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast: - ARCH=$(ARCH) ./build-cargo-dep.sh nc-broadcast - cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep: - ARCH=$(ARCH) ./build-cargo-dep.sh pi-beep \ No newline at end of file + ARCH=aarch64 ./build-cargo-dep.sh pi-beep \ No newline at end of file diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 24dd1d969..a20c56936 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -557,6 +557,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "chumsky" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "ciborium" version = "0.2.1" @@ -1236,6 +1245,12 @@ dependencies = [ "text_lines", ] +[[package]] +name = "dyn-clone" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" + [[package]] name = "ecdsa" version = "0.14.8" @@ -1881,6 +1896,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hifijson" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ef6b41c333e6dd2a4aaa59125a19b633cd17e7aaf372b2260809777bcdef4a" + [[package]] name = "hkdf" version = "0.12.3" @@ -2284,6 +2305,44 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jaq-core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab6f495d0fe97460255aef70fbd77d3b5a7701930163ad69c963b9c6d5d5726" +dependencies = [ + "ahash 0.7.6", + "dyn-clone", + "hifijson", + "indexmap 1.9.3", + "itertools 0.10.5", + "jaq-parse", + "log", + "once_cell", + "regex", + "serde_json", +] + +[[package]] +name = "jaq-parse" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0f97f01eb9e87af3cbcc843b0dfe693fc6b0a2b9093dc8980dd9fc682826b0" +dependencies = [ + "chumsky", + "serde", +] + +[[package]] +name = "jaq-std" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b261109851c8687bc55eab26e6d81e96f3fdab367e2d3d5706947c218ddaf22" +dependencies = [ + "bincode", + "jaq-parse", +] + [[package]] name = "josekit" version = "0.8.3" @@ -4639,6 +4698,8 @@ dependencies = [ "iprange", "isocountry", "itertools 0.10.5", + "jaq-core", + "jaq-std", "josekit", "js_engine", "jsonpath_lib", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 901466606..9179f33c3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -84,6 +84,8 @@ ipnet = { version = "2.7.1", features = ["serde"] } iprange = { version = "0.6.7", features = ["serde"] } isocountry = "0.3.2" itertools = "0.10.3" +jaq-core = "0.10.0" +jaq-std = "0.10.0" josekit = "0.8.1" js_engine = { path = '../libs/js_engine', optional = true } jsonpath_lib = "0.3.0" diff --git a/backend/src/db/mod.rs b/backend/src/db/mod.rs index f51c8d156..a8cc11d1f 100644 --- a/backend/src/db/mod.rs +++ b/backend/src/db/mod.rs @@ -4,9 +4,10 @@ pub mod package; use std::future::Future; use std::sync::Arc; +use color_eyre::eyre::eyre; use futures::{FutureExt, SinkExt, StreamExt}; use patch_db::json_ptr::JsonPointer; -use patch_db::{Dump, Revision}; +use patch_db::{DbHandle, Dump, LockType, Revision}; use rpc_toolkit::command; use rpc_toolkit::hyper::upgrade::Upgraded; use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response}; @@ -24,6 +25,7 @@ use tracing::instrument; pub use self::model::DatabaseModel; use crate::context::RpcContext; use crate::middleware::auth::{HasValidSession, HashSessionToken}; +use crate::util::display_none; use crate::util::serde::{display_serializable, IoFormat}; use crate::{Error, ResultExt}; @@ -163,7 +165,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request) -> Result Result<(), RpcError> { Ok(()) } @@ -199,6 +201,85 @@ pub async fn dump( Ok(ctx.db.dump().await?) } +fn apply_expr(input: jaq_core::Val, expr: &str) -> Result { + let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main()); + + let Some(expr) = expr else { + return Err(Error::new( + eyre!("Failed to parse expression: {:?}", errs), + crate::ErrorKind::InvalidRequest, + )); + }; + + let mut errs = Vec::new(); + + let mut defs = jaq_core::Definitions::core(); + for def in jaq_std::std() { + defs.insert(def, &mut errs); + } + + let filter = defs.finish(expr, Vec::new(), &mut errs); + + if !errs.is_empty() { + return Err(Error::new( + eyre!("Failed to compile expression: {:?}", errs), + crate::ErrorKind::InvalidRequest, + )); + }; + + let inputs = jaq_core::RcIter::new(std::iter::empty()); + let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input); + + let Some(res) = res_iter + .next() + .transpose() + .map_err(|e| eyre!("{e}")) + .with_kind(crate::ErrorKind::Deserialization)? + else { + return Err(Error::new( + eyre!("expr returned no results"), + crate::ErrorKind::InvalidRequest, + )); + }; + + if res_iter.next().is_some() { + return Err(Error::new( + eyre!("expr returned too many results"), + crate::ErrorKind::InvalidRequest, + )); + } + + Ok(res) +} + +#[command(display(display_none))] +pub async fn apply(#[context] ctx: RpcContext, #[arg] expr: String) -> Result<(), Error> { + let mut db = ctx.db.handle(); + + DatabaseModel::new().lock(&mut db, LockType::Write).await?; + + let root_ptr = JsonPointer::::default(); + + let input = db.get_value(&root_ptr, None).await?; + + let res = (|| { + let res = apply_expr(input.into(), &expr)?; + + serde_json::from_value::(res.clone().into()).with_ctx(|_| { + ( + crate::ErrorKind::Deserialization, + "result does not match database model", + ) + })?; + + Ok::(res.into()) + })()?; + + db.put_value(&root_ptr, &res).await?; + + Ok(()) +} + #[command(subcommands(ui))] pub fn put() -> Result<(), RpcError> { Ok(()) diff --git a/system-images/compat/Cargo.lock b/system-images/compat/Cargo.lock index fe2d17395..b278458a3 100644 --- a/system-images/compat/Cargo.lock +++ b/system-images/compat/Cargo.lock @@ -478,6 +478,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "chumsky" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d" +dependencies = [ + "hashbrown", +] + [[package]] name = "ciborium" version = "0.2.0" @@ -1089,6 +1098,12 @@ version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0" +[[package]] +name = "dyn-clone" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" + [[package]] name = "ecdsa" version = "0.14.8" @@ -1708,6 +1723,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hifijson" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ef6b41c333e6dd2a4aaa59125a19b633cd17e7aaf372b2260809777bcdef4a" + [[package]] name = "hkdf" version = "0.12.3" @@ -2079,6 +2100,44 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "jaq-core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb52eeac20f256459e909bd4a03bb8c4fab6a1fdbb8ed52d00f644152df48ece" +dependencies = [ + "ahash", + "dyn-clone", + "hifijson", + "indexmap", + "itertools 0.10.5", + "jaq-parse", + "log", + "once_cell", + "regex", + "serde_json", +] + +[[package]] +name = "jaq-parse" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0f97f01eb9e87af3cbcc843b0dfe693fc6b0a2b9093dc8980dd9fc682826b0" +dependencies = [ + "chumsky", + "serde", +] + +[[package]] +name = "jaq-std" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b261109851c8687bc55eab26e6d81e96f3fdab367e2d3d5706947c218ddaf22" +dependencies = [ + "bincode", + "jaq-parse", +] + [[package]] name = "josekit" version = "0.8.1" @@ -4176,6 +4235,8 @@ dependencies = [ "iprange", "isocountry", "itertools 0.10.5", + "jaq-core", + "jaq-std", "josekit", "jsonpath_lib", "lazy_static",