add cli & rpc to edit db with jq syntax (#2372)

* add cli & rpc to edit db with jq syntax

* build fixes

* fix build

* fix build

* update cargo.lock

.github/workflows/startos-iso.yaml

@@ -67,8 +67,8 @@ jobs:
             fromJson('{
               "x86_64": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
               "x86_64-nonfree": ["buildjet-32vcpu-ubuntu-2204", "buildjet-32vcpu-ubuntu-2204"],
-              "aarch64": ["buildjet-8vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "aarch64": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
-              "aarch64-nonfree": ["buildjet-8vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
+              "aarch64-nonfree": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
               "raspberrypi": ["buildjet-16vcpu-ubuntu-2204-arm", "buildjet-32vcpu-ubuntu-2204-arm"],
             }')[matrix.platform][github.event.inputs.platform == matrix.platform]
           )
@@ -77,6 +77,7 @@ jobs:
     steps:
       - name: Free space
         run: df -h && rm -rf /opt/hostedtoolcache* && df -h
+        if: ${{ github.event.inputs.runner != 'fast' }}
 
       - run: |
           sudo mount -t tmpfs tmpfs .

Makefile

@@ -185,8 +185,5 @@ ui: frontend/dist/raw/ui
 # used by github actions
 backend: $(EMBASSY_BINS)
 
-cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
-	ARCH=$(ARCH) ./build-cargo-dep.sh nc-broadcast
-
 cargo-deps/aarch64-unknown-linux-gnu/release/pi-beep:
-	ARCH=$(ARCH) ./build-cargo-dep.sh pi-beep
+	ARCH=aarch64 ./build-cargo-dep.sh pi-beep

backend/Cargo.lock

@@ -557,6 +557,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "chumsky"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d"
+dependencies = [
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "ciborium"
 version = "0.2.1"
@@ -1236,6 +1245,12 @@ dependencies = [
  "text_lines",
 ]
 
+[[package]]
+name = "dyn-clone"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272"
+
 [[package]]
 name = "ecdsa"
 version = "0.14.8"
@@ -1881,6 +1896,12 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hifijson"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85ef6b41c333e6dd2a4aaa59125a19b633cd17e7aaf372b2260809777bcdef4a"
+
 [[package]]
 name = "hkdf"
 version = "0.12.3"
@@ -2284,6 +2305,44 @@ version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
+[[package]]
+name = "jaq-core"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ab6f495d0fe97460255aef70fbd77d3b5a7701930163ad69c963b9c6d5d5726"
+dependencies = [
+ "ahash 0.7.6",
+ "dyn-clone",
+ "hifijson",
+ "indexmap 1.9.3",
+ "itertools 0.10.5",
+ "jaq-parse",
+ "log",
+ "once_cell",
+ "regex",
+ "serde_json",
+]
+
+[[package]]
+name = "jaq-parse"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f0f97f01eb9e87af3cbcc843b0dfe693fc6b0a2b9093dc8980dd9fc682826b0"
+dependencies = [
+ "chumsky",
+ "serde",
+]
+
+[[package]]
+name = "jaq-std"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b261109851c8687bc55eab26e6d81e96f3fdab367e2d3d5706947c218ddaf22"
+dependencies = [
+ "bincode",
+ "jaq-parse",
+]
+
 [[package]]
 name = "josekit"
 version = "0.8.3"
@@ -4639,6 +4698,8 @@ dependencies = [
  "iprange",
  "isocountry",
  "itertools 0.10.5",
+ "jaq-core",
+ "jaq-std",
  "josekit",
  "js_engine",
  "jsonpath_lib",

backend/Cargo.toml

@@ -84,6 +84,8 @@ ipnet = { version = "2.7.1", features = ["serde"] }
 iprange = { version = "0.6.7", features = ["serde"] }
 isocountry = "0.3.2"
 itertools = "0.10.3"
+jaq-core = "0.10.0"
+jaq-std = "0.10.0"
 josekit = "0.8.1"
 js_engine = { path = '../libs/js_engine', optional = true }
 jsonpath_lib = "0.3.0"

backend/src/db/mod.rs

@@ -4,9 +4,10 @@ pub mod package;
 use std::future::Future;
 use std::sync::Arc;
 
+use color_eyre::eyre::eyre;
 use futures::{FutureExt, SinkExt, StreamExt};
 use patch_db::json_ptr::JsonPointer;
-use patch_db::{Dump, Revision};
+use patch_db::{DbHandle, Dump, LockType, Revision};
 use rpc_toolkit::command;
 use rpc_toolkit::hyper::upgrade::Upgraded;
 use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
@@ -24,6 +25,7 @@ use tracing::instrument;
 pub use self::model::DatabaseModel;
 use crate::context::RpcContext;
 use crate::middleware::auth::{HasValidSession, HashSessionToken};
+use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::{Error, ResultExt};
 
@@ -163,7 +165,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
     Ok(res)
 }
 
-#[command(subcommands(revisions, dump, put))]
+#[command(subcommands(revisions, dump, put, apply))]
 pub fn db() -> Result<(), RpcError> {
     Ok(())
 }
@@ -199,6 +201,85 @@ pub async fn dump(
     Ok(ctx.db.dump().await?)
 }
 
+fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
+    let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
+
+    let Some(expr) = expr else {
+        return Err(Error::new(
+            eyre!("Failed to parse expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let mut errs = Vec::new();
+
+    let mut defs = jaq_core::Definitions::core();
+    for def in jaq_std::std() {
+        defs.insert(def, &mut errs);
+    }
+
+    let filter = defs.finish(expr, Vec::new(), &mut errs);
+
+    if !errs.is_empty() {
+        return Err(Error::new(
+            eyre!("Failed to compile expression: {:?}", errs),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    let inputs = jaq_core::RcIter::new(std::iter::empty());
+    let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
+
+    let Some(res) = res_iter
+        .next()
+        .transpose()
+        .map_err(|e| eyre!("{e}"))
+        .with_kind(crate::ErrorKind::Deserialization)?
+    else {
+        return Err(Error::new(
+            eyre!("expr returned no results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    };
+
+    if res_iter.next().is_some() {
+        return Err(Error::new(
+            eyre!("expr returned too many results"),
+            crate::ErrorKind::InvalidRequest,
+        ));
+    }
+
+    Ok(res)
+}
+
+#[command(display(display_none))]
+pub async fn apply(#[context] ctx: RpcContext, #[arg] expr: String) -> Result<(), Error> {
+    let mut db = ctx.db.handle();
+
+    DatabaseModel::new().lock(&mut db, LockType::Write).await?;
+
+    let root_ptr = JsonPointer::<String>::default();
+
+    let input = db.get_value(&root_ptr, None).await?;
+
+    let res = (|| {
+        let res = apply_expr(input.into(), &expr)?;
+
+        serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
+            (
+                crate::ErrorKind::Deserialization,
+                "result does not match database model",
+            )
+        })?;
+
+        Ok::<serde_json::Value, Error>(res.into())
+    })()?;
+
+    db.put_value(&root_ptr, &res).await?;
+
+    Ok(())
+}
+
 #[command(subcommands(ui))]
 pub fn put() -> Result<(), RpcError> {
     Ok(())

system-images/compat/Cargo.lock

@@ -478,6 +478,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "chumsky"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d"
+dependencies = [
+ "hashbrown",
+]
+
 [[package]]
 name = "ciborium"
 version = "0.2.0"
@@ -1089,6 +1098,12 @@ version = "0.15.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0"
 
+[[package]]
+name = "dyn-clone"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272"
+
 [[package]]
 name = "ecdsa"
 version = "0.14.8"
@@ -1708,6 +1723,12 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hifijson"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85ef6b41c333e6dd2a4aaa59125a19b633cd17e7aaf372b2260809777bcdef4a"
+
 [[package]]
 name = "hkdf"
 version = "0.12.3"
@@ -2079,6 +2100,44 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
+[[package]]
+name = "jaq-core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb52eeac20f256459e909bd4a03bb8c4fab6a1fdbb8ed52d00f644152df48ece"
+dependencies = [
+ "ahash",
+ "dyn-clone",
+ "hifijson",
+ "indexmap",
+ "itertools 0.10.5",
+ "jaq-parse",
+ "log",
+ "once_cell",
+ "regex",
+ "serde_json",
+]
+
+[[package]]
+name = "jaq-parse"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f0f97f01eb9e87af3cbcc843b0dfe693fc6b0a2b9093dc8980dd9fc682826b0"
+dependencies = [
+ "chumsky",
+ "serde",
+]
+
+[[package]]
+name = "jaq-std"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b261109851c8687bc55eab26e6d81e96f3fdab367e2d3d5706947c218ddaf22"
+dependencies = [
+ "bincode",
+ "jaq-parse",
+]
+
 [[package]]
 name = "josekit"
 version = "0.8.1"
@@ -4176,6 +4235,8 @@ dependencies = [
  "iprange",
  "isocountry",
  "itertools 0.10.5",
+ "jaq-core",
+ "jaq-std",
  "josekit",
  "jsonpath_lib",
  "lazy_static",