Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-26 02:11:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Chore/version 0 3 1 0 (#1475)

Browse Source 
* feat: move over to workspaces

* chore: Move to libs

* chore:fix(build): Compat

* chore: fixing pr
This commit is contained in:
J M
2022-06-01 10:22:00 -06:00
committed by GitHub
parent 37344f99a7
commit b8751e7add
49 changed files with 4586 additions and 1837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Makefile
View File

@@ -1,7 +1,7 @@
EMBASSY_BINS := backend/target/aarch64-unknown-linux-gnu/release/embassyd backend/target/aarch64-unknown-linux-gnu/release/embassy-init backend/target/aarch64-unknown-linux-gnu/release/embassy-cli backend/target/aarch64-unknown-linux-gnu/release/embassy-sdk
EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui
EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build)
EMBASSY_V8_SNAPSHOTS := backend/src/procedure/js_scripts/ARM_JS_SNAPSHOT.bin
EMBASSY_V8_SNAPSHOTS := libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
COMPAT_SRC := $(shell find system-images/compat/src)
UTILS_SRC := $(shell find system-images/utils/Dockerfile)
BACKEND_SRC := $(shell find backend/src) $(shell find patch-db/*/src) $(shell find rpc-toolkit/*/src) backend/Cargo.toml backend/Cargo.lock
@@ -29,6 +29,10 @@ clean:
rm -rf patch-db/client/node_modules
rm -rf patch-db/client/dist
rm libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin
rm libs/js_engine/src/artifacts/JS_SNAPSHOT.bin
touch libs/snapshot-creator/Cargo.toml
eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar $(EMBASSY_V8_SNAPSHOTS)
! test -f eos.img || rm eos.img
if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi
@@ -51,10 +55,11 @@ product_key.txt:
if [ "$(KEY)" != "" ]; then $(shell which echo) -n "$(KEY)" > product_key.txt; fi
echo >> product_key.txt
$(EMBASSY_V8_SNAPSHOTS): $(BACKEND_SRC) $(EMBASSY_BINS)
cd backend && ./build-arm-v8-snapshot.sh
$(EMBASSY_V8_SNAPSHOTS): libs/snapshot-creator/Cargo.toml
cd libs/ && ./build-v8-snapshot.sh
cd libs/ && ./build-arm-v8-snapshot.sh
$(EMBASSY_BINS): $(BACKEND_SRC)
$(EMBASSY_BINS): $(BACKEND_SRC) $(EMBASSY_V8_SNAPSHOTS)
cd backend && ./build-prod.sh
frontend/node_modules: frontend/package.json

244
backend/Cargo.lock generated
View File

@@ -347,7 +347,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"thiserror",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-util 0.6.10",
"url",
"winapi",
@@ -806,16 +806,15 @@ dependencies = [
[[package]]
name = "deno_ast"
version = "0.14.1"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0691d08faa4f1fd9898ecfbe82c32623f89969e92e7b97b5119a18399609d25"
checksum = "a2989afff97ba7da10f186e9a45e946b4ef943b9d4babd2ee7b4b24cc9906b69"
dependencies = [
"anyhow",
"base64 0.13.0",
"data-url",
"dprint-swc-ext",
"serde",
"swc_atoms",
"swc_common",
"swc_ecmascript",
"text_lines",
"url",
@@ -823,9 +822,9 @@ dependencies = [
[[package]]
name = "deno_core"
version = "0.135.0"
version = "0.136.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cd837520179a6f8063fe542b98dacec14b44ce647990be476b6eca8e6125e5"
checksum = "07ced67ffe84c64aee6e84e40558835752b6f12807f84d15da8f5954e2b670c5"
dependencies = [
"anyhow",
"deno_ops",
@@ -846,9 +845,9 @@ dependencies = [
[[package]]
name = "deno_ops"
version = "0.13.0"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ab6a5a7c3d5b9fbd43064996bbe61799db5e0bfb0f46672b2f85c0192d15a9"
checksum = "05520711837dd592d2861319ea3cf2dfd81e39bb92e41758ee9172f3623daebd"
dependencies = [
"proc-macro-crate",
"proc-macro2 1.0.39",
@@ -944,6 +943,21 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dprint-swc-ext"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df529037ff02f1c43ae8c6cce54d9ad85546ff89cb5c1988f56130c16e16a48"
dependencies = [
"bumpalo",
"num-bigint",
"rustc-hash",
"swc_atoms",
"swc_common",
"swc_ecmascript",
"text_lines",
]
[[package]]
name = "ed25519"
version = "1.5.2"
@@ -994,8 +1008,6 @@ dependencies = [
"clap",
"color-eyre",
"cookie_store",
"deno_ast",
"deno_core",
"digest 0.9.0",
"divrem",
"ed25519-dalek",
@@ -1003,6 +1015,7 @@ dependencies = [
"fd-lock-rs",
"futures",
"git-version",
"helpers",
"hex",
"hmac",
"http",
@@ -1012,10 +1025,12 @@ dependencies = [
"indexmap",
"isocountry",
"itertools 0.10.3",
"js_engine",
"jsonpath_lib",
"lazy_static",
"libc",
"log",
"models",
"nix 0.23.1",
"nom 7.1.1",
"num",
@@ -1047,7 +1062,7 @@ dependencies = [
"stderrlog",
"tar",
"thiserror",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-compat-02",
"tokio-stream",
"tokio-tar",
@@ -1449,7 +1464,7 @@ dependencies = [
"http",
"indexmap",
"slab",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-util 0.7.2",
"tracing",
]
@@ -1502,6 +1517,15 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "helpers"
version = "0.1.0"
dependencies = [
"models",
"pin-project",
"tokio 1.15.0",
]
[[package]]
name = "hermit-abi"
version = "0.1.19"
@@ -1594,7 +1618,7 @@ dependencies = [
"itoa 1.0.2",
"pin-project-lite 0.2.9",
"socket2",
"tokio 1.18.2",
"tokio 1.15.0",
"tower-service",
"tracing",
"want",
@@ -1609,7 +1633,7 @@ dependencies = [
"bytes 1.1.0",
"hyper",
"native-tls",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-native-tls",
]
@@ -1625,7 +1649,7 @@ dependencies = [
"hyper",
"log",
"sha-1 0.10.0",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-tungstenite",
]
@@ -1639,7 +1663,7 @@ dependencies = [
"hex",
"hyper",
"pin-project",
"tokio 1.18.2",
"tokio 1.15.0",
]
[[package]]
@@ -1782,6 +1806,21 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "js_engine"
version = "0.1.0"
dependencies = [
"dashmap",
"deno_ast",
"deno_core",
"helpers",
"models",
"serde",
"serde_json",
"tokio 1.15.0",
"tracing",
]
[[package]]
name = "json-patch"
version = "0.2.7-alpha.0"
@@ -2064,14 +2103,35 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.3"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799"
checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys",
"miow",
"ntapi",
"winapi",
]
[[package]]
name = "miow"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
dependencies = [
"winapi",
]
[[package]]
name = "models"
version = "0.1.0"
dependencies = [
"emver",
"patch-db",
"rand 0.7.3",
"serde",
"thiserror",
]
[[package]]
@@ -2156,6 +2216,15 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "ntapi"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
dependencies = [
"winapi",
]
[[package]]
name = "num"
version = "0.4.0"
@@ -2441,7 +2510,7 @@ dependencies = [
"serde_cbor 0.11.1",
"serde_json",
"thiserror",
"tokio 1.18.2",
"tokio 1.15.0",
"tracing",
"tracing-error",
]
@@ -2946,7 +3015,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-native-tls",
"tokio-socks",
"tokio-util 0.6.10",
@@ -3009,7 +3078,7 @@ dependencies = [
"serde_cbor 0.11.2",
"serde_json",
"thiserror",
"tokio 1.18.2",
"tokio 1.15.0",
"url",
"yajrc",
]
@@ -3260,9 +3329,9 @@ dependencies = [
[[package]]
name = "serde_v8"
version = "0.46.0"
version = "0.47.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7797d56c9575ced9175e22366e5bd4cc8f3d571cd8c75be510f410ab97a54f6"
checksum = "3ca1daa2506c9d62744fff84d3534192f2e1c70cdf3bed95f298d89156c00b06"
dependencies = [
"bytes 1.1.0",
"derive_more",
@@ -3588,7 +3657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae"
dependencies = [
"once_cell",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-rustls",
]
@@ -3754,9 +3823,9 @@ dependencies = [
[[package]]
name = "swc_common"
version = "0.17.25"
version = "0.18.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "766ad22c1cb8586c038ccba7371a4903a6074b53ee4ba8980a52f502413f120e"
checksum = "e4516bf4969a924bfd1801aed5c4b214687665898c14b7584d227827faff9d6c"
dependencies = [
"ahash",
"ast_node",
@@ -3780,13 +3849,40 @@ dependencies = [
]
[[package]]
name = "swc_ecma_ast"
version = "0.75.0"
name = "swc_config"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72961898fbe56591997e667a1ec6a268383582810351c279a15ec710b6177d33"
checksum = "b8bb05ef56c14b95dd7e62e95960153af811b9a447287f1f6ca59f1337fb83d4"
dependencies = [
"anyhow",
"indexmap",
"serde",
"serde_json",
"swc_config_macro",
]
[[package]]
name = "swc_config_macro"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb64bc03d90fd5c90d6ab917bb2b1d7fbd31957df39e31ea24a3f554b4372251"
dependencies = [
"pmutil",
"proc-macro2 1.0.39",
"quote 1.0.18",
"swc_macros_common",
"syn 1.0.95",
]
[[package]]
name = "swc_ecma_ast"
version = "0.78.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f40169fe465e9a93cda5fe397c3afcb69be5ba2f76c4ab22137af6effaebcc"
dependencies = [
"is-macro",
"num-bigint",
"scoped-tls",
"serde",
"string_enum",
"swc_atoms",
@@ -3796,9 +3892,9 @@ dependencies = [
[[package]]
name = "swc_ecma_codegen"
version = "0.103.0"
version = "0.108.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99ca430d8ea2c8791d1341c4035431c90b87330e39479b4a6dabb4fded124e30"
checksum = "5eec1d30c8f85e8267a8efc66d680aa777902d567c3a05b7dfd42965a4872243"
dependencies = [
"bitflags",
"memchr",
@@ -3828,9 +3924,9 @@ dependencies = [
[[package]]
name = "swc_ecma_parser"
version = "0.100.2"
version = "0.104.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "890d967031e3e7330cd7892f27d826b7b4f37c7caa19db85c78a0862e1fe3974"
checksum = "a5fea08aeb2eb1469928ac7ca2d208fe7816871787e4d93e34924495e724bb25"
dependencies = [
"either",
"enum_kind",
@@ -3843,14 +3939,13 @@ dependencies = [
"swc_ecma_ast",
"tracing",
"typed-arena",
"unicode-id",
]
[[package]]
name = "swc_ecma_transforms"
version = "0.142.0"
version = "0.154.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f20e5e2d8ab843fa0454e049f73f6d99c444a8c0e2320f77028361ab75e2d18e"
checksum = "2bce21d9e8ff785aaf9b4ac11375d9f5767630fcaf882f72e6af0516224085a6"
dependencies = [
"swc_atoms",
"swc_common",
@@ -3865,13 +3960,14 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_base"
version = "0.75.0"
version = "0.85.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "404c6ea7ca61ceb2ce1f4ed448d1436a38c31b8c572850f04541c0229c966bbf"
checksum = "528c99be91500ed393e04e5cfc37763b4b68b71bc4f9b54ff0cd21d714920130"
dependencies = [
"better_scoped_tls",
"once_cell",
"phf",
"rustc-hash",
"serde",
"smallvec",
"swc_atoms",
@@ -3885,9 +3981,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_classes"
version = "0.63.0"
version = "0.73.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "503f2f6bd0f9e6363a93406753bf64675163423774256a267c85a5d9b5b44b08"
checksum = "e74a27c29def9db5ff03db4d3ab3d37701fb6d100951162223b71132908451eb"
dependencies = [
"swc_atoms",
"swc_common",
@@ -3912,9 +4008,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_proposal"
version = "0.97.0"
version = "0.107.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93d08411e517736b0167f3c9784fe9b98cc09308ae12e6072abd2bb2c2236da2"
checksum = "47fc0f3b336764f89adf1899830321c3f5a7e845ede3ad5949eeb7468aa260ab"
dependencies = [
"either",
"serde",
@@ -3931,9 +4027,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_react"
version = "0.104.0"
version = "0.114.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43cda44270dfcc95d61582981baddaf53d96c5233ea7384e81cd6e462816c58e"
checksum = "2fbfcd197ebeb0547b59dee39a164633bcf4fb0edbae886f8046e471e6a10502"
dependencies = [
"ahash",
"base64 0.13.0",
@@ -3946,6 +4042,7 @@ dependencies = [
"string_enum",
"swc_atoms",
"swc_common",
"swc_config",
"swc_ecma_ast",
"swc_ecma_parser",
"swc_ecma_transforms_base",
@@ -3956,9 +4053,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_typescript"
version = "0.107.0"
version = "0.117.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a09397169ed7ce0751a82cb71655f3a4a1fb00d8863aabd5cca9b46eff3dd5f2"
checksum = "96bf410ffcf91d85dc1f8f1bb969fa2637f9430a6917f2174ad76458c776cb89"
dependencies = [
"serde",
"swc_atoms",
@@ -3972,9 +4069,9 @@ dependencies = [
[[package]]
name = "swc_ecma_utils"
version = "0.79.1"
version = "0.85.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44ee8d60b9977f58214af7102dc30855a6754e742afe6d6e26e5bf13883c7b91"
checksum = "031ac49cf598f00f048fecd87b3bda5e14b86f6ccd561ada7fce461e0a3ea8d1"
dependencies = [
"indexmap",
"once_cell",
@@ -3987,9 +4084,9 @@ dependencies = [
[[package]]
name = "swc_ecma_visit"
version = "0.61.0"
version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5ea00a52ba2b971955c62275696d5c59f3cf0cd06db74a66dec378ec9843c78"
checksum = "f2d3783a0dd1e301ae2945ab1241405f913427f9512ec62756d3d2072f7c21bb"
dependencies = [
"num-bigint",
"swc_atoms",
@@ -4001,9 +4098,9 @@ dependencies = [
[[package]]
name = "swc_ecmascript"
version = "0.143.0"
version = "0.157.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebda93aa6422956c184a9eb5fdb0f0f0ff433169fa15e55ef445e5ad0b5e0abe"
checksum = "bd35679e1dc392f776b691b125692d90a7bebd5d23ec96699cfe37d8ae8633b1"
dependencies = [
"swc_ecma_ast",
"swc_ecma_codegen",
@@ -4302,9 +4399,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.18.2"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395"
checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838"
dependencies = [
"bytes 1.1.0",
"libc",
@@ -4312,10 +4409,9 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
"parking_lot 0.12.0",
"parking_lot 0.11.2",
"pin-project-lite 0.2.9",
"signal-hook-registry",
"socket2",
"tokio-macros",
"winapi",
]
@@ -4330,7 +4426,7 @@ dependencies = [
"once_cell",
"pin-project-lite 0.2.9",
"tokio 0.2.25",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-stream",
]
@@ -4352,7 +4448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio 1.18.2",
"tokio 1.15.0",
]
[[package]]
@@ -4362,7 +4458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
"rustls",
"tokio 1.18.2",
"tokio 1.15.0",
"webpki",
]
@@ -4375,7 +4471,7 @@ dependencies = [
"either",
"futures-util",
"thiserror",
"tokio 1.18.2",
"tokio 1.15.0",
]
[[package]]
@@ -4386,7 +4482,7 @@ checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
dependencies = [
"futures-core",
"pin-project-lite 0.2.9",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-util 0.6.10",
]
@@ -4399,7 +4495,7 @@ dependencies = [
"futures-core",
"libc",
"redox_syscall 0.2.13",
"tokio 1.18.2",
"tokio 1.15.0",
"tokio-stream",
"xattr",
]
@@ -4413,7 +4509,7 @@ dependencies = [
"futures-util",
"log",
"pin-project",
"tokio 1.18.2",
"tokio 1.15.0",
"tungstenite",
]
@@ -4428,7 +4524,7 @@ dependencies = [
"futures-sink",
"log",
"pin-project-lite 0.2.9",
"tokio 1.18.2",
"tokio 1.15.0",
]
[[package]]
@@ -4441,7 +4537,7 @@ dependencies = [
"futures-core",
"futures-sink",
"pin-project-lite 0.2.9",
"tokio 1.18.2",
"tokio 1.15.0",
"tracing",
]
@@ -4471,7 +4567,7 @@ dependencies = [
"serde_derive",
"sha2 0.9.9",
"sha3",
"tokio 1.18.2",
"tokio 1.15.0",
]
[[package]]
@@ -4727,9 +4823,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "v8"
version = "0.42.1"
version = "0.43.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854740dcc66681c3e10d15a1ebe1e0dfed77a7e2e58b97913dd3a111cf4cdb5f"
checksum = "1c87ec36fec9ea2cd5a368ae9d0a662a7c5e8caa8768ec1fcc02bea623681b98"
dependencies = [
"bitflags",
"fslock",
@@ -4799,12 +4895,6 @@ version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.80"

13
backend/Cargo.toml
View File

@@ -24,10 +24,6 @@ path = "src/lib.rs"
name = "embassyd"
path = "src/bin/embassyd.rs"
[[bin]]
name = "create-js-snapshots"
path = "src/bin/build-js-snapshots.rs"
[[bin]]
name = "embassy-init"
path = "src/bin/embassy-init.rs"
@@ -43,7 +39,7 @@ path = "src/bin/embassy-cli.rs"
[features]
avahi = ["avahi-sys"]
beta = []
default = ["avahi", "sound", "metal"]
default = ["avahi", "sound", "metal", "js_engine"]
dev = []
metal = []
sound = []
@@ -65,8 +61,6 @@ color-eyre = "0.5"
cookie_store = "0.15.0"
digest = "0.9.0"
divrem = "1.0.0"
deno_core = "0.135.0"
deno_ast = { version = "0.14.1", features = ["transpiling"] }
ed25519-dalek = { version = "1.0.1", features = ["serde"] }
emver = { version = "0.1.6", features = ["serde"] }
fd-lock-rs = "0.1.3"
@@ -78,7 +72,7 @@ http = "0.2.5"
hyper = "0.14.13"
hyper-ws-listener = { git = "https://github.com/Start9Labs/hyper-ws-listener.git", branch = "main" }
imbl = "1.0.1"
indexmap = { version = "1.7.0", features = ["serde"] }
indexmap = { version = "1.8.1", features = ["serde"] }
isocountry = "0.3.2"
itertools = "0.10.1"
jsonpath_lib = "0.3.0"
@@ -87,8 +81,11 @@ libc = "0.2.103"
log = "0.4.14"
nix = "0.23.0"
nom = "7.0.0"
helpers = {path = "../libs/helpers"}
num = "0.4.0"
num_enum = "0.5.4"
models = {version = "*", path = "../libs/models"}
js_engine = {path = '../libs/js_engine', optional = true}
openssh-keys = "0.5.0"
openssl = { version = "0.10.36", features = ["vendored"] }
patch-db = { version = "*", path = "../patch-db/patch-db", features = [

51
backend/src/action.rs
View File

@@ -1,6 +1,4 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::str::FromStr;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
@@ -11,7 +9,7 @@ use tracing::instrument;
use crate::config::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::id::{Id, ImageId, InvalidId};
use crate::id::{ ImageId};
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
@@ -19,52 +17,7 @@ use crate::util::Version;
use crate::volume::Volumes;
use crate::{Error, ResultExt};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct ActionId<S: AsRef<str> = String>(Id<S>);
impl FromStr for ActionId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(ActionId(Id::try_from(s.to_owned())?))
}
}
impl From<ActionId> for String {
fn from(value: ActionId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> AsRef<ActionId<S>> for ActionId<S> {
fn as_ref(&self) -> &ActionId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for ActionId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for ActionId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for ActionId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for ActionId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(ActionId(Deserialize::deserialize(deserializer)?))
}
}
pub use models::ActionId;
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Actions(pub BTreeMap<ActionId, Action>);

7
backend/src/error.rs
View File

@@ -1,6 +1,7 @@
use std::fmt::Display;
use color_eyre::eyre::eyre;
use models::InvalidId;
use patch_db::Revision;
use rpc_toolkit::yajrc::RpcError;
@@ -144,6 +145,7 @@ pub struct Error {
pub kind: ErrorKind,
pub revision: Option<Revision>,
}
impl Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: {}", self.kind.as_str(), self.source)
@@ -158,6 +160,11 @@ impl Error {
}
}
}
impl From<InvalidId> for Error {
fn from(err: InvalidId) -> Self {
Error::new(err, crate::error::ErrorKind::InvalidPackageId)
}
}
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Self {
Error::new(e, ErrorKind::Filesystem)

133
backend/src/id.rs
View File

@@ -1,142 +1,13 @@
use std::borrow::{Borrow, Cow};
use std::fmt::Debug;
use std::str::FromStr;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde::{Deserialize, Deserializer, Serialize};
use crate::util::Version;
use crate::Error;
pub const SYSTEM_ID: Id<&'static str> = Id("x_system");
pub use models::{Id, InvalidId, IdUnchecked, SYSTEM_ID};
#[derive(Debug, thiserror::Error)]
#[error("Invalid ID")]
pub struct InvalidId;
impl From<InvalidId> for Error {
fn from(err: InvalidId) -> Self {
Error::new(err, crate::error::ErrorKind::InvalidPackageId)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct IdUnchecked<S: AsRef<str>>(pub S);
impl<'de> Deserialize<'de> for IdUnchecked<Cow<'de, str>> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = IdUnchecked<Cow<'de, str>>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a valid ID")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v.to_owned())))
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v)))
}
fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Borrowed(v)))
}
}
deserializer.deserialize_any(Visitor)
}
}
impl<'de> Deserialize<'de> for IdUnchecked<String> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(String::deserialize(deserializer)?))
}
}
impl<'de> Deserialize<'de> for IdUnchecked<&'de str> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(<&'de str>::deserialize(deserializer)?))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Id<S: AsRef<str> = String>(S);
impl<S: AsRef<str>> Id<S> {
pub fn try_from(value: S) -> Result<Self, InvalidId> {
if value
.as_ref()
.chars()
.all(|c| c.is_ascii_lowercase() || c == '-')
{
Ok(Id(value))
} else {
Err(InvalidId)
}
}
}
impl<'a> Id<&'a str> {
pub fn owned(&self) -> Id {
Id(self.0.to_owned())
}
}
impl From<Id> for String {
fn from(value: Id) -> Self {
value.0
}
}
impl<S: AsRef<str>> std::ops::Deref for Id<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<S: AsRef<str>> std::fmt::Display for Id<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_ref())
}
}
impl<S: AsRef<str>> AsRef<str> for Id<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for Id<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for Id<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Id::try_from(unchecked.0).map_err(serde::de::Error::custom)
}
}
impl<S: AsRef<str>> Serialize for Id<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
serializer.serialize_str(self.as_ref())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct ImageId<S: AsRef<str> = String>(Id<S>);

1
backend/src/install/mod.rs
View File

@@ -59,7 +59,6 @@ pub const PKG_ARCHIVE_DIR: &str = "package-data/archive";
pub const PKG_PUBLIC_DIR: &str = "package-data/public";
pub const PKG_DOCKER_DIR: &str = "package-data/docker";
pub const PKG_WASM_DIR: &str = "package-data/wasm";
pub const PKG_SCRIPT_DIR: &str = "package-data/scripts";
#[command(display(display_serializable))]
pub async fn list(#[context] ctx: RpcContext) -> Result<Vec<(PackageId, Version)>, Error> {

2
backend/src/manager/mod.rs
View File

@@ -393,6 +393,7 @@ impl Manager {
let _ = self.shared.on_stop.send(OnStop::Exit);
let action = match &self.shared.manifest.main {
PackageProcedure::Docker(a) => a,
#[cfg(feature = "js_engine")]
PackageProcedure::Script(_) => return Ok(()),
};
match self
@@ -546,6 +547,7 @@ async fn stop(shared: &ManagerSharedState) -> Result<(), Error> {
}
let action = match &shared.manifest.main {
PackageProcedure::Docker(a) => a,
#[cfg(feature = "js_engine")]
PackageProcedure::Script(_) => return Ok(()),
};
match shared

43
backend/src/net/interface.rs
View File

@@ -1,5 +1,4 @@
use std::collections::BTreeMap;
use std::path::Path;
use color_eyre::eyre::eyre;
use futures::TryStreamExt;
@@ -16,6 +15,8 @@ use crate::s9pk::manifest::PackageId;
use crate::util::serde::Port;
use crate::{Error, ResultExt};
pub use models::InterfaceId;
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Interfaces(pub BTreeMap<InterfaceId, Interface>); // TODO
@@ -113,46 +114,6 @@ impl Interfaces {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct InterfaceId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> From<Id<S>> for InterfaceId<S> {
fn from(id: Id<S>) -> Self {
Self(id)
}
}
impl<S: AsRef<str>> std::fmt::Display for InterfaceId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> std::ops::Deref for InterfaceId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<str> for InterfaceId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for InterfaceId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(InterfaceId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for InterfaceId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]

591
backend/src/procedure/js_scripts.rs
View File

@@ -1,5 +1,6 @@
use std::{path::PathBuf, time::Duration};
use std::{path::{PathBuf, Path}, time::Duration};
use models::VolumeId;
use serde::{Deserialize, Serialize};
use tracing::instrument;
@@ -7,22 +8,24 @@ use crate::{
context::RpcContext, s9pk::manifest::PackageId, util::Version, volume::Volumes, Error,
};
use self::js_runtime::JsExecutionEnvironment;
use js_engine::{JsExecutionEnvironment, PathForVolumeId};
use super::ProcedureName;
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
pub struct JsCode(String);
pub use js_engine::{JsError};
impl PathForVolumeId for Volumes {
fn path_for(&self, data_dir: &Path, package_id: &PackageId, version: &Version, volume_id: &VolumeId) -> Option<PathBuf> {
let volume = self.get(volume_id)?;
Some(volume.path_for(data_dir, package_id, version, volume_id))
}
fn readonly(&self,volume_id: &VolumeId) -> bool {
self.get(volume_id).map(|x| x.readonly()).unwrap_or(false)
}
#[derive(Debug, Clone, Copy)]
pub enum JsError {
Unknown = 1,
Javascript = 2,
Engine = 3,
BoundryLayerSerDe = 4,
Tokio = 5,
FileSystem = 6,
Timeout = 143,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
@@ -50,7 +53,7 @@ impl JsProcedure {
directory,
pkg_id,
pkg_version,
volumes.clone(),
Box::new(volumes.clone()),
)
.await?
.run_action(name, input);
@@ -82,7 +85,7 @@ impl JsProcedure {
&ctx.datadir,
pkg_id,
pkg_version,
volumes.clone(),
Box::new(volumes.clone()),
)
.await?
.read_only_effects()
@@ -100,564 +103,6 @@ impl JsProcedure {
}
}
mod js_runtime {
use deno_core::anyhow::{anyhow, bail};
use deno_core::error::AnyError;
use deno_core::resolve_import;
use deno_core::JsRuntime;
use deno_core::ModuleLoader;
use deno_core::ModuleSource;
use deno_core::ModuleSourceFuture;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::RuntimeOptions;
use deno_core::Snapshot;
use deno_core::{Extension, OpDecl};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::Arc;
use std::{path::PathBuf, pin::Pin};
use tokio::io::AsyncReadExt;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::volume::script_dir;
use crate::volume::Volumes;
use super::super::ProcedureName;
use super::{JsCode, JsError};
#[cfg(target_arch = "x86_64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./js_scripts/JS_SNAPSHOT.bin");
#[cfg(target_arch = "aarch64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./js_scripts/ARM_JS_SNAPSHOT.bin");
#[derive(Clone, Deserialize, Serialize)]
struct JsContext {
sandboxed: bool,
datadir: PathBuf,
run_function: String,
version: Version,
package_id: PackageId,
volumes: Arc<Volumes>,
input: Value,
}
#[derive(Clone, Default)]
struct AnswerState(std::sync::Arc<deno_core::parking_lot::Mutex<Value>>);
#[derive(Clone, Debug)]
struct ModsLoader {
code: JsCode,
}
impl ModuleLoader for ModsLoader {
fn resolve(
&self,
specifier: &str,
referrer: &str,
_is_main: bool,
) -> Result<ModuleSpecifier, AnyError> {
if referrer.contains("embassy") {
bail!("Embassy.js cannot import anything else");
}
let s = resolve_import(specifier, referrer).unwrap();
Ok(s)
}
fn load(
&self,
module_specifier: &ModuleSpecifier,
maybe_referrer: Option<ModuleSpecifier>,
is_dyn_import: bool,
) -> Pin<Box<ModuleSourceFuture>> {
let module_specifier = module_specifier.as_str().to_owned();
let module = match &*module_specifier {
"file:///deno_global.js" => Ok(ModuleSource {
module_url_specified: "file:///deno_global.js".to_string(),
module_url_found: "file:///deno_global.js".to_string(),
code: "const old_deno = Deno; Deno = null; export default old_deno"
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///loadModule.js" => Ok(ModuleSource {
module_url_specified: "file:///loadModule.js".to_string(),
module_url_found: "file:///loadModule.js".to_string(),
code: include_str!("./js_scripts/loadModule.js")
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///embassy.js" => Ok(ModuleSource {
module_url_specified: "file:///embassy.js".to_string(),
module_url_found: "file:///embassy.js".to_string(),
code: self.code.0.as_bytes().to_vec().into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
x => Err(anyhow!("Not allowed to import: {}", x)),
};
Box::pin(async move {
if is_dyn_import {
bail!("Will not import dynamic");
}
match &maybe_referrer {
Some(x) if x.as_str() == "file:///embassy.js" => {
bail!("Embassy is not allowed to import")
}
_ => (),
}
module
})
}
}
pub struct JsExecutionEnvironment {
sandboxed: bool,
base_directory: PathBuf,
module_loader: ModsLoader,
package_id: PackageId,
version: Version,
volumes: Arc<Volumes>,
}
impl JsExecutionEnvironment {
pub async fn load_from_package(
data_directory: impl AsRef<std::path::Path>,
package_id: &crate::s9pk::manifest::PackageId,
version: &crate::util::Version,
volumes: Volumes,
) -> Result<Self, (JsError, String)> {
let data_dir = data_directory.as_ref();
let base_directory = data_dir;
let js_code = JsCode({
let file_path = script_dir(data_dir, package_id, version).join("embassy.js");
let mut file = match tokio::fs::File::open(file_path.clone()).await {
Ok(x) => x,
Err(e) => {
tracing::debug!("path: {:?}", file_path);
tracing::debug!("{:?}", e);
return Err((
JsError::FileSystem,
format!("The file opening '{:?}' created error: {}", file_path, e),
));
}
};
let mut buffer = Default::default();
if let Err(err) = file.read_to_string(&mut buffer).await {
tracing::debug!("{:?}", err);
return Err((
JsError::FileSystem,
format!("The file reading created error: {}", err),
));
};
buffer
});
Ok(Self {
base_directory: base_directory.to_owned(),
module_loader: ModsLoader { code: js_code },
package_id: package_id.clone(),
version: version.clone(),
volumes: Arc::new(volumes),
sandboxed: false,
})
}
pub fn read_only_effects(mut self) -> Self {
self.sandboxed = true;
self
}
pub async fn run_action<I: Serialize, O: for<'de> Deserialize<'de>>(
self,
procedure_name: ProcedureName,
input: Option<I>,
) -> Result<O, (JsError, String)> {
let input = match serde_json::to_value(input) {
Ok(a) => a,
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
"Couldn't convert input".to_string(),
));
}
};
let safer_handle: crate::util::NonDetachingJoinHandle<_> =
tokio::task::spawn_blocking(move || self.execute(procedure_name, input)).into();
let output = safer_handle
.await
.map_err(|err| (JsError::Tokio, format!("Tokio gave us the error: {}", err)))??;
match serde_json::from_value(output.clone()) {
Ok(x) => Ok(x),
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
format!(
"Couldn't convert output = {:#?} to the correct type",
serde_json::to_string_pretty(&output).unwrap_or_default()
),
));
}
}
}
fn declarations() -> Vec<OpDecl> {
vec![
fns::read_file::decl(),
fns::write_file::decl(),
fns::remove_file::decl(),
fns::create_dir::decl(),
fns::remove_dir::decl(),
fns::current_function::decl(),
fns::log_trace::decl(),
fns::log_warn::decl(),
fns::log_error::decl(),
fns::log_debug::decl(),
fns::log_info::decl(),
fns::get_context::decl(),
fns::get_input::decl(),
fns::set_value::decl(),
fns::is_sandboxed::decl(),
]
}
fn execute(
&self,
procedure_name: ProcedureName,
input: Value,
) -> Result<Value, (JsError, String)> {
let base_directory = self.base_directory.clone();
let answer_state = AnswerState::default();
let ext_answer_state = answer_state.clone();
let js_ctx = JsContext {
datadir: base_directory,
run_function: procedure_name.js_function_name(),
package_id: self.package_id.clone(),
volumes: self.volumes.clone(),
version: self.version.clone(),
sandboxed: self.sandboxed,
input,
};
let ext = Extension::builder()
.ops(Self::declarations())
.state(move |state| {
state.put(ext_answer_state.clone());
state.put(js_ctx.clone());
Ok(())
})
.build();
let loader = std::rc::Rc::new(self.module_loader.clone());
let runtime_options = RuntimeOptions {
module_loader: Some(loader),
extensions: vec![ext],
startup_snapshot: Some(Snapshot::Static(SNAPSHOT_BYTES)),
..Default::default()
};
let mut runtime = JsRuntime::new(runtime_options);
let future = async move {
let mod_id = runtime
.load_main_module(&"file:///loadModule.js".parse().unwrap(), None)
.await?;
let evaluated = runtime.mod_evaluate(mod_id);
let res = runtime.run_event_loop(false).await;
res?;
evaluated.await??;
Ok::<_, AnyError>(())
};
tokio::runtime::Handle::current()
.block_on(future)
.map_err(|e| {
tracing::debug!("{:?}", e);
(JsError::Javascript, format!("{}", e))
})?;
let answer = answer_state.0.lock().clone();
Ok(answer)
}
}
/// Note: Make sure that we have the assumption that all these methods are callable at any time, and all call restrictions should be in rust
mod fns {
use deno_core::{
anyhow::{anyhow, bail},
error::AnyError,
*,
};
use serde_json::Value;
use std::{
convert::TryFrom,
path::{Path, PathBuf},
};
use crate::volume::VolumeId;
use super::{AnswerState, JsContext};
#[op]
async fn read_file(
ctx: JsContext,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<String, AnyError> {
let volume = match ctx.volumes.get(&volume_id) {
Some(a) => a,
None => {
bail!("There is no {} in volumes", volume_id);
}
};
let volume_path =
volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
//get_path_for in volume.rs
let new_file = volume_path.join(path_in);
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
let answer = tokio::fs::read_to_string(new_file).await?;
Ok(answer)
}
#[op]
async fn write_file(
ctx: JsContext,
volume_id: VolumeId,
path_in: PathBuf,
write: String,
) -> Result<(), AnyError> {
if ctx.sandboxed {
bail!("Cannot write in sandbox mode");
}
let volume = match ctx.volumes.get(&volume_id) {
Some(a) => a,
None => {
bail!("There is no {} in volumes", volume_id);
}
};
if volume.readonly() {
bail!("Volume {} is readonly", volume_id);
}
let volume_path =
volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::write(new_file, write).await?;
Ok(())
}
#[op]
async fn remove_file(
ctx: JsContext,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
if ctx.sandboxed {
bail!("Cannot write in sandbox mode");
}
let volume = match ctx.volumes.get(&volume_id) {
Some(a) => a,
None => {
bail!("There is no {} in volumes", volume_id);
}
};
if volume.readonly() {
bail!("Volume {} is readonly", volume_id);
}
let volume_path =
volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_file(new_file).await?;
Ok(())
}
#[op]
async fn remove_dir(
ctx: JsContext,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
if ctx.sandboxed {
bail!("Cannot write in sandbox mode");
}
let volume = match ctx.volumes.get(&volume_id) {
Some(a) => a,
None => {
bail!("There is no {} in volumes", volume_id);
}
};
if volume.readonly() {
bail!("Volume {} is readonly", volume_id);
}
let volume_path =
volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_dir_all(new_file).await?;
Ok(())
}
#[op]
async fn create_dir(
ctx: JsContext,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
if ctx.sandboxed {
bail!("Cannot write in sandbox mode");
}
let volume = match ctx.volumes.get(&volume_id) {
Some(a) => a,
None => {
bail!("There is no {} in volumes", volume_id);
}
};
if volume.readonly() {
bail!("Volume {} is readonly", volume_id);
}
let volume_path =
volume.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id);
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::create_dir_all(new_file).await?;
Ok(())
}
#[op]
fn current_function(state: &mut OpState) -> Result<String, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.run_function.clone())
}
#[op]
fn log_trace(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::trace!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_warn(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::warn!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_error(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::error!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_debug(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::debug!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_info(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::info!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn get_context(state: &mut OpState) -> Result<JsContext, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.clone())
}
#[op]
fn get_input(state: &mut OpState) -> Result<Value, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.input.clone())
}
#[op]
fn set_value(state: &mut OpState, value: Value) -> Result<(), AnyError> {
let mut answer = state.borrow::<AnswerState>().0.lock();
*answer = value;
Ok(())
}
#[op]
fn is_sandboxed(state: &mut OpState) -> Result<bool, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.sandboxed)
}
/// We need to make sure that during the file accessing, we don't reach beyond our scope of control
async fn is_subset(
parent: impl AsRef<Path>,
child: impl AsRef<Path>,
) -> Result<bool, AnyError> {
let child = tokio::fs::canonicalize(child).await?;
let parent = tokio::fs::canonicalize(parent).await?;
Ok(child.starts_with(parent))
}
}
}
#[tokio::test]
async fn js_action_execute() {
let js_action = JsProcedure {};

BIN
backend/src/procedure/js_scripts/ARM_JS_SNAPSHOT.bin
View File

Binary file not shown.

BIN
backend/src/procedure/js_scripts/JS_SNAPSHOT.bin
View File

Binary file not shown.

61
backend/src/procedure/mod.rs
View File

@@ -6,66 +6,17 @@ use serde::{Deserialize, Serialize};
use tracing::instrument;
use self::docker::DockerProcedure;
use self::js_scripts::JsProcedure;
use crate::action::ActionId;
use crate::context::RpcContext;
use crate::id::ImageId;
use crate::s9pk::manifest::PackageId;
use crate::status::health_check::HealthCheckId;
use crate::util::Version;
use crate::volume::Volumes;
use crate::Error;
pub mod docker;
#[cfg(feature = "js_engine")]
pub mod js_scripts;
#[derive(Debug, Clone)]
pub enum ProcedureName {
Main, // Usually just run container
CreateBackup,
RestoreBackup,
GetConfig,
SetConfig,
Migration,
Properties,
Check(PackageId),
AutoConfig(PackageId),
Health(HealthCheckId),
Action(ActionId),
}
impl ProcedureName {
fn docker_name(&self) -> Option<String> {
match self {
ProcedureName::Main => None,
ProcedureName::CreateBackup => Some("CreateBackup".to_string()),
ProcedureName::RestoreBackup => Some("RestoreBackup".to_string()),
ProcedureName::GetConfig => Some("GetConfig".to_string()),
ProcedureName::SetConfig => Some("SetConfig".to_string()),
ProcedureName::Migration => Some("Migration".to_string()),
ProcedureName::Properties => Some(format!("Properties-{}", rand::random::<u64>())),
ProcedureName::Health(id) => Some(format!("{}Health", id)),
ProcedureName::Action(id) => Some(format!("{}Action", id)),
ProcedureName::Check(_) => None,
ProcedureName::AutoConfig(_) => None,
}
}
fn js_function_name(&self) -> String {
match self {
ProcedureName::Main => "/main".to_string(),
ProcedureName::CreateBackup => "/createBackup".to_string(),
ProcedureName::RestoreBackup => "/restoreBackup".to_string(),
ProcedureName::GetConfig => "/getConfig".to_string(),
ProcedureName::SetConfig => "/setConfig".to_string(),
ProcedureName::Migration => "/migration".to_string(),
ProcedureName::Properties => "/properties".to_string(),
ProcedureName::Health(id) => format!("/health/{}", id),
ProcedureName::Action(id) => format!("/action/{}", id),
ProcedureName::Check(id) => format!("/dependencies/{}/check", id),
ProcedureName::AutoConfig(id) => format!("/dependencies/{}/autoConfigure", id),
}
}
}
pub use models::ProcedureName;
// TODO: create RPC endpoint that looks up the appropriate action and calls `execute`
@@ -74,11 +25,14 @@ impl ProcedureName {
#[serde(tag = "type")]
pub enum PackageProcedure {
Docker(DockerProcedure),
Script(JsProcedure),
#[cfg(feature = "js_engine")]
Script(js_scripts::JsProcedure),
}
impl PackageProcedure {
pub fn is_script(&self) -> bool {
match self {
#[cfg(feature = "js_engine")]
Self::Script(_) => true,
_ => false,
}
@@ -93,6 +47,7 @@ impl PackageProcedure {
match self {
PackageProcedure::Docker(action) => action.validate(volumes, image_ids, expected_io),
#[cfg(feature = "js_engine")]
PackageProcedure::Script(action) => action.validate(volumes),
}
}
@@ -124,6 +79,7 @@ impl PackageProcedure {
)
.await
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
procedure
.execute(
@@ -156,6 +112,7 @@ impl PackageProcedure {
.sandboxed(ctx, pkg_id, pkg_version, volumes, input, timeout)
.await
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
procedure
.sandboxed(ctx, pkg_id, pkg_version, volumes, input, timeout, name)

85
backend/src/s9pk/manifest.rs
View File

@@ -1,17 +1,14 @@
use std::borrow::Borrow;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use color_eyre::eyre::eyre;
use patch_db::HasModel;
use serde::{Deserialize, Serialize, Serializer};
use serde::{Deserialize, Serialize};
use url::Url;
use crate::action::Actions;
use crate::backup::BackupActions;
use crate::config::action::ConfigActions;
use crate::dependencies::Dependencies;
use crate::id::{Id, InvalidId, SYSTEM_ID};
use crate::migration::Migrations;
use crate::net::interface::Interfaces;
use crate::procedure::PackageProcedure;
@@ -21,85 +18,7 @@ use crate::version::{Current, VersionT};
use crate::volume::Volumes;
use crate::Error;
pub const SYSTEM_PACKAGE_ID: PackageId<&'static str> = PackageId(SYSTEM_ID);
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct PackageId<S: AsRef<str> = String>(Id<S>);
impl<'a> PackageId<&'a str> {
pub fn owned(&self) -> PackageId {
PackageId(self.0.owned())
}
}
impl FromStr for PackageId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(PackageId(Id::try_from(s.to_owned())?))
}
}
impl From<PackageId> for String {
fn from(value: PackageId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> From<Id<S>> for PackageId<S> {
fn from(id: Id<S>) -> Self {
PackageId(id)
}
}
impl<S: AsRef<str>> std::ops::Deref for PackageId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<PackageId<S>> for PackageId<S> {
fn as_ref(&self) -> &PackageId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for PackageId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for PackageId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for PackageId<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for PackageId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for PackageId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(PackageId(Deserialize::deserialize(deserializer)?))
}
}
impl<S> Serialize for PackageId<S>
where
S: AsRef<str>,
{
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
Serialize::serialize(&self.0, serializer)
}
}
pub use models::{PackageId, SYSTEM_PACKAGE_ID};
fn current_version() -> Version {
Current::new().semver().into()

35
backend/src/status/health_check.rs
View File

@@ -1,12 +1,11 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::id::{Id, ImageId};
use crate::id::{ ImageId};
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::Duration;
@@ -14,35 +13,7 @@ use crate::util::Version;
use crate::volume::Volumes;
use crate::{Error, ResultExt};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct HealthCheckId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> std::fmt::Display for HealthCheckId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for HealthCheckId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for HealthCheckId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(HealthCheckId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for HealthCheckId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
pub use models::HealthCheckId;
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct HealthChecks(pub BTreeMap<HealthCheckId, HealthCheck>);

137
backend/src/util/mod.rs
View File

@@ -1,16 +1,12 @@
use std::collections::BTreeMap;
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::process::Stdio;
use std::str::FromStr;
use std::sync::Arc;
use std::task::{Context, Poll};
use ::serde::{Deserialize, Deserializer, Serialize, Serializer};
use async_trait::async_trait;
use clap::ArgMatches;
use color_eyre::eyre::{self, eyre};
@@ -19,16 +15,17 @@ use fd_lock_rs::FdLock;
use futures::future::BoxFuture;
use futures::FutureExt;
use lazy_static::lazy_static;
use patch_db::{HasModel, Model};
use pin_project::pin_project;
use tokio::fs::File;
use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
use tokio::task::{JoinError, JoinHandle};
use tracing::instrument;
use crate::shutdown::Shutdown;
use crate::{Error, ResultExt as _};
pub use helpers::NonDetachingJoinHandle;
pub use models::Version;
pub mod io;
pub mod logger;
pub mod serde;
@@ -125,110 +122,6 @@ impl<T> SNone<T> {
}
impl<T> SOption<T> for SNone<T> {}
#[derive(Debug, Clone)]
pub struct Version {
version: emver::Version,
string: String,
}
impl Version {
pub fn as_str(&self) -> &str {
self.string.as_str()
}
pub fn into_version(self) -> emver::Version {
self.version
}
}
impl std::fmt::Display for Version {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.string)
}
}
impl std::str::FromStr for Version {
type Err = <emver::Version as FromStr>::Err;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Version {
string: s.to_owned(),
version: s.parse()?,
})
}
}
impl From<emver::Version> for Version {
fn from(v: emver::Version) -> Self {
Version {
string: v.to_string(),
version: v,
}
}
}
impl From<Version> for emver::Version {
fn from(v: Version) -> Self {
v.version
}
}
impl Default for Version {
fn default() -> Self {
Self::from(emver::Version::default())
}
}
impl Deref for Version {
type Target = emver::Version;
fn deref(&self) -> &Self::Target {
&self.version
}
}
impl AsRef<emver::Version> for Version {
fn as_ref(&self) -> &emver::Version {
&self.version
}
}
impl AsRef<str> for Version {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl PartialEq for Version {
fn eq(&self, other: &Version) -> bool {
self.version.eq(&other.version)
}
}
impl Eq for Version {}
impl PartialOrd for Version {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.version.partial_cmp(&other.version)
}
}
impl Ord for Version {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.version.cmp(&other.version)
}
}
impl Hash for Version {
fn hash<H: Hasher>(&self, state: &mut H) {
self.version.hash(state)
}
}
impl<'de> Deserialize<'de> for Version {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let string = String::deserialize(deserializer)?;
let version = emver::Version::from_str(&string).map_err(::serde::de::Error::custom)?;
Ok(Self { string, version })
}
}
impl Serialize for Version {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
self.string.serialize(serializer)
}
}
impl HasModel for Version {
type Model = Model<Version>;
}
#[async_trait]
pub trait AsyncFileExt: Sized {
async fn maybe_open<P: AsRef<Path> + Send + Sync>(path: P) -> std::io::Result<Option<Self>>;
@@ -358,30 +251,6 @@ where
}
}
#[pin_project::pin_project(PinnedDrop)]
pub struct NonDetachingJoinHandle<T>(#[pin] JoinHandle<T>);
impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
fn from(t: JoinHandle<T>) -> Self {
NonDetachingJoinHandle(t)
}
}
#[pin_project::pinned_drop]
impl<T> PinnedDrop for NonDetachingJoinHandle<T> {
fn drop(self: std::pin::Pin<&mut Self>) {
let this = self.project();
this.0.into_ref().get_ref().abort()
}
}
impl<T> Future for NonDetachingJoinHandle<T> {
type Output = Result<T, JoinError>;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = self.project();
this.0.poll(cx)
}
}
pub struct GeneralGuard<F: FnOnce() -> T, T = ()>(Option<F>);
impl<F: FnOnce() -> T, T> GeneralGuard<F, T> {

7
backend/src/version/mod.rs
View File

@@ -11,8 +11,9 @@ mod v0_3_0;
mod v0_3_0_1;
mod v0_3_0_2;
mod v0_3_0_3;
mod v0_3_1;
pub type Current = v0_3_0_3::Version;
pub type Current = v0_3_1::Version;
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
#[serde(untagged)]
@@ -21,6 +22,7 @@ enum Version {
V0_3_0_1(Wrapper<v0_3_0_1::Version>),
V0_3_0_2(Wrapper<v0_3_0_2::Version>),
V0_3_0_3(Wrapper<v0_3_0_3::Version>),
V0_3_1(Wrapper<v0_3_1::Version>),
Other(emver::Version),
}
@@ -40,6 +42,7 @@ impl Version {
Version::V0_3_0_1(Wrapper(x)) => x.semver(),
Version::V0_3_0_2(Wrapper(x)) => x.semver(),
Version::V0_3_0_3(Wrapper(x)) => x.semver(),
Version::V0_3_1(Wrapper(x)) => x.semver(),
Version::Other(x) => x.clone(),
}
}
@@ -154,6 +157,7 @@ pub async fn init<Db: DbHandle>(
Version::V0_3_0_1(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
Version::V0_3_0_2(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
Version::V0_3_0_3(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
Version::V0_3_1(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
Version::Other(_) => {
return Err(Error::new(
eyre!("Cannot downgrade"),
@@ -189,6 +193,7 @@ mod tests {
Just(Version::V0_3_0_1(Wrapper(v0_3_0_1::Version::new()))),
Just(Version::V0_3_0_2(Wrapper(v0_3_0_2::Version::new()))),
Just(Version::V0_3_0_3(Wrapper(v0_3_0_3::Version::new()))),
Just(Version::V0_3_1(Wrapper(v0_3_1::Version::new()))),
em_version().prop_map(Version::Other),
]
}

36
backend/src/version/v0_3_1.rs Normal file
View File

@@ -0,0 +1,36 @@
use emver::VersionRange;
use super::*;
const V0_3_1: emver::Version = emver::Version::new(0, 3, 1, 0);
lazy_static::lazy_static! {
pub static ref V0_3_1_COMPAT: VersionRange = VersionRange::Conj(
Box::new(VersionRange::Anchor(
emver::GTE,
emver::Version::new(0, 3, 0, 0),
)),
Box::new(VersionRange::Anchor(emver::LTE, Current::new().semver())),
);
}
#[derive(Clone, Debug)]
pub struct Version;
#[async_trait]
impl VersionT for Version {
type Previous = v0_3_0_3::Version;
fn new() -> Self {
Version
}
fn semver(&self) -> emver::Version {
V0_3_1
}
fn compat(&self) -> &'static VersionRange {
&*V0_3_1_COMPAT
}
async fn up<Db: DbHandle>(&self, _db: &mut Db) -> Result<(), Error> {
Ok(())
}
async fn down<Db: DbHandle>(&self, _db: &mut Db) -> Result<(), Error> {
Ok(())
}
}

74
backend/src/volume.rs
View File

@@ -1,79 +1,23 @@
use std::borrow::Borrow;
use std::collections::BTreeMap;
use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf};
use patch_db::{HasModel, Map, MapModel};
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::id::{Id, IdUnchecked};
use crate::install::PKG_SCRIPT_DIR;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::net::NetController;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::{Error, ResultExt};
pub const PKG_VOLUME_DIR: &'static str = "package-data/volumes";
pub const BACKUP_DIR: &'static str = "/media/embassy-os/backups";
pub use helpers::script_dir;
pub use models::VolumeId;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum VolumeId<S: AsRef<str> = String> {
Backup,
Custom(Id<S>),
}
impl<S: AsRef<str>> std::fmt::Display for VolumeId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VolumeId::Backup => write!(f, "BACKUP"),
VolumeId::Custom(id) => write!(f, "{}", id),
}
}
}
impl<S: AsRef<str>> AsRef<str> for VolumeId<S> {
fn as_ref(&self) -> &str {
match self {
VolumeId::Backup => "BACKUP",
VolumeId::Custom(id) => id.as_ref(),
}
}
}
impl<S: AsRef<str>> Borrow<str> for VolumeId<S> {
fn borrow(&self) -> &str {
self.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for VolumeId<S> {
fn as_ref(&self) -> &Path {
AsRef::<str>::as_ref(self).as_ref()
}
}
impl<'de, S> Deserialize<'de> for VolumeId<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Ok(match unchecked.0.as_ref() {
"BACKUP" => VolumeId::Backup,
_ => VolumeId::Custom(Id::try_from(unchecked.0).map_err(serde::de::Error::custom)?),
})
}
}
impl<S: AsRef<str>> Serialize for VolumeId<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: serde::Serializer,
{
serializer.serialize_str(self.as_ref())
}
}
pub const PKG_VOLUME_DIR: &str = "package-data/volumes";
pub const BACKUP_DIR: &str = "/media/embassy-os/backups";
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Volumes(BTreeMap<VolumeId, Volume>);
@@ -166,14 +110,6 @@ pub fn asset_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, version: &Versi
.join(version.as_str())
}
pub fn script_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, version: &Version) -> PathBuf {
datadir
.as_ref()
.join(&*PKG_SCRIPT_DIR)
.join(pkg_id)
.join(version.as_str())
}
pub fn backup_dir(pkg_id: &PackageId) -> PathBuf {
Path::new(BACKUP_DIR).join(pkg_id).join("data")
}

10
libs/.gitignore vendored Normal file
View File

@@ -0,0 +1,10 @@
/target
**/*.rs.bk
.DS_Store
.vscode
secrets.db
*.s9pk
*.sqlite3
.env
.editorconfig
proptest-regressions/*

2287
libs/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

8
libs/Cargo.toml Normal file
View File

@@ -0,0 +1,8 @@
[workspace]
members = [
"snapshot-creator",
"models",
"js_engine",
"helpers"
]

14
backend/build-arm-v8-snapshot.sh → libs/build-arm-v8-snapshot.sh
View File

@@ -4,10 +4,18 @@
set -e
if [ "$0" != "./build-arm-v8-snapshot.sh" ]; then
>&2 echo "Must be run from backend directory"
>&2 echo "Must be run from backend/workspace directory"
exit 1
fi
echo "Building "
cd ..
docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64 sh -c "(cd libs/ && cargo build -p snapshot-creator --release )"
cd -
echo "Creating Arm v8 Snapshot"
docker run --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/create-js-snapshots"
mv JS_SNAPSHOT.bin src/procedure/js_scripts/ARM_JS_SNAPSHOT.bin
docker run --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot-creator"
sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin
sudo chmod 0644 JS_SNAPSHOT.bin
sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin

16
libs/build-v8-snapshot.sh Executable file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Reason for this being is that we need to create a snapshot for the deno runtime. It wants to pull 3 files from build, and during the creation it gets embedded, but for some
# reason during the actual runtime it is looking for them. So this will create a docker in arm that creates the snaphot needed for the arm
set -e
if [ "$0" != "./build-v8-snapshot.sh" ]; then
>&2 echo "Must be run from backend/workspace directory"
exit 1
fi
echo "Creating v8 Snapshot"
cargo run -p snapshot-creator --release
sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin
sudo chmod 0644 JS_SNAPSHOT.bin
sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/JS_SNAPSHOT.bin

11
libs/helpers/Cargo.toml Normal file
View File

@@ -0,0 +1,11 @@
[package]
name = "helpers"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
pin-project = "1.0.8"
tokio = { version = "1.15.*", features = ["full"] }
models = {path = "../models"}

31
libs/helpers/src/lib.rs Normal file
View File

@@ -0,0 +1,31 @@
use std::future::Future;
use tokio::task::{JoinError, JoinHandle};
mod script_dir;
pub use script_dir::*;
#[pin_project::pin_project(PinnedDrop)]
pub struct NonDetachingJoinHandle<T>(#[pin] JoinHandle<T>);
impl<T> From<JoinHandle<T>> for NonDetachingJoinHandle<T> {
fn from(t: JoinHandle<T>) -> Self {
NonDetachingJoinHandle(t)
}
}
#[pin_project::pinned_drop]
impl<T> PinnedDrop for NonDetachingJoinHandle<T> {
fn drop(self: std::pin::Pin<&mut Self>) {
let this = self.project();
this.0.into_ref().get_ref().abort()
}
}
impl<T> Future for NonDetachingJoinHandle<T> {
type Output = Result<T, JoinError>;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = self.project();
this.0.poll(cx)
}
}

13
libs/helpers/src/script_dir.rs Normal file
View File

@@ -0,0 +1,13 @@
use std::path::{Path, PathBuf};
use models::{PackageId, Version};
pub const PKG_SCRIPT_DIR: &str = "package-data/scripts";
pub fn script_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, version: &Version) -> PathBuf {
datadir
.as_ref()
.join(&*PKG_SCRIPT_DIR)
.join(pkg_id)
.join(version.as_str())
}

17
libs/js_engine/Cargo.toml Normal file
View File

@@ -0,0 +1,17 @@
[package]
name = "js_engine"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dashmap = "5.1.0"
deno_core = "0.136.0"
deno_ast = {version="0.15.0", features = ["transpiling"]}
models = {path = "../models"}
helpers = {path = "../helpers"}
serde = { version = "1.0.*", features = ["derive", "rc"] }
serde_json = "1.0.*"
tokio = { version = "1.*.*", features = ["full"] }
tracing = "0.1"

BIN
libs/js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin Normal file
View File

Binary file not shown.

BIN
libs/js_engine/src/artifacts/JS_SNAPSHOT.bin Normal file
View File

Binary file not shown.

12
backend/src/procedure/js_scripts/loadModule.js → libs/js_engine/src/artifacts/loadModule.js
View File

@@ -21,14 +21,12 @@ function jsonPointerValue(obj, pointer) {
}
// @ts-ignore
const context = Deno.core.opSync("get_context");
// @ts-ignore
const writeFile = ({ path, volumeId, toWrite }) => Deno.core.opAsync("write_file", context, volumeId, path, toWrite);
const writeFile = ({ path, volumeId, toWrite }) => Deno.core.opAsync("write_file", volumeId, path, toWrite);
// @ts-ignore
const readFile = ({ volumeId, path }) => Deno.core.opAsync("read_file", context, volumeId, path);
const readFile = ({ volumeId, path }) => Deno.core.opAsync("read_file", volumeId, path);
// @ts-ignore
const removeFile = ({ volumeId, path }) => Deno.core.opAsync("remove_file", context, volumeId, path);
const removeFile = ({ volumeId, path }) => Deno.core.opAsync("remove_file", volumeId, path);
// @ts-ignore
const isSandboxed = () => Deno.core.opSync("is_sandboxed");
@@ -42,9 +40,9 @@ const writeJsonFile = ({ volumeId, path, toWrite }) =>
// @ts-ignore
const readJsonFile = async ({ volumeId, path }) => JSON.parse(await readFile({ volumeId, path }));
// @ts-ignore
const createDir = ({ volumeId, path }) => Deno.core.opAsync("create_dir", context, volumeId, path);
const createDir = ({ volumeId, path }) => Deno.core.opAsync("create_dir", volumeId, path);
// @ts-ignore
const removeDir = ({ volumeId, path }) => Deno.core.opAsync("remove_dir", context, volumeId, path);
const removeDir = ({ volumeId, path }) => Deno.core.opAsync("remove_dir", volumeId, path);
// @ts-ignore
const trace = (x) => Deno.core.opSync("log_trace", x);
// @ts-ignore

550
libs/js_engine/src/lib.rs Normal file
View File

@@ -0,0 +1,550 @@
use deno_core::anyhow::{anyhow, bail};
use deno_core::error::AnyError;
use deno_core::resolve_import;
use deno_core::JsRuntime;
use deno_core::ModuleLoader;
use deno_core::ModuleSource;
use deno_core::ModuleSourceFuture;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::RuntimeOptions;
use deno_core::Snapshot;
use deno_core::{Extension, OpDecl};
use helpers::script_dir;
use helpers::NonDetachingJoinHandle;
use models::{PackageId, ProcedureName, Version, VolumeId};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{path::Path, sync::Arc};
use std::{path::PathBuf, pin::Pin};
use tokio::io::AsyncReadExt;
pub trait PathForVolumeId: Send + Sync {
fn path_for(
&self,
data_dir: &Path,
package_id: &PackageId,
version: &Version,
volume_id: &VolumeId,
) -> Option<PathBuf>;
fn readonly(&self, volume_id: &VolumeId) -> bool;
}
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
pub struct JsCode(String);
#[derive(Debug, Clone, Copy)]
pub enum JsError {
Unknown = 1,
Javascript = 2,
Engine = 3,
BoundryLayerSerDe = 4,
Tokio = 5,
FileSystem = 6,
Timeout = 143,
}
#[cfg(target_arch = "x86_64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/JS_SNAPSHOT.bin");
#[cfg(target_arch = "aarch64")]
const SNAPSHOT_BYTES: &[u8] = include_bytes!("./artifacts/ARM_JS_SNAPSHOT.bin");
#[derive(Clone)]
struct JsContext {
sandboxed: bool,
datadir: PathBuf,
run_function: String,
version: Version,
package_id: PackageId,
volumes: Arc<dyn PathForVolumeId>,
input: Value,
}
#[derive(Clone, Default)]
struct AnswerState(std::sync::Arc<deno_core::parking_lot::Mutex<Value>>);
#[derive(Clone, Debug)]
struct ModsLoader {
code: JsCode,
}
impl ModuleLoader for ModsLoader {
fn resolve(
&self,
specifier: &str,
referrer: &str,
_is_main: bool,
) -> Result<ModuleSpecifier, AnyError> {
if referrer.contains("embassy") {
bail!("Embassy.js cannot import anything else");
}
let s = resolve_import(specifier, referrer).unwrap();
Ok(s)
}
fn load(
&self,
module_specifier: &ModuleSpecifier,
maybe_referrer: Option<ModuleSpecifier>,
is_dyn_import: bool,
) -> Pin<Box<ModuleSourceFuture>> {
let module_specifier = module_specifier.as_str().to_owned();
let module = match &*module_specifier {
"file:///deno_global.js" => Ok(ModuleSource {
module_url_specified: "file:///deno_global.js".to_string(),
module_url_found: "file:///deno_global.js".to_string(),
code: "const old_deno = Deno; Deno = null; export default old_deno"
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///loadModule.js" => Ok(ModuleSource {
module_url_specified: "file:///loadModule.js".to_string(),
module_url_found: "file:///loadModule.js".to_string(),
code: include_str!("./artifacts/loadModule.js")
.as_bytes()
.to_vec()
.into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
"file:///embassy.js" => Ok(ModuleSource {
module_url_specified: "file:///embassy.js".to_string(),
module_url_found: "file:///embassy.js".to_string(),
code: self.code.0.as_bytes().to_vec().into_boxed_slice(),
module_type: ModuleType::JavaScript,
}),
x => Err(anyhow!("Not allowed to import: {}", x)),
};
Box::pin(async move {
if is_dyn_import {
bail!("Will not import dynamic");
}
match &maybe_referrer {
Some(x) if x.as_str() == "file:///embassy.js" => {
bail!("Embassy is not allowed to import")
}
_ => (),
}
module
})
}
}
pub struct JsExecutionEnvironment {
sandboxed: bool,
base_directory: PathBuf,
module_loader: ModsLoader,
package_id: PackageId,
version: Version,
volumes: Arc<dyn PathForVolumeId>,
}
impl JsExecutionEnvironment {
pub async fn load_from_package(
data_directory: impl AsRef<std::path::Path>,
package_id: &PackageId,
version: &Version,
volumes: Box<dyn PathForVolumeId>,
) -> Result<Self, (JsError, String)> {
let data_dir = data_directory.as_ref();
let base_directory = data_dir;
let js_code = JsCode({
let file_path = script_dir(data_dir, package_id, version).join("embassy.js");
let mut file = match tokio::fs::File::open(file_path.clone()).await {
Ok(x) => x,
Err(e) => {
tracing::debug!("path: {:?}", file_path);
tracing::debug!("{:?}", e);
return Err((
JsError::FileSystem,
format!("The file opening '{:?}' created error: {}", file_path, e),
));
}
};
let mut buffer = Default::default();
if let Err(err) = file.read_to_string(&mut buffer).await {
tracing::debug!("{:?}", err);
return Err((
JsError::FileSystem,
format!("The file reading created error: {}", err),
));
};
buffer
});
Ok(Self {
base_directory: base_directory.to_owned(),
module_loader: ModsLoader { code: js_code },
package_id: package_id.clone(),
version: version.clone(),
volumes: volumes.into(),
sandboxed: false,
})
}
pub fn read_only_effects(mut self) -> Self {
self.sandboxed = true;
self
}
pub async fn run_action<I: Serialize, O: for<'de> Deserialize<'de>>(
self,
procedure_name: ProcedureName,
input: Option<I>,
) -> Result<O, (JsError, String)> {
let input = match serde_json::to_value(input) {
Ok(a) => a,
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
"Couldn't convert input".to_string(),
));
}
};
let safer_handle: NonDetachingJoinHandle<_> =
tokio::task::spawn_blocking(move || self.execute(procedure_name, input)).into();
let output = safer_handle
.await
.map_err(|err| (JsError::Tokio, format!("Tokio gave us the error: {}", err)))??;
match serde_json::from_value(output.clone()) {
Ok(x) => Ok(x),
Err(err) => {
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
return Err((
JsError::BoundryLayerSerDe,
format!(
"Couldn't convert output = {:#?} to the correct type",
serde_json::to_string_pretty(&output).unwrap_or_default()
),
));
}
}
}
fn declarations() -> Vec<OpDecl> {
vec![
fns::read_file::decl(),
fns::write_file::decl(),
fns::remove_file::decl(),
fns::create_dir::decl(),
fns::remove_dir::decl(),
fns::current_function::decl(),
fns::log_trace::decl(),
fns::log_warn::decl(),
fns::log_error::decl(),
fns::log_debug::decl(),
fns::log_info::decl(),
fns::get_input::decl(),
fns::set_value::decl(),
fns::is_sandboxed::decl(),
]
}
fn execute(
&self,
procedure_name: ProcedureName,
input: Value,
) -> Result<Value, (JsError, String)> {
let base_directory = self.base_directory.clone();
let answer_state = AnswerState::default();
let ext_answer_state = answer_state.clone();
let js_ctx = JsContext {
datadir: base_directory,
run_function: procedure_name.js_function_name(),
package_id: self.package_id.clone(),
volumes: self.volumes.clone(),
version: self.version.clone(),
sandboxed: self.sandboxed,
input,
};
let ext = Extension::builder()
.ops(Self::declarations())
.state(move |state| {
state.put(ext_answer_state.clone());
state.put(js_ctx.clone());
Ok(())
})
.build();
let loader = std::rc::Rc::new(self.module_loader.clone());
let runtime_options = RuntimeOptions {
module_loader: Some(loader),
extensions: vec![ext],
startup_snapshot: Some(Snapshot::Static(SNAPSHOT_BYTES)),
..Default::default()
};
let mut runtime = JsRuntime::new(runtime_options);
let future = async move {
let mod_id = runtime
.load_main_module(&"file:///loadModule.js".parse().unwrap(), None)
.await?;
let evaluated = runtime.mod_evaluate(mod_id);
let res = runtime.run_event_loop(false).await;
res?;
evaluated.await??;
Ok::<_, AnyError>(())
};
tokio::runtime::Handle::current()
.block_on(future)
.map_err(|e| {
tracing::debug!("{:?}", e);
(JsError::Javascript, format!("{}", e))
})?;
let answer = answer_state.0.lock().clone();
Ok(answer)
}
}
/// Note: Make sure that we have the assumption that all these methods are callable at any time, and all call restrictions should be in rust
mod fns {
use deno_core::{
anyhow::{anyhow, bail},
error::AnyError,
*,
};
use serde_json::Value;
use std::{
cell::RefCell,
convert::TryFrom,
path::{Path, PathBuf},
rc::Rc,
};
use models::VolumeId;
use super::{AnswerState, JsContext};
#[op]
async fn read_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<String, AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
//get_path_for in volume.rs
let new_file = volume_path.join(path_in);
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
let answer = tokio::fs::read_to_string(new_file).await?;
Ok(answer)
}
#[op]
async fn write_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
write: String,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::write(new_file, write).await?;
Ok(())
}
#[op]
async fn remove_file(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_file(new_file).await?;
Ok(())
}
#[op]
async fn remove_dir(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
// With the volume check
if !is_subset(&volume_path, &new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::remove_dir_all(new_file).await?;
Ok(())
}
#[op]
async fn create_dir(
state: Rc<RefCell<OpState>>,
volume_id: VolumeId,
path_in: PathBuf,
) -> Result<(), AnyError> {
let state = state.borrow();
let ctx: &JsContext = state.borrow();
let volume_path = ctx
.volumes
.path_for(&ctx.datadir, &ctx.package_id, &ctx.version, &volume_id)
.ok_or_else(|| anyhow!("There is no {} in volumes", volume_id))?;
if ctx.volumes.readonly(&volume_id) {
bail!("Volume {} is readonly", volume_id);
}
let new_file = volume_path.join(path_in);
let parent_new_file = new_file
.parent()
.ok_or_else(|| anyhow!("Expecting that file is not root"))?;
// With the volume check
if !is_subset(&volume_path, &parent_new_file).await? {
bail!(
"Path '{}' has broken away from parent '{}'",
new_file.to_string_lossy(),
volume_path.to_string_lossy(),
);
}
tokio::fs::create_dir_all(new_file).await?;
Ok(())
}
#[op]
fn current_function(state: &mut OpState) -> Result<String, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.run_function.clone())
}
#[op]
fn log_trace(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::trace!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_warn(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::warn!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_error(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::error!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_debug(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::debug!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn log_info(state: &mut OpState, input: String) -> Result<(), AnyError> {
let ctx = state.borrow::<JsContext>();
tracing::info!(
package_id = tracing::field::display(&ctx.package_id),
run_function = tracing::field::display(&ctx.run_function),
"{}",
input
);
Ok(())
}
#[op]
fn get_input(state: &mut OpState) -> Result<Value, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.input.clone())
}
#[op]
fn set_value(state: &mut OpState, value: Value) -> Result<(), AnyError> {
let mut answer = state.borrow::<AnswerState>().0.lock();
*answer = value;
Ok(())
}
#[op]
fn is_sandboxed(state: &mut OpState) -> Result<bool, AnyError> {
let ctx = state.borrow::<JsContext>();
Ok(ctx.sandboxed)
}
/// We need to make sure that during the file accessing, we don't reach beyond our scope of control
async fn is_subset(
parent: impl AsRef<Path>,
child: impl AsRef<Path>,
) -> Result<bool, AnyError> {
let child = tokio::fs::canonicalize(child).await?;
let parent = tokio::fs::canonicalize(parent).await?;
Ok(child.starts_with(parent))
}
}

15
libs/models/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "models"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
"trace",
] }
serde = { version = "1.0.*", features = ["derive", "rc"] }
thiserror = "1.0.*"
emver = { version = "0.1.6", features = ["serde"] }
rand = "0.7.*"

52
libs/models/src/action_id.rs Normal file
View File

@@ -0,0 +1,52 @@
use std::{str::FromStr, path::Path};
use serde::{Serialize, Deserialize};
use crate::{Id, InvalidId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct ActionId<S: AsRef<str> = String>(Id<S>);
impl FromStr for ActionId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(ActionId(Id::try_from(s.to_owned())?))
}
}
impl From<ActionId> for String {
fn from(value: ActionId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> AsRef<ActionId<S>> for ActionId<S> {
fn as_ref(&self) -> &ActionId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for ActionId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for ActionId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for ActionId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for ActionId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(ActionId(serde::Deserialize::deserialize(deserializer)?))
}
}

35
libs/models/src/health_check_id.rs Normal file
View File

@@ -0,0 +1,35 @@
use std::path::Path;
use serde::{Serialize, Deserialize, Deserializer};
use crate::Id;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct HealthCheckId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> std::fmt::Display for HealthCheckId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for HealthCheckId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for HealthCheckId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(HealthCheckId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for HealthCheckId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}

75
libs/models/src/id.rs Normal file
View File

@@ -0,0 +1,75 @@
use std::borrow::Borrow;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use crate::{invalid_id::InvalidId, id_unchecked::IdUnchecked};
pub const SYSTEM_ID: Id<&'static str> = Id("x_system");
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Id<S: AsRef<str> = String>(S);
impl<S: AsRef<str>> Id<S> {
pub fn try_from(value: S) -> Result<Self, InvalidId> {
if value
.as_ref()
.chars()
.all(|c| c.is_ascii_lowercase() || c == '-')
{
Ok(Id(value))
} else {
Err(InvalidId)
}
}
}
impl<'a> Id<&'a str> {
pub fn owned(&self) -> Id {
Id(self.0.to_owned())
}
}
impl From<Id> for String {
fn from(value: Id) -> Self {
value.0
}
}
impl<S: AsRef<str>> std::ops::Deref for Id<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<S: AsRef<str>> std::fmt::Display for Id<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_ref())
}
}
impl<S: AsRef<str>> AsRef<str> for Id<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for Id<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for Id<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Id::try_from(unchecked.0).map_err(serde::de::Error::custom)
}
}
impl<S: AsRef<str>> Serialize for Id<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
serializer.serialize_str(self.as_ref())
}
}

56
libs/models/src/id_unchecked.rs Normal file
View File

@@ -0,0 +1,56 @@
use std::borrow::Cow;
use serde::{Deserialize, Deserializer};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct IdUnchecked<S: AsRef<str>>(pub S);
impl<'de> Deserialize<'de> for IdUnchecked<Cow<'de, str>> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = IdUnchecked<Cow<'de, str>>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a valid ID")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v.to_owned())))
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Owned(v)))
}
fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(IdUnchecked(Cow::Borrowed(v)))
}
}
deserializer.deserialize_any(Visitor)
}
}
impl<'de> Deserialize<'de> for IdUnchecked<String> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(String::deserialize(deserializer)?))
}
}
impl<'de> Deserialize<'de> for IdUnchecked<&'de str> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(IdUnchecked(<&'de str>::deserialize(deserializer)?))
}
}

47
libs/models/src/interface_id.rs Normal file
View File

@@ -0,0 +1,47 @@
use std::path::Path;
use serde::{Serialize, Deserialize, Deserializer};
use crate::Id;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct InterfaceId<S: AsRef<str> = String>(Id<S>);
impl<S: AsRef<str>> From<Id<S>> for InterfaceId<S> {
fn from(id: Id<S>) -> Self {
Self(id)
}
}
impl<S: AsRef<str>> std::fmt::Display for InterfaceId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> std::ops::Deref for InterfaceId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<str> for InterfaceId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<'de, S> Deserialize<'de> for InterfaceId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Ok(InterfaceId(Deserialize::deserialize(deserializer)?))
}
}
impl<S: AsRef<str>> AsRef<Path> for InterfaceId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}

4
libs/models/src/invalid_id.rs Normal file
View File

@@ -0,0 +1,4 @@
#[derive(Debug, thiserror::Error)]
#[error("Invalid ID")]
pub struct InvalidId;

21
libs/models/src/lib.rs Normal file
View File

@@ -0,0 +1,21 @@
mod id;
mod invalid_id;
mod id_unchecked;
mod version;
mod volume_id;
mod interface_id;
mod package_id;
mod procedure_name;
mod health_check_id;
mod action_id;
pub use id::*;
pub use invalid_id::*;
pub use id_unchecked::*;
pub use version::*;
pub use volume_id::*;
pub use interface_id::*;
pub use package_id::*;
pub use procedure_name::*;
pub use health_check_id::*;
pub use action_id::*;

85
libs/models/src/package_id.rs Normal file
View File

@@ -0,0 +1,85 @@
use std::{str::FromStr, borrow::Borrow, path::Path};
use serde::{Deserialize, Serialize, Serializer};
use crate::{Id, InvalidId, SYSTEM_ID};
pub const SYSTEM_PACKAGE_ID: PackageId<&'static str> = PackageId(SYSTEM_ID);
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct PackageId<S: AsRef<str> = String>(Id<S>);
impl<'a> PackageId<&'a str> {
pub fn owned(&self) -> PackageId {
PackageId(self.0.owned())
}
}
impl FromStr for PackageId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(PackageId(Id::try_from(s.to_owned())?))
}
}
impl From<PackageId> for String {
fn from(value: PackageId) -> Self {
value.0.into()
}
}
impl<S: AsRef<str>> From<Id<S>> for PackageId<S> {
fn from(id: Id<S>) -> Self {
PackageId(id)
}
}
impl<S: AsRef<str>> std::ops::Deref for PackageId<S> {
type Target = S;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl<S: AsRef<str>> AsRef<PackageId<S>> for PackageId<S> {
fn as_ref(&self) -> &PackageId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for PackageId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for PackageId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for PackageId<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for PackageId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for PackageId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(PackageId(Deserialize::deserialize(deserializer)?))
}
}
impl<S> Serialize for PackageId<S>
where
S: AsRef<str>,
{
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
Serialize::serialize(&self.0, serializer)
}
}

50
libs/models/src/procedure_name.rs Normal file
View File

@@ -0,0 +1,50 @@
use crate::{PackageId, HealthCheckId, ActionId};
#[derive(Debug, Clone)]
pub enum ProcedureName {
Main, // Usually just run container
CreateBackup,
RestoreBackup,
GetConfig,
SetConfig,
Migration,
Properties,
Check(PackageId),
AutoConfig(PackageId),
Health(HealthCheckId),
Action(ActionId),
}
impl ProcedureName {
pub fn docker_name(&self) -> Option<String> {
match self {
ProcedureName::Main => None,
ProcedureName::CreateBackup => Some("CreateBackup".to_string()),
ProcedureName::RestoreBackup => Some("RestoreBackup".to_string()),
ProcedureName::GetConfig => Some("GetConfig".to_string()),
ProcedureName::SetConfig => Some("SetConfig".to_string()),
ProcedureName::Migration => Some("Migration".to_string()),
ProcedureName::Properties => Some(format!("Properties-{}", rand::random::<u64>())),
ProcedureName::Health(id) => Some(format!("{}Health", id)),
ProcedureName::Action(id) => Some(format!("{}Action", id)),
ProcedureName::Check(_) => None,
ProcedureName::AutoConfig(_) => None,
}
}
pub fn js_function_name(&self) -> String {
match self {
ProcedureName::Main => "/main".to_string(),
ProcedureName::CreateBackup => "/createBackup".to_string(),
ProcedureName::RestoreBackup => "/restoreBackup".to_string(),
ProcedureName::GetConfig => "/getConfig".to_string(),
ProcedureName::SetConfig => "/setConfig".to_string(),
ProcedureName::Migration => "/migration".to_string(),
ProcedureName::Properties => "/properties".to_string(),
ProcedureName::Health(id) => format!("/health/{}", id),
ProcedureName::Action(id) => format!("/action/{}", id),
ProcedureName::Check(id) => format!("/dependencies/{}/check", id),
ProcedureName::AutoConfig(id) => format!("/dependencies/{}/autoConfigure", id),
}
}
}

110
libs/models/src/version.rs Normal file
View File

@@ -0,0 +1,110 @@
use std::{str::FromStr, ops::Deref};
use std::hash::{Hash, Hasher};
use patch_db::{HasModel, Model};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Debug, Clone)]
pub struct Version {
version: emver::Version,
string: String,
}
impl Version {
pub fn as_str(&self) -> &str {
self.string.as_str()
}
pub fn into_version(self) -> emver::Version {
self.version
}
}
impl std::fmt::Display for Version {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.string)
}
}
impl std::str::FromStr for Version {
type Err = <emver::Version as FromStr>::Err;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Version {
string: s.to_owned(),
version: s.parse()?,
})
}
}
impl From<emver::Version> for Version {
fn from(v: emver::Version) -> Self {
Version {
string: v.to_string(),
version: v,
}
}
}
impl From<Version> for emver::Version {
fn from(v: Version) -> Self {
v.version
}
}
impl Default for Version {
fn default() -> Self {
Self::from(emver::Version::default())
}
}
impl Deref for Version {
type Target = emver::Version;
fn deref(&self) -> &Self::Target {
&self.version
}
}
impl AsRef<emver::Version> for Version {
fn as_ref(&self) -> &emver::Version {
&self.version
}
}
impl AsRef<str> for Version {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl PartialEq for Version {
fn eq(&self, other: &Version) -> bool {
self.version.eq(&other.version)
}
}
impl Eq for Version {}
impl PartialOrd for Version {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.version.partial_cmp(&other.version)
}
}
impl Ord for Version {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.version.cmp(&other.version)
}
}
impl Hash for Version {
fn hash<H: Hasher>(&self, state: &mut H) {
self.version.hash(state)
}
}
impl<'de> Deserialize<'de> for Version {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let string = String::deserialize(deserializer)?;
let version = emver::Version::from_str(&string).map_err(::serde::de::Error::custom)?;
Ok(Self { string, version })
}
}
impl Serialize for Version {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
self.string.serialize(serializer)
}
}
impl HasModel for Version {
type Model = Model<Version>;
}

62
libs/models/src/volume_id.rs Normal file
View File

@@ -0,0 +1,62 @@
use std::{borrow::Borrow, path::Path};
use serde::{Deserialize, Serialize, Deserializer};
use crate::{Id, IdUnchecked};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum VolumeId<S: AsRef<str> = String> {
Backup,
Custom(Id<S>),
}
impl<S: AsRef<str>> std::fmt::Display for VolumeId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VolumeId::Backup => write!(f, "BACKUP"),
VolumeId::Custom(id) => write!(f, "{}", id),
}
}
}
impl<S: AsRef<str>> AsRef<str> for VolumeId<S> {
fn as_ref(&self) -> &str {
match self {
VolumeId::Backup => "BACKUP",
VolumeId::Custom(id) => id.as_ref(),
}
}
}
impl<S: AsRef<str>> Borrow<str> for VolumeId<S> {
fn borrow(&self) -> &str {
self.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for VolumeId<S> {
fn as_ref(&self) -> &Path {
AsRef::<str>::as_ref(self).as_ref()
}
}
impl<'de, S> Deserialize<'de> for VolumeId<S>
where
S: AsRef<str>,
IdUnchecked<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
Ok(match unchecked.0.as_ref() {
"BACKUP" => VolumeId::Backup,
_ => VolumeId::Custom(Id::try_from(unchecked.0).map_err(serde::de::Error::custom)?),
})
}
}
impl<S: AsRef<str>> Serialize for VolumeId<S> {
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: serde::Serializer,
{
serializer.serialize_str(self.as_ref())
}
}

11
libs/snapshot-creator/Cargo.toml Normal file
View File

@@ -0,0 +1,11 @@
[package]
name = "snapshot-creator"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
dashmap = "5.1.0"
deno_core = "0.136.0"
deno_ast = {version="0.15.0", features = ["transpiling"]}

4
backend/src/bin/build-js-snapshots.rs → libs/snapshot-creator/src/main.rs
View File

@@ -1,5 +1,7 @@
use deno_core::{JsRuntime, RuntimeOptions};
fn main() {
let mut runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
let mut runtime = JsRuntime::new(RuntimeOptions {
will_snapshot: true,
..Default::default()
});

6
package-lock.json generated Normal file
View File

@@ -0,0 +1,6 @@
{
"name": "embassy-os",
"lockfileVersion": 2,
"requires": true,
"packages": {}
}

1287
system-images/compat/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

1
system-images/compat/Cargo.toml
View File

@@ -10,6 +10,7 @@ version = "0.1.0"
anyhow = { version = "1.0.40", features = ["backtrace"] }
beau_collector = "0.2.1"
clap = "2.33.3"
dashmap = "5.3.2"
embassy-os = { path = "../../backend", default-features = false }
emver = { version = "0.1.2", features = ["serde"] }
failure = "0.1.8"