diff --git a/Makefile b/Makefile index 34064e799..4915101e3 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PLATFORM := $(shell if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo ARCH := $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo aarch64; else echo $(PLATFORM) | sed 's/-nonfree$$//g'; fi) IMAGE_TYPE=$(shell if [ "$(PLATFORM)" = raspberrypi ]; then echo img; else echo iso; fi) BINS := core/target/$(ARCH)-unknown-linux-musl/release/startbox core/target/$(ARCH)-unknown-linux-musl/release/containerbox -WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/diagnostic-ui web/dist/raw/install-wizard +WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/install-wizard FIRMWARE_ROMS := ./firmware/$(PLATFORM) $(shell jq --raw-output '.[] | select(.platform[] | contains("$(PLATFORM)")) | "./firmware/$(PLATFORM)/" + .id + ".rom.gz"' build/lib/firmware.json) BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts $(FIRMWARE_ROMS) DEBIAN_SRC := $(shell git ls-files debian/) @@ -20,7 +20,6 @@ CORE_SRC := $(shell git ls-files core) $(shell git ls-files --recurse-submodules WEB_SHARED_SRC := $(shell git ls-files web/projects/shared) $(shell ls -p web/ | grep -v / | sed 's/^/web\//g') web/node_modules/.package-lock.json web/config.json patch-db/client/dist web/patchdb-ui-seed.json WEB_UI_SRC := $(shell git ls-files web/projects/ui) WEB_SETUP_WIZARD_SRC := $(shell git ls-files web/projects/setup-wizard) -WEB_DIAGNOSTIC_UI_SRC := $(shell git ls-files web/projects/diagnostic-ui) WEB_INSTALL_WIZARD_SRC := $(shell git ls-files web/projects/install-wizard) PATCH_DB_CLIENT_SRC := $(shell git ls-files --recurse-submodules patch-db/client) GZIP_BIN := $(shell which pigz || which gzip) @@ -244,10 +243,6 @@ web/dist/raw/setup-wizard: $(WEB_SETUP_WIZARD_SRC) $(WEB_SHARED_SRC) npm --prefix web run build:setup touch web/dist/raw/setup-wizard -web/dist/raw/diagnostic-ui: $(WEB_DIAGNOSTIC_UI_SRC) $(WEB_SHARED_SRC) - npm --prefix web run build:dui - touch web/dist/raw/diagnostic-ui - web/dist/raw/install-wizard: $(WEB_INSTALL_WIZARD_SRC) $(WEB_SHARED_SRC) npm --prefix web run build:install-wiz touch web/dist/raw/install-wizard diff --git a/core/Cargo.lock b/core/Cargo.lock index cf10152c2..031ec41ab 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -42,7 +42,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -54,7 +54,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy", @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -156,9 +156,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arrayref" @@ -187,16 +187,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ - "concurrent-queue", + "concurrent-queue 2.5.0", "event-listener", "futures-core", ] [[package]] name = "async-compression" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "brotli", "flate2", @@ -225,7 +225,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -236,7 +236,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -248,6 +248,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atty" version = "0.2.14" @@ -278,7 +284,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -418,6 +424,17 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "barrage" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be5951c75bdabb58753d140dd5802f12ff3a483cb2e16fb5276e111b94b19e87" +dependencies = [ + "concurrent-queue 1.2.4", + "event-listener", + "spin 0.9.8", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -562,9 +579,9 @@ dependencies = [ [[package]] name = "brotli" -version = "5.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -573,9 +590,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -600,10 +617,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] -name = "cc" -version = "1.0.96" +name = "cache-padded" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" +checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" + +[[package]] +name = "cc" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -688,9 +711,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive", @@ -698,9 +721,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstream", "anstyle", @@ -710,21 +733,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color-eyre" @@ -759,6 +782,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -945,18 +977,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -991,9 +1023,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" @@ -1117,14 +1149,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "darling" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ "darling_core", "darling_macro", @@ -1132,27 +1164,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.60", + "strsim 0.11.1", + "syn 2.0.66", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1183,7 +1215,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1206,7 +1238,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1274,6 +1306,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "divrem" version = "1.0.0" @@ -1367,9 +1410,9 @@ dependencies = [ [[package]] name = "either" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" dependencies = [ "serde", ] @@ -1407,9 +1450,9 @@ dependencies = [ [[package]] name = "ena" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" dependencies = [ "log", ] @@ -1444,7 +1487,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1455,9 +1498,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1517,9 +1560,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38793c55593b33412e3ae40c2c9781ffaa6f438f6f8c10f24e71846fbd7ae01e" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" @@ -1533,12 +1576,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "fixedbitset" version = "0.4.2" @@ -1678,7 +1715,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1735,9 +1772,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1794,15 +1831,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap 2.2.6", "slab", @@ -2005,12 +2042,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -2018,9 +2055,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -2036,9 +2073,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -2067,7 +2104,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -2085,7 +2122,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2109,9 +2146,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -2150,6 +2187,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "id-pool" version = "0.2.2" @@ -2187,12 +2342,14 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", ] [[package]] @@ -2302,9 +2459,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] @@ -2535,7 +2692,7 @@ dependencies = [ "petgraph", "pico-args", "regex", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "string_cache", "term", "tiny-keccak", @@ -2549,7 +2706,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" dependencies = [ - "regex-automata 0.4.6", + "regex-automata 0.4.7", ] [[package]] @@ -2579,9 +2736,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -2618,9 +2775,15 @@ checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "lock_api" @@ -2734,9 +2897,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -2786,11 +2949,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -2885,9 +3047,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -2899,11 +3061,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -2927,9 +3088,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -2962,11 +3123,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -3010,7 +3170,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3076,7 +3236,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3087,9 +3247,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.3+3.2.1" +version = "300.3.1+3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" dependencies = [ "cc", ] @@ -3159,9 +3319,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -3182,9 +3342,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "patch-db" @@ -3254,9 +3414,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -3294,7 +3454,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3400,9 +3560,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -3421,7 +3581,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "rusty-fork", "tempfile", "unarray", @@ -3440,9 +3600,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -3450,22 +3610,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -3566,7 +3726,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", ] [[package]] @@ -3655,21 +3815,21 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -3683,13 +3843,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -3700,9 +3860,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" @@ -3717,7 +3877,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -3781,7 +3941,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted", @@ -3802,7 +3962,7 @@ dependencies = [ [[package]] name = "rpc-toolkit" version = "0.2.3" -source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#f5566840bb9af0743612fede4034f5a390cd1eee" +source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#5a24903031e72ac75fd23889215361edc7b20842" dependencies = [ "async-stream", "async-trait", @@ -3871,9 +4031,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -3923,7 +4083,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -3949,9 +4109,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -3965,9 +4125,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.3" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", @@ -4064,11 +4224,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -4077,9 +4237,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -4087,9 +4247,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -4128,7 +4288,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -4155,9 +4315,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -4201,7 +4361,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -4396,11 +4556,10 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" dependencies = [ - "itertools 0.12.1", "nom", "unicode_categories", ] @@ -4629,7 +4788,7 @@ dependencies = [ "quote", "regex-syntax 0.6.29", "strsim 0.10.0", - "syn 2.0.60", + "syn 2.0.66", "unicode-width", ] @@ -4675,6 +4834,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "start-os" version = "0.3.5-rev.2" @@ -4686,6 +4851,7 @@ dependencies = [ "axum 0.7.5", "axum-server", "backhand", + "barrage", "base32", "base64 0.21.7", "base64ct", @@ -4783,8 +4949,9 @@ dependencies = [ "tokio-tar", "tokio-tungstenite", "tokio-util", - "toml 0.8.12", + "toml 0.8.14", "torut", + "tower-service", "tracing", "tracing-error", "tracing-futures", @@ -4827,13 +4994,13 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -4867,9 +5034,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -4888,6 +5055,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4917,9 +5095,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" dependencies = [ "filetime", "libc", @@ -4970,22 +5148,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5049,6 +5227,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -5066,9 +5254,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -5096,13 +5284,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5180,16 +5368,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -5206,21 +5393,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.12", + "toml_edit 0.22.14", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] @@ -5251,15 +5438,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.7", + "winnow 0.6.13", ] [[package]] @@ -5276,7 +5463,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -5360,7 +5547,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5512,7 +5699,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "termcolor", ] @@ -5553,7 +5740,7 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5598,6 +5785,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -5606,9 +5799,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unicode-xid" @@ -5630,12 +5823,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.0", "percent-encoding", "serde", ] @@ -5653,10 +5846,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "utf8parse" -version = "0.2.1" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -5664,7 +5869,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", ] [[package]] @@ -5752,7 +5957,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -5786,7 +5991,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6026,9 +6231,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.7" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b9415ee827af173ebb3f15f9083df5a122eb93572ec28741fb153356ea2578" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" dependencies = [ "memchr", ] @@ -6043,6 +6248,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -6106,30 +6323,75 @@ dependencies = [ ] [[package]] -name = "zerocopy" -version = "0.7.32" +name = "yoke" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -6142,7 +6404,29 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", +] + +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 3ad4f9eeb..a8707bf65 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -59,6 +59,7 @@ async-stream = "0.3.5" async-trait = "0.1.74" axum = { version = "0.7.3", features = ["ws"] } axum-server = "0.6.0" +barrage = "0.2.3" backhand = "0.18.0" base32 = "0.4.0" base64 = "0.21.4" @@ -102,7 +103,7 @@ id-pool = { version = "0.2.2", default-features = false, features = [ ] } imbl = "2.0.2" imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" } -include_dir = "0.7.3" +include_dir = { version = "0.7.3", features = ["metadata"] } indexmap = { version = "2.0.2", features = ["serde"] } indicatif = { version = "0.17.7", features = ["tokio"] } integer-encoding = { version = "4.0.0", features = ["tokio_async"] } @@ -178,6 +179,7 @@ tokio-util = { version = "0.7.9", features = ["io"] } torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies", features = [ "serialize", ] } +tower-service = "0.3.2" tracing = "0.1.39" tracing-error = "0.2.0" tracing-futures = "0.2.5" diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index 4753a4290..556f750ec 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -4,25 +4,25 @@ use std::sync::Arc; use clap::Parser; use futures::{stream, StreamExt}; use models::PackageId; -use openssl::x509::X509; use patch_db::json_ptr::ROOT; use serde::{Deserialize, Serialize}; -use torut::onion::OnionAddressV3; +use tokio::sync::Mutex; use tracing::instrument; use ts_rs::TS; use super::target::BackupTargetId; use crate::backup::os::OsBackup; +use crate::context::setup::SetupResult; use crate::context::{RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::mount::backup::BackupMountGuard; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; -use crate::hostname::Hostname; -use crate::init::init; +use crate::init::{init, InitResult}; use crate::prelude::*; use crate::s9pk::S9pk; use crate::service::service_map::DownloadInstallFuture; +use crate::setup::SetupExecuteProgress; use crate::util::serde::IoFormat; #[derive(Deserialize, Serialize, Parser, TS)] @@ -67,14 +67,21 @@ pub async fn restore_packages_rpc( Ok(()) } -#[instrument(skip(ctx))] +#[instrument(skip_all)] pub async fn recover_full_embassy( - ctx: SetupContext, + ctx: &SetupContext, disk_guid: Arc, start_os_password: String, recovery_source: TmpMountGuard, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { + SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { + let mut restore_phase = restore_phase.or_not_found("restore progress")?; + let backup_guard = BackupMountGuard::mount( recovery_source, recovery_password.as_deref().unwrap_or_default(), @@ -99,10 +106,17 @@ pub async fn recover_full_embassy( db.put(&ROOT, &Database::init(&os_backup.account)?).await?; drop(db); - init(&ctx.config).await?; + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; - let rpc_ctx = RpcContext::init(&ctx.config, disk_guid.clone()).await?; + let rpc_ctx = RpcContext::init( + &ctx.config, + disk_guid.clone(), + Some(net_ctrl), + rpc_ctx_phases, + ) + .await?; + restore_phase.start(); let ids: Vec<_> = backup_guard .metadata .package_backups @@ -110,26 +124,26 @@ pub async fn recover_full_embassy( .cloned() .collect(); let tasks = restore_packages(&rpc_ctx, backup_guard, ids).await?; + restore_phase.set_total(tasks.len() as u64); + let restore_phase = Arc::new(Mutex::new(restore_phase)); stream::iter(tasks) - .for_each_concurrent(5, |(id, res)| async move { - match async { res.await?.await }.await { - Ok(_) => (), - Err(err) => { - tracing::error!("Error restoring package {}: {}", id, err); - tracing::debug!("{:?}", err); + .for_each_concurrent(5, |(id, res)| { + let restore_phase = restore_phase.clone(); + async move { + match async { res.await?.await }.await { + Ok(_) => (), + Err(err) => { + tracing::error!("Error restoring package {}: {}", id, err); + tracing::debug!("{:?}", err); + } } + *restore_phase.lock().await += 1; } }) .await; + restore_phase.lock().await.complete(); - rpc_ctx.shutdown().await?; - - Ok(( - disk_guid, - os_backup.account.hostname, - os_backup.account.tor_key.public().get_onion_address(), - os_backup.account.root_ca_cert, - )) + Ok(((&os_backup.account).try_into()?, rpc_ctx)) } #[instrument(skip(ctx, backup_guard))] diff --git a/core/startos/src/bins/registry.rs b/core/startos/src/bins/registry.rs index 3028d1766..132e0984a 100644 --- a/core/startos/src/bins/registry.rs +++ b/core/startos/src/bins/registry.rs @@ -14,7 +14,8 @@ use crate::util::logger::EmbassyLogger; async fn inner_main(config: &RegistryConfig) -> Result<(), Error> { let server = async { let ctx = RegistryContext::init(config).await?; - let server = WebServer::registry(ctx.listen, ctx.clone()); + let mut server = WebServer::new(ctx.listen); + server.serve_registry(ctx.clone()); let mut shutdown_recv = ctx.shutdown.subscribe(); diff --git a/core/startos/src/bins/start_init.rs b/core/startos/src/bins/start_init.rs index 8e60884f1..f4aa411b5 100644 --- a/core/startos/src/bins/start_init.rs +++ b/core/startos/src/bins/start_init.rs @@ -1,47 +1,56 @@ -use std::net::{Ipv6Addr, SocketAddr}; -use std::path::Path; use std::sync::Arc; -use std::time::Duration; -use helpers::NonDetachingJoinHandle; use tokio::process::Command; use tracing::instrument; use crate::context::config::ServerConfig; -use crate::context::{DiagnosticContext, InstallContext, SetupContext}; -use crate::disk::fsck::{RepairStrategy, RequiresReboot}; +use crate::context::rpc::InitRpcContextPhases; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; +use crate::disk::fsck::RepairStrategy; use crate::disk::main::DEFAULT_PASSWORD; use crate::disk::REPAIR_DISK_PATH; -use crate::firmware::update_firmware; -use crate::init::STANDBY_MODE_PATH; +use crate::firmware::{check_for_firmware_update, update_firmware}; +use crate::init::{InitPhases, InitResult, STANDBY_MODE_PATH}; use crate::net::web_server::WebServer; +use crate::prelude::*; +use crate::progress::FullProgressTracker; use crate::shutdown::Shutdown; -use crate::sound::{BEP, CHIME}; use crate::util::Invoke; -use crate::{Error, ErrorKind, ResultExt, PLATFORM}; +use crate::PLATFORM; #[instrument(skip_all)] -async fn setup_or_init(config: &ServerConfig) -> Result, Error> { - let song = NonDetachingJoinHandle::from(tokio::spawn(async { - loop { - BEP.play().await.unwrap(); - BEP.play().await.unwrap(); - tokio::time::sleep(Duration::from_secs(30)).await; - } - })); +async fn setup_or_init( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { + if let Some(firmware) = check_for_firmware_update() + .await + .map_err(|e| { + tracing::warn!("Error checking for firmware update: {e}"); + tracing::debug!("{e:?}"); + }) + .ok() + .and_then(|a| a) + { + let init_ctx = InitContext::init(config).await?; + let handle = &init_ctx.progress; + let mut update_phase = handle.add_phase("Updating Firmware".into(), Some(10)); + let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); - match update_firmware().await { - Ok(RequiresReboot(true)) => { - return Ok(Some(Shutdown { - export_args: None, - restart: true, - })) - } - Err(e) => { + server.serve_init(init_ctx); + + update_phase.start(); + if let Err(e) = update_firmware(firmware).await { tracing::warn!("Error performing firmware update: {e}"); tracing::debug!("{e:?}"); + } else { + update_phase.complete(); + reboot_phase.start(); + return Ok(Err(Shutdown { + export_args: None, + restart: true, + })); } - _ => (), } Command::new("ln") @@ -84,14 +93,7 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> let ctx = InstallContext::init().await?; - let server = WebServer::install( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; - - drop(song); - tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this - CHIME.play().await?; + server.serve_install(ctx.clone()); ctx.shutdown .subscribe() @@ -99,33 +101,23 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> .await .expect("context dropped"); - server.shutdown().await; + return Ok(Err(Shutdown { + export_args: None, + restart: true, + })); + } - Command::new("reboot") - .invoke(crate::ErrorKind::Unknown) - .await?; - } else if tokio::fs::metadata("/media/startos/config/disk.guid") + if tokio::fs::metadata("/media/startos/config/disk.guid") .await .is_err() { let ctx = SetupContext::init(config)?; - let server = WebServer::setup( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; - - drop(song); - tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this - CHIME.play().await?; + server.serve_setup(ctx.clone()); let mut shutdown = ctx.shutdown.subscribe(); shutdown.recv().await.expect("context dropped"); - server.shutdown().await; - - drop(shutdown); - tokio::task::yield_now().await; if let Err(e) = Command::new("killall") .arg("firefox-esr") @@ -135,19 +127,40 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> tracing::error!("Failed to kill kiosk: {}", e); tracing::debug!("{:?}", e); } + + Ok(Ok(match ctx.result.get() { + Some(Ok((_, rpc_ctx))) => (rpc_ctx.clone(), ctx.progress.clone()), + Some(Err(e)) => return Err(e.clone_output()), + None => { + return Err(Error::new( + eyre!("Setup mode exited before setup completed"), + ErrorKind::Unknown, + )) + } + })) } else { + let init_ctx = InitContext::init(config).await?; + let handle = init_ctx.progress.clone(); + + let mut disk_phase = handle.add_phase("Opening data drive".into(), Some(10)); + let init_phases = InitPhases::new(&handle); + let rpc_ctx_phases = InitRpcContextPhases::new(&handle); + + server.serve_init(init_ctx); + + disk_phase.start(); let guid_string = tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await?; - let guid = guid_string.trim(); + let disk_guid = Arc::new(String::from(guid_string.trim())); let requires_reboot = crate::disk::main::import( - guid, + &**disk_guid, config.datadir(), if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen }, - if guid.ends_with("_UNENC") { + if disk_guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) @@ -159,40 +172,31 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> .await .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?; } - if requires_reboot.0 { - crate::disk::main::export(guid, config.datadir()).await?; - Command::new("reboot") - .invoke(crate::ErrorKind::Unknown) - .await?; - } + disk_phase.complete(); tracing::info!("Loaded Disk"); - crate::init::init(config).await?; - drop(song); - } - Ok(None) -} - -async fn run_script_if_exists>(path: P) { - let script = path.as_ref(); - if script.exists() { - match Command::new("/bin/bash").arg(script).spawn() { - Ok(mut c) => { - if let Err(e) = c.wait().await { - tracing::error!("Error Running {}: {}", script.display(), e); - tracing::debug!("{:?}", e); - } - } - Err(e) => { - tracing::error!("Error Running {}: {}", script.display(), e); - tracing::debug!("{:?}", e); - } + if requires_reboot.0 { + let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); + reboot_phase.start(); + return Ok(Err(Shutdown { + export_args: Some((disk_guid, config.datadir().to_owned())), + restart: true, + })); } + + let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?; + + let rpc_ctx = RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(Ok((rpc_ctx, handle))) } } #[instrument(skip_all)] -async fn inner_main(config: &ServerConfig) -> Result, Error> { +pub async fn main( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() { tokio::fs::remove_file(STANDBY_MODE_PATH).await?; Command::new("sync").invoke(ErrorKind::Filesystem).await?; @@ -200,16 +204,11 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { futures::future::pending::<()>().await; } - crate::sound::BEP.play().await?; - - run_script_if_exists("/media/startos/config/preinit.sh").await; - - let res = match setup_or_init(config).await { + let res = match setup_or_init(server, config).await { Err(e) => { async move { - tracing::error!("{}", e.source); - tracing::debug!("{}", e.source); - crate::sound::BEETHOVEN.play().await?; + tracing::error!("{e}"); + tracing::debug!("{e:?}"); let ctx = DiagnosticContext::init( config, @@ -229,44 +228,16 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { e, )?; - let server = WebServer::diagnostic( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; + server.serve_diagnostic(ctx.clone()); let shutdown = ctx.shutdown.subscribe().recv().await.unwrap(); - server.shutdown().await; - - Ok(shutdown) + Ok(Err(shutdown)) } .await } Ok(s) => Ok(s), }; - run_script_if_exists("/media/startos/config/postinit.sh").await; - res } - -pub fn main(config: &ServerConfig) { - let res = { - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("failed to initialize runtime"); - rt.block_on(inner_main(config)) - }; - - match res { - Ok(Some(shutdown)) => shutdown.execute(), - Ok(None) => (), - Err(e) => { - eprintln!("{}", e.source); - tracing::debug!("{:?}", e.source); - drop(e.source); - std::process::exit(e.kind as i32) - } - } -} diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index f0bc428be..7576c41e9 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -1,6 +1,5 @@ use std::ffi::OsString; use std::net::{Ipv6Addr, SocketAddr}; -use std::path::Path; use std::sync::Arc; use clap::Parser; @@ -10,7 +9,8 @@ use tokio::signal::unix::signal; use tracing::instrument; use crate::context::config::ServerConfig; -use crate::context::{DiagnosticContext, RpcContext}; +use crate::context::rpc::InitRpcContextPhases; +use crate::context::{DiagnosticContext, InitContext, RpcContext}; use crate::net::web_server::WebServer; use crate::shutdown::Shutdown; use crate::system::launch_metrics_task; @@ -18,9 +18,31 @@ use crate::util::logger::EmbassyLogger; use crate::{Error, ErrorKind, ResultExt}; #[instrument(skip_all)] -async fn inner_main(config: &ServerConfig) -> Result, Error> { - let (rpc_ctx, server, shutdown) = async { - let rpc_ctx = RpcContext::init( +async fn inner_main( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { + let rpc_ctx = if !tokio::fs::metadata("/run/startos/initialized") + .await + .is_ok() + { + let (ctx, handle) = match super::start_init::main(server, &config).await? { + Err(s) => return Ok(Some(s)), + Ok(ctx) => ctx, + }; + tokio::fs::write("/run/startos/initialized", "").await?; + + server.serve_main(ctx.clone()); + handle.complete(); + + ctx + } else { + let init_ctx = InitContext::init(config).await?; + let handle = init_ctx.progress.clone(); + let rpc_ctx_phases = InitRpcContextPhases::new(&handle); + server.serve_init(init_ctx); + + let ctx = RpcContext::init( config, Arc::new( tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy @@ -28,13 +50,19 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { .trim() .to_owned(), ), + None, + rpc_ctx_phases, ) .await?; + + server.serve_main(ctx.clone()); + handle.complete(); + + ctx + }; + + let (rpc_ctx, shutdown) = async { crate::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?; - let server = WebServer::main( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - rpc_ctx.clone(), - )?; let mut shutdown_recv = rpc_ctx.shutdown.subscribe(); @@ -74,8 +102,6 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { .await }); - crate::sound::CHIME.play().await?; - metrics_task .map_err(|e| { Error::new( @@ -93,10 +119,9 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { sig_handler.abort(); - Ok::<_, Error>((rpc_ctx, server, shutdown)) + Ok::<_, Error>((rpc_ctx, shutdown)) } .await?; - server.shutdown().await; rpc_ctx.shutdown().await?; tracing::info!("RPC Context is dropped"); @@ -109,24 +134,22 @@ pub fn main(args: impl IntoIterator) { let config = ServerConfig::parse_from(args).load().unwrap(); - if !Path::new("/run/embassy/initialized").exists() { - super::start_init::main(&config); - std::fs::write("/run/embassy/initialized", "").unwrap(); - } - let res = { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .expect("failed to initialize runtime"); rt.block_on(async { - match inner_main(&config).await { - Ok(a) => Ok(a), + let mut server = WebServer::new(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80)); + match inner_main(&mut server, &config).await { + Ok(a) => { + server.shutdown().await; + Ok(a) + } Err(e) => { async { - tracing::error!("{}", e.source); - tracing::debug!("{:?}", e.source); - crate::sound::BEETHOVEN.play().await?; + tracing::error!("{e}"); + tracing::debug!("{e:?}"); let ctx = DiagnosticContext::init( &config, if tokio::fs::metadata("/media/startos/config/disk.guid") @@ -145,10 +168,7 @@ pub fn main(args: impl IntoIterator) { e, )?; - let server = WebServer::diagnostic( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; + server.serve_diagnostic(ctx.clone()); let mut shutdown = ctx.shutdown.subscribe(); @@ -157,7 +177,7 @@ pub fn main(args: impl IntoIterator) { server.shutdown().await; - Ok::<_, Error>(shutdown) + Ok::<_, Error>(Some(shutdown)) } .await } diff --git a/core/startos/src/context/cli.rs b/core/startos/src/context/cli.rs index 014457b67..22084bbe1 100644 --- a/core/startos/src/context/cli.rs +++ b/core/startos/src/context/cli.rs @@ -18,7 +18,7 @@ use tracing::instrument; use super::setup::CURRENT_SECRET; use crate::context::config::{local_config_path, ClientConfig}; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; use crate::prelude::*; use crate::rpc_continuations::Guid; @@ -271,6 +271,11 @@ impl CallRemote for CliContext { call_remote_http(&self.client, self.rpc_url.clone(), method, params).await } } +impl CallRemote for CliContext { + async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result { + call_remote_http(&self.client, self.rpc_url.clone(), method, params).await + } +} impl CallRemote for CliContext { async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result { call_remote_http(&self.client, self.rpc_url.clone(), method, params).await diff --git a/core/startos/src/context/diagnostic.rs b/core/startos/src/context/diagnostic.rs index 10379dcf3..0bf67e172 100644 --- a/core/startos/src/context/diagnostic.rs +++ b/core/startos/src/context/diagnostic.rs @@ -14,7 +14,7 @@ use crate::Error; pub struct DiagnosticContextSeed { pub datadir: PathBuf, - pub shutdown: Sender>, + pub shutdown: Sender, pub error: Arc, pub disk_guid: Option>, pub rpc_continuations: RpcContinuations, diff --git a/core/startos/src/context/init.rs b/core/startos/src/context/init.rs new file mode 100644 index 000000000..f5f4a5430 --- /dev/null +++ b/core/startos/src/context/init.rs @@ -0,0 +1,47 @@ +use std::ops::Deref; +use std::sync::Arc; + +use rpc_toolkit::Context; +use tokio::sync::broadcast::Sender; +use tracing::instrument; + +use crate::context::config::ServerConfig; +use crate::progress::FullProgressTracker; +use crate::rpc_continuations::RpcContinuations; +use crate::Error; + +pub struct InitContextSeed { + pub config: ServerConfig, + pub progress: FullProgressTracker, + pub shutdown: Sender<()>, + pub rpc_continuations: RpcContinuations, +} + +#[derive(Clone)] +pub struct InitContext(Arc); +impl InitContext { + #[instrument(skip_all)] + pub async fn init(cfg: &ServerConfig) -> Result { + let (shutdown, _) = tokio::sync::broadcast::channel(1); + Ok(Self(Arc::new(InitContextSeed { + config: cfg.clone(), + progress: FullProgressTracker::new(), + shutdown, + rpc_continuations: RpcContinuations::new(), + }))) + } +} + +impl AsRef for InitContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } +} + +impl Context for InitContext {} +impl Deref for InitContext { + type Target = InitContextSeed; + fn deref(&self) -> &Self::Target { + &*self.0 + } +} diff --git a/core/startos/src/context/install.rs b/core/startos/src/context/install.rs index d4717d2b0..c0c564b34 100644 --- a/core/startos/src/context/install.rs +++ b/core/startos/src/context/install.rs @@ -6,11 +6,13 @@ use tokio::sync::broadcast::Sender; use tracing::instrument; use crate::net::utils::find_eth_iface; +use crate::rpc_continuations::RpcContinuations; use crate::Error; pub struct InstallContextSeed { pub ethernet_interface: String, pub shutdown: Sender<()>, + pub rpc_continuations: RpcContinuations, } #[derive(Clone)] @@ -22,10 +24,17 @@ impl InstallContext { Ok(Self(Arc::new(InstallContextSeed { ethernet_interface: find_eth_iface().await?, shutdown, + rpc_continuations: RpcContinuations::new(), }))) } } +impl AsRef for InstallContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } +} + impl Context for InstallContext {} impl Deref for InstallContext { type Target = InstallContextSeed; diff --git a/core/startos/src/context/mod.rs b/core/startos/src/context/mod.rs index 77f54f26c..efe261b0c 100644 --- a/core/startos/src/context/mod.rs +++ b/core/startos/src/context/mod.rs @@ -1,12 +1,14 @@ pub mod cli; pub mod config; pub mod diagnostic; +pub mod init; pub mod install; pub mod rpc; pub mod setup; pub use cli::CliContext; pub use diagnostic::DiagnosticContext; +pub use init::InitContext; pub use install::InstallContext; pub use rpc::RpcContext; pub use setup::SetupContext; diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index a3e77a62c..3dd3354aa 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -6,11 +6,12 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use imbl_value::InternedString; use josekit::jwk::Jwk; use reqwest::{Client, Proxy}; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{CallRemote, Context, Empty}; -use tokio::sync::{broadcast, oneshot, Mutex, RwLock}; +use tokio::sync::{broadcast, Mutex, RwLock}; use tokio::time::Instant; use tracing::instrument; @@ -22,12 +23,12 @@ use crate::dependencies::compute_dependency_config_errs; use crate::disk::OsPartitionInfo; use crate::init::check_time_is_synchronized; use crate::lxc::{ContainerId, LxcContainer, LxcManager}; -use crate::middleware::auth::HashSessionToken; -use crate::net::net_controller::NetController; +use crate::net::net_controller::{NetController, PreInitNetController}; use crate::net::utils::{find_eth_iface, find_wifi_iface}; use crate::net::wifi::WpaCli; use crate::prelude::*; -use crate::rpc_continuations::RpcContinuations; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle}; +use crate::rpc_continuations::{OpenAuthedContinuations, RpcContinuations}; use crate::service::ServiceMap; use crate::shutdown::Shutdown; use crate::system::get_mem_info; @@ -49,7 +50,7 @@ pub struct RpcContextSeed { pub shutdown: broadcast::Sender>, pub tor_socks: SocketAddr, pub lxc_manager: Arc, - pub open_authed_websockets: Mutex>>>, + pub open_authed_continuations: OpenAuthedContinuations, pub rpc_continuations: RpcContinuations, pub wifi_manager: Option>>, pub current_secret: Arc, @@ -68,45 +69,103 @@ pub struct Hardware { pub ram: u64, } +pub struct InitRpcContextPhases { + load_db: PhaseProgressTrackerHandle, + init_net_ctrl: PhaseProgressTrackerHandle, + read_device_info: PhaseProgressTrackerHandle, + cleanup_init: CleanupInitPhases, +} +impl InitRpcContextPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + load_db: handle.add_phase("Loading database".into(), Some(5)), + init_net_ctrl: handle.add_phase("Initializing network".into(), Some(1)), + read_device_info: handle.add_phase("Reading device information".into(), Some(1)), + cleanup_init: CleanupInitPhases::new(handle), + } + } +} + +pub struct CleanupInitPhases { + init_services: PhaseProgressTrackerHandle, + check_dependencies: PhaseProgressTrackerHandle, +} +impl CleanupInitPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + init_services: handle.add_phase("Initializing services".into(), Some(10)), + check_dependencies: handle.add_phase("Checking dependencies".into(), Some(1)), + } + } +} + #[derive(Clone)] pub struct RpcContext(Arc); impl RpcContext { #[instrument(skip_all)] - pub async fn init(config: &ServerConfig, disk_guid: Arc) -> Result { - tracing::info!("Loaded Config"); + pub async fn init( + config: &ServerConfig, + disk_guid: Arc, + net_ctrl: Option, + InitRpcContextPhases { + mut load_db, + mut init_net_ctrl, + mut read_device_info, + cleanup_init, + }: InitRpcContextPhases, + ) -> Result { let tor_proxy = config.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new( Ipv4Addr::new(127, 0, 0, 1), 9050, ))); let (shutdown, _) = tokio::sync::broadcast::channel(1); - let db = TypedPatchDb::::load(config.db().await?).await?; + load_db.start(); + let db = if let Some(net_ctrl) = &net_ctrl { + net_ctrl.db.clone() + } else { + TypedPatchDb::::load(config.db().await?).await? + }; let peek = db.peek().await; let account = AccountInfo::load(&peek)?; + load_db.complete(); tracing::info!("Opened PatchDB"); + + init_net_ctrl.start(); let net_controller = Arc::new( NetController::init( - db.clone(), - config - .tor_control - .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), - tor_proxy, + if let Some(net_ctrl) = net_ctrl { + net_ctrl + } else { + PreInitNetController::init( + db.clone(), + config + .tor_control + .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), + tor_proxy, + &account.hostname, + account.tor_key.clone(), + ) + .await? + }, config .dns_bind .as_deref() .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]), - &account.hostname, - account.tor_key.clone(), ) .await?, ); + init_net_ctrl.complete(); tracing::info!("Initialized Net Controller"); + let services = ServiceMap::default(); let metrics_cache = RwLock::>::new(None); - tracing::info!("Initialized Notification Manager"); let tor_proxy_url = format!("socks5h://{tor_proxy}"); + + read_device_info.start(); let devices = lshw().await?; let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024; + read_device_info.complete(); if !db .peek() @@ -163,7 +222,7 @@ impl RpcContext { shutdown, tor_socks: tor_proxy, lxc_manager: Arc::new(LxcManager::new()), - open_authed_websockets: Mutex::new(BTreeMap::new()), + open_authed_continuations: OpenAuthedContinuations::new(), rpc_continuations: RpcContinuations::new(), wifi_manager: wifi_interface .clone() @@ -196,7 +255,7 @@ impl RpcContext { }); let res = Self(seed.clone()); - res.cleanup_and_initialize().await?; + res.cleanup_and_initialize(cleanup_init).await?; tracing::info!("Cleaned up transient states"); Ok(res) } @@ -210,11 +269,18 @@ impl RpcContext { Ok(()) } - #[instrument(skip(self))] - pub async fn cleanup_and_initialize(&self) -> Result<(), Error> { - self.services.init(&self).await?; + #[instrument(skip_all)] + pub async fn cleanup_and_initialize( + &self, + CleanupInitPhases { + init_services, + mut check_dependencies, + }: CleanupInitPhases, + ) -> Result<(), Error> { + self.services.init(&self, init_services).await?; tracing::info!("Initialized Package Managers"); + check_dependencies.start(); let mut updated_current_dependents = BTreeMap::new(); let peek = self.db.peek().await; for (package_id, package) in peek.as_public().as_package_data().as_entries()?.into_iter() { @@ -238,6 +304,7 @@ impl RpcContext { Ok(()) }) .await?; + check_dependencies.complete(); Ok(()) } @@ -274,6 +341,11 @@ impl AsRef for RpcContext { &self.rpc_continuations } } +impl AsRef> for RpcContext { + fn as_ref(&self) -> &OpenAuthedContinuations { + &self.open_authed_continuations + } +} impl Context for RpcContext {} impl Deref for RpcContext { type Target = RpcContextSeed; diff --git a/core/startos/src/context/setup.rs b/core/startos/src/context/setup.rs index 013dc060b..6041f49b9 100644 --- a/core/startos/src/context/setup.rs +++ b/core/startos/src/context/setup.rs @@ -1,23 +1,31 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; +use futures::{Future, StreamExt}; +use helpers::NonDetachingJoinHandle; use josekit::jwk::Jwk; use patch_db::PatchDb; -use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::Context; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgConnectOptions; use sqlx::PgPool; use tokio::sync::broadcast::Sender; -use tokio::sync::RwLock; +use tokio::sync::OnceCell; use tracing::instrument; +use ts_rs::TS; +use crate::account::AccountInfo; use crate::context::config::ServerConfig; +use crate::context::RpcContext; use crate::disk::OsPartitionInfo; use crate::init::init_postgres; use crate::prelude::*; -use crate::setup::SetupStatus; +use crate::progress::FullProgressTracker; +use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations}; +use crate::setup::SetupProgress; +use crate::util::net::WebSocketExt; lazy_static::lazy_static! { pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| { @@ -27,30 +35,35 @@ lazy_static::lazy_static! { }); } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct SetupResult { pub tor_address: String, pub lan_address: String, pub root_ca: String, } +impl TryFrom<&AccountInfo> for SetupResult { + type Error = Error; + fn try_from(value: &AccountInfo) -> Result { + Ok(Self { + tor_address: format!("https://{}", value.tor_key.public().get_onion_address()), + lan_address: value.hostname.lan_address(), + root_ca: String::from_utf8(value.root_ca_cert.to_pem()?)?, + }) + } +} pub struct SetupContextSeed { pub config: ServerConfig, pub os_partitions: OsPartitionInfo, pub disable_encryption: bool, + pub progress: FullProgressTracker, + pub task: OnceCell>, + pub result: OnceCell>, pub shutdown: Sender<()>, pub datadir: PathBuf, - pub selected_v2_drive: RwLock>, - pub cached_product_key: RwLock>>, - pub setup_status: RwLock>>, - pub setup_result: RwLock, SetupResult)>>, -} - -impl AsRef for SetupContextSeed { - fn as_ref(&self) -> &Jwk { - &*CURRENT_SECRET - } + pub rpc_continuations: RpcContinuations, } #[derive(Clone)] @@ -69,12 +82,12 @@ impl SetupContext { ) })?, disable_encryption: config.disable_encryption.unwrap_or(false), + progress: FullProgressTracker::new(), + task: OnceCell::new(), + result: OnceCell::new(), shutdown, datadir, - selected_v2_drive: RwLock::new(None), - cached_product_key: RwLock::new(None), - setup_status: RwLock::new(None), - setup_result: RwLock::new(None), + rpc_continuations: RpcContinuations::new(), }))) } #[instrument(skip_all)] @@ -97,6 +110,104 @@ impl SetupContext { .with_kind(crate::ErrorKind::Database)?; Ok(secret_store) } + + pub fn run_setup(&self, f: F) -> Result<(), Error> + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send, + { + let local_ctx = self.clone(); + self.task + .set( + tokio::spawn(async move { + local_ctx + .result + .get_or_init(|| async { + match f().await { + Ok(res) => { + tracing::info!("Setup complete!"); + Ok(res) + } + Err(e) => { + tracing::error!("Setup failed: {e}"); + tracing::debug!("{e:?}"); + Err(e) + } + } + }) + .await; + local_ctx.progress.complete(); + }) + .into(), + ) + .map_err(|_| { + if self.result.initialized() { + Error::new(eyre!("Setup already complete"), ErrorKind::InvalidRequest) + } else { + Error::new( + eyre!("Setup already in progress"), + ErrorKind::InvalidRequest, + ) + } + })?; + Ok(()) + } + + pub async fn progress(&self) -> SetupProgress { + use axum::extract::ws; + + let guid = Guid::new(); + let progress_tracker = self.progress.clone(); + let progress = progress_tracker.snapshot(); + self.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws( + |mut ws| async move { + if let Err(e) = async { + let mut stream = + progress_tracker.stream(Some(Duration::from_millis(100))); + while let Some(progress) = stream.next().await { + ws.send(ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + if progress.overall.is_complete() { + break; + } + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error in setup progress websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + + SetupProgress { progress, guid } + } +} + +impl AsRef for SetupContext { + fn as_ref(&self) -> &Jwk { + &*CURRENT_SECRET + } +} + +impl AsRef for SetupContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } } impl Context for SetupContext {} diff --git a/core/startos/src/db/mod.rs b/core/startos/src/db/mod.rs index 0bb8a23db..e59161e9b 100644 --- a/core/startos/src/db/mod.rs +++ b/core/startos/src/db/mod.rs @@ -3,175 +3,40 @@ pub mod prelude; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; -use axum::extract::ws::{self, WebSocket}; -use axum::extract::WebSocketUpgrade; -use axum::response::Response; +use axum::extract::ws; use clap::Parser; -use futures::{FutureExt, StreamExt}; -use http::header::COOKIE; -use http::HeaderMap; +use imbl_value::InternedString; use itertools::Itertools; use patch_db::json_ptr::{JsonPointer, ROOT}; use patch_db::{Dump, Revision}; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tokio::sync::oneshot; use tracing::instrument; use ts_rs::TS; use crate::context::{CliContext, RpcContext}; -use crate::middleware::auth::{HasValidSession, HashSessionToken}; use crate::prelude::*; +use crate::rpc_continuations::{Guid, RpcContinuation}; +use crate::util::net::WebSocketExt; use crate::util::serde::{apply_expr, HandlerExtSerde}; lazy_static::lazy_static! { static ref PUBLIC: JsonPointer = "/public".parse().unwrap(); } -#[instrument(skip_all)] -async fn ws_handler( - ctx: RpcContext, - session: Option<(HasValidSession, HashSessionToken)>, - mut stream: WebSocket, -) -> Result<(), Error> { - let (dump, sub) = ctx.db.dump_and_sub(PUBLIC.clone()).await; - - if let Some((session, token)) = session { - let kill = subscribe_to_session_kill(&ctx, token).await; - send_dump(session.clone(), &mut stream, dump).await?; - - deal_with_messages(session, kill, sub, stream).await?; - } else { - stream - .send(ws::Message::Close(Some(ws::CloseFrame { - code: ws::close_code::ERROR, - reason: "UNAUTHORIZED".into(), - }))) - .await - .with_kind(ErrorKind::Network)?; - drop(stream); - } - - Ok(()) -} - -async fn subscribe_to_session_kill( - ctx: &RpcContext, - token: HashSessionToken, -) -> oneshot::Receiver<()> { - let (send, recv) = oneshot::channel(); - let mut guard = ctx.open_authed_websockets.lock().await; - if !guard.contains_key(&token) { - guard.insert(token, vec![send]); - } else { - guard.get_mut(&token).unwrap().push(send); - } - recv -} - -#[instrument(skip_all)] -async fn deal_with_messages( - _has_valid_authentication: HasValidSession, - mut kill: oneshot::Receiver<()>, - mut sub: patch_db::Subscriber, - mut stream: WebSocket, -) -> Result<(), Error> { - let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(5)); - - loop { - futures::select! { - _ = (&mut kill).fuse() => { - tracing::info!("Closing WebSocket: Reason: Session Terminated"); - stream - .send(ws::Message::Close(Some(ws::CloseFrame { - code: ws::close_code::ERROR, - reason: "UNAUTHORIZED".into(), - }))).await - .with_kind(ErrorKind::Network)?; - drop(stream); - return Ok(()) - } - new_rev = sub.recv().fuse() => { - let rev = new_rev.expect("UNREACHABLE: patch-db is dropped"); - stream - .send(ws::Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?)) - .await - .with_kind(ErrorKind::Network)?; - } - message = stream.next().fuse() => { - let message = message.transpose().with_kind(ErrorKind::Network)?; - match message { - None => { - tracing::info!("Closing WebSocket: Stream Finished"); - return Ok(()) - } - _ => (), - } - } - // This is trying to give a health checks to the home to keep the ui alive. - _ = timer.tick().fuse() => { - stream - .send(ws::Message::Ping(vec![])) - .await - .with_kind(crate::ErrorKind::Network)?; - } - } - } -} - -async fn send_dump( - _has_valid_authentication: HasValidSession, - stream: &mut WebSocket, - dump: Dump, -) -> Result<(), Error> { - stream - .send(ws::Message::Text( - serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?, - )) - .await - .with_kind(ErrorKind::Network)?; - Ok(()) -} - -pub async fn subscribe( - ctx: RpcContext, - headers: HeaderMap, - ws: WebSocketUpgrade, -) -> Result { - let session = match async { - let token = HashSessionToken::from_header(headers.get(COOKIE))?; - let session = HasValidSession::from_header(headers.get(COOKIE), &ctx).await?; - Ok::<_, Error>((session, token)) - } - .await - { - Ok(a) => Some(a), - Err(e) => { - if e.kind != ErrorKind::Authorization { - tracing::error!("Error Authenticating Websocket: {}", e); - tracing::debug!("{:?}", e); - } - None - } - }; - Ok(ws.on_upgrade(|ws| async move { - match ws_handler(ctx, session, ws).await { - Ok(()) => (), - Err(e) => { - tracing::error!("WebSocket Closed: {}", e); - tracing::debug!("{:?}", e); - } - } - })) -} - pub fn db() -> ParentHandler { ParentHandler::new() .subcommand("dump", from_fn_async(cli_dump).with_display_serializable()) .subcommand("dump", from_fn_async(dump).no_cli()) + .subcommand( + "subscribe", + from_fn_async(subscribe) + .with_metadata("get_session", Value::Bool(true)) + .no_cli(), + ) .subcommand("put", put::()) .subcommand("apply", from_fn_async(cli_apply).no_display()) .subcommand("apply", from_fn_async(apply).no_cli()) @@ -215,7 +80,13 @@ async fn cli_dump( context .call_remote::( &method, - imbl_value::json!({ "includePrivate":include_private }), + imbl_value::json!({ + "pointer": if include_private { + AsRef::::as_ref(&ROOT) + } else { + AsRef::::as_ref(&*PUBLIC) + } + }), ) .await?, )? @@ -224,25 +95,76 @@ async fn cli_dump( Ok(dump) } -#[derive(Deserialize, Serialize, Parser, TS)] +#[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -#[command(rename_all = "kebab-case")] pub struct DumpParams { - #[arg(long = "include-private", short = 'p')] - #[serde(default)] - #[ts(skip)] - include_private: bool, + #[ts(type = "string | null")] + pointer: Option, } -pub async fn dump( +pub async fn dump(ctx: RpcContext, DumpParams { pointer }: DumpParams) -> Result { + Ok(ctx.db.dump(pointer.as_ref().unwrap_or(&*PUBLIC)).await) +} + +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SubscribeParams { + #[ts(type = "string | null")] + pointer: Option, + #[ts(skip)] + #[serde(rename = "__auth_session")] + session: InternedString, +} + +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SubscribeRes { + #[ts(type = "{ id: number; value: unknown }")] + pub dump: Dump, + pub guid: Guid, +} + +pub async fn subscribe( ctx: RpcContext, - DumpParams { include_private }: DumpParams, -) -> Result { - Ok(if include_private { - ctx.db.dump(&ROOT).await - } else { - ctx.db.dump(&PUBLIC).await - }) + SubscribeParams { pointer, session }: SubscribeParams, +) -> Result { + let (dump, mut sub) = ctx + .db + .dump_and_sub(pointer.unwrap_or_else(|| PUBLIC.clone())) + .await; + let guid = Guid::new(); + ctx.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws_authed( + &ctx, + session, + |mut ws| async move { + if let Err(e) = async { + while let Some(rev) = sub.recv().await { + ws.send(ws::Message::Text( + serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error in db websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + + Ok(SubscribeRes { dump, guid }) } #[derive(Deserialize, Serialize, Parser)] diff --git a/core/startos/src/diagnostic.rs b/core/startos/src/diagnostic.rs index 485f2359f..5e99580e9 100644 --- a/core/startos/src/diagnostic.rs +++ b/core/startos/src/diagnostic.rs @@ -27,10 +27,6 @@ pub fn diagnostic() -> ParentHandler { "kernel-logs", from_fn_async(crate::logs::cli_logs::).no_display(), ) - .subcommand( - "exit", - from_fn(exit).no_display().with_call_remote::(), - ) .subcommand( "restart", from_fn(restart) @@ -51,20 +47,15 @@ pub fn error(ctx: DiagnosticContext) -> Result, Error> { Ok(ctx.error.clone()) } -pub fn exit(ctx: DiagnosticContext) -> Result<(), Error> { - ctx.shutdown.send(None).expect("receiver dropped"); - Ok(()) -} - pub fn restart(ctx: DiagnosticContext) -> Result<(), Error> { ctx.shutdown - .send(Some(Shutdown { + .send(Shutdown { export_args: ctx .disk_guid .clone() .map(|guid| (guid, ctx.datadir.clone())), restart: true, - })) + }) .expect("receiver dropped"); Ok(()) } diff --git a/core/startos/src/disk/main.rs b/core/startos/src/disk/main.rs index d414c247e..ee807a938 100644 --- a/core/startos/src/disk/main.rs +++ b/core/startos/src/disk/main.rs @@ -13,7 +13,7 @@ use crate::disk::mount::util::unmount; use crate::util::Invoke; use crate::{Error, ErrorKind, ResultExt}; -pub const PASSWORD_PATH: &'static str = "/run/embassy/password"; +pub const PASSWORD_PATH: &'static str = "/run/startos/password"; pub const DEFAULT_PASSWORD: &'static str = "password"; pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8); diff --git a/core/startos/src/firmware.rs b/core/startos/src/firmware.rs index 20347bcff..a9d5ced79 100644 --- a/core/startos/src/firmware.rs +++ b/core/startos/src/firmware.rs @@ -9,6 +9,7 @@ use tokio::process::Command; use crate::disk::fsck::RequiresReboot; use crate::prelude::*; +use crate::progress::PhaseProgressTrackerHandle; use crate::util::Invoke; use crate::PLATFORM; @@ -49,12 +50,7 @@ pub fn display_firmware_update_result(result: RequiresReboot) { } } -/// We wanted to make sure during every init -/// that the firmware was the correct and updated for -/// systems like the Pure System that a new firmware -/// was released and the updates where pushed through the pure os. -// #[command(rename = "update-firmware", display(display_firmware_update_result))] -pub async fn update_firmware() -> Result { +pub async fn check_for_firmware_update() -> Result, Error> { let system_product_name = String::from_utf8( Command::new("dmidecode") .arg("-s") @@ -74,22 +70,21 @@ pub async fn update_firmware() -> Result { .trim() .to_owned(); if system_product_name.is_empty() || bios_version.is_empty() { - return Ok(RequiresReboot(false)); + return Ok(None); } - let firmware_dir = Path::new("/usr/lib/startos/firmware"); - for firmware in serde_json::from_str::>( &tokio::fs::read_to_string("/usr/lib/startos/firmware.json").await?, ) .with_kind(ErrorKind::Deserialization)? { - let id = firmware.id; let matches_product_name = firmware .system_product_name - .map_or(true, |spn| spn == system_product_name); + .as_ref() + .map_or(true, |spn| spn == &system_product_name); let matches_bios_version = firmware .bios_version + .as_ref() .map_or(Some(true), |bv| { let mut semver_str = bios_version.as_str(); if let Some(prefix) = &bv.semver_prefix { @@ -113,35 +108,45 @@ pub async fn update_firmware() -> Result { }) .unwrap_or(false); if firmware.platform.contains(&*PLATFORM) && matches_product_name && matches_bios_version { - let filename = format!("{id}.rom.gz"); - let firmware_path = firmware_dir.join(&filename); - Command::new("sha256sum") - .arg("-c") - .input(Some(&mut std::io::Cursor::new(format!( - "{} {}", - firmware.shasum, - firmware_path.display() - )))) - .invoke(ErrorKind::Filesystem) - .await?; - let mut rdr = if tokio::fs::metadata(&firmware_path).await.is_ok() { - GzipDecoder::new(BufReader::new(File::open(&firmware_path).await?)) - } else { - return Err(Error::new( - eyre!("Firmware {id}.rom.gz not found in {firmware_dir:?}"), - ErrorKind::NotFound, - )); - }; - Command::new("flashrom") - .arg("-p") - .arg("internal") - .arg("-w-") - .input(Some(&mut rdr)) - .invoke(ErrorKind::Firmware) - .await?; - return Ok(RequiresReboot(true)); + return Ok(Some(firmware)); } } - Ok(RequiresReboot(false)) + Ok(None) +} + +/// We wanted to make sure during every init +/// that the firmware was the correct and updated for +/// systems like the Pure System that a new firmware +/// was released and the updates where pushed through the pure os. +pub async fn update_firmware(firmware: Firmware) -> Result<(), Error> { + let id = &firmware.id; + let firmware_dir = Path::new("/usr/lib/startos/firmware"); + let filename = format!("{id}.rom.gz"); + let firmware_path = firmware_dir.join(&filename); + Command::new("sha256sum") + .arg("-c") + .input(Some(&mut std::io::Cursor::new(format!( + "{} {}", + firmware.shasum, + firmware_path.display() + )))) + .invoke(ErrorKind::Filesystem) + .await?; + let mut rdr = if tokio::fs::metadata(&firmware_path).await.is_ok() { + GzipDecoder::new(BufReader::new(File::open(&firmware_path).await?)) + } else { + return Err(Error::new( + eyre!("Firmware {id}.rom.gz not found in {firmware_dir:?}"), + ErrorKind::NotFound, + )); + }; + Command::new("flashrom") + .arg("-p") + .arg("internal") + .arg("-w-") + .input(Some(&mut rdr)) + .invoke(ErrorKind::Firmware) + .await?; + Ok(()) } diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 97c674ac5..cdc444c32 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -1,25 +1,40 @@ use std::fs::Permissions; +use std::io::Cursor; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::time::{Duration, SystemTime}; +use axum::extract::ws::{self, CloseFrame}; use color_eyre::eyre::eyre; +use futures::{StreamExt, TryStreamExt}; +use itertools::Itertools; use models::ResultExt; use rand::random; +use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler}; +use serde::{Deserialize, Serialize}; use tokio::process::Command; use tracing::instrument; +use ts_rs::TS; use crate::account::AccountInfo; use crate::context::config::ServerConfig; +use crate::context::{CliContext, InitContext}; use crate::db::model::public::ServerStatus; use crate::db::model::Database; use crate::disk::mount::util::unmount; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; +use crate::net::net_controller::PreInitNetController; use crate::prelude::*; +use crate::progress::{ + FullProgress, FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar, +}; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::ssh::SSH_AUTHORIZED_KEYS_FILE; -use crate::util::cpupower::{get_available_governors, get_preferred_governor, set_governor}; -use crate::util::Invoke; -use crate::{Error, ARCH}; +use crate::util::io::IOHook; +use crate::util::net::WebSocketExt; +use crate::util::{cpupower, Invoke}; +use crate::Error; pub const SYSTEM_REBUILD_PATH: &str = "/media/startos/config/system-rebuild"; pub const STANDBY_MODE_PATH: &str = "/media/startos/config/standby"; @@ -180,14 +195,114 @@ pub async fn init_postgres(datadir: impl AsRef) -> Result<(), Error> { } pub struct InitResult { - pub db: TypedPatchDb, + pub net_ctrl: PreInitNetController, +} + +pub struct InitPhases { + preinit: Option, + local_auth: PhaseProgressTrackerHandle, + load_database: PhaseProgressTrackerHandle, + load_ssh_keys: PhaseProgressTrackerHandle, + start_net: PhaseProgressTrackerHandle, + mount_logs: PhaseProgressTrackerHandle, + load_ca_cert: PhaseProgressTrackerHandle, + load_wifi: PhaseProgressTrackerHandle, + init_tmp: PhaseProgressTrackerHandle, + set_governor: PhaseProgressTrackerHandle, + sync_clock: PhaseProgressTrackerHandle, + enable_zram: PhaseProgressTrackerHandle, + update_server_info: PhaseProgressTrackerHandle, + launch_service_network: PhaseProgressTrackerHandle, + run_migrations: PhaseProgressTrackerHandle, + validate_db: PhaseProgressTrackerHandle, + postinit: Option, +} +impl InitPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + preinit: if Path::new("/media/startos/config/preinit.sh").exists() { + Some(handle.add_phase("Running preinit.sh".into(), Some(5))) + } else { + None + }, + local_auth: handle.add_phase("Enabling local authentication".into(), Some(1)), + load_database: handle.add_phase("Loading database".into(), Some(5)), + load_ssh_keys: handle.add_phase("Loading SSH Keys".into(), Some(1)), + start_net: handle.add_phase("Starting network controller".into(), Some(1)), + mount_logs: handle.add_phase("Switching logs to write to data drive".into(), Some(1)), + load_ca_cert: handle.add_phase("Loading CA certificate".into(), Some(1)), + load_wifi: handle.add_phase("Loading WiFi configuration".into(), Some(1)), + init_tmp: handle.add_phase("Initializing temporary files".into(), Some(1)), + set_governor: handle.add_phase("Setting CPU performance profile".into(), Some(1)), + sync_clock: handle.add_phase("Synchronizing system clock".into(), Some(10)), + enable_zram: handle.add_phase("Enabling ZRAM".into(), Some(1)), + update_server_info: handle.add_phase("Updating server info".into(), Some(1)), + launch_service_network: handle.add_phase("Launching service intranet".into(), Some(10)), + run_migrations: handle.add_phase("Running migrations".into(), Some(10)), + validate_db: handle.add_phase("Validating database".into(), Some(1)), + postinit: if Path::new("/media/startos/config/postinit.sh").exists() { + Some(handle.add_phase("Running postinit.sh".into(), Some(5))) + } else { + None + }, + } + } +} + +pub async fn run_script>(path: P, mut progress: PhaseProgressTrackerHandle) { + let script = path.as_ref(); + progress.start(); + if let Err(e) = async { + let script = tokio::fs::read_to_string(script).await?; + progress.set_total(script.as_bytes().iter().filter(|b| **b == b'\n').count() as u64); + let mut reader = IOHook::new(Cursor::new(script.as_bytes())); + reader.post_read(|buf| progress += buf.iter().filter(|b| **b == b'\n').count() as u64); + Command::new("/bin/bash") + .input(Some(&mut reader)) + .invoke(ErrorKind::Unknown) + .await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error Running {}: {}", script.display(), e); + tracing::debug!("{:?}", e); + } + progress.complete(); } #[instrument(skip_all)] -pub async fn init(cfg: &ServerConfig) -> Result { - tokio::fs::create_dir_all("/run/embassy") +pub async fn init( + cfg: &ServerConfig, + InitPhases { + preinit, + mut local_auth, + mut load_database, + mut load_ssh_keys, + mut start_net, + mut mount_logs, + mut load_ca_cert, + mut load_wifi, + mut init_tmp, + mut set_governor, + mut sync_clock, + mut enable_zram, + mut update_server_info, + mut launch_service_network, + run_migrations, + mut validate_db, + postinit, + }: InitPhases, +) -> Result { + if let Some(progress) = preinit { + run_script("/media/startos/config/preinit.sh", progress).await; + } + + local_auth.start(); + tokio::fs::create_dir_all("/run/startos") .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/embassy"))?; + .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/startos"))?; if tokio::fs::metadata(LOCAL_AUTH_COOKIE_PATH).await.is_err() { tokio::fs::write( LOCAL_AUTH_COOKIE_PATH, @@ -207,43 +322,41 @@ pub async fn init(cfg: &ServerConfig) -> Result { .invoke(crate::ErrorKind::Filesystem) .await?; } + local_auth.complete(); + load_database.start(); let db = TypedPatchDb::::load_unchecked(cfg.db().await?); let peek = db.peek().await; + load_database.complete(); tracing::info!("Opened PatchDB"); + load_ssh_keys.start(); crate::ssh::sync_keys( &peek.as_private().as_ssh_pubkeys().de()?, SSH_AUTHORIZED_KEYS_FILE, ) .await?; + load_ssh_keys.complete(); tracing::info!("Synced SSH Keys"); let account = AccountInfo::load(&peek)?; - let mut server_info = peek.as_public().as_server_info().de()?; - - // write to ca cert store - tokio::fs::write( - "/usr/local/share/ca-certificates/startos-root-ca.crt", - account.root_ca_cert.to_pem()?, + start_net.start(); + let net_ctrl = PreInitNetController::init( + db.clone(), + cfg.tor_control + .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), + cfg.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::new(127, 0, 0, 1), + 9050, + ))), + &account.hostname, + account.tor_key, ) .await?; - Command::new("update-ca-certificates") - .invoke(crate::ErrorKind::OpenSsl) - .await?; - - crate::net::wifi::synchronize_wpa_supplicant_conf( - &cfg.datadir().join("main"), - &mut server_info.wifi, - ) - .await?; - tracing::info!("Synchronized WiFi"); - - let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok() - || &*server_info.version < &emver::Version::new(0, 3, 2, 0) - || (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0)); + start_net.complete(); + mount_logs.start(); let log_dir = cfg.datadir().join("main/logs"); if tokio::fs::metadata(&log_dir).await.is_err() { tokio::fs::create_dir_all(&log_dir).await?; @@ -272,10 +385,35 @@ pub async fn init(cfg: &ServerConfig) -> Result { .arg("systemd-journald") .invoke(crate::ErrorKind::Journald) .await?; + mount_logs.complete(); tracing::info!("Mounted Logs"); + let mut server_info = peek.as_public().as_server_info().de()?; + + load_ca_cert.start(); + // write to ca cert store + tokio::fs::write( + "/usr/local/share/ca-certificates/startos-root-ca.crt", + account.root_ca_cert.to_pem()?, + ) + .await?; + Command::new("update-ca-certificates") + .invoke(crate::ErrorKind::OpenSsl) + .await?; + load_ca_cert.complete(); + + load_wifi.start(); + crate::net::wifi::synchronize_wpa_supplicant_conf( + &cfg.datadir().join("main"), + &mut server_info.wifi, + ) + .await?; + load_wifi.complete(); + tracing::info!("Synchronized WiFi"); + + init_tmp.start(); let tmp_dir = cfg.datadir().join("package-data/tmp"); - if should_rebuild && tokio::fs::metadata(&tmp_dir).await.is_ok() { + if tokio::fs::metadata(&tmp_dir).await.is_ok() { tokio::fs::remove_dir_all(&tmp_dir).await?; } if tokio::fs::metadata(&tmp_dir).await.is_err() { @@ -286,23 +424,30 @@ pub async fn init(cfg: &ServerConfig) -> Result { tokio::fs::remove_dir_all(&tmp_var).await?; } crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?; + init_tmp.complete(); + set_governor.start(); let governor = if let Some(governor) = &server_info.governor { - if get_available_governors().await?.contains(governor) { + if cpupower::get_available_governors() + .await? + .contains(governor) + { Some(governor) } else { tracing::warn!("CPU Governor \"{governor}\" Not Available"); None } } else { - get_preferred_governor().await? + cpupower::get_preferred_governor().await? }; if let Some(governor) = governor { tracing::info!("Setting CPU Governor to \"{governor}\""); - set_governor(governor).await?; + cpupower::set_governor(governor).await?; tracing::info!("Set CPU Governor"); } + set_governor.complete(); + sync_clock.start(); server_info.ntp_synced = false; let mut not_made_progress = 0u32; for _ in 0..1800 { @@ -329,10 +474,15 @@ pub async fn init(cfg: &ServerConfig) -> Result { } else { tracing::info!("Syncronized system clock"); } + sync_clock.complete(); + enable_zram.start(); if server_info.zram { crate::system::enable_zram().await? } + enable_zram.complete(); + + update_server_info.start(); server_info.ip_info = crate::net::dhcp::init_ips().await?; server_info.status_info = ServerStatus { updated: false, @@ -341,36 +491,129 @@ pub async fn init(cfg: &ServerConfig) -> Result { shutting_down: false, restarting: false, }; - db.mutate(|v| { v.as_public_mut().as_server_info_mut().ser(&server_info)?; Ok(()) }) .await?; + update_server_info.complete(); + launch_service_network.start(); Command::new("systemctl") .arg("start") .arg("lxc-net.service") .invoke(ErrorKind::Lxc) .await?; + launch_service_network.complete(); - crate::version::init(&db).await?; + crate::version::init(&db, run_migrations).await?; + validate_db.start(); db.mutate(|d| { let model = d.de()?; d.ser(&model) }) .await?; + validate_db.complete(); - if should_rebuild { - match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e), - }?; + if let Some(progress) = postinit { + run_script("/media/startos/config/postinit.sh", progress).await; } tracing::info!("System initialized."); - Ok(InitResult { db }) + Ok(InitResult { net_ctrl }) +} + +pub fn init_api() -> ParentHandler { + ParentHandler::new() + .subcommand("logs", crate::system::logs::()) + .subcommand( + "logs", + from_fn_async(crate::logs::cli_logs::).no_display(), + ) + .subcommand("kernel-logs", crate::system::kernel_logs::()) + .subcommand( + "kernel-logs", + from_fn_async(crate::logs::cli_logs::).no_display(), + ) + .subcommand("subscribe", from_fn_async(init_progress).no_cli()) + .subcommand("subscribe", from_fn_async(cli_init_progress).no_display()) +} + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct InitProgressRes { + pub progress: FullProgress, + pub guid: Guid, +} + +pub async fn init_progress(ctx: InitContext) -> Result { + let progress_tracker = ctx.progress.clone(); + let progress = progress_tracker.snapshot(); + let guid = Guid::new(); + ctx.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws( + |mut ws| async move { + if let Err(e) = async { + let mut stream = progress_tracker.stream(Some(Duration::from_millis(100))); + while let Some(progress) = stream.next().await { + ws.send(ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + if progress.overall.is_complete() { + break; + } + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("error in init progress websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + Ok(InitProgressRes { progress, guid }) +} + +pub async fn cli_init_progress( + HandlerArgs { + context: ctx, + parent_method, + method, + raw_params, + .. + }: HandlerArgs, +) -> Result<(), Error> { + let res: InitProgressRes = from_value( + ctx.call_remote::( + &parent_method + .into_iter() + .chain(method.into_iter()) + .join("."), + raw_params, + ) + .await?, + )?; + let mut ws = ctx.ws_continuation(res.guid).await?; + let mut bar = PhasedProgressBar::new("Initializing..."); + while let Some(msg) = ws.try_next().await.with_kind(ErrorKind::Network)? { + if let tokio_tungstenite::tungstenite::Message::Text(msg) = msg { + bar.update(&serde_json::from_str(&msg).with_kind(ErrorKind::Deserialization)?); + } + } + Ok(()) } diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index 707503615..5d50da27d 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -6,7 +6,8 @@ use clap::builder::ValueParserFactory; use clap::{value_parser, CommandFactory, FromArgMatches, Parser}; use color_eyre::eyre::eyre; use emver::VersionRange; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; +use imbl_value::InternedString; use itertools::Itertools; use patch_db::json_ptr::JsonPointer; use reqwest::header::{HeaderMap, CONTENT_LENGTH}; @@ -29,6 +30,7 @@ use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::S9pk; use crate::upload::upload; use crate::util::clap::FromStrParser; +use crate::util::net::WebSocketExt; use crate::util::Never; pub const PKG_ARCHIVE_DIR: &str = "package-data/archive"; @@ -170,7 +172,15 @@ pub async fn install( Ok(()) } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SideloadParams { + #[ts(skip)] + #[serde(rename = "__auth_session")] + session: InternedString, +} + +#[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] pub struct SideloadResponse { pub upload: Guid, @@ -178,8 +188,11 @@ pub struct SideloadResponse { } #[instrument(skip_all)] -pub async fn sideload(ctx: RpcContext) -> Result { - let (upload, file) = upload(&ctx).await?; +pub async fn sideload( + ctx: RpcContext, + SideloadParams { session }: SideloadParams, +) -> Result { + let (upload, file) = upload(&ctx, session.clone()).await?; let (id_send, id_recv) = oneshot::channel(); let (err_send, err_recv) = oneshot::channel(); let progress = Guid::new(); @@ -193,8 +206,8 @@ pub async fn sideload(ctx: RpcContext) -> Result { .await; ctx.rpc_continuations.add( progress.clone(), - RpcContinuation::ws( - Box::new(|mut ws| { + RpcContinuation::ws_authed(&ctx, session, + |mut ws| { use axum::extract::ws::Message; async move { if let Err(e) = async { @@ -251,7 +264,7 @@ pub async fn sideload(ctx: RpcContext) -> Result { } } - ws.close().await.with_kind(ErrorKind::Network)?; + ws.normal_close("complete").await?; Ok::<_, Error>(()) } @@ -261,8 +274,7 @@ pub async fn sideload(ctx: RpcContext) -> Result { tracing::debug!("{e:?}"); } } - .boxed() - }), + }, Duration::from_secs(600), ), ) diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index 0e125af98..6e7cc8f8b 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -1,8 +1,5 @@ pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com"; // pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com"; -pub const CAP_1_KiB: usize = 1024; -pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; -pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; pub const HOST_IP: [u8; 4] = [172, 18, 0, 1]; pub use std::env::consts::ARCH; lazy_static::lazy_static! { @@ -18,6 +15,15 @@ lazy_static::lazy_static! { }; } +mod cap { + #![allow(non_upper_case_globals)] + + pub const CAP_1_KiB: usize = 1024; + pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; + pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; +} +pub use cap::*; + pub mod account; pub mod action; pub mod auth; @@ -75,13 +81,17 @@ use rpc_toolkit::{ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::context::{CliContext, DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{ + CliContext, DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext, +}; +use crate::disk::fsck::RequiresReboot; use crate::registry::context::{RegistryContext, RegistryUrlParams}; use crate::util::serde::HandlerExtSerde; #[derive(Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[command(rename_all = "kebab-case")] +#[ts(export)] pub struct EchoParams { message: String, } @@ -90,6 +100,20 @@ pub fn echo(_: C, EchoParams { message }: EchoParams) -> Result) -> std::fmt::Result { + std::fmt::Debug::fmt(&self, f) + } +} + pub fn main_api() -> ParentHandler { ParentHandler::new() .subcommand::("git-info", from_fn(version::git_info)) @@ -99,6 +123,12 @@ pub fn main_api() -> ParentHandler { .with_metadata("authenticated", Value::Bool(false)) .with_call_remote::(), ) + .subcommand( + "state", + from_fn(|_: RpcContext| Ok::<_, Error>(ApiState::Running)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) .subcommand("server", server::()) .subcommand("package", package::()) .subcommand("net", net::net::()) @@ -179,11 +209,18 @@ pub fn server() -> ParentHandler { ) .subcommand( "update-firmware", - from_fn_async(|_: RpcContext| firmware::update_firmware()) - .with_custom_display_fn(|_handle, result| { - Ok(firmware::display_firmware_update_result(result)) - }) - .with_call_remote::(), + from_fn_async(|_: RpcContext| async { + if let Some(firmware) = firmware::check_for_firmware_update().await? { + firmware::update_firmware(firmware).await?; + Ok::<_, Error>(RequiresReboot(true)) + } else { + Ok(RequiresReboot(false)) + } + }) + .with_custom_display_fn(|_handle, result| { + Ok(firmware::display_firmware_update_result(result)) + }) + .with_call_remote::(), ) } @@ -204,7 +241,12 @@ pub fn package() -> ParentHandler { .with_metadata("sync_db", Value::Bool(true)) .no_cli(), ) - .subcommand("sideload", from_fn_async(install::sideload).no_cli()) + .subcommand( + "sideload", + from_fn_async(install::sideload) + .with_metadata("get_session", Value::Bool(true)) + .no_cli(), + ) .subcommand("install", from_fn_async(install::cli_install).no_display()) .subcommand( "uninstall", @@ -273,9 +315,34 @@ pub fn diagnostic_api() -> ParentHandler { "echo", from_fn(echo::).with_call_remote::(), ) + .subcommand( + "state", + from_fn(|_: DiagnosticContext| Ok::<_, Error>(ApiState::Error)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) .subcommand("diagnostic", diagnostic::diagnostic::()) } +pub fn init_api() -> ParentHandler { + ParentHandler::new() + .subcommand::( + "git-info", + from_fn(version::git_info).with_metadata("authenticated", Value::Bool(false)), + ) + .subcommand( + "echo", + from_fn(echo::).with_call_remote::(), + ) + .subcommand( + "state", + from_fn(|_: InitContext| Ok::<_, Error>(ApiState::Initializing)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) + .subcommand("init", init::init_api::()) +} + pub fn setup_api() -> ParentHandler { ParentHandler::new() .subcommand::( diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index 8d37120ba..1f1424338 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -7,7 +7,7 @@ use std::time::Duration; use clap::builder::ValueParserFactory; use clap::Parser; -use futures::{AsyncWriteExt, FutureExt, StreamExt}; +use futures::{AsyncWriteExt, StreamExt}; use imbl_value::{InOMap, InternedString}; use models::InvalidId; use rpc_toolkit::yajrc::{RpcError, RpcResponse}; @@ -456,51 +456,49 @@ pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result break, - Some(Ok(Message::Text(txt))) => { - let mut id = None; - let result = async { - let req: RpcRequest = serde_json::from_str(&txt) - .map_err(|e| RpcError { - data: Some(serde_json::Value::String( - e.to_string(), - )), - ..rpc_toolkit::yajrc::PARSE_ERROR - })?; - id = req.id; - rpc.request(req.method, req.params).await - } - .await; - ws.send(Message::Text( - serde_json::to_string( - &RpcResponse:: { id, result }, - ) - .with_kind(ErrorKind::Serialization)?, - )) - .await - .with_kind(ErrorKind::Network)?; - } - Some(Ok(_)) => (), - Some(Err(e)) => { - return Err(Error::new(e, ErrorKind::Network)); + |mut ws| async move { + if let Err(e) = async { + loop { + match ws.next().await { + None => break, + Some(Ok(Message::Text(txt))) => { + let mut id = None; + let result = async { + let req: RpcRequest = + serde_json::from_str(&txt).map_err(|e| RpcError { + data: Some(serde_json::Value::String( + e.to_string(), + )), + ..rpc_toolkit::yajrc::PARSE_ERROR + })?; + id = req.id; + rpc.request(req.method, req.params).await } + .await; + ws.send(Message::Text( + serde_json::to_string(&RpcResponse:: { + id, + result, + }) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + } + Some(Ok(_)) => (), + Some(Err(e)) => { + return Err(Error::new(e, ErrorKind::Network)); } } - Ok::<_, Error>(()) - } - .await - { - tracing::error!("{e}"); - tracing::debug!("{e:?}"); } + Ok::<_, Error>(()) } - .boxed() - }), + .await + { + tracing::error!("{e}"); + tracing::debug!("{e:?}"); + } + }, Duration::from_secs(30), ), ) diff --git a/core/startos/src/middleware/auth.rs b/core/startos/src/middleware/auth.rs index 1852c4890..ecf88dba7 100644 --- a/core/startos/src/middleware/auth.rs +++ b/core/startos/src/middleware/auth.rs @@ -23,7 +23,7 @@ use tokio::sync::Mutex; use crate::context::RpcContext; use crate::prelude::*; -pub const LOCAL_AUTH_COOKIE_PATH: &str = "/run/embassy/rpc.authcookie"; +pub const LOCAL_AUTH_COOKIE_PATH: &str = "/run/startos/rpc.authcookie"; #[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -48,19 +48,9 @@ impl HasLoggedOutSessions { .into_iter() .map(|s| s.as_logout_session_id()) .collect(); - ctx.open_authed_websockets - .lock() - .await - .retain(|session, sockets| { - if to_log_out.contains(session.hashed()) { - for socket in std::mem::take(sockets) { - let _ = socket.send(()); - } - false - } else { - true - } - }); + for sid in &to_log_out { + ctx.open_authed_continuations.kill(sid) + } ctx.db .mutate(|db| { let sessions = db.as_private_mut().as_sessions_mut(); diff --git a/core/startos/src/middleware/diagnostic.rs b/core/startos/src/middleware/diagnostic.rs deleted file mode 100644 index 2a7467e34..000000000 --- a/core/startos/src/middleware/diagnostic.rs +++ /dev/null @@ -1,42 +0,0 @@ -use rpc_toolkit::yajrc::RpcMethod; -use rpc_toolkit::{Empty, Middleware, RpcRequest, RpcResponse}; - -use crate::context::DiagnosticContext; -use crate::prelude::*; - -#[derive(Clone)] -pub struct DiagnosticMode { - method: Option, -} -impl DiagnosticMode { - pub fn new() -> Self { - Self { method: None } - } -} - -impl Middleware for DiagnosticMode { - type Metadata = Empty; - async fn process_rpc_request( - &mut self, - _: &DiagnosticContext, - _: Self::Metadata, - request: &mut RpcRequest, - ) -> Result<(), RpcResponse> { - self.method = Some(request.method.as_str().to_owned()); - Ok(()) - } - async fn process_rpc_response(&mut self, _: &DiagnosticContext, response: &mut RpcResponse) { - if let Err(e) = &mut response.result { - if e.code == -32601 { - *e = Error::new( - eyre!( - "{} is not available on the Diagnostic API", - self.method.as_ref().map(|s| s.as_str()).unwrap_or_default() - ), - crate::ErrorKind::DiagnosticMode, - ) - .into(); - } - } - } -} diff --git a/core/startos/src/middleware/mod.rs b/core/startos/src/middleware/mod.rs index 3af0cb5a4..3438dc3db 100644 --- a/core/startos/src/middleware/mod.rs +++ b/core/startos/src/middleware/mod.rs @@ -1,4 +1,3 @@ pub mod auth; pub mod cors; pub mod db; -pub mod diagnostic; diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 69c5c7940..270c7ca09 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -23,22 +23,18 @@ use crate::prelude::*; use crate::util::serde::MaybeUtf8String; use crate::HOST_IP; -pub struct NetController { - db: TypedPatchDb, - pub(super) tor: TorController, - pub(super) vhost: VHostController, - pub(super) dns: DnsController, - pub(super) forward: LanPortForwardController, - pub(super) os_bindings: Vec>, +pub struct PreInitNetController { + pub db: TypedPatchDb, + tor: TorController, + vhost: VHostController, + os_bindings: Vec>, } - -impl NetController { +impl PreInitNetController { #[instrument(skip_all)] pub async fn init( db: TypedPatchDb, tor_control: SocketAddr, tor_socks: SocketAddr, - dns_bind: &[SocketAddr], hostname: &Hostname, os_tor_key: TorSecretKeyV3, ) -> Result { @@ -46,8 +42,6 @@ impl NetController { db: db.clone(), tor: TorController::new(tor_control, tor_socks), vhost: VHostController::new(db), - dns: DnsController::init(dns_bind).await?, - forward: LanPortForwardController::new(), os_bindings: Vec::new(), }; res.add_os_bindings(hostname, os_tor_key).await?; @@ -73,8 +67,6 @@ impl NetController { alpn.clone(), ) .await?; - self.os_bindings - .push(self.dns.add(None, HOST_IP.into()).await?); // LAN IP self.os_bindings.push( @@ -142,6 +134,39 @@ impl NetController { Ok(()) } +} + +pub struct NetController { + db: TypedPatchDb, + pub(super) tor: TorController, + pub(super) vhost: VHostController, + pub(super) dns: DnsController, + pub(super) forward: LanPortForwardController, + pub(super) os_bindings: Vec>, +} + +impl NetController { + pub async fn init( + PreInitNetController { + db, + tor, + vhost, + os_bindings, + }: PreInitNetController, + dns_bind: &[SocketAddr], + ) -> Result { + let mut res = Self { + db, + tor, + vhost, + dns: DnsController::init(dns_bind).await?, + forward: LanPortForwardController::new(), + os_bindings, + }; + res.os_bindings + .push(res.dns.add(None, HOST_IP.into()).await?); + Ok(res) + } #[instrument(skip_all)] pub async fn create_service( diff --git a/core/startos/src/net/refresher.html b/core/startos/src/net/refresher.html new file mode 100644 index 000000000..445c6b5be --- /dev/null +++ b/core/startos/src/net/refresher.html @@ -0,0 +1,11 @@ + + + StartOS: Loading... + + + + Loading... + + \ No newline at end of file diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index fff1731ce..7e8034d99 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -1,4 +1,3 @@ -use std::fs::Metadata; use std::future::Future; use std::path::{Path, PathBuf}; use std::time::UNIX_EPOCH; @@ -13,25 +12,26 @@ use digest::Digest; use futures::future::ready; use http::header::ACCEPT_ENCODING; use http::request::Parts as RequestParts; -use http::{HeaderMap, Method, StatusCode}; +use http::{Method, StatusCode}; +use imbl_value::InternedString; use include_dir::Dir; use new_mime_guess::MimeGuess; use openssl::hash::MessageDigest; use openssl::x509::X509; -use rpc_toolkit::Server; +use rpc_toolkit::{Context, HttpServer, Server}; use tokio::fs::File; use tokio::io::BufReader; use tokio_util::io::ReaderStream; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; -use crate::db::subscribe; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::hostname::Hostname; use crate::middleware::auth::{Auth, HasValidSession}; use crate::middleware::cors::Cors; use crate::middleware::db::SyncDb; -use crate::middleware::diagnostic::DiagnosticMode; -use crate::rpc_continuations::Guid; -use crate::{diagnostic_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt}; +use crate::rpc_continuations::{Guid, RpcContinuations}; +use crate::{ + diagnostic_api, init_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt, +}; const NOT_FOUND: &[u8] = b"Not Found"; const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed"; @@ -49,7 +49,6 @@ const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "u #[derive(Clone)] pub enum UiMode { Setup, - Diag, Install, Main, } @@ -58,128 +57,46 @@ impl UiMode { fn path(&self, path: &str) -> PathBuf { match self { Self::Setup => Path::new("setup-wizard").join(path), - Self::Diag => Path::new("diagnostic-ui").join(path), Self::Install => Path::new("install-wizard").join(path), Self::Main => Path::new("ui").join(path), } } } -pub fn setup_ui_file_router(ctx: SetupContext) -> Router { - Router::new() - .route_service( - "/rpc/*path", - post(Server::new(move || ready(Ok(ctx.clone())), setup_api()).middleware(Cors::new())), - ) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Setup) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn diag_ui_file_router(ctx: DiagnosticContext) -> Router { +pub fn rpc_router>( + ctx: C, + server: HttpServer, +) -> Router { Router::new() + .route("/rpc/*path", post(server)) .route( - "/rpc/*path", - post( - Server::new(move || ready(Ok(ctx.clone())), diagnostic_api()) - .middleware(Cors::new()) - .middleware(DiagnosticMode::new()), - ), - ) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Diag) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn install_ui_file_router(ctx: InstallContext) -> Router { - Router::new() - .route("/rpc/*path", { - let ctx = ctx.clone(); - post(Server::new(move || ready(Ok(ctx.clone())), install_api()).middleware(Cors::new())) - }) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Install) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn main_ui_server_router(ctx: RpcContext) -> Router { - Router::new() - .route("/rpc/*path", { - let ctx = ctx.clone(); - post( - Server::new(move || ready(Ok(ctx.clone())), main_api::()) - .middleware(Cors::new()) - .middleware(Auth::new()) - .middleware(SyncDb::new()), - ) - }) - .route( - "/ws/db", - any({ - let ctx = ctx.clone(); - move |headers: HeaderMap, ws: x::WebSocketUpgrade| async move { - subscribe(ctx, headers, ws) - .await - .unwrap_or_else(server_error) - } - }), - ) - .route( - "/ws/rpc/*path", + "/ws/rpc/:guid", get({ let ctx = ctx.clone(); - move |x::Path(path): x::Path, + move |x::Path(guid): x::Path, ws: axum::extract::ws::WebSocketUpgrade| async move { - match Guid::from(&path) { - None => { - tracing::debug!("No Guid Path"); - bad_request() - } - Some(guid) => match ctx.rpc_continuations.get_ws_handler(&guid).await { - Some(cont) => ws.on_upgrade(cont), - _ => not_found(), - }, + match AsRef::::as_ref(&ctx).get_ws_handler(&guid).await { + Some(cont) => ws.on_upgrade(cont), + _ => not_found(), } } }), ) .route( - "/rest/rpc/*path", + "/rest/rpc/:guid", any({ let ctx = ctx.clone(); - move |request: x::Request| async move { - let path = request - .uri() - .path() - .strip_prefix("/rest/rpc/") - .unwrap_or_default(); - match Guid::from(&path) { - None => { - tracing::debug!("No Guid Path"); - bad_request() - } - Some(guid) => match ctx.rpc_continuations.get_rest_handler(&guid).await { - None => not_found(), - Some(cont) => cont(request).await.unwrap_or_else(server_error), - }, + move |x::Path(guid): x::Path, request: x::Request| async move { + match AsRef::::as_ref(&ctx).get_rest_handler(&guid).await { + None => not_found(), + Some(cont) => cont(request).await.unwrap_or_else(server_error), } } }), ) - .fallback(any(move |request: Request| async move { - main_start_os_ui(request, ctx) - .await - .unwrap_or_else(server_error) - })) } -async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { +fn serve_ui(req: Request, ui_mode: UiMode) -> Result { let (request_parts, _body) = req.into_parts(); match &request_parts.method { &Method::GET => { @@ -196,9 +113,7 @@ async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { .or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html"))); if let Some(file) = file { - FileData::from_embedded(&request_parts, file) - .into_response(&request_parts) - .await + FileData::from_embedded(&request_parts, file).into_response(&request_parts) } else { Ok(not_found()) } @@ -207,6 +122,75 @@ async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { } } +pub fn setup_ui_router(ctx: SetupContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), setup_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Setup).unwrap_or_else(server_error) + })) +} + +pub fn diagnostic_ui_router(ctx: DiagnosticContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), diagnostic_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn install_ui_router(ctx: InstallContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), install_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Install).unwrap_or_else(server_error) + })) +} + +pub fn init_ui_router(ctx: InitContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), init_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn main_ui_router(ctx: RpcContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), main_api::()) + .middleware(Cors::new()) + .middleware(Auth::new()) + .middleware(SyncDb::new()), + ) + // TODO: cert + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn refresher() -> Router { + Router::new().fallback(get(|request: Request| async move { + let res = include_bytes!("./refresher.html"); + FileData { + data: Body::from(&res[..]), + e_tag: None, + encoding: None, + len: Some(res.len() as u64), + mime: Some("text/html".into()), + } + .into_response(&request.into_parts().0) + .unwrap_or_else(server_error) + })) +} + async fn if_authorized< F: FnOnce() -> Fut, Fut: Future> + Send + Sync, @@ -223,89 +207,6 @@ async fn if_authorized< } } -async fn main_start_os_ui(req: Request, ctx: RpcContext) -> Result { - let (request_parts, _body) = req.into_parts(); - match ( - &request_parts.method, - request_parts - .uri - .path() - .strip_prefix('/') - .unwrap_or(request_parts.uri.path()) - .split_once('/'), - ) { - (&Method::GET, Some(("public", path))) => { - todo!("pull directly from s9pk") - } - (&Method::GET, Some(("proxy", target))) => { - if_authorized(&ctx, &request_parts, || async { - let target = urlencoding::decode(target)?; - let res = ctx - .client - .get(target.as_ref()) - .headers( - request_parts - .headers - .iter() - .filter(|(h, _)| { - !PROXY_STRIP_HEADERS - .iter() - .any(|bad| h.as_str().eq_ignore_ascii_case(bad)) - }) - .flat_map(|(h, v)| { - Some(( - reqwest::header::HeaderName::from_lowercase( - h.as_str().as_bytes(), - ) - .ok()?, - reqwest::header::HeaderValue::from_bytes(v.as_bytes()).ok()?, - )) - }) - .collect(), - ) - .send() - .await - .with_kind(crate::ErrorKind::Network)?; - let mut hres = Response::builder().status(res.status().as_u16()); - for (h, v) in res.headers().clone() { - if let Some(h) = h { - hres = hres.header(h.to_string(), v.as_bytes()); - } - } - hres.body(Body::from_stream(res.bytes_stream())) - .with_kind(crate::ErrorKind::Network) - }) - .await - } - (&Method::GET, Some(("eos", "local.crt"))) => { - let account = ctx.account.read().await; - cert_send(&account.root_ca_cert, &account.hostname) - } - (&Method::GET, _) => { - let uri_path = UiMode::Main.path( - request_parts - .uri - .path() - .strip_prefix('/') - .unwrap_or(request_parts.uri.path()), - ); - - let file = EMBEDDED_UIS - .get_file(&*uri_path) - .or_else(|| EMBEDDED_UIS.get_file(&*UiMode::Main.path("index.html"))); - - if let Some(file) = file { - FileData::from_embedded(&request_parts, file) - .into_response(&request_parts) - .await - } else { - Ok(not_found()) - } - } - _ => Ok(method_not_allowed()), - } -} - pub fn unauthorized(err: Error, path: &str) -> Response { tracing::warn!("unauthorized for {} @{:?}", err, path); tracing::debug!("{:?}", err); @@ -373,8 +274,8 @@ struct FileData { data: Body, len: Option, encoding: Option<&'static str>, - e_tag: String, - mime: Option, + e_tag: Option, + mime: Option, } impl FileData { fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self { @@ -407,10 +308,23 @@ impl FileData { len: Some(data.len() as u64), encoding, data: data.into(), - e_tag: e_tag(path, None), + e_tag: file.metadata().map(|metadata| { + e_tag( + path, + format!( + "{}", + metadata + .modified() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1), + ) + .as_bytes(), + ) + }), mime: MimeGuess::from_path(path) .first() - .map(|m| m.essence_str().to_owned()), + .map(|m| m.essence_str().into()), } } @@ -434,7 +348,18 @@ impl FileData { .await .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?; - let e_tag = e_tag(path, Some(&metadata)); + let e_tag = Some(e_tag( + path, + format!( + "{}", + metadata + .modified()? + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1) + ) + .as_bytes(), + )); let (len, data) = if encoding == Some("gzip") { ( @@ -455,16 +380,18 @@ impl FileData { e_tag, mime: MimeGuess::from_path(path) .first() - .map(|m| m.essence_str().to_owned()), + .map(|m| m.essence_str().into()), }) } - async fn into_response(self, req: &RequestParts) -> Result { + fn into_response(self, req: &RequestParts) -> Result { let mut builder = Response::builder(); if let Some(mime) = self.mime { builder = builder.header(http::header::CONTENT_TYPE, &*mime); } - builder = builder.header(http::header::ETAG, &*self.e_tag); + if let Some(e_tag) = &self.e_tag { + builder = builder.header(http::header::ETAG, &**e_tag); + } builder = builder.header( http::header::CACHE_CONTROL, "public, max-age=21000000, immutable", @@ -481,11 +408,12 @@ impl FileData { builder = builder.header(http::header::CONNECTION, "keep-alive"); } - if req - .headers - .get("if-none-match") - .and_then(|h| h.to_str().ok()) - == Some(self.e_tag.as_ref()) + if self.e_tag.is_some() + && req + .headers + .get("if-none-match") + .and_then(|h| h.to_str().ok()) + == self.e_tag.as_deref() { builder = builder.status(StatusCode::NOT_MODIFIED); builder.body(Body::empty()) @@ -503,21 +431,14 @@ impl FileData { } } -fn e_tag(path: &Path, metadata: Option<&Metadata>) -> String { +lazy_static::lazy_static! { + static ref INSTANCE_NONCE: u64 = rand::random(); +} + +fn e_tag(path: &Path, modified: impl AsRef<[u8]>) -> String { let mut hasher = sha2::Sha256::new(); hasher.update(format!("{:?}", path).as_bytes()); - if let Some(modified) = metadata.and_then(|m| m.modified().ok()) { - hasher.update( - format!( - "{}", - modified - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - ) - .as_bytes(), - ); - } + hasher.update(modified.as_ref()); let res = hasher.finalize(); format!( "\"{}\"", diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index a89aae92f..a9cfdf046 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -1,23 +1,84 @@ +use std::convert::Infallible; use std::net::SocketAddr; +use std::task::Poll; use std::time::Duration; +use axum::extract::Request; use axum::Router; use axum_server::Handle; +use bytes::Bytes; +use futures::future::ready; +use futures::FutureExt; use helpers::NonDetachingJoinHandle; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::net::static_server::{ - diag_ui_file_router, install_ui_file_router, main_ui_server_router, setup_ui_file_router, + diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, refresher, + setup_ui_router, }; -use crate::Error; +use crate::prelude::*; + +#[derive(Clone)] +pub struct SwappableRouter(watch::Sender); +impl SwappableRouter { + pub fn new(router: Router) -> Self { + Self(watch::channel(router).0) + } + pub fn swap(&self, router: Router) { + let _ = self.0.send_replace(router); + } +} + +#[derive(Clone)] +pub struct SwappableRouterService(watch::Receiver); +impl tower_service::Service> for SwappableRouterService +where + B: axum::body::HttpBody + Send + 'static, + B::Error: Into, +{ + type Response = >>::Response; + type Error = >>::Error; + type Future = >>::Future; + #[inline] + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + let mut changed = self.0.changed().boxed(); + if changed.poll_unpin(cx).is_ready() { + return Poll::Ready(Ok(())); + } + drop(changed); + tower_service::Service::>::poll_ready(&mut self.0.borrow().clone(), cx) + } + fn call(&mut self, req: Request) -> Self::Future { + self.0.borrow().clone().call(req) + } +} + +impl tower_service::Service for SwappableRouter { + type Response = SwappableRouterService; + type Error = Infallible; + type Future = futures::future::Ready>; + #[inline] + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, _: T) -> Self::Future { + ready(Ok(SwappableRouterService(self.0.subscribe()))) + } +} pub struct WebServer { shutdown: oneshot::Sender<()>, + router: SwappableRouter, thread: NonDetachingJoinHandle<()>, } impl WebServer { - pub fn new(bind: SocketAddr, router: Router) -> Self { + pub fn new(bind: SocketAddr) -> Self { + let router = SwappableRouter::new(refresher()); + let thread_router = router.clone(); let (shutdown, shutdown_recv) = oneshot::channel(); let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { let handle = Handle::new(); @@ -25,14 +86,18 @@ impl WebServer { server.http_builder().http1().preserve_header_case(true); server.http_builder().http1().title_case_headers(true); - if let (Err(e), _) = tokio::join!(server.serve(router.into_make_service()), async { + if let (Err(e), _) = tokio::join!(server.serve(thread_router), async { let _ = shutdown_recv.await; handle.graceful_shutdown(Some(Duration::from_secs(0))); }) { tracing::error!("Spawning hyper server error: {}", e); } })); - Self { shutdown, thread } + Self { + shutdown, + router, + thread, + } } pub async fn shutdown(self) { @@ -40,19 +105,27 @@ impl WebServer { self.thread.await.unwrap() } - pub fn main(bind: SocketAddr, ctx: RpcContext) -> Result { - Ok(Self::new(bind, main_ui_server_router(ctx))) + pub fn serve_router(&mut self, router: Router) { + self.router.swap(router) } - pub fn setup(bind: SocketAddr, ctx: SetupContext) -> Result { - Ok(Self::new(bind, setup_ui_file_router(ctx))) + pub fn serve_main(&mut self, ctx: RpcContext) { + self.serve_router(main_ui_router(ctx)) } - pub fn diagnostic(bind: SocketAddr, ctx: DiagnosticContext) -> Result { - Ok(Self::new(bind, diag_ui_file_router(ctx))) + pub fn serve_setup(&mut self, ctx: SetupContext) { + self.serve_router(setup_ui_router(ctx)) } - pub fn install(bind: SocketAddr, ctx: InstallContext) -> Result { - Ok(Self::new(bind, install_ui_file_router(ctx))) + pub fn serve_diagnostic(&mut self, ctx: DiagnosticContext) { + self.serve_router(diagnostic_ui_router(ctx)) + } + + pub fn serve_install(&mut self, ctx: InstallContext) { + self.serve_router(install_ui_router(ctx)) + } + + pub fn serve_init(&mut self, ctx: InitContext) { + self.serve_router(init_ui_router(ctx)) } } diff --git a/core/startos/src/progress.rs b/core/startos/src/progress.rs index 9a22405ca..f70a5adbc 100644 --- a/core/startos/src/progress.rs +++ b/core/startos/src/progress.rs @@ -1,14 +1,16 @@ use std::panic::UnwindSafe; -use std::sync::Arc; use std::time::Duration; -use futures::Future; +use futures::future::pending; +use futures::stream::BoxStream; +use futures::{Future, FutureExt, StreamExt, TryFutureExt}; +use helpers::NonDetachingJoinHandle; use imbl_value::{InOMap, InternedString}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use itertools::Itertools; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncSeek, AsyncWrite}; -use tokio::sync::{mpsc, watch}; +use tokio::sync::watch; use ts_rs::TS; use crate::db::model::{Database, DatabaseModel}; @@ -168,39 +170,23 @@ impl FullProgress { } } +#[derive(Clone)] pub struct FullProgressTracker { - overall: Arc>, - overall_recv: watch::Receiver, - phases: InOMap>, - new_phase: ( - mpsc::UnboundedSender<(InternedString, watch::Receiver)>, - mpsc::UnboundedReceiver<(InternedString, watch::Receiver)>, - ), + overall: watch::Sender, + phases: watch::Sender>>, } impl FullProgressTracker { pub fn new() -> Self { - let (overall, overall_recv) = watch::channel(Progress::new()); - Self { - overall: Arc::new(overall), - overall_recv, - phases: InOMap::new(), - new_phase: mpsc::unbounded_channel(), - } + let (overall, _) = watch::channel(Progress::new()); + let (phases, _) = watch::channel(InOMap::new()); + Self { overall, phases } } - fn fill_phases(&mut self) -> bool { - let mut changed = false; - while let Ok((name, phase)) = self.new_phase.1.try_recv() { - self.phases.insert(name, phase); - changed = true; - } - changed - } - pub fn snapshot(&mut self) -> FullProgress { - self.fill_phases(); + pub fn snapshot(&self) -> FullProgress { FullProgress { overall: *self.overall.borrow(), phases: self .phases + .borrow() .iter() .map(|(name, progress)| NamedProgress { name: name.clone(), @@ -209,28 +195,75 @@ impl FullProgressTracker { .collect(), } } - pub async fn changed(&mut self) { - if self.fill_phases() { - return; - } - let phases = self - .phases - .iter_mut() - .map(|(_, p)| Box::pin(p.changed())) - .collect_vec(); - tokio::select! { - _ = self.overall_recv.changed() => (), - _ = futures::future::select_all(phases) => (), - } - } - pub fn handle(&self) -> FullProgressTrackerHandle { - FullProgressTrackerHandle { - overall: self.overall.clone(), - new_phase: self.new_phase.0.clone(), + pub fn stream(&self, min_interval: Option) -> BoxStream<'static, FullProgress> { + struct StreamState { + overall: watch::Receiver, + phases_recv: watch::Receiver>>, + phases: InOMap>, } + let mut overall = self.overall.subscribe(); + overall.mark_changed(); // make sure stream starts with a value + let phases_recv = self.phases.subscribe(); + let phases = phases_recv.borrow().clone(); + let state = StreamState { + overall, + phases_recv, + phases, + }; + futures::stream::unfold( + state, + move |StreamState { + mut overall, + mut phases_recv, + mut phases, + }| async move { + let changed = phases + .iter_mut() + .map(|(_, p)| async move { p.changed().or_else(|_| pending()).await }.boxed()) + .chain([overall.changed().boxed()]) + .chain([phases_recv.changed().boxed()]) + .map(|fut| fut.map(|r| r.unwrap_or_default())) + .collect_vec(); + if let Some(min_interval) = min_interval { + tokio::join!( + tokio::time::sleep(min_interval), + futures::future::select_all(changed), + ); + } else { + futures::future::select_all(changed).await; + } + + for (name, phase) in &*phases_recv.borrow_and_update() { + if !phases.contains_key(name) { + phases.insert(name.clone(), phase.clone()); + } + } + + let o = *overall.borrow_and_update(); + + Some(( + FullProgress { + overall: o, + phases: phases + .iter_mut() + .map(|(name, progress)| NamedProgress { + name: name.clone(), + progress: *progress.borrow_and_update(), + }) + .collect(), + }, + StreamState { + overall, + phases_recv, + phases, + }, + )) + }, + ) + .boxed() } pub fn sync_to_db( - mut self, + &self, db: TypedPatchDb, deref: DerefFn, min_interval: Option, @@ -239,9 +272,9 @@ impl FullProgressTracker { DerefFn: Fn(&mut DatabaseModel) -> Option<&mut Model> + 'static, for<'a> &'a DerefFn: UnwindSafe + Send, { + let mut stream = self.stream(min_interval); async move { - loop { - let progress = self.snapshot(); + while let Some(progress) = stream.next().await { if db .mutate(|v| { if let Some(p) = deref(v) { @@ -255,25 +288,23 @@ impl FullProgressTracker { { break; } - tokio::join!(self.changed(), async { - if let Some(interval) = min_interval { - tokio::time::sleep(interval).await - } else { - futures::future::ready(()).await - } - }); } Ok(()) } } -} - -#[derive(Clone)] -pub struct FullProgressTrackerHandle { - overall: Arc>, - new_phase: mpsc::UnboundedSender<(InternedString, watch::Receiver)>, -} -impl FullProgressTrackerHandle { + pub fn progress_bar_task(&self, name: &str) -> NonDetachingJoinHandle<()> { + let mut stream = self.stream(None); + let mut bar = PhasedProgressBar::new(name); + tokio::spawn(async move { + while let Some(progress) = stream.next().await { + bar.update(&progress); + if progress.overall.is_complete() { + break; + } + } + }) + .into() + } pub fn add_phase( &self, name: InternedString, @@ -284,7 +315,9 @@ impl FullProgressTrackerHandle { .send_modify(|o| o.add_total(overall_contribution)); } let (send, recv) = watch::channel(Progress::new()); - let _ = self.new_phase.send((name, recv)); + self.phases.send_modify(|p| { + p.insert(name, recv); + }); PhaseProgressTrackerHandle { overall: self.overall.clone(), overall_contribution, @@ -298,7 +331,7 @@ impl FullProgressTrackerHandle { } pub struct PhaseProgressTrackerHandle { - overall: Arc>, + overall: watch::Sender, overall_contribution: Option, contributed: u64, progress: watch::Sender, diff --git a/core/startos/src/registry/context.rs b/core/startos/src/registry/context.rs index 99d60307b..e5beabca7 100644 --- a/core/startos/src/registry/context.rs +++ b/core/startos/src/registry/context.rs @@ -169,7 +169,8 @@ impl CallRemote for CliContext { &AnySigningKey::Ed25519(self.developer_key()?.clone()), &body, &host, - )?.to_header(), + )? + .to_header(), ) .body(body) .send() diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index 656edf337..9a53d6338 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -70,7 +70,7 @@ pub fn registry_api() -> ParentHandler { .subcommand("db", db::db_api::()) } -pub fn registry_server_router(ctx: RegistryContext) -> Router { +pub fn registry_router(ctx: RegistryContext) -> Router { use axum::extract as x; use axum::routing::{any, get, post}; Router::new() @@ -128,7 +128,7 @@ pub fn registry_server_router(ctx: RegistryContext) -> Router { } impl WebServer { - pub fn registry(bind: SocketAddr, ctx: RegistryContext) -> Self { - Self::new(bind, registry_server_router(ctx)) + pub fn serve_registry(&mut self, ctx: RegistryContext) { + self.serve_router(registry_router(ctx)) } } diff --git a/core/startos/src/registry/os/asset/add.rs b/core/startos/src/registry/os/asset/add.rs index 6ca495547..33f5ef90f 100644 --- a/core/startos/src/registry/os/asset/add.rs +++ b/core/startos/src/registry/os/asset/add.rs @@ -186,29 +186,16 @@ pub async fn cli_add_asset( let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = - progress_handle.add_phase(InternedString::intern("Signing File"), Some(10)); - let mut verify_phase = - progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10)); + let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding File to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", path.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", path.display())); sign_phase.start(); let blake3 = file.blake3_mmap().await?; @@ -252,7 +239,7 @@ pub async fn cli_add_asset( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/os/asset/get.rs b/core/startos/src/registry/os/asset/get.rs index e099a50cf..29ff24da6 100644 --- a/core/startos/src/registry/os/asset/get.rs +++ b/core/startos/src/registry/os/asset/get.rs @@ -3,7 +3,7 @@ use std::panic::UnwindSafe; use std::path::{Path, PathBuf}; use clap::Parser; -use helpers::{AtomicFile, NonDetachingJoinHandle}; +use helpers::AtomicFile; use imbl_value::{json, InternedString}; use itertools::Itertools; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; @@ -12,7 +12,7 @@ use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; @@ -135,29 +135,17 @@ async fn cli_get_os_asset( .await .with_kind(ErrorKind::Filesystem)?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); + let progress = FullProgressTracker::new(); let mut download_phase = - progress_handle.add_phase(InternedString::intern("Downloading File"), Some(100)); + progress.add_phase(InternedString::intern("Downloading File"), Some(100)); download_phase.set_total(res.commitment.size); let reverify_phase = if reverify { - Some(progress_handle.add_phase(InternedString::intern("Reverifying File"), Some(10))) + Some(progress.add_phase(InternedString::intern("Reverifying File"), Some(10))) } else { None }; - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new("Downloading..."); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = progress.progress_bar_task("Downloading..."); download_phase.start(); let mut download_writer = download_phase.writer(&mut *file); @@ -177,7 +165,7 @@ async fn cli_get_os_asset( reverify_phase.complete(); } - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; } diff --git a/core/startos/src/registry/os/asset/sign.rs b/core/startos/src/registry/os/asset/sign.rs index 0cb657bef..50c583593 100644 --- a/core/startos/src/registry/os/asset/sign.rs +++ b/core/startos/src/registry/os/asset/sign.rs @@ -3,7 +3,6 @@ use std::panic::UnwindSafe; use std::path::PathBuf; use clap::Parser; -use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; @@ -12,7 +11,7 @@ use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; @@ -169,27 +168,15 @@ pub async fn cli_sign_asset( let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = - progress_handle.add_phase(InternedString::intern("Signing File"), Some(10)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding Signature to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", path.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", path.display())); sign_phase.start(); let blake3 = file.blake3_mmap().await?; @@ -220,7 +207,7 @@ pub async fn cli_sign_asset( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/package/add.rs b/core/startos/src/registry/package/add.rs index d28aeaaa4..9bc772f78 100644 --- a/core/startos/src/registry/package/add.rs +++ b/core/startos/src/registry/package/add.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; use std::sync::Arc; use clap::Parser; -use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; use rpc_toolkit::HandlerArgs; @@ -12,7 +11,7 @@ use url::Url; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::context::RegistryContext; use crate::registry::package::index::PackageVersionInfo; use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; @@ -110,28 +109,16 @@ pub async fn cli_add_package( ) -> Result<(), Error> { let s9pk = S9pk::open(&file, None).await?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = progress_handle.add_phase(InternedString::intern("Signing File"), Some(1)); - let mut verify_phase = - progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1)); + let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding File to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", file.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", file.display())); sign_phase.start(); let commitment = s9pk.as_archive().commitment().await?; @@ -160,7 +147,7 @@ pub async fn cli_add_package( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/signer/commitment/request.rs b/core/startos/src/registry/signer/commitment/request.rs index 62d59163f..ce60b7f88 100644 --- a/core/startos/src/registry/signer/commitment/request.rs +++ b/core/startos/src/registry/signer/commitment/request.rs @@ -1,5 +1,5 @@ -use std::time::{SystemTime, UNIX_EPOCH}; use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; use axum::body::Body; use axum::extract::Request; diff --git a/core/startos/src/rpc_continuations.rs b/core/startos/src/rpc_continuations.rs index e6b823ef9..043130b69 100644 --- a/core/startos/src/rpc_continuations.rs +++ b/core/startos/src/rpc_continuations.rs @@ -1,5 +1,8 @@ use std::collections::BTreeMap; +use std::pin::Pin; use std::str::FromStr; +use std::sync::Mutex as SyncMutex; +use std::task::{Context, Poll}; use std::time::Duration; use axum::extract::ws::WebSocket; @@ -7,9 +10,10 @@ use axum::extract::Request; use axum::response::Response; use clap::builder::ValueParserFactory; use futures::future::BoxFuture; +use futures::{Future, FutureExt}; use helpers::TimedResource; use imbl_value::InternedString; -use tokio::sync::Mutex; +use tokio::sync::{broadcast, Mutex as AsyncMutex}; use ts_rs::TS; #[allow(unused_imports)] @@ -73,21 +77,103 @@ impl std::fmt::Display for Guid { } } -pub type RestHandler = - Box BoxFuture<'static, Result> + Send>; +pub struct RestFuture { + kill: Option>, + fut: BoxFuture<'static, Result>, +} +impl Future for RestFuture { + type Output = Result; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.kill.as_ref().map_or(false, |k| !k.is_empty()) { + Poll::Ready(Err(Error::new( + eyre!("session killed"), + ErrorKind::Authorization, + ))) + } else { + self.fut.poll_unpin(cx) + } + } +} +pub type RestHandler = Box RestFuture + Send>; -pub type WebSocketHandler = Box BoxFuture<'static, ()> + Send>; +pub struct WebSocketFuture { + kill: Option>, + fut: BoxFuture<'static, ()>, +} +impl Future for WebSocketFuture { + type Output = (); + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.kill.as_ref().map_or(false, |k| !k.is_empty()) { + Poll::Ready(()) + } else { + self.fut.poll_unpin(cx) + } + } +} +pub type WebSocketHandler = Box WebSocketFuture + Send>; pub enum RpcContinuation { Rest(TimedResource), WebSocket(TimedResource), } impl RpcContinuation { - pub fn rest(handler: RestHandler, timeout: Duration) -> Self { - RpcContinuation::Rest(TimedResource::new(handler, timeout)) + pub fn rest(handler: F, timeout: Duration) -> Self + where + F: FnOnce(Request) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + RpcContinuation::Rest(TimedResource::new( + Box::new(|req| RestFuture { + kill: None, + fut: handler(req).boxed(), + }), + timeout, + )) } - pub fn ws(handler: WebSocketHandler, timeout: Duration) -> Self { - RpcContinuation::WebSocket(TimedResource::new(handler, timeout)) + pub fn ws(handler: F, timeout: Duration) -> Self + where + F: FnOnce(WebSocket) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + RpcContinuation::WebSocket(TimedResource::new( + Box::new(|ws| WebSocketFuture { + kill: None, + fut: handler(ws).boxed(), + }), + timeout, + )) + } + pub fn rest_authed(ctx: Ctx, session: T, handler: F, timeout: Duration) -> Self + where + Ctx: AsRef>, + T: Eq + Ord, + F: FnOnce(Request) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let kill = Some(ctx.as_ref().subscribe_to_kill(session)); + RpcContinuation::Rest(TimedResource::new( + Box::new(|req| RestFuture { + kill, + fut: handler(req).boxed(), + }), + timeout, + )) + } + pub fn ws_authed(ctx: Ctx, session: T, handler: F, timeout: Duration) -> Self + where + Ctx: AsRef>, + T: Eq + Ord, + F: FnOnce(WebSocket) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let kill = Some(ctx.as_ref().subscribe_to_kill(session)); + RpcContinuation::WebSocket(TimedResource::new( + Box::new(|ws| WebSocketFuture { + kill, + fut: handler(ws).boxed(), + }), + timeout, + )) } pub fn is_timed_out(&self) -> bool { match self { @@ -97,10 +183,10 @@ impl RpcContinuation { } } -pub struct RpcContinuations(Mutex>); +pub struct RpcContinuations(AsyncMutex>); impl RpcContinuations { pub fn new() -> Self { - RpcContinuations(Mutex::new(BTreeMap::new())) + RpcContinuations(AsyncMutex::new(BTreeMap::new())) } #[instrument(skip_all)] @@ -146,3 +232,28 @@ impl RpcContinuations { x.get().await } } + +pub struct OpenAuthedContinuations(SyncMutex>>); +impl OpenAuthedContinuations +where + T: Eq + Ord, +{ + pub fn new() -> Self { + Self(SyncMutex::new(BTreeMap::new())) + } + pub fn kill(&self, session: &T) { + if let Some(channel) = self.0.lock().unwrap().remove(session) { + channel.send(()).ok(); + } + } + fn subscribe_to_kill(&self, session: T) -> broadcast::Receiver<()> { + let mut map = self.0.lock().unwrap(); + if let Some(send) = map.get(&session) { + send.subscribe() + } else { + let (send, recv) = broadcast::channel(1); + map.insert(session, send); + recv + } + } +} diff --git a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs index 92eb40f9d..4b7d5736d 100644 --- a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs +++ b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs @@ -97,6 +97,7 @@ impl ArchiveSource for MultiCursorFile { .ok() .map(|m| m.len()) } + #[allow(refining_impl_trait)] async fn fetch_all(&self) -> Result { use tokio::io::AsyncSeekExt; diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 6588de836..2eecc565e 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -354,7 +354,7 @@ impl Service { .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation if let Some(mut progress) = progress { progress.finalization_progress.complete(); - progress.progress_handle.complete(); + progress.progress.complete(); tokio::task::yield_now().await; } ctx.db diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index 1474ea35e..f8874d21a 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -18,10 +18,7 @@ use crate::disk::mount::guard::GenericMountGuard; use crate::install::PKG_ARCHIVE_DIR; use crate::notifications::{notify, NotificationLevel}; use crate::prelude::*; -use crate::progress::{ - FullProgressTracker, FullProgressTrackerHandle, PhaseProgressTrackerHandle, - ProgressTrackerWriter, -}; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle, ProgressTrackerWriter}; use crate::s9pk::manifest::PackageId; use crate::s9pk::merkle_archive::source::FileSource; use crate::s9pk::S9pk; @@ -34,7 +31,7 @@ pub type InstallFuture = BoxFuture<'static, Result<(), Error>>; pub struct InstallProgressHandles { pub finalization_progress: PhaseProgressTrackerHandle, - pub progress_handle: FullProgressTrackerHandle, + pub progress: FullProgressTracker, } /// This is the structure to contain all the services @@ -59,13 +56,22 @@ impl ServiceMap { } #[instrument(skip_all)] - pub async fn init(&self, ctx: &RpcContext) -> Result<(), Error> { - for id in ctx.db.peek().await.as_public().as_package_data().keys()? { + pub async fn init( + &self, + ctx: &RpcContext, + mut progress: PhaseProgressTrackerHandle, + ) -> Result<(), Error> { + progress.start(); + let ids = ctx.db.peek().await.as_public().as_package_data().keys()?; + progress.set_total(ids.len() as u64); + for id in ids { if let Err(e) = self.load(ctx, &id, LoadDisposition::Retry).await { tracing::error!("Error loading installed package as service: {e}"); tracing::debug!("{e:?}"); } + progress += 1; } + progress.complete(); Ok(()) } @@ -112,17 +118,16 @@ impl ServiceMap { }; let size = s9pk.size(); - let mut progress = FullProgressTracker::new(); + let progress = FullProgressTracker::new(); let download_progress_contribution = size.unwrap_or(60); - let progress_handle = progress.handle(); - let mut download_progress = progress_handle.add_phase( + let mut download_progress = progress.add_phase( InternedString::intern("Download"), Some(download_progress_contribution), ); if let Some(size) = size { download_progress.set_total(size); } - let mut finalization_progress = progress_handle.add_phase( + let mut finalization_progress = progress.add_phase( InternedString::intern(op_name), Some(download_progress_contribution / 2), ); @@ -194,7 +199,7 @@ impl ServiceMap { let deref_id = id.clone(); let sync_progress_task = - NonDetachingJoinHandle::from(tokio::spawn(progress.sync_to_db( + NonDetachingJoinHandle::from(tokio::spawn(progress.clone().sync_to_db( ctx.db.clone(), move |v| { v.as_public_mut() @@ -248,7 +253,7 @@ impl ServiceMap { service .uninstall(Some(s9pk.as_manifest().version.clone())) .await?; - progress_handle.complete(); + progress.complete(); Some(version) } else { None @@ -261,7 +266,7 @@ impl ServiceMap { recovery_source, Some(InstallProgressHandles { finalization_progress, - progress_handle, + progress, }), ) .await? @@ -275,7 +280,7 @@ impl ServiceMap { prev, Some(InstallProgressHandles { finalization_progress, - progress_handle, + progress, }), ) .await? diff --git a/core/startos/src/setup.rs b/core/startos/src/setup.rs index a035e932f..2b701c01f 100644 --- a/core/startos/src/setup.rs +++ b/core/startos/src/setup.rs @@ -4,7 +4,6 @@ use std::time::Duration; use color_eyre::eyre::eyre; use josekit::jwk::Jwk; -use openssl::x509::X509; use patch_db::json_ptr::ROOT; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; @@ -12,15 +11,15 @@ use serde::{Deserialize, Serialize}; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::try_join; -use torut::onion::OnionAddressV3; use tracing::instrument; use ts_rs::TS; use crate::account::AccountInfo; use crate::backup::restore::recover_full_embassy; use crate::backup::target::BackupTargetFS; +use crate::context::rpc::InitRpcContextPhases; use crate::context::setup::SetupResult; -use crate::context::SetupContext; +use crate::context::{RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::fsck::RepairStrategy; use crate::disk::main::DEFAULT_PASSWORD; @@ -29,10 +28,12 @@ use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; use crate::disk::util::{pvscan, recovery_info, DiskInfo, EmbassyOsRecoveryInfo}; use crate::disk::REPAIR_DISK_PATH; -use crate::hostname::Hostname; -use crate::init::{init, InitResult}; +use crate::init::{init, InitPhases, InitResult}; +use crate::net::net_controller::PreInitNetController; use crate::net::ssl::root_ca_start_time; use crate::prelude::*; +use crate::progress::{FullProgress, PhaseProgressTrackerHandle}; +use crate::rpc_continuations::Guid; use crate::util::crypto::EncryptedWire; use crate::util::io::{dir_copy, dir_size, Counter}; use crate::{Error, ErrorKind, ResultExt}; @@ -75,10 +76,12 @@ pub async fn list_disks(ctx: SetupContext) -> Result, Error> { async fn setup_init( ctx: &SetupContext, password: Option, -) -> Result<(Hostname, OnionAddressV3, X509), Error> { - let InitResult { db } = init(&ctx.config).await?; + init_phases: InitPhases, +) -> Result<(AccountInfo, PreInitNetController), Error> { + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; - let account = db + let account = net_ctrl + .db .mutate(|m| { let mut account = AccountInfo::load(m)?; if let Some(password) = password { @@ -93,15 +96,12 @@ async fn setup_init( }) .await?; - Ok(( - account.hostname, - account.tor_key.public().get_onion_address(), - account.root_ca_cert, - )) + Ok((account, net_ctrl)) } #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct AttachParams { #[serde(rename = "startOsPassword")] password: Option, @@ -110,25 +110,20 @@ pub struct AttachParams { pub async fn attach( ctx: SetupContext, - AttachParams { password, guid }: AttachParams, -) -> Result<(), Error> { - let mut status = ctx.setup_status.write().await; - if status.is_some() { - return Err(Error::new( - eyre!("Setup already in progress"), - ErrorKind::InvalidRequest, - )); - } - *status = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); - drop(status); - tokio::task::spawn(async move { - if let Err(e) = async { + AttachParams { + password, + guid: disk_guid, + }: AttachParams, +) -> Result { + let setup_ctx = ctx.clone(); + ctx.run_setup(|| async move { + let progress = &setup_ctx.progress; + let mut disk_phase = progress.add_phase("Opening data drive".into(), Some(10)); + let init_phases = InitPhases::new(&progress); + let rpc_ctx_phases = InitRpcContextPhases::new(&progress); + let password: Option = match password { - Some(a) => match a.decrypt(&*ctx) { + Some(a) => match a.decrypt(&setup_ctx) { a @ Some(_) => a, None => { return Err(Error::new( @@ -139,15 +134,17 @@ pub async fn attach( }, None => None, }; + + disk_phase.start(); let requires_reboot = crate::disk::main::import( - &*guid, - &ctx.datadir, + &*disk_guid, + &setup_ctx.datadir, if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen }, - if guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) }, + if disk_guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) }, ) .await?; if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { @@ -156,7 +153,7 @@ pub async fn attach( .with_ctx(|_| (ErrorKind::Filesystem, REPAIR_DISK_PATH))?; } if requires_reboot.0 { - crate::disk::main::export(&*guid, &ctx.datadir).await?; + crate::disk::main::export(&*disk_guid, &setup_ctx.datadir).await?; return Err(Error::new( eyre!( "Errors were corrected with your disk, but the server must be restarted in order to proceed" @@ -164,37 +161,48 @@ pub async fn attach( ErrorKind::DiskManagement, )); } - let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?; - *ctx.setup_result.write().await = Some((guid, SetupResult { - tor_address: format!("https://{}", tor_addr), - lan_address: hostname.lan_address(), - root_ca: String::from_utf8(root_ca.to_pem()?)?, - })); - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: true, - })); - Ok(()) - }.await { - tracing::error!("Error Setting Up Embassy: {}", e); - tracing::debug!("{:?}", e); - *ctx.setup_status.write().await = Some(Err(e.into())); - } - }); - Ok(()) + disk_phase.complete(); + + let (account, net_ctrl) = setup_init(&setup_ctx, password, init_phases).await?; + + let rpc_ctx = RpcContext::init(&setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) + })?; + + Ok(ctx.progress().await) } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -pub struct SetupStatus { - pub bytes_transferred: u64, - pub total_bytes: Option, - pub complete: bool, +#[ts(export)] +#[serde(tag = "status")] +pub enum SetupStatusRes { + Complete(SetupResult), + Running(SetupProgress), } -pub async fn status(ctx: SetupContext) -> Result, RpcError> { - ctx.setup_status.read().await.clone().transpose() +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct SetupProgress { + pub progress: FullProgress, + pub guid: Guid, +} + +pub async fn status(ctx: SetupContext) -> Result, Error> { + if let Some(res) = ctx.result.get() { + match res { + Ok((res, _)) => Ok(Some(SetupStatusRes::Complete(res.clone()))), + Err(e) => Err(e.clone_output()), + } + } else { + if ctx.task.initialized() { + Ok(Some(SetupStatusRes::Running(ctx.progress().await))) + } else { + Ok(None) + } + } } /// We want to be able to get a secret, a shared private key with the frontend @@ -202,7 +210,7 @@ pub async fn status(ctx: SetupContext) -> Result, RpcError> /// without knowing the password over clearnet. We use the public key shared across the network /// since it is fine to share the public, and encrypt against the public. pub async fn get_pubkey(ctx: SetupContext) -> Result { - let secret = ctx.as_ref().clone(); + let secret = AsRef::::as_ref(&ctx).clone(); let pub_key = secret.to_public_key()?; Ok(pub_key) } @@ -213,6 +221,7 @@ pub fn cifs() -> ParentHandler { #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct VerifyCifsParams { hostname: String, path: PathBuf, @@ -230,7 +239,7 @@ pub async fn verify_cifs( password, }: VerifyCifsParams, ) -> Result { - let password: Option = password.map(|x| x.decrypt(&*ctx)).flatten(); + let password: Option = password.map(|x| x.decrypt(&ctx)).flatten(); let guard = TmpMountGuard::mount( &Cifs { hostname, @@ -256,7 +265,8 @@ pub enum RecoverySource { #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -pub struct ExecuteParams { +#[ts(export)] +pub struct SetupExecuteParams { start_os_logicalname: PathBuf, start_os_password: EncryptedWire, recovery_source: Option, @@ -266,104 +276,65 @@ pub struct ExecuteParams { // #[command(rpc_only)] pub async fn execute( ctx: SetupContext, - ExecuteParams { + SetupExecuteParams { start_os_logicalname, start_os_password, recovery_source, recovery_password, - }: ExecuteParams, -) -> Result<(), Error> { - let start_os_password = match start_os_password.decrypt(&*ctx) { + }: SetupExecuteParams, +) -> Result { + let start_os_password = match start_os_password.decrypt(&ctx) { Some(a) => a, None => { return Err(Error::new( - color_eyre::eyre::eyre!("Couldn't decode embassy-password"), + color_eyre::eyre::eyre!("Couldn't decode startOsPassword"), crate::ErrorKind::Unknown, )) } }; let recovery_password: Option = match recovery_password { - Some(a) => match a.decrypt(&*ctx) { + Some(a) => match a.decrypt(&ctx) { Some(a) => Some(a), None => { return Err(Error::new( - color_eyre::eyre::eyre!("Couldn't decode recovery-password"), + color_eyre::eyre::eyre!("Couldn't decode recoveryPassword"), crate::ErrorKind::Unknown, )) } }, None => None, }; - let mut status = ctx.setup_status.write().await; - if status.is_some() { - return Err(Error::new( - eyre!("Setup already in progress"), - ErrorKind::InvalidRequest, - )); - } - *status = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); - drop(status); - tokio::task::spawn({ - async move { - let ctx = ctx.clone(); - match execute_inner( - ctx.clone(), - start_os_logicalname, - start_os_password, - recovery_source, - recovery_password, - ) - .await - { - Ok((guid, hostname, tor_addr, root_ca)) => { - tracing::info!("Setup Complete!"); - *ctx.setup_result.write().await = Some(( - guid, - SetupResult { - tor_address: format!("https://{}", tor_addr), - lan_address: hostname.lan_address(), - root_ca: String::from_utf8( - root_ca.to_pem().expect("failed to serialize root ca"), - ) - .expect("invalid pem string"), - }, - )); - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: true, - })); - } - Err(e) => { - tracing::error!("Error Setting Up Server: {}", e); - tracing::debug!("{:?}", e); - *ctx.setup_status.write().await = Some(Err(e.into())); - } - } - } - }); - Ok(()) + + let setup_ctx = ctx.clone(); + ctx.run_setup(|| { + execute_inner( + setup_ctx, + start_os_logicalname, + start_os_password, + recovery_source, + recovery_password, + ) + })?; + + Ok(ctx.progress().await) } #[instrument(skip_all)] // #[command(rpc_only)] pub async fn complete(ctx: SetupContext) -> Result { - let (guid, setup_result) = if let Some((guid, setup_result)) = &*ctx.setup_result.read().await { - (guid.clone(), setup_result.clone()) - } else { - return Err(Error::new( + match ctx.result.get() { + Some(Ok((res, ctx))) => { + let mut guid_file = File::create("/media/startos/config/disk.guid").await?; + guid_file.write_all(ctx.disk_guid.as_bytes()).await?; + guid_file.sync_all().await?; + Ok(res.clone()) + } + Some(Err(e)) => Err(e.clone_output()), + None => Err(Error::new( eyre!("setup.execute has not completed successfully"), crate::ErrorKind::InvalidRequest, - )); - }; - let mut guid_file = File::create("/media/startos/config/disk.guid").await?; - guid_file.write_all(guid.as_bytes()).await?; - guid_file.sync_all().await?; - Ok(setup_result) + )), + } } #[instrument(skip_all)] @@ -380,7 +351,22 @@ pub async fn execute_inner( start_os_password: String, recovery_source: Option, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { +) -> Result<(SetupResult, RpcContext), Error> { + let progress = &ctx.progress; + let mut disk_phase = progress.add_phase("Formatting data drive".into(), Some(10)); + let restore_phase = match &recovery_source { + Some(RecoverySource::Backup { .. }) => { + Some(progress.add_phase("Restoring backup".into(), Some(100))) + } + Some(RecoverySource::Migrate { .. }) => { + Some(progress.add_phase("Transferring data".into(), Some(100))) + } + None => None, + }; + let init_phases = InitPhases::new(&progress); + let rpc_ctx_phases = InitRpcContextPhases::new(&progress); + + disk_phase.start(); let encryption_password = if ctx.disable_encryption { None } else { @@ -402,41 +388,70 @@ pub async fn execute_inner( encryption_password, ) .await?; + disk_phase.complete(); - if let Some(RecoverySource::Backup { target }) = recovery_source { - recover(ctx, guid, start_os_password, target, recovery_password).await - } else if let Some(RecoverySource::Migrate { guid: old_guid }) = recovery_source { - migrate(ctx, guid, &old_guid, start_os_password).await - } else { - let (hostname, tor_addr, root_ca) = fresh_setup(&ctx, &start_os_password).await?; - Ok((guid, hostname, tor_addr, root_ca)) + let progress = SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }; + + match recovery_source { + Some(RecoverySource::Backup { target }) => { + recover( + &ctx, + guid, + start_os_password, + target, + recovery_password, + progress, + ) + .await + } + Some(RecoverySource::Migrate { guid: old_guid }) => { + migrate(&ctx, guid, &old_guid, start_os_password, progress).await + } + None => fresh_setup(&ctx, guid, &start_os_password, progress).await, } } +pub struct SetupExecuteProgress { + pub init_phases: InitPhases, + pub restore_phase: Option, + pub rpc_ctx_phases: InitRpcContextPhases, +} + async fn fresh_setup( ctx: &SetupContext, + guid: Arc, start_os_password: &str, -) -> Result<(Hostname, OnionAddressV3, X509), Error> { + SetupExecuteProgress { + init_phases, + rpc_ctx_phases, + .. + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { let account = AccountInfo::new(start_os_password, root_ca_start_time().await?)?; let db = ctx.db().await?; db.put(&ROOT, &Database::init(&account)?).await?; drop(db); - init(&ctx.config).await?; - Ok(( - account.hostname, - account.tor_key.public().get_onion_address(), - account.root_ca_cert, - )) + + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; + + let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) } #[instrument(skip_all)] async fn recover( - ctx: SetupContext, + ctx: &SetupContext, guid: Arc, start_os_password: String, recovery_source: BackupTargetFS, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { + progress: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { let recovery_source = TmpMountGuard::mount(&recovery_source, ReadWrite).await?; recover_full_embassy( ctx, @@ -444,23 +459,26 @@ async fn recover( start_os_password, recovery_source, recovery_password, + progress, ) .await } #[instrument(skip_all)] async fn migrate( - ctx: SetupContext, + ctx: &SetupContext, guid: Arc, old_guid: &str, start_os_password: String, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); + SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { + let mut restore_phase = restore_phase.or_not_found("restore progress")?; + restore_phase.start(); let _ = crate::disk::main::import( &old_guid, "/media/startos/migrate", @@ -500,20 +518,12 @@ async fn migrate( res = async { loop { tokio::time::sleep(Duration::from_secs(1)).await; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: Some(main_transfer_size.load() + package_data_transfer_size.load()), - complete: false, - })); + restore_phase.set_total(main_transfer_size.load() + package_data_transfer_size.load()); } } => res, }; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: Some(size), - complete: false, - })); + restore_phase.set_total(size); let main_transfer_progress = Counter::new(0, ordering); let package_data_transfer_progress = Counter::new(0, ordering); @@ -529,18 +539,17 @@ async fn migrate( res = async { loop { tokio::time::sleep(Duration::from_secs(1)).await; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: main_transfer_progress.load() + package_data_transfer_progress.load(), - total_bytes: Some(size), - complete: false, - })); + restore_phase.set_done(main_transfer_progress.load() + package_data_transfer_progress.load()); } } => res, } - let (hostname, tor_addr, root_ca) = setup_init(&ctx, Some(start_os_password)).await?; - crate::disk::main::export(&old_guid, "/media/startos/migrate").await?; + restore_phase.complete(); - Ok((guid, hostname, tor_addr, root_ca)) + let (account, net_ctrl) = setup_init(&ctx, Some(start_os_password), init_phases).await?; + + let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) } diff --git a/core/startos/src/update/mod.rs b/core/startos/src/update/mod.rs index be908e776..dc164d2cd 100644 --- a/core/startos/src/update/mod.rs +++ b/core/startos/src/update/mod.rs @@ -20,9 +20,7 @@ use ts_rs::TS; use crate::context::{CliContext, RpcContext}; use crate::notifications::{notify, NotificationLevel}; use crate::prelude::*; -use crate::progress::{ - FullProgressTracker, FullProgressTrackerHandle, PhaseProgressTrackerHandle, PhasedProgressBar, -}; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar}; use crate::registry::asset::RegistryAsset; use crate::registry::context::{RegistryContext, RegistryUrlParams}; use crate::registry::os::index::OsVersionInfo; @@ -34,6 +32,7 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, }; +use crate::util::net::WebSocketExt; use crate::util::Invoke; use crate::PLATFORM; @@ -91,50 +90,47 @@ pub async fn update_system( .add( guid.clone(), RpcContinuation::ws( - Box::new(|mut ws| { - async move { - if let Err(e) = async { - let mut sub = ctx + |mut ws| async move { + if let Err(e) = async { + let mut sub = ctx + .db + .subscribe( + "/public/serverInfo/statusInfo/updateProgress" + .parse::() + .with_kind(ErrorKind::Database)?, + ) + .await; + while { + let progress = ctx .db - .subscribe( - "/public/serverInfo/statusInfo/updateProgress" - .parse::() - .with_kind(ErrorKind::Database)?, - ) - .await; - while { - let progress = ctx - .db - .peek() - .await - .into_public() - .into_server_info() - .into_status_info() - .into_update_progress() - .de()?; - ws.send(axum::extract::ws::Message::Text( - serde_json::to_string(&progress) - .with_kind(ErrorKind::Serialization)?, - )) + .peek() .await - .with_kind(ErrorKind::Network)?; - progress.is_some() - } { - sub.recv().await; - } - - ws.close().await.with_kind(ErrorKind::Network)?; - - Ok::<_, Error>(()) - } - .await - { - tracing::error!("Error returning progress of update: {e}"); - tracing::debug!("{e:?}") + .into_public() + .into_server_info() + .into_status_info() + .into_update_progress() + .de()?; + ws.send(axum::extract::ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + progress.is_some() + } { + sub.recv().await; } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) } - .boxed() - }), + .await + { + tracing::error!("Error returning progress of update: {e}"); + tracing::debug!("{e:?}") + } + }, Duration::from_secs(30), ), ) @@ -250,13 +246,12 @@ async fn maybe_do_update( asset.validate(SIG_CONTEXT, asset.all_signers())?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut download_phase = progress_handle.add_phase("Downloading File".into(), Some(100)); + let progress = FullProgressTracker::new(); + let mut download_phase = progress.add_phase("Downloading File".into(), Some(100)); download_phase.set_total(asset.commitment.size); - let reverify_phase = progress_handle.add_phase("Reverifying File".into(), Some(10)); - let sync_boot_phase = progress_handle.add_phase("Syncing Boot Files".into(), Some(1)); - let finalize_phase = progress_handle.add_phase("Finalizing Update".into(), Some(1)); + let reverify_phase = progress.add_phase("Reverifying File".into(), Some(10)); + let sync_boot_phase = progress.add_phase("Syncing Boot Files".into(), Some(1)); + let finalize_phase = progress.add_phase("Finalizing Update".into(), Some(1)); let start_progress = progress.snapshot(); @@ -287,7 +282,7 @@ async fn maybe_do_update( )); } - let progress_task = NonDetachingJoinHandle::from(tokio::spawn(progress.sync_to_db( + let progress_task = NonDetachingJoinHandle::from(tokio::spawn(progress.clone().sync_to_db( ctx.db.clone(), |db| { db.as_public_mut() @@ -304,7 +299,7 @@ async fn maybe_do_update( ctx.clone(), asset, UpdateProgressHandles { - progress_handle, + progress, download_phase, reverify_phase, sync_boot_phase, @@ -373,7 +368,7 @@ async fn maybe_do_update( } struct UpdateProgressHandles { - progress_handle: FullProgressTrackerHandle, + progress: FullProgressTracker, download_phase: PhaseProgressTrackerHandle, reverify_phase: PhaseProgressTrackerHandle, sync_boot_phase: PhaseProgressTrackerHandle, @@ -385,7 +380,7 @@ async fn do_update( ctx: RpcContext, asset: RegistryAsset, UpdateProgressHandles { - progress_handle, + progress, mut download_phase, mut reverify_phase, mut sync_boot_phase, @@ -436,7 +431,7 @@ async fn do_update( .await?; finalize_phase.complete(); - progress_handle.complete(); + progress.complete(); Ok(()) } diff --git a/core/startos/src/upload.rs b/core/startos/src/upload.rs index f28d81799..b922ab9d2 100644 --- a/core/startos/src/upload.rs +++ b/core/startos/src/upload.rs @@ -5,9 +5,10 @@ use std::time::Duration; use axum::body::Body; use axum::response::Response; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use http::header::CONTENT_LENGTH; use http::StatusCode; +use imbl_value::InternedString; use tokio::fs::File; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::sync::watch; @@ -19,68 +20,70 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::ArchiveSource; use crate::util::io::TmpDir; -pub async fn upload(ctx: &RpcContext) -> Result<(Guid, UploadingFile), Error> { +pub async fn upload( + ctx: &RpcContext, + session: InternedString, +) -> Result<(Guid, UploadingFile), Error> { let guid = Guid::new(); let (mut handle, file) = UploadingFile::new().await?; ctx.rpc_continuations .add( guid.clone(), - RpcContinuation::rest( - Box::new(|request| { - async move { - let headers = request.headers(); - let content_length = match headers.get(CONTENT_LENGTH).map(|a| a.to_str()) { - None => { - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from("Content-Length is required")) - .with_kind(ErrorKind::Network) - } - Some(Err(_)) => { + RpcContinuation::rest_authed( + ctx, + session, + |request| async move { + let headers = request.headers(); + let content_length = match headers.get(CONTENT_LENGTH).map(|a| a.to_str()) { + None => { + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::from("Content-Length is required")) + .with_kind(ErrorKind::Network) + } + Some(Err(_)) => { + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::from("Invalid Content-Length")) + .with_kind(ErrorKind::Network) + } + Some(Ok(a)) => match a.parse::() { + Err(_) => { return Response::builder() .status(StatusCode::BAD_REQUEST) .body(Body::from("Invalid Content-Length")) .with_kind(ErrorKind::Network) } - Some(Ok(a)) => match a.parse::() { - Err(_) => { - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from("Invalid Content-Length")) - .with_kind(ErrorKind::Network) - } - Ok(a) => a, - }, - }; + Ok(a) => a, + }, + }; - handle - .progress - .send_modify(|p| p.expected_size = Some(content_length)); + handle + .progress + .send_modify(|p| p.expected_size = Some(content_length)); - let mut body = request.into_body().into_data_stream(); - while let Some(next) = body.next().await { - if let Err(e) = async { - handle - .write_all(&next.map_err(|e| { - std::io::Error::new(std::io::ErrorKind::Other, e) - })?) - .await?; - Ok(()) - } - .await - { - handle.progress.send_if_modified(|p| p.handle_error(&e)); - break; - } + let mut body = request.into_body().into_data_stream(); + while let Some(next) = body.next().await { + if let Err(e) = async { + handle + .write_all(&next.map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, e) + })?) + .await?; + Ok(()) + } + .await + { + handle.progress.send_if_modified(|p| p.handle_error(&e)); + break; } - - Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty()) - .with_kind(ErrorKind::Network) } - .boxed() - }), + + Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty()) + .with_kind(ErrorKind::Network) + }, Duration::from_secs(30), ), ) diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index f4476ee2b..9a6bab64b 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -274,6 +274,81 @@ pub fn response_to_reader(response: reqwest::Response) -> impl AsyncRead + Unpin })) } +#[pin_project::pin_project] +pub struct IOHook<'a, T> { + #[pin] + pub io: T, + pre_write: Option Result<(), std::io::Error> + Send + 'a>>, + post_write: Option>, + post_read: Option>, +} +impl<'a, T> IOHook<'a, T> { + pub fn new(io: T) -> Self { + Self { + io, + pre_write: None, + post_write: None, + post_read: None, + } + } + pub fn into_inner(self) -> T { + self.io + } + pub fn pre_write Result<(), std::io::Error> + Send + 'a>(&mut self, f: F) { + self.pre_write = Some(Box::new(f)) + } + pub fn post_write(&mut self, f: F) { + self.post_write = Some(Box::new(f)) + } + pub fn post_read(&mut self, f: F) { + self.post_read = Some(Box::new(f)) + } +} +impl<'a, T: AsyncWrite> AsyncWrite for IOHook<'a, T> { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.project(); + if let Some(pre_write) = this.pre_write { + pre_write(buf)?; + } + let written = futures::ready!(this.io.poll_write(cx, buf)?); + if let Some(post_write) = this.post_write { + post_write(&buf[..written]); + } + Poll::Ready(Ok(written)) + } + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.project().io.poll_flush(cx) + } + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.project().io.poll_shutdown(cx) + } +} +impl<'a, T: AsyncRead> AsyncRead for IOHook<'a, T> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.project(); + let start = buf.filled().len(); + futures::ready!(this.io.poll_read(cx, buf)?); + if let Some(post_read) = this.post_read { + post_read(&buf.filled()[start..]); + } + Poll::Ready(Ok(())) + } +} + #[pin_project::pin_project] pub struct BufferedWriteReader { #[pin] @@ -768,7 +843,7 @@ fn poll_flush_prefix( flush_writer: bool, ) -> Poll> { while let Some(mut cur) = prefix.pop_front() { - let buf = cur.remaining_slice(); + let buf = CursorExt::remaining_slice(&cur); if !buf.is_empty() { match writer.as_mut().poll_write(cx, buf)? { Poll::Ready(n) if n == buf.len() => (), diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index 4346a0b1e..f2334632e 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -36,6 +36,7 @@ pub mod http_reader; pub mod io; pub mod logger; pub mod lshw; +pub mod net; pub mod rpc; pub mod rpc_client; pub mod serde; diff --git a/core/startos/src/util/net.rs b/core/startos/src/util/net.rs new file mode 100644 index 000000000..93131f16e --- /dev/null +++ b/core/startos/src/util/net.rs @@ -0,0 +1,24 @@ +use std::borrow::Cow; + +use axum::extract::ws::{self, CloseFrame}; +use futures::Future; + +use crate::prelude::*; + +pub trait WebSocketExt { + fn normal_close( + self, + msg: impl Into>, + ) -> impl Future>; +} + +impl WebSocketExt for ws::WebSocket { + async fn normal_close(mut self, msg: impl Into>) -> Result<(), Error> { + self.send(ws::Message::Close(Some(CloseFrame { + code: 1000, + reason: msg.into(), + }))) + .await + .with_kind(ErrorKind::Network) + } +} diff --git a/core/startos/src/util/serde.rs b/core/startos/src/util/serde.rs index 382ef2814..5d1289e7e 100644 --- a/core/startos/src/util/serde.rs +++ b/core/startos/src/util/serde.rs @@ -22,8 +22,8 @@ use ts_rs::TS; use super::IntoDoubleEndedIterator; use crate::prelude::*; -use crate::util::Apply; use crate::util::clap::FromStrParser; +use crate::util::Apply; pub fn deserialize_from_str< 'de, @@ -1040,15 +1040,19 @@ impl> std::fmt::Display for Base64 { f.write_str(&base64::encode(self.0.as_ref())) } } -impl>> FromStr for Base64 -{ +impl>> FromStr for Base64 { type Err = Error; fn from_str(s: &str) -> Result { base64::decode(&s) .with_kind(ErrorKind::Deserialization)? .apply(TryFrom::try_from) .map(Self) - .map_err(|_| Error::new(eyre!("failed to create from buffer"), ErrorKind::Deserialization)) + .map_err(|_| { + Error::new( + eyre!("failed to create from buffer"), + ErrorKind::Deserialization, + ) + }) } } impl<'de, T: TryFrom>> Deserialize<'de> for Base64 { diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 7212b8801..d063558e2 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -7,6 +7,7 @@ use imbl_value::InternedString; use crate::db::model::Database; use crate::prelude::*; +use crate::progress::PhaseProgressTrackerHandle; use crate::Error; mod v0_3_5; @@ -85,11 +86,12 @@ where &self, version: &V, db: &TypedPatchDb, + progress: &mut PhaseProgressTrackerHandle, ) -> impl Future> + Send { async { match self.semver().cmp(&version.semver()) { - Ordering::Greater => self.rollback_to_unchecked(version, db).await, - Ordering::Less => version.migrate_from_unchecked(self, db).await, + Ordering::Greater => self.rollback_to_unchecked(version, db, progress).await, + Ordering::Less => version.migrate_from_unchecked(self, db, progress).await, Ordering::Equal => Ok(()), } } @@ -98,11 +100,15 @@ where &'a self, version: &'a V, db: &'a TypedPatchDb, + progress: &'a mut PhaseProgressTrackerHandle, ) -> BoxFuture<'a, Result<(), Error>> { + progress.add_total(1); async { let previous = Self::Previous::new(); if version.semver() < previous.semver() { - previous.migrate_from_unchecked(version, db).await?; + previous + .migrate_from_unchecked(version, db, progress) + .await?; } else if version.semver() > previous.semver() { return Err(Error::new( eyre!( @@ -115,6 +121,7 @@ where tracing::info!("{} -> {}", previous.semver(), self.semver(),); self.up(db).await?; self.commit(db).await?; + *progress += 1; Ok(()) } .boxed() @@ -123,14 +130,18 @@ where &'a self, version: &'a V, db: &'a TypedPatchDb, + progress: &'a mut PhaseProgressTrackerHandle, ) -> BoxFuture<'a, Result<(), Error>> { async { let previous = Self::Previous::new(); tracing::info!("{} -> {}", self.semver(), previous.semver(),); self.down(db).await?; previous.commit(db).await?; + *progress += 1; if version.semver() < previous.semver() { - previous.rollback_to_unchecked(version, db).await?; + previous + .rollback_to_unchecked(version, db, progress) + .await?; } else if version.semver() > previous.semver() { return Err(Error::new( eyre!( @@ -196,7 +207,11 @@ where } } -pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { +pub async fn init( + db: &TypedPatchDb, + mut progress: PhaseProgressTrackerHandle, +) -> Result<(), Error> { + progress.start(); let version = Version::from_util_version( db.peek() .await @@ -213,10 +228,10 @@ pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { ErrorKind::MigrationFailed, )); } - Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_6(v) => v.0.migrate_to(&Current::new(), &db).await?, + Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_6(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, Version::Other(_) => { return Err(Error::new( eyre!("Cannot downgrade"), @@ -224,6 +239,7 @@ pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { )) } } + progress.complete(); Ok(()) } diff --git a/core/startos/src/volume.rs b/core/startos/src/volume.rs index e801da79b..c7165b202 100644 --- a/core/startos/src/volume.rs +++ b/core/startos/src/volume.rs @@ -20,7 +20,11 @@ pub fn data_dir>(datadir: P, pkg_id: &PackageId, volume_id: &Volu .join(volume_id) } -pub fn asset_dir>(datadir: P, pkg_id: &PackageId, version: &VersionString) -> PathBuf { +pub fn asset_dir>( + datadir: P, + pkg_id: &PackageId, + version: &VersionString, +) -> PathBuf { datadir .as_ref() .join(PKG_VOLUME_DIR) diff --git a/patch-db b/patch-db index 88a804f56..7aa53249f 160000 --- a/patch-db +++ b/patch-db @@ -1 +1 @@ -Subproject commit 88a804f56f446d34896ef331d915b821a581cf01 +Subproject commit 7aa53249f9353162475ea347abac92abcfba5493 diff --git a/sdk/lib/osBindings/AttachParams.ts b/sdk/lib/osBindings/AttachParams.ts new file mode 100644 index 000000000..048151d2f --- /dev/null +++ b/sdk/lib/osBindings/AttachParams.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" + +export type AttachParams = { + startOsPassword: EncryptedWire | null + guid: string +} diff --git a/sdk/lib/osBindings/BackupTargetFS.ts b/sdk/lib/osBindings/BackupTargetFS.ts new file mode 100644 index 000000000..0cff2cc4e --- /dev/null +++ b/sdk/lib/osBindings/BackupTargetFS.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BlockDev } from "./BlockDev" +import type { Cifs } from "./Cifs" + +export type BackupTargetFS = + | ({ type: "disk" } & BlockDev) + | ({ type: "cifs" } & Cifs) diff --git a/sdk/lib/osBindings/BlockDev.ts b/sdk/lib/osBindings/BlockDev.ts new file mode 100644 index 000000000..46db81011 --- /dev/null +++ b/sdk/lib/osBindings/BlockDev.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BlockDev = { logicalname: string } diff --git a/sdk/lib/osBindings/Cifs.ts b/sdk/lib/osBindings/Cifs.ts new file mode 100644 index 000000000..f7099bd7f --- /dev/null +++ b/sdk/lib/osBindings/Cifs.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type Cifs = { + hostname: string + path: string + username: string + password: string | null +} diff --git a/sdk/lib/osBindings/InitProgressRes.ts b/sdk/lib/osBindings/InitProgressRes.ts new file mode 100644 index 000000000..38caf7bdb --- /dev/null +++ b/sdk/lib/osBindings/InitProgressRes.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FullProgress } from "./FullProgress" +import type { Guid } from "./Guid" + +export type InitProgressRes = { progress: FullProgress; guid: Guid } diff --git a/sdk/lib/osBindings/RecoverySource.ts b/sdk/lib/osBindings/RecoverySource.ts new file mode 100644 index 000000000..c40ec5132 --- /dev/null +++ b/sdk/lib/osBindings/RecoverySource.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BackupTargetFS } from "./BackupTargetFS" + +export type RecoverySource = + | { type: "migrate"; guid: string } + | { type: "backup"; target: BackupTargetFS } diff --git a/sdk/lib/osBindings/SetupExecuteParams.ts b/sdk/lib/osBindings/SetupExecuteParams.ts new file mode 100644 index 000000000..4593e7667 --- /dev/null +++ b/sdk/lib/osBindings/SetupExecuteParams.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" +import type { RecoverySource } from "./RecoverySource" + +export type SetupExecuteParams = { + startOsLogicalname: string + startOsPassword: EncryptedWire + recoverySource: RecoverySource | null + recoveryPassword: EncryptedWire | null +} diff --git a/sdk/lib/osBindings/SetupProgress.ts b/sdk/lib/osBindings/SetupProgress.ts new file mode 100644 index 000000000..845636da3 --- /dev/null +++ b/sdk/lib/osBindings/SetupProgress.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FullProgress } from "./FullProgress" +import type { Guid } from "./Guid" + +export type SetupProgress = { progress: FullProgress; guid: Guid } diff --git a/sdk/lib/osBindings/SetupResult.ts b/sdk/lib/osBindings/SetupResult.ts new file mode 100644 index 000000000..464aeb4b7 --- /dev/null +++ b/sdk/lib/osBindings/SetupResult.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetupResult = { + torAddress: string + lanAddress: string + rootCa: string +} diff --git a/sdk/lib/osBindings/SetupStatusRes.ts b/sdk/lib/osBindings/SetupStatusRes.ts new file mode 100644 index 000000000..93d10c59b --- /dev/null +++ b/sdk/lib/osBindings/SetupStatusRes.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { SetupProgress } from "./SetupProgress" +import type { SetupResult } from "./SetupResult" + +export type SetupStatusRes = + | ({ status: "complete" } & SetupResult) + | ({ status: "running" } & SetupProgress) diff --git a/sdk/lib/osBindings/VerifyCifsParams.ts b/sdk/lib/osBindings/VerifyCifsParams.ts new file mode 100644 index 000000000..407e6caaa --- /dev/null +++ b/sdk/lib/osBindings/VerifyCifsParams.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" + +export type VerifyCifsParams = { + hostname: string + path: string + username: string + password: EncryptedWire | null +} diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 06a4bed7e..ac2e19e45 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -15,17 +15,21 @@ export { AlpnInfo } from "./AlpnInfo" export { AnySignature } from "./AnySignature" export { AnySigningKey } from "./AnySigningKey" export { AnyVerifyingKey } from "./AnyVerifyingKey" +export { AttachParams } from "./AttachParams" export { BackupProgress } from "./BackupProgress" +export { BackupTargetFS } from "./BackupTargetFS" export { Base64 } from "./Base64" export { BindInfo } from "./BindInfo" export { BindOptions } from "./BindOptions" export { BindParams } from "./BindParams" export { Blake3Commitment } from "./Blake3Commitment" +export { BlockDev } from "./BlockDev" export { Callback } from "./Callback" export { Category } from "./Category" export { CheckDependenciesParam } from "./CheckDependenciesParam" export { CheckDependenciesResult } from "./CheckDependenciesResult" export { ChrootParams } from "./ChrootParams" +export { Cifs } from "./Cifs" export { ContactInfo } from "./ContactInfo" export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams" export { CurrentDependencies } from "./CurrentDependencies" @@ -73,6 +77,7 @@ export { ImageConfig } from "./ImageConfig" export { ImageId } from "./ImageId" export { ImageMetadata } from "./ImageMetadata" export { ImageSource } from "./ImageSource" +export { InitProgressRes } from "./InitProgressRes" export { InstalledState } from "./InstalledState" export { InstallingInfo } from "./InstallingInfo" export { InstallingState } from "./InstallingState" @@ -105,6 +110,7 @@ export { ParamsPackageId } from "./ParamsPackageId" export { PasswordType } from "./PasswordType" export { Progress } from "./Progress" export { Public } from "./Public" +export { RecoverySource } from "./RecoverySource" export { RegistryAsset } from "./RegistryAsset" export { RemoveActionParams } from "./RemoveActionParams" export { RemoveAddressParams } from "./RemoveAddressParams" @@ -127,10 +133,15 @@ export { SetMainStatusStatus } from "./SetMainStatusStatus" export { SetMainStatus } from "./SetMainStatus" export { SetStoreParams } from "./SetStoreParams" export { SetSystemSmtpParams } from "./SetSystemSmtpParams" +export { SetupExecuteParams } from "./SetupExecuteParams" +export { SetupProgress } from "./SetupProgress" +export { SetupResult } from "./SetupResult" +export { SetupStatusRes } from "./SetupStatusRes" export { SignAssetParams } from "./SignAssetParams" export { SignerInfo } from "./SignerInfo" export { Status } from "./Status" export { UpdatingState } from "./UpdatingState" +export { VerifyCifsParams } from "./VerifyCifsParams" export { VersionSignerParams } from "./VersionSignerParams" export { Version } from "./Version" export { VolumeId } from "./VolumeId" diff --git a/web/package-lock.json b/web/package-lock.json index 68156eb21..81b612ac1 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -31,6 +31,7 @@ "@taiga-ui/core": "3.20.0", "@taiga-ui/icons": "3.20.0", "@taiga-ui/kit": "3.20.0", + "@tinkoff/ng-dompurify": "4.0.0", "angular-svg-round-progressbar": "^9.0.0", "ansi-to-html": "^0.7.2", "base64-js": "^1.5.1", @@ -1973,7 +1974,7 @@ }, "../sdk/dist": { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", @@ -5432,6 +5433,20 @@ "rxjs": ">=6.0.0" } }, + "node_modules/@tinkoff/ng-dompurify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tinkoff/ng-dompurify/-/ng-dompurify-4.0.0.tgz", + "integrity": "sha512-BjKUweWLrOx8UOZw+Tl+Dae5keYuSbeMkppcXQdsvwASMrPfmP7d3Q206Q6HDqOV2WnpnFqGUB95IMbLAeRRuw==", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@angular/core": ">=12.0.0", + "@angular/platform-browser": ">=12.0.0", + "@types/dompurify": ">=2.3.0", + "dompurify": ">= 2.3.0" + } + }, "node_modules/@tinkoff/ng-event-plugins": { "version": "3.1.0", "license": "Apache-2.0", @@ -5549,7 +5564,6 @@ }, "node_modules/@types/dompurify": { "version": "2.3.4", - "dev": true, "license": "MIT", "dependencies": { "@types/trusted-types": "*" @@ -5726,7 +5740,6 @@ }, "node_modules/@types/trusted-types": { "version": "2.0.2", - "dev": true, "license": "MIT" }, "node_modules/@types/uuid": { diff --git a/web/package.json b/web/package.json index 9e842455e..3ea29c6fd 100644 --- a/web/package.json +++ b/web/package.json @@ -13,7 +13,6 @@ "check:setup": "tsc --project projects/setup-wizard/tsconfig.json --noEmit --skipLibCheck", "check:ui": "tsc --project projects/ui/tsconfig.json --noEmit --skipLibCheck", "build:deps": "rm -rf .angular/cache && (cd ../patch-db/client && npm ci && npm run build) && (cd ../sdk && make bundle)", - "build:dui": "ng run diagnostic-ui:build", "build:install-wiz": "ng run install-wizard:build", "build:setup": "ng run setup-wizard:build", "build:ui": "ng run ui:build", @@ -25,7 +24,6 @@ "analyze:ui": "webpack-bundle-analyzer dist/raw/ui/stats.json", "publish:shared": "npm run build:shared && npm publish ./dist/shared --access public", "publish:marketplace": "npm run build:marketplace && npm publish ./dist/marketplace --access public", - "start:dui": "npm run-script build-config && ionic serve --project diagnostic-ui --host 0.0.0.0", "start:install-wiz": "npm run-script build-config && ionic serve --project install-wizard --host 0.0.0.0", "start:setup": "npm run-script build-config && ionic serve --project setup-wizard --host 0.0.0.0", "start:ui": "npm run-script build-config && ionic serve --project ui --ip --host 0.0.0.0", @@ -56,6 +54,7 @@ "@taiga-ui/core": "3.20.0", "@taiga-ui/icons": "3.20.0", "@taiga-ui/kit": "3.20.0", + "@tinkoff/ng-dompurify": "4.0.0", "angular-svg-round-progressbar": "^9.0.0", "ansi-to-html": "^0.7.2", "base64-js": "^1.5.1", diff --git a/web/projects/diagnostic-ui/src/app/app-routing.module.ts b/web/projects/diagnostic-ui/src/app/app-routing.module.ts deleted file mode 100644 index f9f009b48..000000000 --- a/web/projects/diagnostic-ui/src/app/app-routing.module.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { NgModule } from '@angular/core' -import { PreloadAllModules, RouterModule, Routes } from '@angular/router' - -const routes: Routes = [ - { - path: '', - loadChildren: () => - import('./pages/home/home.module').then(m => m.HomePageModule), - }, - { - path: 'logs', - loadChildren: () => - import('./pages/logs/logs.module').then(m => m.LogsPageModule), - }, -] - -@NgModule({ - imports: [ - RouterModule.forRoot(routes, { - scrollPositionRestoration: 'enabled', - preloadingStrategy: PreloadAllModules, - useHash: true, - }), - ], - exports: [RouterModule], -}) -export class AppRoutingModule {} diff --git a/web/projects/diagnostic-ui/src/app/app.component.html b/web/projects/diagnostic-ui/src/app/app.component.html deleted file mode 100644 index cd28a7e80..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.html +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/web/projects/diagnostic-ui/src/app/app.component.scss b/web/projects/diagnostic-ui/src/app/app.component.scss deleted file mode 100644 index b528fd9bd..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.scss +++ /dev/null @@ -1,8 +0,0 @@ -:host { - display: block; - height: 100%; -} - -tui-root { - height: 100%; -} diff --git a/web/projects/diagnostic-ui/src/app/app.component.ts b/web/projects/diagnostic-ui/src/app/app.component.ts deleted file mode 100644 index 5ac82a652..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Component } from '@angular/core' - -@Component({ - selector: 'app-root', - templateUrl: 'app.component.html', - styleUrls: ['app.component.scss'], -}) -export class AppComponent { - constructor() {} -} diff --git a/web/projects/diagnostic-ui/src/app/app.module.ts b/web/projects/diagnostic-ui/src/app/app.module.ts deleted file mode 100644 index 1abde53a3..000000000 --- a/web/projects/diagnostic-ui/src/app/app.module.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { NgModule } from '@angular/core' -import { BrowserAnimationsModule } from '@angular/platform-browser/animations' -import { RouteReuseStrategy } from '@angular/router' -import { IonicModule, IonicRouteStrategy } from '@ionic/angular' -import { TuiRootModule } from '@taiga-ui/core' -import { AppComponent } from './app.component' -import { AppRoutingModule } from './app-routing.module' -import { HttpClientModule } from '@angular/common/http' -import { ApiService } from './services/api/api.service' -import { MockApiService } from './services/api/mock-api.service' -import { LiveApiService } from './services/api/live-api.service' -import { RELATIVE_URL, WorkspaceConfig } from '@start9labs/shared' - -const { - useMocks, - ui: { api }, -} = require('../../../../config.json') as WorkspaceConfig - -@NgModule({ - declarations: [AppComponent], - imports: [ - HttpClientModule, - BrowserAnimationsModule, - IonicModule.forRoot({ - mode: 'md', - }), - AppRoutingModule, - TuiRootModule, - ], - providers: [ - { provide: RouteReuseStrategy, useClass: IonicRouteStrategy }, - { - provide: ApiService, - useClass: useMocks ? MockApiService : LiveApiService, - }, - { - provide: RELATIVE_URL, - useValue: `/${api.url}/${api.version}`, - }, - ], - bootstrap: [AppComponent], -}) -export class AppModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts b/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts deleted file mode 100644 index efb1977dc..000000000 --- a/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { NgModule } from '@angular/core' -import { RouterModule, Routes } from '@angular/router' -import { HomePage } from './home.page' - -const routes: Routes = [ - { - path: '', - component: HomePage, - }, -] - -@NgModule({ - imports: [RouterModule.forChild(routes)], - exports: [RouterModule], -}) -export class HomePageRoutingModule {} diff --git a/web/projects/diagnostic-ui/src/app/services/api/api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/api.service.ts deleted file mode 100644 index 562d486c3..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/api.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { LogsRes, ServerLogsReq } from '@start9labs/shared' - -export abstract class ApiService { - abstract getError(): Promise - abstract restart(): Promise - abstract forgetDrive(): Promise - abstract repairDisk(): Promise - abstract systemRebuild(): Promise - abstract getLogs(params: ServerLogsReq): Promise -} - -export interface GetErrorRes { - code: number - message: string - data: { details: string } -} diff --git a/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts deleted file mode 100644 index bbde6e5ba..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { Injectable } from '@angular/core' -import { - HttpService, - isRpcError, - RpcError, - RPCOptions, -} from '@start9labs/shared' -import { ApiService, GetErrorRes } from './api.service' -import { LogsRes, ServerLogsReq } from '@start9labs/shared' - -@Injectable() -export class LiveApiService implements ApiService { - constructor(private readonly http: HttpService) {} - - async getError(): Promise { - return this.rpcRequest({ - method: 'diagnostic.error', - params: {}, - }) - } - - async restart(): Promise { - return this.rpcRequest({ - method: 'diagnostic.restart', - params: {}, - }) - } - - async forgetDrive(): Promise { - return this.rpcRequest({ - method: 'diagnostic.disk.forget', - params: {}, - }) - } - - async repairDisk(): Promise { - return this.rpcRequest({ - method: 'diagnostic.disk.repair', - params: {}, - }) - } - - async systemRebuild(): Promise { - return this.rpcRequest({ - method: 'diagnostic.rebuild', - params: {}, - }) - } - - async getLogs(params: ServerLogsReq): Promise { - return this.rpcRequest({ - method: 'diagnostic.logs', - params, - }) - } - - private async rpcRequest(opts: RPCOptions): Promise { - const res = await this.http.rpcRequest(opts) - - const rpcRes = res.body - - if (isRpcError(rpcRes)) { - throw new RpcError(rpcRes.error) - } - - return rpcRes.result - } -} diff --git a/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts deleted file mode 100644 index d991edd32..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Injectable } from '@angular/core' -import { pauseFor } from '@start9labs/shared' -import { ApiService, GetErrorRes } from './api.service' -import { LogsRes, ServerLogsReq, Log } from '@start9labs/shared' - -@Injectable() -export class MockApiService implements ApiService { - async getError(): Promise { - await pauseFor(1000) - return { - code: 15, - message: 'Unknown server', - data: { details: 'Some details about the error here' }, - } - } - - async restart(): Promise { - await pauseFor(1000) - } - - async forgetDrive(): Promise { - await pauseFor(1000) - } - - async repairDisk(): Promise { - await pauseFor(1000) - } - - async systemRebuild(): Promise { - await pauseFor(1000) - } - - async getLogs(params: ServerLogsReq): Promise { - await pauseFor(1000) - let entries: Log[] - if (Math.random() < 0.2) { - entries = packageLogs - } else { - const arrLength = params.limit - ? Math.ceil(params.limit / packageLogs.length) - : 10 - entries = new Array(arrLength) - .fill(packageLogs) - .reduce((acc, val) => acc.concat(val), []) - } - return { - entries, - startCursor: 'start-cursor', - endCursor: 'end-cursor', - } - } -} - -const packageLogs = [ - { - timestamp: '2019-12-26T14:20:30.872Z', - message: '****** START *****', - }, - { - timestamp: '2019-12-26T14:21:30.872Z', - message: 'ServerLogs ServerLogs ServerLogs ServerLogs ServerLogs', - }, - { - timestamp: '2019-12-26T14:22:30.872Z', - message: '****** FINISH *****', - }, -] diff --git a/web/projects/diagnostic-ui/src/environments/environment.prod.ts b/web/projects/diagnostic-ui/src/environments/environment.prod.ts deleted file mode 100644 index 970e25bd7..000000000 --- a/web/projects/diagnostic-ui/src/environments/environment.prod.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const environment = { - production: true, -} diff --git a/web/projects/diagnostic-ui/src/environments/environment.ts b/web/projects/diagnostic-ui/src/environments/environment.ts deleted file mode 100644 index 5c68c17ab..000000000 --- a/web/projects/diagnostic-ui/src/environments/environment.ts +++ /dev/null @@ -1,16 +0,0 @@ -// This file can be replaced during build by using the `fileReplacements` array. -// `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. -// The list of file replacements can be found in `angular.json`. - -export const environment = { - production: false, -} - -/* - * For easier debugging in development mode, you can import the following file - * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. - * - * This import should be commented out in production mode because it will have a negative impact - * on performance if an error is thrown. - */ -// import 'zone.js/dist/zone-error'; // Included with Angular CLI. diff --git a/web/projects/diagnostic-ui/src/index.html b/web/projects/diagnostic-ui/src/index.html deleted file mode 100644 index 1822018f3..000000000 --- a/web/projects/diagnostic-ui/src/index.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - - StartOS Diagnostic UI - - - - - - - - - - - - - - - diff --git a/web/projects/diagnostic-ui/src/main.ts b/web/projects/diagnostic-ui/src/main.ts deleted file mode 100644 index 21499c3cd..000000000 --- a/web/projects/diagnostic-ui/src/main.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { enableProdMode } from '@angular/core' -import { platformBrowserDynamic } from '@angular/platform-browser-dynamic' -import { AppModule } from './app/app.module' -import { environment } from './environments/environment' - -if (environment.production) { - enableProdMode() -} - -platformBrowserDynamic() - .bootstrapModule(AppModule) - .catch(err => console.error(err)) diff --git a/web/projects/diagnostic-ui/src/polyfills.ts b/web/projects/diagnostic-ui/src/polyfills.ts deleted file mode 100644 index 4437ced44..000000000 --- a/web/projects/diagnostic-ui/src/polyfills.ts +++ /dev/null @@ -1,64 +0,0 @@ -/** - * This file includes polyfills needed by Angular and is loaded before the app. - * You can add your own extra polyfills to this file. - * - * This file is divided into 2 sections: - * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. - * 2. Application imports. Files imported after ZoneJS that should be loaded before your main - * file. - * - * The current setup is for so-called "evergreen" browsers; the last versions of browsers that - * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), - * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. - * - * Learn more in https://angular.io/guide/browser-support - */ - -/*************************************************************************************************** - * BROWSER POLYFILLS - */ - -/** IE11 requires the following for NgClass support on SVG elements */ -// import 'classlist.js'; // Run `npm install --save classlist.js`. - -/** - * Web Animations `@angular/platform-browser/animations` - * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. - * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). - */ -// import 'web-animations-js'; // Run `npm install --save web-animations-js`. - -/** - * By default, zone.js will patch all possible macroTask and DomEvents - * user can disable parts of macroTask/DomEvents patch by setting following flags - * because those flags need to be set before `zone.js` being loaded, and webpack - * will put import in the top of bundle, so user need to create a separate file - * in this directory (for example: zone-flags.ts), and put the following flags - * into that file, and then add the following code before importing zone.js. - * import './zone-flags'; - * - * The flags allowed in zone-flags.ts are listed here. - * - * The following flags will work for all browsers. - * - * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame - * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick - * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames - * - * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js - * with the following flag, it will bypass `zone.js` patch for IE/Edge - * - * (window as any).__Zone_enable_cross_context_check = true; - * - */ - -import './zone-flags' - -/*************************************************************************************************** - * Zone JS is required by default for Angular itself. - */ -import 'zone.js/dist/zone' // Included with Angular CLI. - -/*************************************************************************************************** - * APPLICATION IMPORTS - */ diff --git a/web/projects/diagnostic-ui/src/styles.scss b/web/projects/diagnostic-ui/src/styles.scss deleted file mode 100644 index ac0aadb69..000000000 --- a/web/projects/diagnostic-ui/src/styles.scss +++ /dev/null @@ -1,41 +0,0 @@ -@font-face { - font-family: 'Montserrat'; - font-style: normal; - font-weight: normal; - src: url('/assets/fonts/Montserrat/Montserrat-Regular.ttf'); -} - -/** Ionic CSS Variables overrides **/ -:root { - --ion-font-family: 'Montserrat'; - - --ion-color-primary: #0075e1; - - --ion-color-medium: #989aa2; - --ion-color-medium-rgb: 152,154,162; - --ion-color-medium-contrast: #000000; - --ion-color-medium-contrast-rgb: 0,0,0; - --ion-color-medium-shade: #86888f; - --ion-color-medium-tint: #a2a4ab; - - --ion-color-light: #222428; - --ion-color-light-rgb: 34,36,40; - --ion-color-light-contrast: #ffffff; - --ion-color-light-contrast-rgb: 255,255,255; - --ion-color-light-shade: #1e2023; - --ion-color-light-tint: #383a3e; - - --ion-item-background: #2b2b2b; - --ion-toolbar-background: #2b2b2b; - --ion-card-background: #2b2b2b; - - --ion-background-color: #282828; - --ion-background-color-rgb: 30,30,30; - --ion-text-color: var(--ion-color-dark); - --ion-text-color-rgb: var(--ion-color-dark-rgb); -} - -.loader { - --spinner-color: var(--ion-color-warning) !important; - z-index: 40000 !important; -} diff --git a/web/projects/diagnostic-ui/src/zone-flags.ts b/web/projects/diagnostic-ui/src/zone-flags.ts deleted file mode 100644 index 24ca60fe2..000000000 --- a/web/projects/diagnostic-ui/src/zone-flags.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Prevents Angular change detection from - * running with certain Web Component callbacks - */ -// eslint-disable-next-line no-underscore-dangle -(window as any).__Zone_disable_customElements = true diff --git a/web/projects/diagnostic-ui/tsconfig.json b/web/projects/diagnostic-ui/tsconfig.json deleted file mode 100644 index f642f09b3..000000000 --- a/web/projects/diagnostic-ui/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -/* To learn more about this file see: https://angular.io/config/tsconfig. */ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "baseUrl": "./" - }, - "files": ["src/main.ts", "src/polyfills.ts"], - "include": ["src/**/*.d.ts"] -} diff --git a/web/projects/setup-wizard/src/app/app.component.ts b/web/projects/setup-wizard/src/app/app.component.ts index e4bf41f5c..e10d672e5 100644 --- a/web/projects/setup-wizard/src/app/app.component.ts +++ b/web/projects/setup-wizard/src/app/app.component.ts @@ -21,7 +21,7 @@ export class AppComponent { let route = '/home' if (inProgress) { - route = inProgress.complete ? '/success' : '/loading' + route = inProgress.status === 'complete' ? '/success' : '/loading' } await this.navCtrl.navigateForward(route) diff --git a/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts b/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts index 1a5dd042d..7f7ee6241 100644 --- a/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts +++ b/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts @@ -5,12 +5,7 @@ import { ModalController, NavController, } from '@ionic/angular' -import { - ApiService, - BackupRecoverySource, - DiskRecoverySource, - DiskMigrateSource, -} from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/api.service' import { DiskInfo, ErrorToastService, GuidPipe } from '@start9labs/shared' import { StateService } from 'src/app/services/state.service' import { PasswordPage } from '../../modals/password/password.page' @@ -58,18 +53,17 @@ export class EmbassyPage { } else if (this.stateService.setupType === 'restore') { this.storageDrives = disks.filter( d => + this.stateService.recoverySource?.type === 'backup' && + this.stateService.recoverySource.target?.type === 'disk' && !d.partitions .map(p => p.logicalname) - .includes( - ( - (this.stateService.recoverySource as BackupRecoverySource) - ?.target as DiskRecoverySource - )?.logicalname, - ), + .includes(this.stateService.recoverySource.target.logicalname), ) - } else if (this.stateService.setupType === 'transfer') { - const guid = (this.stateService.recoverySource as DiskMigrateSource) - .guid + } else if ( + this.stateService.setupType === 'transfer' && + this.stateService.recoverySource?.type === 'migrate' + ) { + const guid = this.stateService.recoverySource.guid this.storageDrives = disks.filter(d => { return ( d.guid !== guid && !d.partitions.map(p => p.guid).includes(guid) diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts b/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts index e937a7e19..2f3507941 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts @@ -2,11 +2,11 @@ import { NgModule } from '@angular/core' import { CommonModule } from '@angular/common' import { IonicModule } from '@ionic/angular' import { FormsModule } from '@angular/forms' -import { LoadingPage, ToMessagePipe } from './loading.page' +import { LoadingPage } from './loading.page' import { LoadingPageRoutingModule } from './loading-routing.module' @NgModule({ imports: [CommonModule, FormsModule, IonicModule, LoadingPageRoutingModule], - declarations: [LoadingPage, ToMessagePipe], + declarations: [LoadingPage], }) export class LoadingPageModule {} diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.html b/web/projects/setup-wizard/src/app/pages/loading/loading.page.html index fd7fcc24c..6c9ca41ab 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.html +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.html @@ -1,39 +1,17 @@ - - - - - - - Initializing StartOS -
- - {{ progress.transferred | toMessage }} - -
- +
+

+ Setting up your server +

+
+ Progress: {{ (progress.total * 100).toFixed(0) }}% +
- - -

- - - Progress: {{ (transferred * 100).toFixed() }}% - - - {{ (progress.totalBytes / 1073741824).toFixed(2) }} GB - - -

- - - - - - + +

{{ progress.message }}

+
diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss b/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss index 87bfffa33..e69de29bb 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss @@ -1,3 +0,0 @@ -ion-card-title { - font-size: 42px; -} \ No newline at end of file diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts b/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts index ce1a1b3c0..459be5c7a 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts @@ -1,15 +1,23 @@ import { Component } from '@angular/core' import { NavController } from '@ionic/angular' -import { StateService } from 'src/app/services/state.service' import { Pipe, PipeTransform } from '@angular/core' -import { BehaviorSubject } from 'rxjs' +import { + EMPTY, + Observable, + catchError, + filter, + from, + interval, + map, + of, + startWith, + switchMap, + take, + tap, +} from 'rxjs' import { ApiService } from 'src/app/services/api/api.service' -import { ErrorToastService, pauseFor } from '@start9labs/shared' - -type Progress = { - totalBytes: number | null - transferred: number -} +import { ErrorToastService } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' @Component({ selector: 'app-loading', @@ -17,10 +25,46 @@ type Progress = { styleUrls: ['loading.page.scss'], }) export class LoadingPage { - readonly progress$ = new BehaviorSubject({ - totalBytes: null, - transferred: 0, - }) + readonly progress$ = this.getRunningStatus$().pipe( + switchMap(res => + this.api.openProgressWebsocket$(res.guid).pipe( + startWith(res.progress), + catchError((_, watch$) => { + return interval(2000).pipe( + switchMap(() => + from(this.api.getStatus()).pipe(catchError(() => EMPTY)), + ), + take(1), + switchMap(() => watch$), + ) + }), + tap(progress => { + if (progress.overall === true) { + this.getStatus() + } + }), + ), + ), + map(({ phases, overall }) => { + return { + total: getDecimal(overall), + message: phases + .filter( + ( + p, + ): p is { + name: string + progress: { + done: number + total: number | null + } + } => p.progress !== true && p.progress !== null, + ) + .map(p => `${p.name}${getPhaseBytes(p.progress)}`) + .join(','), + } + }), + ) constructor( private readonly navCtrl: NavController, @@ -28,55 +72,55 @@ export class LoadingPage { private readonly errorToastService: ErrorToastService, ) {} - ngOnInit() { - this.poll() - } + private async getStatus(): Promise<{ + status: 'running' + guid: string + progress: T.FullProgress + } | void> { + const res = await this.api.getStatus() - async poll() { - try { - const progress = await this.api.getStatus() - - if (!progress) return - - const { totalBytes, bytesTransferred } = progress - - this.progress$.next({ - totalBytes, - transferred: totalBytes ? bytesTransferred / totalBytes : 0, - }) - - if (progress.complete) { - this.navCtrl.navigateForward(`/success`) - this.progress$.complete() - return - } - - await pauseFor(250) - - setTimeout(() => this.poll(), 0) // prevent call stack from growing - } catch (e: any) { - this.errorToastService.present(e) - } - } -} - -@Pipe({ - name: 'toMessage', -}) -export class ToMessagePipe implements PipeTransform { - constructor(private readonly stateService: StateService) {} - - transform(progress: number | null): string { - if (['fresh', 'attach'].includes(this.stateService.setupType || '')) { - return 'Setting up your server' - } - - if (!progress) { - return 'Calculating size' - } else if (progress < 1) { - return 'Copying data' + if (!res) { + this.navCtrl.navigateRoot('/home') + } else if (res.status === 'complete') { + this.navCtrl.navigateForward(`/success`) } else { - return 'Finalizing' + return res } } + + private getRunningStatus$(): Observable<{ + status: 'running' + guid: string + progress: T.FullProgress + }> { + return from(this.getStatus()).pipe( + filter(Boolean), + catchError(e => { + this.errorToastService.present(e) + return of(e) + }), + take(1), + ) + } +} + +function getDecimal(progress: T.Progress): number { + if (progress === true) { + return 1 + } else if (!progress || !progress.total) { + return 0 + } else { + return progress.total && progress.done / progress.total + } +} + +function getPhaseBytes( + progress: + | false + | { + done: number + total: number | null + }, +): string { + return progress === false ? '' : `: (${progress.done}/${progress.total})` } diff --git a/web/projects/setup-wizard/src/app/services/api/api.service.ts b/web/projects/setup-wizard/src/app/services/api/api.service.ts index 375e64a78..6719ce859 100644 --- a/web/projects/setup-wizard/src/app/services/api/api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/api.service.ts @@ -1,16 +1,21 @@ import * as jose from 'node-jose' import { DiskListResponse, StartOSDiskInfo } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' +import { WebSocketSubjectConfig } from 'rxjs/webSocket' +import { Observable } from 'rxjs' + export abstract class ApiService { pubkey?: jose.JWK.Key - abstract getStatus(): Promise // setup.status + abstract getStatus(): Promise // setup.status abstract getPubKey(): Promise // setup.get-pubkey abstract getDrives(): Promise // setup.disk.list - abstract verifyCifs(cifs: CifsRecoverySource): Promise // setup.cifs.verify - abstract attach(importInfo: AttachReq): Promise // setup.attach - abstract execute(setupInfo: ExecuteReq): Promise // setup.execute - abstract complete(): Promise // setup.complete + abstract verifyCifs(cifs: T.VerifyCifsParams): Promise // setup.cifs.verify + abstract attach(importInfo: T.AttachParams): Promise // setup.attach + abstract execute(setupInfo: T.SetupExecuteParams): Promise // setup.execute + abstract complete(): Promise // setup.complete abstract exit(): Promise // setup.exit + abstract openProgressWebsocket$(guid: string): Observable async encrypt(toEncrypt: string): Promise { if (!this.pubkey) throw new Error('No pubkey found!') @@ -27,29 +32,7 @@ type Encrypted = { encrypted: string } -export type StatusRes = { - bytesTransferred: number - totalBytes: number | null - complete: boolean -} | null - -export type AttachReq = { - guid: string - startOsPassword: Encrypted -} - -export type ExecuteReq = { - startOsLogicalname: string - startOsPassword: Encrypted - recoverySource: RecoverySource | null - recoveryPassword: Encrypted | null -} - -export type CompleteRes = { - torAddress: string - lanAddress: string - rootCa: string -} +export type WebsocketConfig = Omit, 'url'> export type DiskBackupTarget = { vendor: string | null @@ -68,27 +51,3 @@ export type CifsBackupTarget = { mountable: boolean startOs: StartOSDiskInfo | null } - -export type DiskRecoverySource = { - type: 'disk' - logicalname: string // partition logicalname -} - -export type BackupRecoverySource = { - type: 'backup' - target: CifsRecoverySource | DiskRecoverySource -} -export type RecoverySource = BackupRecoverySource | DiskMigrateSource - -export type DiskMigrateSource = { - type: 'migrate' - guid: string -} - -export type CifsRecoverySource = { - type: 'cifs' - hostname: string - path: string - username: string - password: Encrypted | null -} diff --git a/web/projects/setup-wizard/src/app/services/api/live-api.service.ts b/web/projects/setup-wizard/src/app/services/api/live-api.service.ts index 566cc84cf..f431f5151 100644 --- a/web/projects/setup-wizard/src/app/services/api/live-api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/live-api.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@angular/core' +import { Inject, Injectable } from '@angular/core' import { DiskListResponse, StartOSDiskInfo, @@ -8,27 +8,35 @@ import { RpcError, RPCOptions, } from '@start9labs/shared' -import { - ApiService, - CifsRecoverySource, - DiskRecoverySource, - StatusRes, - AttachReq, - ExecuteReq, - CompleteRes, -} from './api.service' +import { T } from '@start9labs/start-sdk' +import { ApiService, WebsocketConfig } from './api.service' import * as jose from 'node-jose' +import { Observable } from 'rxjs' +import { DOCUMENT } from '@angular/common' +import { webSocket } from 'rxjs/webSocket' @Injectable({ providedIn: 'root', }) export class LiveApiService extends ApiService { - constructor(private readonly http: HttpService) { + constructor( + private readonly http: HttpService, + @Inject(DOCUMENT) private readonly document: Document, + ) { super() } - async getStatus() { - return this.rpcRequest({ + openProgressWebsocket$(guid: string): Observable { + const { location } = this.document.defaultView! + const host = location.host + + return webSocket({ + url: `ws://${host}/ws/rpc/${guid}`, + }) + } + + async getStatus(): Promise { + return this.rpcRequest({ method: 'setup.status', params: {}, }) @@ -41,7 +49,7 @@ export class LiveApiService extends ApiService { * this wil all public/private key, which means that there is no information loss * through the network. */ - async getPubKey() { + async getPubKey(): Promise { const response: jose.JWK.Key = await this.rpcRequest({ method: 'setup.get-pubkey', params: {}, @@ -50,14 +58,14 @@ export class LiveApiService extends ApiService { this.pubkey = response } - async getDrives() { + async getDrives(): Promise { return this.rpcRequest({ method: 'setup.disk.list', params: {}, }) } - async verifyCifs(source: CifsRecoverySource) { + async verifyCifs(source: T.VerifyCifsParams): Promise { source.path = source.path.replace('/\\/g', '/') return this.rpcRequest({ method: 'setup.cifs.verify', @@ -65,14 +73,14 @@ export class LiveApiService extends ApiService { }) } - async attach(params: AttachReq) { - await this.rpcRequest({ + async attach(params: T.AttachParams): Promise { + return this.rpcRequest({ method: 'setup.attach', params, }) } - async execute(setupInfo: ExecuteReq) { + async execute(setupInfo: T.SetupExecuteParams): Promise { if (setupInfo.recoverySource?.type === 'backup') { if (isCifsSource(setupInfo.recoverySource.target)) { setupInfo.recoverySource.target.path = @@ -80,14 +88,14 @@ export class LiveApiService extends ApiService { } } - await this.rpcRequest({ + return this.rpcRequest({ method: 'setup.execute', params: setupInfo, }) } - async complete() { - const res = await this.rpcRequest({ + async complete(): Promise { + const res = await this.rpcRequest({ method: 'setup.complete', params: {}, }) @@ -98,7 +106,7 @@ export class LiveApiService extends ApiService { } } - async exit() { + async exit(): Promise { await this.rpcRequest({ method: 'setup.exit', params: {}, @@ -119,7 +127,7 @@ export class LiveApiService extends ApiService { } function isCifsSource( - source: CifsRecoverySource | DiskRecoverySource | null, -): source is CifsRecoverySource { - return !!(source as CifsRecoverySource)?.hostname + source: T.BackupTargetFS | null, +): source is T.Cifs & { type: 'cifs' } { + return !!(source as T.Cifs)?.hostname } diff --git a/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts b/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts index df32bd09e..0a1c221f7 100644 --- a/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts @@ -1,42 +1,151 @@ import { Injectable } from '@angular/core' -import { encodeBase64, pauseFor } from '@start9labs/shared' import { - ApiService, - AttachReq, - CifsRecoverySource, - CompleteRes, - ExecuteReq, -} from './api.service' + DiskListResponse, + StartOSDiskInfo, + encodeBase64, + pauseFor, +} from '@start9labs/shared' +import { ApiService } from './api.service' import * as jose from 'node-jose' - -let tries: number +import { T } from '@start9labs/start-sdk' +import { + Observable, + concatMap, + delay, + from, + interval, + map, + mergeScan, + of, + startWith, + switchMap, + switchScan, + takeWhile, +} from 'rxjs' @Injectable({ providedIn: 'root', }) export class MockApiService extends ApiService { - async getStatus() { - const restoreOrMigrate = true + // fullProgress$(): Observable { + // const phases = [ + // { + // name: 'Preparing Data', + // progress: null, + // }, + // { + // name: 'Transferring Data', + // progress: null, + // }, + // { + // name: 'Finalizing Setup', + // progress: null, + // }, + // ] + + // return from(phases).pipe( + // switchScan((acc, val, i) => {}, { overall: null, phases }), + // ) + // } + + // namedProgress$(namedProgress: T.NamedProgress): Observable { + // return of(namedProgress).pipe(startWith(namedProgress)) + // } + + // progress$(progress: T.Progress): Observable {} + + // websocket + + openProgressWebsocket$(guid: string): Observable { + return of(PROGRESS) + // const numPhases = PROGRESS.phases.length + + // return of(PROGRESS).pipe( + // switchMap(full => + // from(PROGRESS.phases).pipe( + // mergeScan((full, phase, i) => { + // if ( + // !phase.progress || + // typeof phase.progress !== 'object' || + // !phase.progress.total + // ) { + // full.phases[i].progress = true + + // if ( + // full.overall && + // typeof full.overall === 'object' && + // full.overall.total + // ) { + // const step = full.overall.total / numPhases + // full.overall.done += step + // } + + // return of(full).pipe(delay(2000)) + // } else { + // const total = phase.progress.total + // const step = total / 4 + // let done = phase.progress.done + + // return interval(1000).pipe( + // takeWhile(() => done < total), + // map(() => { + // done += step + + // console.error(done) + + // if ( + // full.overall && + // typeof full.overall === 'object' && + // full.overall.total + // ) { + // const step = full.overall.total / numPhases / 4 + + // full.overall.done += step + // } + + // if (done === total) { + // full.phases[i].progress = true + + // if (i === numPhases - 1) { + // full.overall = true + // } + // } + // return full + // }), + // ) + // } + // }, full), + // ), + // ), + // ) + } + + private statusIndex = 0 + async getStatus(): Promise { await pauseFor(1000) - if (tries === undefined) { - tries = 0 - return null - } + this.statusIndex++ - tries++ - - const total = tries <= 4 ? tries * 268435456 : 1073741824 - const progress = tries > 4 ? (tries - 4) * 268435456 : 0 - - return { - bytesTransferred: restoreOrMigrate ? progress : 0, - totalBytes: restoreOrMigrate ? total : null, - complete: progress === total, + switch (this.statusIndex) { + case 2: + return { + status: 'running', + progress: PROGRESS, + guid: 'progress-guid', + } + case 3: + return { + status: 'complete', + torAddress: 'https://asdafsadasdasasdasdfasdfasdf.onion', + lanAddress: 'https://adjective-noun.local', + rootCa: encodeBase64(rootCA), + } + default: + return null } } - async getPubKey() { + async getPubKey(): Promise { await pauseFor(1000) // randomly generated @@ -52,7 +161,7 @@ export class MockApiService extends ApiService { }) } - async getDrives() { + async getDrives(): Promise { await pauseFor(1000) return [ { @@ -127,7 +236,7 @@ export class MockApiService extends ApiService { ] } - async verifyCifs(params: CifsRecoverySource) { + async verifyCifs(params: T.VerifyCifsParams): Promise { await pauseFor(1000) return { version: '0.3.0', @@ -138,15 +247,25 @@ export class MockApiService extends ApiService { } } - async attach(params: AttachReq) { + async attach(params: T.AttachParams): Promise { await pauseFor(1000) + + return { + progress: PROGRESS, + guid: 'progress-guid', + } } - async execute(setupInfo: ExecuteReq) { + async execute(setupInfo: T.SetupExecuteParams): Promise { await pauseFor(1000) + + return { + progress: PROGRESS, + guid: 'progress-guid', + } } - async complete(): Promise { + async complete(): Promise { await pauseFor(1000) return { torAddress: 'https://asdafsadasdasasdasdfasdfasdf.onion', @@ -155,7 +274,7 @@ export class MockApiService extends ApiService { } } - async exit() { + async exit(): Promise { await pauseFor(1000) } } @@ -182,3 +301,8 @@ Rf3ZOPm9QP92YpWyYDkfAU04xdDo1vR0MYjKPkl4LjRqSU/tcCJnPMbJiwq+bWpX 2WJoEBXB/p15Kn6JxjI0ze2SnSI48JZ8it4fvxrhOo0VoLNIuCuNXJOwU17Rdl1W YJidaq7je6k18AdgPA0Kh8y1XtfUH3fTaVw4 -----END CERTIFICATE-----` + +const PROGRESS = { + overall: null, + phases: [], +} diff --git a/web/projects/setup-wizard/src/app/services/state.service.ts b/web/projects/setup-wizard/src/app/services/state.service.ts index 916b066ee..8c653a088 100644 --- a/web/projects/setup-wizard/src/app/services/state.service.ts +++ b/web/projects/setup-wizard/src/app/services/state.service.ts @@ -1,13 +1,13 @@ import { Injectable } from '@angular/core' -import { ApiService, RecoverySource } from './api/api.service' +import { ApiService } from './api/api.service' +import { T } from '@start9labs/start-sdk' @Injectable({ providedIn: 'root', }) export class StateService { setupType?: 'fresh' | 'restore' | 'attach' | 'transfer' - - recoverySource?: RecoverySource + recoverySource?: T.RecoverySource recoveryPassword?: string constructor(private readonly api: ApiService) {} diff --git a/web/projects/shared/src/types/api.ts b/web/projects/shared/src/types/api.ts index 26cd00218..743ea6ac8 100644 --- a/web/projects/shared/src/types/api.ts +++ b/web/projects/shared/src/types/api.ts @@ -13,6 +13,7 @@ export type LogsRes = { export interface Log { timestamp: string message: string + bootId: string } export type DiskListResponse = DiskInfo[] diff --git a/web/projects/ui/src/app/app-routing.module.ts b/web/projects/ui/src/app/app-routing.module.ts index ddb66d3c5..e7a2036ec 100644 --- a/web/projects/ui/src/app/app-routing.module.ts +++ b/web/projects/ui/src/app/app-routing.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core' import { PreloadAllModules, RouterModule, Routes } from '@angular/router' +import { stateNot } from 'src/app/services/state.service' import { AuthGuard } from './guards/auth.guard' import { UnauthGuard } from './guards/unauth.guard' @@ -15,15 +16,29 @@ const routes: Routes = [ loadChildren: () => import('./pages/login/login.module').then(m => m.LoginPageModule), }, + { + path: 'diagnostic', + canActivate: [stateNot(['initializing', 'running'])], + loadChildren: () => + import('./pages/diagnostic-routes/diagnostic-routing.module').then( + m => m.DiagnosticModule, + ), + }, + { + path: 'initializing', + canActivate: [stateNot(['error', 'running'])], + loadChildren: () => + import('./pages/init/init.module').then(m => m.InitPageModule), + }, { path: 'home', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], loadChildren: () => import('./pages/home/home.module').then(m => m.HomePageModule), }, { path: 'system', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/server-routes/server-routing.module').then( @@ -32,14 +47,14 @@ const routes: Routes = [ }, { path: 'updates', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/updates/updates.module').then(m => m.UpdatesPageModule), }, { path: 'marketplace', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/marketplace-routes/marketplace-routing.module').then( @@ -48,7 +63,7 @@ const routes: Routes = [ }, { path: 'notifications', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], loadChildren: () => import('./pages/notifications/notifications.module').then( m => m.NotificationsPageModule, @@ -56,7 +71,7 @@ const routes: Routes = [ }, { path: 'services', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/apps-routes/apps-routing.module').then( diff --git a/web/projects/ui/src/app/app.component.html b/web/projects/ui/src/app/app.component.html index 0506d5214..0d9fb860f 100644 --- a/web/projects/ui/src/app/app.component.html +++ b/web/projects/ui/src/app/app.component.html @@ -15,6 +15,7 @@ type="overlay" side="start" class="left-menu" + [class.left-menu_hidden]="withoutMenu" > diff --git a/web/projects/ui/src/app/app.component.scss b/web/projects/ui/src/app/app.component.scss index 55135b1e5..aedbdc6c4 100644 --- a/web/projects/ui/src/app/app.component.scss +++ b/web/projects/ui/src/app/app.component.scss @@ -9,11 +9,15 @@ tui-root { .left-menu { --side-max-width: 280px; + + &_hidden { + display: none; + } } .menu { :host-context(body[data-theme='Light']) & { - --ion-color-base: #F4F4F5 !important; + --ion-color-base: #f4f4f5 !important; } } diff --git a/web/projects/ui/src/app/app.component.ts b/web/projects/ui/src/app/app.component.ts index 1210eba7a..ddf8c074f 100644 --- a/web/projects/ui/src/app/app.component.ts +++ b/web/projects/ui/src/app/app.component.ts @@ -1,4 +1,5 @@ import { Component, inject, OnDestroy } from '@angular/core' +import { IsActiveMatchOptions, Router } from '@angular/router' import { combineLatest, map, merge, startWith } from 'rxjs' import { AuthService } from './services/auth.service' import { SplitPaneTracker } from './services/split-pane.service' @@ -15,6 +16,13 @@ import { THEME } from '@start9labs/shared' import { PatchDB } from 'patch-db-client' import { DataModel } from './services/patch-db/data-model' +const OPTIONS: IsActiveMatchOptions = { + paths: 'subset', + queryParams: 'exact', + fragment: 'ignored', + matrixParams: 'ignored', +} + @Component({ selector: 'app-root', templateUrl: 'app.component.html', @@ -27,7 +35,7 @@ export class AppComponent implements OnDestroy { readonly theme$ = inject(THEME) readonly offline$ = combineLatest([ this.authService.isVerified$, - this.connection.connected$, + this.connection$, this.patch .watch$('serverInfo', 'statusInfo') .pipe(startWith({ restarting: false, shuttingDown: false })), @@ -44,8 +52,9 @@ export class AppComponent implements OnDestroy { private readonly patchMonitor: PatchMonitorService, private readonly splitPane: SplitPaneTracker, private readonly patch: PatchDB, + private readonly router: Router, readonly authService: AuthService, - readonly connection: ConnectionService, + readonly connection$: ConnectionService, readonly clientStorageService: ClientStorageService, readonly themeSwitcher: ThemeSwitcherService, ) {} @@ -56,6 +65,13 @@ export class AppComponent implements OnDestroy { .subscribe(name => this.titleService.setTitle(name || 'StartOS')) } + get withoutMenu(): boolean { + return ( + this.router.isActive('initializing', OPTIONS) || + this.router.isActive('diagnostic', OPTIONS) + ) + } + splitPaneVisible({ detail }: any) { this.splitPane.sidebarOpen$.next(detail.visible) } diff --git a/web/projects/ui/src/app/app.module.ts b/web/projects/ui/src/app/app.module.ts index 324300851..c0264f064 100644 --- a/web/projects/ui/src/app/app.module.ts +++ b/web/projects/ui/src/app/app.module.ts @@ -1,4 +1,5 @@ import { + TuiAlertModule, TuiDialogModule, TuiModeModule, TuiRootModule, @@ -58,6 +59,7 @@ import { environment } from '../environments/environment' ConnectionBarComponentModule, TuiRootModule, TuiDialogModule, + TuiAlertModule, TuiModeModule, TuiThemeNightModule, WidgetsPageModule, diff --git a/web/projects/ui/src/app/app.providers.ts b/web/projects/ui/src/app/app.providers.ts index 5d0ccc4f2..bf26a8cb9 100644 --- a/web/projects/ui/src/app/app.providers.ts +++ b/web/projects/ui/src/app/app.providers.ts @@ -10,6 +10,7 @@ import { AuthService } from './services/auth.service' import { ClientStorageService } from './services/client-storage.service' import { FilterPackagesPipe } from '../../../marketplace/src/pipes/filter-packages.pipe' import { ThemeSwitcherService } from './services/theme-switcher.service' +import { StorageService } from './services/storage.service' const { useMocks, @@ -30,7 +31,7 @@ export const APP_PROVIDERS: Provider[] = [ }, { provide: APP_INITIALIZER, - deps: [AuthService, ClientStorageService, Router], + deps: [StorageService, AuthService, ClientStorageService, Router], useFactory: appInitializer, multi: true, }, @@ -45,13 +46,15 @@ export const APP_PROVIDERS: Provider[] = [ ] export function appInitializer( + storage: StorageService, auth: AuthService, localStorage: ClientStorageService, router: Router, ): () => void { return () => { + storage.migrate036() auth.init() - localStorage.init() + localStorage.init() // @TODO pretty sure we can navigate before this step router.initialNavigation() } } diff --git a/web/projects/ui/src/app/app/menu/menu.component.ts b/web/projects/ui/src/app/app/menu/menu.component.ts index 5c1fbe8bd..b2ab62368 100644 --- a/web/projects/ui/src/app/app/menu/menu.component.ts +++ b/web/projects/ui/src/app/app/menu/menu.component.ts @@ -70,7 +70,7 @@ export class MenuComponent { readonly showEOSUpdate$ = this.eosService.showUpdate$ - private readonly local$ = this.connectionService.connected$.pipe( + private readonly local$ = this.connection$.pipe( filter(Boolean), switchMap(() => this.patch.watch$('packageData').pipe(first())), switchMap(outer => @@ -126,6 +126,6 @@ export class MenuComponent { private readonly marketplaceService: MarketplaceService, private readonly splitPane: SplitPaneTracker, private readonly emver: Emver, - private readonly connectionService: ConnectionService, + private readonly connection$: ConnectionService, ) {} } diff --git a/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts b/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts index da28f805f..cf0eab598 100644 --- a/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts +++ b/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts @@ -1,8 +1,9 @@ import { ChangeDetectionStrategy, Component } from '@angular/core' import { PatchDB } from 'patch-db-client' import { combineLatest, map, Observable, startWith } from 'rxjs' -import { ConnectionService } from 'src/app/services/connection.service' +import { NetworkService } from 'src/app/services/network.service' import { DataModel } from 'src/app/services/patch-db/data-model' +import { StateService } from 'src/app/services/state.service' @Component({ selector: 'connection-bar', @@ -11,16 +12,14 @@ import { DataModel } from 'src/app/services/patch-db/data-model' changeDetection: ChangeDetectionStrategy.OnPush, }) export class ConnectionBarComponent { - private readonly websocket$ = this.connectionService.websocketConnected$ - readonly connection$: Observable<{ message: string color: string icon: string dots: boolean }> = combineLatest([ - this.connectionService.networkConnected$, - this.websocket$.pipe(startWith(false)), + this.network$, + this.state$.pipe(map(Boolean)), this.patch .watch$('serverInfo', 'statusInfo') .pipe(startWith({ restarting: false, shuttingDown: false })), @@ -65,7 +64,8 @@ export class ConnectionBarComponent { ) constructor( - private readonly connectionService: ConnectionService, + private readonly network$: NetworkService, + private readonly state$: StateService, private readonly patch: PatchDB, ) {} } diff --git a/web/projects/ui/src/app/components/logs/logs.component.ts b/web/projects/ui/src/app/components/logs/logs.component.ts index 5d033fc83..3d31313cb 100644 --- a/web/projects/ui/src/app/components/logs/logs.component.ts +++ b/web/projects/ui/src/app/components/logs/logs.component.ts @@ -11,7 +11,6 @@ import { takeUntil, tap, } from 'rxjs' -import { WebSocketSubjectConfig } from 'rxjs/webSocket' import { LogsRes, ServerLogsReq, @@ -72,7 +71,7 @@ export class LogsComponent { private readonly api: ApiService, private readonly loadingCtrl: LoadingController, private readonly downloadHtml: DownloadHTMLService, - private readonly connectionService: ConnectionService, + private readonly connection$: ConnectionService, ) {} async ngOnInit() { @@ -149,43 +148,42 @@ export class LogsComponent { private reconnect$(): Observable { return from(this.followLogs({})).pipe( tap(_ => this.recordConnectionChange()), - switchMap(({ guid }) => this.connect$(guid, true)), + switchMap(({ guid }) => this.connect$(guid)), ) } - private connect$(guid: string, reconnect = false) { - const config: WebSocketSubjectConfig = { - url: `/rpc/${guid}`, - openObserver: { - next: () => { - this.websocketStatus = 'connected' + private connect$(guid: string) { + return this.api + .openWebsocket$(guid, { + openObserver: { + next: () => { + this.websocketStatus = 'connected' + }, }, - }, - } - - return this.api.openLogsWebsocket$(config).pipe( - tap(_ => this.count++), - bufferTime(1000), - tap(msgs => { - this.loading = false - this.processRes({ entries: msgs }) - if (this.infiniteStatus === 0 && this.count >= this.limit) - this.infiniteStatus = 1 - }), - catchError(() => { - this.recordConnectionChange(false) - return this.connectionService.connected$.pipe( - tap( - connected => - (this.websocketStatus = connected - ? 'reconnecting' - : 'disconnected'), - ), - filter(Boolean), - switchMap(() => this.reconnect$()), - ) - }), - ) + }) + .pipe( + tap(_ => this.count++), + bufferTime(1000), + tap(msgs => { + this.loading = false + this.processRes({ entries: msgs }) + if (this.infiniteStatus === 0 && this.count >= this.limit) + this.infiniteStatus = 1 + }), + catchError(() => { + this.recordConnectionChange(false) + return this.connection$.pipe( + tap( + connected => + (this.websocketStatus = connected + ? 'reconnecting' + : 'disconnected'), + ), + filter(Boolean), + switchMap(() => this.reconnect$()), + ) + }), + ) } private recordConnectionChange(success = true) { diff --git a/web/projects/ui/src/app/components/status/status.component.html b/web/projects/ui/src/app/components/status/status.component.html index db9e8f8f7..65c142f4a 100644 --- a/web/projects/ui/src/app/components/status/status.component.html +++ b/web/projects/ui/src/app/components/status/status.component.html @@ -1,12 +1,12 @@

- {{ (connected$ | async) ? rendering.display : 'Unknown' }} + {{ (connection$ | async) ? rendering.display : 'Unknown' }} . This may take a while diff --git a/web/projects/ui/src/app/components/status/status.component.ts b/web/projects/ui/src/app/components/status/status.component.ts index c9fec4968..45f66f291 100644 --- a/web/projects/ui/src/app/components/status/status.component.ts +++ b/web/projects/ui/src/app/components/status/status.component.ts @@ -21,7 +21,5 @@ export class StatusComponent { @Input() installingInfo?: InstallingInfo @Input() sigtermTimeout?: string | null = null - readonly connected$ = this.connectionService.connected$ - - constructor(private readonly connectionService: ConnectionService) {} + constructor(readonly connection$: ConnectionService) {} } diff --git a/web/projects/ui/src/app/modals/os-update/os-update.page.ts b/web/projects/ui/src/app/modals/os-update/os-update.page.ts index 9c49b72bf..9900bbb47 100644 --- a/web/projects/ui/src/app/modals/os-update/os-update.page.ts +++ b/web/projects/ui/src/app/modals/os-update/os-update.page.ts @@ -20,9 +20,8 @@ export class OSUpdatePage { private readonly embassyApi: ApiService, private readonly eosService: EOSService, ) {} - ngOnInit() { - const releaseNotes = this.eosService.eos?.releaseNotes! + const releaseNotes = this.eosService.osUpdate?.releaseNotes! this.versions = Object.keys(releaseNotes) .sort() diff --git a/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html b/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html index 8f1af1470..b0bfbbb24 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html +++ b/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html @@ -1,4 +1,4 @@ - + Health Checks - + diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts index f3db08063..fef84a5ba 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts @@ -12,9 +12,7 @@ export class AppShowHealthChecksComponent { @Input() healthChecks!: Record - readonly connected$ = this.connectionService.connected$ - - constructor(private readonly connectionService: ConnectionService) {} + constructor(readonly connection$: ConnectionService) {} isLoading(result: T.HealthCheckResult['result']): boolean { return result === 'starting' || result === 'loading' diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html index f9db34c73..be7bc1c30 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html @@ -11,7 +11,7 @@ - + diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts index 4bc1bc464..ba60c331d 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts @@ -39,8 +39,6 @@ export class AppShowStatusComponent { isInstalled = isInstalled - readonly connected$ = this.connectionService.connected$ - constructor( private readonly alertCtrl: AlertController, private readonly errToast: ErrorToastService, @@ -48,7 +46,7 @@ export class AppShowStatusComponent { private readonly embassyApi: ApiService, private readonly launcherService: UiLauncherService, private readonly modalService: ModalService, - private readonly connectionService: ConnectionService, + readonly connection$: ConnectionService, private readonly patch: PatchDB, ) {} diff --git a/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts new file mode 100644 index 000000000..4409288c1 --- /dev/null +++ b/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts @@ -0,0 +1,21 @@ +import { NgModule } from '@angular/core' +import { RouterModule, Routes } from '@angular/router' + +const ROUTES: Routes = [ + { + path: '', + loadChildren: () => + import('./home/home.module').then(m => m.HomePageModule), + }, + { + path: 'logs', + loadChildren: () => + import('./logs/logs.module').then(m => m.LogsPageModule), + }, +] + +@NgModule({ + imports: [RouterModule.forChild(ROUTES)], + exports: [RouterModule], +}) +export class DiagnosticModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts similarity index 55% rename from web/projects/diagnostic-ui/src/app/pages/home/home.module.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts index 1664b7c72..9565220ae 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.module.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts @@ -1,12 +1,18 @@ import { NgModule } from '@angular/core' import { CommonModule } from '@angular/common' +import { Routes, RouterModule } from '@angular/router' import { IonicModule } from '@ionic/angular' -import { FormsModule } from '@angular/forms' import { HomePage } from './home.page' -import { HomePageRoutingModule } from './home-routing.module' + +const routes: Routes = [ + { + path: '', + component: HomePage, + }, +] @NgModule({ - imports: [CommonModule, FormsModule, IonicModule, HomePageRoutingModule], + imports: [CommonModule, IonicModule, RouterModule.forChild(routes)], declarations: [HomePage], }) export class HomePageModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.html b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html similarity index 92% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.html rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html index 9cba08258..69a58a3aa 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.page.html +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html @@ -51,12 +51,6 @@ }} -

- - System Rebuild - -
-
Repair Drive diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.scss b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.scss similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.scss rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.scss diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.ts b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts similarity index 74% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts index bbda6939f..9bb7376bc 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.page.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts @@ -1,9 +1,9 @@ import { Component } from '@angular/core' import { AlertController, LoadingController } from '@ionic/angular' -import { ApiService } from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/embassy-api.service' @Component({ - selector: 'app-home', + selector: 'diagnostic-home', templateUrl: 'home.page.html', styleUrls: ['home.page.scss'], }) @@ -25,7 +25,7 @@ export class HomePage { async ngOnInit() { try { - const error = await this.api.getError() + const error = await this.api.diagnosticGetError() // incorrect drive if (error.code === 15) { this.error = { @@ -92,7 +92,7 @@ export class HomePage { await loader.present() try { - await this.api.restart() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) @@ -108,8 +108,8 @@ export class HomePage { await loader.present() try { - await this.api.forgetDrive() - await this.api.restart() + await this.api.diagnosticForgetDrive() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) @@ -118,32 +118,6 @@ export class HomePage { } } - async presentAlertSystemRebuild() { - const alert = await this.alertCtrl.create({ - header: 'Warning', - message: - '

This action will tear down all service containers and rebuild them from scratch. No data will be deleted.

A system rebuild can be useful if your system gets into a bad state, and it should only be performed if you are experiencing general performance or reliability issues.

It may take up to an hour to complete. During this time, you will lose all connectivity to your Start9 server.

', - buttons: [ - { - text: 'Cancel', - role: 'cancel', - }, - { - text: 'Rebuild', - handler: () => { - try { - this.systemRebuild() - } catch (e) { - console.error(e) - } - }, - }, - ], - cssClass: 'alert-warning-message', - }) - await alert.present() - } - async presentAlertRepairDisk() { const alert = await this.alertCtrl.create({ header: 'Warning', @@ -174,23 +148,6 @@ export class HomePage { window.location.reload() } - private async systemRebuild(): Promise { - const loader = await this.loadingCtrl.create({ - cssClass: 'loader', - }) - await loader.present() - - try { - await this.api.systemRebuild() - await this.api.restart() - this.restarted = true - } catch (e) { - console.error(e) - } finally { - loader.dismiss() - } - } - private async repairDisk(): Promise { const loader = await this.loadingCtrl.create({ cssClass: 'loader', @@ -198,8 +155,8 @@ export class HomePage { await loader.present() try { - await this.api.repairDisk() - await this.api.restart() + await this.api.diagnosticRepairDisk() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.module.ts similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.module.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.module.ts diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.html b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.html similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.html rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.html diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.scss b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.scss similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.scss rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.scss diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts similarity index 93% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts index 44c314a96..5119b9d93 100644 --- a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts @@ -1,6 +1,6 @@ import { Component, ViewChild } from '@angular/core' import { IonContent } from '@ionic/angular' -import { ApiService } from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/embassy-api.service' import { ErrorToastService, toLocalIsoString } from '@start9labs/shared' var Convert = require('ansi-to-html') @@ -49,7 +49,7 @@ export class LogsPage { private async getLogs() { try { - const { startCursor, entries } = await this.api.getLogs({ + const { startCursor, entries } = await this.api.diagnosticGetLogs({ cursor: this.startCursor, before: !!this.startCursor, limit: this.limit, diff --git a/web/projects/ui/src/app/pages/init/init.module.ts b/web/projects/ui/src/app/pages/init/init.module.ts new file mode 100644 index 000000000..07dd71185 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.module.ts @@ -0,0 +1,24 @@ +import { CommonModule } from '@angular/common' +import { NgModule } from '@angular/core' +import { RouterModule, Routes } from '@angular/router' +import { TuiProgressModule } from '@taiga-ui/kit' +import { LogsModule } from 'src/app/pages/init/logs/logs.module' +import { InitPage } from './init.page' + +const routes: Routes = [ + { + path: '', + component: InitPage, + }, +] + +@NgModule({ + imports: [ + CommonModule, + LogsModule, + TuiProgressModule, + RouterModule.forChild(routes), + ], + declarations: [InitPage], +}) +export class InitPageModule {} diff --git a/web/projects/ui/src/app/pages/init/init.page.html b/web/projects/ui/src/app/pages/init/init.page.html new file mode 100644 index 000000000..bd3467bbb --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.html @@ -0,0 +1,18 @@ +
+

+ Initializing StartOS +

+
+ Progress: {{ (progress.total * 100).toFixed(0) }}% +
+ + +

+
+ diff --git a/web/projects/ui/src/app/pages/init/init.page.scss b/web/projects/ui/src/app/pages/init/init.page.scss new file mode 100644 index 000000000..9fbf7098a --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.scss @@ -0,0 +1,23 @@ +section { + border-radius: 0.25rem; + padding: 1rem; + margin: 1.5rem; + text-align: center; + /* TODO: Theme */ + background: #e0e0e0; + color: #333; + --tui-clear-inverse: rgba(0, 0, 0, 0.1); +} + +logs-window { + display: flex; + flex-direction: column; + height: 18rem; + padding: 1rem; + margin: 0 1.5rem auto; + text-align: left; + overflow: hidden; + border-radius: 2rem; + /* TODO: Theme */ + background: #181818; +} diff --git a/web/projects/ui/src/app/pages/init/init.page.ts b/web/projects/ui/src/app/pages/init/init.page.ts new file mode 100644 index 000000000..318881223 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.ts @@ -0,0 +1,11 @@ +import { Component, inject } from '@angular/core' +import { InitService } from 'src/app/pages/init/init.service' + +@Component({ + selector: 'init-page', + templateUrl: 'init.page.html', + styleUrls: ['init.page.scss'], +}) +export class InitPage { + readonly progress$ = inject(InitService) +} diff --git a/web/projects/ui/src/app/pages/init/init.service.ts b/web/projects/ui/src/app/pages/init/init.service.ts new file mode 100644 index 000000000..3cca42a58 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.service.ts @@ -0,0 +1,91 @@ +import { inject, Injectable } from '@angular/core' +import { ErrorToastService } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' +import { + catchError, + defer, + EMPTY, + from, + map, + Observable, + startWith, + switchMap, + tap, +} from 'rxjs' +import { ApiService } from 'src/app/services/api/embassy-api.service' +import { StateService } from 'src/app/services/state.service' + +interface MappedProgress { + readonly total: number | null + readonly message: string +} + +@Injectable({ providedIn: 'root' }) +export class InitService extends Observable { + private readonly state = inject(StateService) + private readonly api = inject(ApiService) + private readonly errorService = inject(ErrorToastService) + private readonly progress$ = defer(() => + from(this.api.initGetProgress()), + ).pipe( + switchMap(({ guid, progress }) => + this.api + .openWebsocket$(guid, {}) + .pipe(startWith(progress)), + ), + map(({ phases, overall }) => { + return { + total: getOverallDecimal(overall), + message: phases + .filter( + ( + p, + ): p is { + name: string + progress: { + done: number + total: number | null + } + } => p.progress !== true && p.progress !== null, + ) + .map(p => `${p.name}${getPhaseBytes(p.progress)}`) + .join(', '), + } + }), + tap(({ total }) => { + if (total === 1) { + this.state.syncState() + } + }), + catchError(e => { + this.errorService.present(e) + + return EMPTY + }), + ) + + constructor() { + super(subscriber => this.progress$.subscribe(subscriber)) + } +} + +function getOverallDecimal(progress: T.Progress): number { + if (progress === true) { + return 1 + } else if (!progress || !progress.total) { + return 0 + } else { + return progress.total && progress.done / progress.total + } +} + +function getPhaseBytes( + progress: + | false + | { + done: number + total: number | null + }, +): string { + return progress === false ? '' : `: (${progress.done}/${progress.total})` +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.component.ts b/web/projects/ui/src/app/pages/init/logs/logs.component.ts new file mode 100644 index 000000000..edce1f282 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.component.ts @@ -0,0 +1,33 @@ +import { Component, ElementRef, inject } from '@angular/core' +import { INTERSECTION_ROOT } from '@ng-web-apis/intersection-observer' +import { LogsService } from 'src/app/pages/init/logs/logs.service' + +@Component({ + selector: 'logs-window', + templateUrl: 'logs.template.html', + styles: [ + ` + pre { + margin: 0; + } + `, + ], + providers: [ + { + provide: INTERSECTION_ROOT, + useExisting: ElementRef, + }, + ], +}) +export class LogsComponent { + readonly logs$ = inject(LogsService) + scroll = true + + scrollTo(bottom: HTMLElement) { + if (this.scroll) bottom.scrollIntoView({ behavior: 'smooth' }) + } + + onBottom([{ isIntersecting }]: readonly IntersectionObserverEntry[]) { + this.scroll = isIntersecting + } +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.module.ts b/web/projects/ui/src/app/pages/init/logs/logs.module.ts new file mode 100644 index 000000000..ee4a1bc1d --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.module.ts @@ -0,0 +1,20 @@ +import { CommonModule } from '@angular/common' +import { NgModule } from '@angular/core' +import { IntersectionObserverModule } from '@ng-web-apis/intersection-observer' +import { MutationObserverModule } from '@ng-web-apis/mutation-observer' +import { TuiScrollbarModule } from '@taiga-ui/core' +import { NgDompurifyModule } from '@tinkoff/ng-dompurify' +import { LogsComponent } from './logs.component' + +@NgModule({ + imports: [ + CommonModule, + MutationObserverModule, + IntersectionObserverModule, + NgDompurifyModule, + TuiScrollbarModule, + ], + declarations: [LogsComponent], + exports: [LogsComponent], +}) +export class LogsModule {} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.service.ts b/web/projects/ui/src/app/pages/init/logs/logs.service.ts new file mode 100644 index 000000000..e06d56b42 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.service.ts @@ -0,0 +1,49 @@ +import { inject, Injectable } from '@angular/core' +import { Log, toLocalIsoString } from '@start9labs/shared' +import { + bufferTime, + defer, + filter, + map, + Observable, + scan, + switchMap, +} from 'rxjs' +import { ApiService } from 'src/app/services/api/embassy-api.service' + +var Convert = require('ansi-to-html') +var convert = new Convert({ + newline: true, + bg: 'transparent', + colors: { + 4: 'Cyan', + }, + escapeXML: true, +}) + +function convertAnsi(entries: readonly any[]): string { + return entries + .map( + ({ timestamp, message }) => + `${toLocalIsoString( + new Date(timestamp), + )}  ${convert.toHtml(message)}`, + ) + .join('
') +} + +@Injectable({ providedIn: 'root' }) +export class LogsService extends Observable { + private readonly api = inject(ApiService) + private readonly log$ = defer(() => this.api.initFollowLogs({})).pipe( + switchMap(({ guid }) => this.api.openWebsocket$(guid, {})), + bufferTime(250), + filter(logs => !!logs.length), + map(convertAnsi), + scan((logs: readonly string[], log) => [...logs, log], []), + ) + + constructor() { + super(subscriber => this.log$.subscribe(subscriber)) + } +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.template.html b/web/projects/ui/src/app/pages/init/logs/logs.template.html new file mode 100644 index 000000000..24ea6d0c1 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.template.html @@ -0,0 +1,9 @@ + + 

+  

+
diff --git a/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts b/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
index fde1c968f..2c0e9c7fe 100644
--- a/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
+++ b/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
@@ -42,7 +42,7 @@ export class CAWizardComponent {
 
   private async testHttps() {
     const url = `https://${this.document.location.host}${this.relativeUrl}`
-    await this.api.echo({ message: 'ping' }, url).then(() => {
+    await this.api.getState().then(() => {
       this.caTrusted = true
     })
   }
diff --git a/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html b/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
index 70b977ecc..859d48129 100644
--- a/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
+++ b/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
@@ -74,7 +74,7 @@
       Memory
       
         Percentage Used
-        {{ memory.percentageUsed }} %
+        {{ memory.percentageUsed.value }} %
       
       
         Total
@@ -98,7 +98,7 @@
       
       
         zram Total
-        {{ memory.zramTotal }} MiB
+        {{ memory.zramTotal.value }} MiB
       
       
         zram Available
diff --git a/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts b/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
index 73763578e..f375b0e86 100644
--- a/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
+++ b/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
@@ -319,30 +319,6 @@ export class ServerShowPage {
     await alert.present()
   }
 
-  async presentAlertSystemRebuild() {
-    const localPkgs = await getAllPackages(this.patch)
-    const minutes = Object.keys(localPkgs).length * 2
-    const alert = await this.alertCtrl.create({
-      header: 'Warning',
-      message: `This action will tear down all service containers and rebuild them from scratch. No data will be deleted. This action is useful if your system gets into a bad state, and it should only be performed if you are experiencing general performance or reliability issues. It may take up to ${minutes} minutes to complete. During this time, you will lose all connectivity to your server.`,
-      buttons: [
-        {
-          text: 'Cancel',
-          role: 'cancel',
-        },
-        {
-          text: 'Rebuild',
-          handler: () => {
-            this.systemRebuild()
-          },
-          cssClass: 'enter-click',
-        },
-      ],
-      cssClass: 'alert-warning-message',
-    })
-    await alert.present()
-  }
-
   async presentAlertRepairDisk() {
     const alert = await this.alertCtrl.create({
       header: 'Warning',
@@ -437,23 +413,6 @@ export class ServerShowPage {
     }
   }
 
-  private async systemRebuild() {
-    const action = 'System Rebuild'
-
-    const loader = await this.loadingCtrl.create({
-      message: `Beginning ${action}...`,
-    })
-    await loader.present()
-
-    try {
-      await this.embassyApi.systemRebuild({})
-    } catch (e: any) {
-      this.errToast.present(e)
-    } finally {
-      loader.dismiss()
-    }
-  }
-
   private async checkForEosUpdate(): Promise {
     const loader = await this.loadingCtrl.create({
       message: 'Checking for updates',
@@ -718,14 +677,6 @@ export class ServerShowPage {
         detail: false,
         disabled$: of(false),
       },
-      {
-        title: 'System Rebuild',
-        description: '',
-        icon: 'construct-outline',
-        action: () => this.presentAlertSystemRebuild(),
-        detail: false,
-        disabled$: of(false),
-      },
       {
         title: 'Repair Disk',
         description: '',
diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts
index 5642eece5..38ee4774b 100644
--- a/web/projects/ui/src/app/services/api/api.fixures.ts
+++ b/web/projects/ui/src/app/services/api/api.fixures.ts
@@ -16,7 +16,7 @@ export module Mock {
     restarting: false,
     shuttingDown: false,
   }
-  export const MarketplaceEos: RR.GetMarketplaceEosRes = {
+  export const MarketplaceEos: RR.CheckOSUpdateRes = {
     version: '0.3.5.2',
     headline: 'Our biggest release ever.',
     releaseNotes: {
@@ -493,30 +493,23 @@ export module Mock {
     {
       timestamp: '2022-07-28T03:52:54.808769Z',
       message: '****** START *****',
+      bootId: 'hsjnfdklasndhjasvbjamsksajbndjn',
     },
     {
       timestamp: '2019-12-26T14:21:30.872Z',
       message:
         '\u001b[34mPOST \u001b[0;32;49m200\u001b[0m photoview.startos/api/graphql \u001b[0;36;49m1.169406ms\u001b',
+      bootId: 'hsjnfdklasndhjasvbjamsksajbndjn',
     },
     {
       timestamp: '2019-12-26T14:22:30.872Z',
       message: '****** FINISH *****',
-    },
-  ]
-
-  export const PackageLogs: Log[] = [
-    {
-      timestamp: '2022-07-28T03:52:54.808769Z',
-      message: '****** START *****',
+      bootId: 'gvbwfiuasokdasjndasnjdmfvbahjdmdkfm',
     },
     {
-      timestamp: '2019-12-26T14:21:30.872Z',
-      message: 'PackageLogs PackageLogs PackageLogs PackageLogs PackageLogs',
-    },
-    {
-      timestamp: '2019-12-26T14:22:30.872Z',
-      message: '****** FINISH *****',
+      timestamp: '2019-12-26T15:22:30.872Z',
+      message: '****** AGAIN *****',
+      bootId: 'gvbwfiuasokdasjndasnjdmfvbahjdmdkfm',
     },
   ]
 
diff --git a/web/projects/ui/src/app/services/api/api.types.ts b/web/projects/ui/src/app/services/api/api.types.ts
index 3539164f5..a5c52e9a2 100644
--- a/web/projects/ui/src/app/services/api/api.types.ts
+++ b/web/projects/ui/src/app/services/api/api.types.ts
@@ -1,17 +1,28 @@
-import { Dump, Revision } from 'patch-db-client'
+import { Dump } from 'patch-db-client'
 import { MarketplacePkg, StoreInfo } from '@start9labs/marketplace'
 import { PackagePropertiesVersioned } from 'src/app/util/properties.util'
 import { ConfigSpec } from 'src/app/pkg-config/config-types'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { StartOSDiskInfo, LogsRes, ServerLogsReq } from '@start9labs/shared'
 import { T } from '@start9labs/start-sdk'
+import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 
 export module RR {
+  // websocket
+
+  export type WebsocketConfig = Omit, 'url'>
+
+  // server state
+
+  export type ServerState = 'initializing' | 'error' | 'running'
+
   // DB
 
-  export type GetRevisionsRes = Revision[] | Dump
-
-  export type GetDumpRes = Dump
+  export type SubscribePatchReq = {}
+  export type SubscribePatchRes = {
+    dump: Dump
+    guid: string
+  }
 
   export type SetDBValueReq = { pointer: string; value: T } // db.put.ui
   export type SetDBValueRes = null
@@ -33,10 +44,22 @@ export module RR {
   } // auth.reset-password
   export type ResetPasswordRes = null
 
-  // server
+  // diagnostic
 
-  export type EchoReq = { message: string; timeout?: number } // server.echo
-  export type EchoRes = string
+  export type DiagnosticErrorRes = {
+    code: number
+    message: string
+    data: { details: string }
+  }
+
+  // init
+
+  export type InitGetProgressRes = {
+    progress: T.FullProgress
+    guid: string
+  }
+
+  // server
 
   export type GetSystemTimeReq = {} // server.time
   export type GetSystemTimeRes = {
@@ -65,8 +88,8 @@ export module RR {
   export type ShutdownServerReq = {} // server.shutdown
   export type ShutdownServerRes = null
 
-  export type SystemRebuildReq = {} // server.rebuild
-  export type SystemRebuildRes = null
+  export type DiskRepairReq = {} // server.disk.repair
+  export type DiskRepairRes = null
 
   export type ResetTorReq = {
     wipeState: boolean
@@ -254,8 +277,8 @@ export module RR {
   export type GetMarketplaceInfoReq = { serverId: string }
   export type GetMarketplaceInfoRes = StoreInfo
 
-  export type GetMarketplaceEosReq = { serverId: string }
-  export type GetMarketplaceEosRes = MarketplaceEOS
+  export type CheckOSUpdateReq = { serverId: string }
+  export type CheckOSUpdateRes = OSUpdate
 
   export type GetMarketplacePackagesReq = {
     ids?: { id: string; version: string }[]
@@ -271,7 +294,7 @@ export module RR {
   export type GetReleaseNotesRes = { [version: string]: string }
 }
 
-export interface MarketplaceEOS {
+export interface OSUpdate {
   version: string
   headline: string
   releaseNotes: { [version: string]: string }
diff --git a/web/projects/ui/src/app/services/api/embassy-api.service.ts b/web/projects/ui/src/app/services/api/embassy-api.service.ts
index 3f1d9881d..d6bb11632 100644
--- a/web/projects/ui/src/app/services/api/embassy-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-api.service.ts
@@ -1,9 +1,5 @@
 import { Observable } from 'rxjs'
-import { Update } from 'patch-db-client'
 import { RR } from './api.types'
-import { DataModel } from 'src/app/services/patch-db/data-model'
-import { Log } from '@start9labs/shared'
-import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 
 export abstract class ApiService {
   // http
@@ -14,8 +10,23 @@ export abstract class ApiService {
   // for sideloading packages
   abstract uploadPackage(guid: string, body: Blob): Promise
 
+  // websocket
+
+  abstract openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable
+
+  // server state
+
+  abstract getState(): Promise
+
   // db
 
+  abstract subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise
+
   abstract setDbValue(
     pathArr: Array,
     value: T,
@@ -35,16 +46,26 @@ export abstract class ApiService {
     params: RR.ResetPasswordReq,
   ): Promise
 
+  // diagnostic
+
+  abstract diagnosticGetError(): Promise
+  abstract diagnosticRestart(): Promise
+  abstract diagnosticForgetDrive(): Promise
+  abstract diagnosticRepairDisk(): Promise
+  abstract diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise
+
+  // init
+
+  abstract initGetProgress(): Promise
+
+  abstract initFollowLogs(
+    params: RR.FollowServerLogsReq,
+  ): Promise
+
   // server
 
-  abstract echo(params: RR.EchoReq, urlOverride?: string): Promise
-
-  abstract openPatchWebsocket$(): Observable>
-
-  abstract openLogsWebsocket$(
-    config: WebSocketSubjectConfig,
-  ): Observable
-
   abstract getSystemTime(
     params: RR.GetSystemTimeReq,
   ): Promise
@@ -89,11 +110,7 @@ export abstract class ApiService {
     params: RR.ShutdownServerReq,
   ): Promise
 
-  abstract systemRebuild(
-    params: RR.SystemRebuildReq,
-  ): Promise
-
-  abstract repairDisk(params: RR.SystemRebuildReq): Promise
+  abstract repairDisk(params: RR.DiskRepairReq): Promise
 
   abstract resetTor(params: RR.ResetTorReq): Promise
 
@@ -105,7 +122,7 @@ export abstract class ApiService {
     url: string,
   ): Promise
 
-  abstract getEos(): Promise
+  abstract checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise
 
   // notification
 
diff --git a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
index 631c12e6b..8826b2883 100644
--- a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
@@ -3,7 +3,6 @@ import {
   HttpOptions,
   HttpService,
   isRpcError,
-  Log,
   Method,
   RpcError,
   RPCOptions,
@@ -12,13 +11,12 @@ import { ApiService } from './embassy-api.service'
 import { RR } from './api.types'
 import { parsePropertiesPermissive } from 'src/app/util/properties.util'
 import { ConfigService } from '../config.service'
-import { webSocket, WebSocketSubjectConfig } from 'rxjs/webSocket'
+import { webSocket } from 'rxjs/webSocket'
 import { Observable, filter, firstValueFrom } from 'rxjs'
 import { AuthService } from '../auth.service'
 import { DOCUMENT } from '@angular/common'
 import { DataModel } from '../patch-db/data-model'
-import { PatchDB, pathFromArray, Update } from 'patch-db-client'
-import { getServerInfo } from 'src/app/util/get-server-info'
+import { PatchDB, pathFromArray } from 'patch-db-client'
 
 @Injectable()
 export class LiveApiService extends ApiService {
@@ -30,10 +28,11 @@ export class LiveApiService extends ApiService {
     private readonly patch: PatchDB,
   ) {
     super()
-    ;(window as any).rpcClient = this
+    ; (window as any).rpcClient = this
   }
 
   // for getting static files: ex icons, instructions, licenses
+
   async getStatic(url: string): Promise {
     return this.httpRequest({
       method: Method.GET,
@@ -43,6 +42,7 @@ export class LiveApiService extends ApiService {
   }
 
   // for sideloading packages
+
   async uploadPackage(guid: string, body: Blob): Promise {
     return this.httpRequest({
       method: Method.POST,
@@ -52,8 +52,36 @@ export class LiveApiService extends ApiService {
     })
   }
 
+  // websocket
+
+  openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable {
+    const { location } = this.document.defaultView!
+    const protocol = location.protocol === 'http:' ? 'ws' : 'wss'
+    const host = location.host
+
+    return webSocket({
+      url: `${protocol}://${host}/ws/rpc/${guid}`,
+      ...config,
+    })
+  }
+
+  // state
+
+  async getState(): Promise {
+    return this.rpcRequest({ method: 'state', params: {} })
+  }
+
   // db
 
+  async subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise {
+    return this.rpcRequest({ method: 'db.subscribe', params })
+  }
+
   async setDbValue(
     pathArr: Array,
     value: T,
@@ -87,29 +115,57 @@ export class LiveApiService extends ApiService {
     return this.rpcRequest({ method: 'auth.reset-password', params })
   }
 
+  // diagnostic
+
+  async diagnosticGetError(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.error',
+      params: {},
+    })
+  }
+
+  async diagnosticRestart(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.restart',
+      params: {},
+    })
+  }
+
+  async diagnosticForgetDrive(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.disk.forget',
+      params: {},
+    })
+  }
+
+  async diagnosticRepairDisk(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.disk.repair',
+      params: {},
+    })
+  }
+
+  async diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.logs',
+      params,
+    })
+  }
+
+  // init
+
+  async initGetProgress(): Promise {
+    return this.rpcRequest({ method: 'init.subscribe', params: {} })
+  }
+
+  async initFollowLogs(): Promise {
+    return this.rpcRequest({ method: 'init.logs.follow', params: {} })
+  }
+
   // server
 
-  async echo(params: RR.EchoReq, urlOverride?: string): Promise {
-    return this.rpcRequest({ method: 'echo', params }, urlOverride)
-  }
-
-  openPatchWebsocket$(): Observable> {
-    const config: WebSocketSubjectConfig> = {
-      url: `/db`,
-      closeObserver: {
-        next: val => {
-          if (val.reason === 'UNAUTHORIZED') this.auth.setUnverified()
-        },
-      },
-    }
-
-    return this.openWebsocket(config)
-  }
-
-  openLogsWebsocket$(config: WebSocketSubjectConfig): Observable {
-    return this.openWebsocket(config)
-  }
-
   async getSystemTime(
     params: RR.GetSystemTimeReq,
   ): Promise {
@@ -175,12 +231,6 @@ export class LiveApiService extends ApiService {
     return this.rpcRequest({ method: 'server.shutdown', params })
   }
 
-  async systemRebuild(
-    params: RR.RestartServerReq,
-  ): Promise {
-    return this.rpcRequest({ method: 'server.rebuild', params })
-  }
-
   async repairDisk(params: RR.RestartServerReq): Promise {
     return this.rpcRequest({ method: 'disk.repair', params })
   }
@@ -203,10 +253,7 @@ export class LiveApiService extends ApiService {
     })
   }
 
-  async getEos(): Promise {
-    const { id } = await getServerInfo(this.patch)
-    const qp: RR.GetMarketplaceEosReq = { serverId: id }
-
+  async checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise {
     return this.marketplaceProxy(
       '/eos/v0/latest',
       qp,
@@ -417,16 +464,6 @@ export class LiveApiService extends ApiService {
     })
   }
 
-  private openWebsocket(config: WebSocketSubjectConfig): Observable {
-    const { location } = this.document.defaultView!
-    const protocol = location.protocol === 'http:' ? 'ws' : 'wss'
-    const host = location.host
-
-    config.url = `${protocol}://${host}/ws${config.url}`
-
-    return webSocket(config)
-  }
-
   private async rpcRequest(
     options: RPCOptions,
     urlOverride?: string,
@@ -445,9 +482,7 @@ export class LiveApiService extends ApiService {
     const patchSequence = res.headers.get('x-patch-sequence')
     if (patchSequence)
       await firstValueFrom(
-        this.patch.cache$.pipe(
-          filter(({ sequence }) => sequence >= Number(patchSequence)),
-        ),
+        this.patch.cache$.pipe(filter(({ id }) => id >= Number(patchSequence))),
       )
 
     return body.result
diff --git a/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts b/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
index f623f6db3..728b4ff35 100644
--- a/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
@@ -1,15 +1,14 @@
 import { Injectable } from '@angular/core'
-import { Log, pauseFor } from '@start9labs/shared'
+import { Log, RPCErrorDetails, pauseFor } from '@start9labs/shared'
 import { ApiService } from './embassy-api.service'
 import {
   Operation,
   PatchOp,
   pathFromArray,
   RemoveOperation,
-  Update,
+  Revision,
 } from 'patch-db-client'
 import {
-  DataModel,
   InstallingState,
   PackageDataEntry,
   StateInfo,
@@ -20,22 +19,17 @@ import { parsePropertiesPermissive } from 'src/app/util/properties.util'
 import { Mock } from './api.fixures'
 import markdown from 'raw-loader!../../../../../shared/assets/markdown/md-sample.md'
 import {
-  EMPTY,
-  iif,
+  from,
   interval,
   map,
   Observable,
   shareReplay,
+  startWith,
   Subject,
-  switchMap,
   tap,
-  timer,
 } from 'rxjs'
-import { LocalStorageBootstrap } from '../patch-db/local-storage-bootstrap'
 import { mockPatchData } from './mock-patch'
-import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 import { AuthService } from '../auth.service'
-import { ConnectionService } from '../connection.service'
 import { StoreInfo } from '@start9labs/marketplace'
 import { T } from '@start9labs/start-sdk'
 
@@ -71,32 +65,17 @@ const PROGRESS: T.FullProgress = {
 
 @Injectable()
 export class MockApiService extends ApiService {
-  readonly mockWsSource$ = new Subject>()
+  readonly mockWsSource$ = new Subject()
   private readonly revertTime = 1800
   sequence = 0
 
-  constructor(
-    private readonly bootstrapper: LocalStorageBootstrap,
-    private readonly connectionService: ConnectionService,
-    private readonly auth: AuthService,
-  ) {
+  constructor(private readonly auth: AuthService) {
     super()
     this.auth.isVerified$
       .pipe(
         tap(() => {
           this.sequence = 0
         }),
-        switchMap(verified =>
-          iif(
-            () => verified,
-            timer(2000).pipe(
-              tap(() => {
-                this.connectionService.websocketConnected$.next(true)
-              }),
-            ),
-            EMPTY,
-          ),
-        ),
       )
       .subscribe()
   }
@@ -111,8 +90,57 @@ export class MockApiService extends ApiService {
     return 'success'
   }
 
+  // websocket
+
+  openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable {
+    if (guid === 'db-guid') {
+      return this.mockWsSource$.pipe(
+        shareReplay({ bufferSize: 1, refCount: true }),
+      )
+    } else if (guid === 'logs-guid') {
+      return interval(50).pipe(
+        map((_, index) => {
+          // mock fire open observer
+          if (index === 0) config.openObserver?.next(new Event(''))
+          if (index === 100) throw new Error('HAAHHA')
+          return Mock.ServerLogs[0]
+        }),
+      )
+    } else if (guid === 'init-progress-guid') {
+      return from(this.initProgress()).pipe(
+        startWith(PROGRESS),
+      ) as Observable
+    } else {
+      throw new Error('invalid guid type')
+    }
+  }
+
+  // server state
+
+  private stateIndex = 0
+  async getState(): Promise {
+    await pauseFor(1000)
+
+    this.stateIndex++
+
+    return this.stateIndex === 1 ? 'initializing' : 'running'
+  }
+
   // db
 
+  async subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise {
+    await pauseFor(2000)
+    return {
+      dump: { id: 1, value: mockPatchData },
+      guid: 'db-guid',
+    }
+  }
+
   async setDbValue(
     pathArr: Array,
     value: T,
@@ -136,11 +164,6 @@ export class MockApiService extends ApiService {
 
   async login(params: RR.LoginReq): Promise {
     await pauseFor(2000)
-
-    setTimeout(() => {
-      this.mockWsSource$.next({ id: 1, value: mockPatchData })
-    }, 2000)
-
     return null
   }
 
@@ -166,34 +189,63 @@ export class MockApiService extends ApiService {
     return null
   }
 
-  // server
+  // diagnostic
 
-  async echo(params: RR.EchoReq, url?: string): Promise {
-    if (url) {
-      const num = Math.floor(Math.random() * 10) + 1
-      if (num > 8) return params.message
-      throw new Error()
+  async getError(): Promise {
+    await pauseFor(1000)
+    return {
+      code: 15,
+      message: 'Unknown server',
+      data: { details: 'Some details about the error here' },
     }
+  }
+
+  async diagnosticGetError(): Promise {
+    await pauseFor(1000)
+    return {
+      code: 15,
+      message: 'Unknown server',
+      data: { details: 'Some details about the error here' },
+    }
+  }
+
+  async diagnosticRestart(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticForgetDrive(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticRepairDisk(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise {
+    return this.getServerLogs(params)
+  }
+
+  // init
+
+  async initGetProgress(): Promise {
+    await pauseFor(250)
+    return {
+      progress: PROGRESS,
+      guid: 'init-progress-guid',
+    }
+  }
+
+  async initFollowLogs(): Promise {
     await pauseFor(2000)
-    return params.message
+    return {
+      startCursor: 'start-cursor',
+      guid: 'logs-guid',
+    }
   }
 
-  openPatchWebsocket$(): Observable> {
-    return this.mockWsSource$.pipe(
-      shareReplay({ bufferSize: 1, refCount: true }),
-    )
-  }
-
-  openLogsWebsocket$(config: WebSocketSubjectConfig): Observable {
-    return interval(50).pipe(
-      map((_, index) => {
-        // mock fire open observer
-        if (index === 0) config.openObserver?.next(new Event(''))
-        if (index === 100) throw new Error('HAAHHA')
-        return Mock.ServerLogs[0]
-      }),
-    )
-  }
+  // server
 
   async getSystemTime(
     params: RR.GetSystemTimeReq,
@@ -248,7 +300,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -258,7 +310,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -268,11 +320,11 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
-  randomLogs(limit = 1): Log[] {
+  private randomLogs(limit = 1): Log[] {
     const arrLength = Math.ceil(limit / Mock.ServerLogs.length)
     const logs = new Array(arrLength)
       .fill(Mock.ServerLogs)
@@ -374,12 +426,6 @@ export class MockApiService extends ApiService {
     return null
   }
 
-  async systemRebuild(
-    params: RR.SystemRebuildReq,
-  ): Promise {
-    return this.restartServer(params)
-  }
-
   async repairDisk(params: RR.RestartServerReq): Promise {
     await pauseFor(2000)
     return null
@@ -422,7 +468,7 @@ export class MockApiService extends ApiService {
     }
   }
 
-  async getEos(): Promise {
+  async checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise {
     await pauseFor(2000)
     return Mock.MarketplaceEos
   }
@@ -641,13 +687,13 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     let entries
     if (Math.random() < 0.2) {
-      entries = Mock.PackageLogs
+      entries = Mock.ServerLogs
     } else {
       const arrLength = params.limit
-        ? Math.ceil(params.limit / Mock.PackageLogs.length)
+        ? Math.ceil(params.limit / Mock.ServerLogs.length)
         : 10
       entries = new Array(arrLength)
-        .fill(Mock.PackageLogs)
+        .fill(Mock.ServerLogs)
         .reduce((acc, val) => acc.concat(val), [])
     }
     return {
@@ -663,7 +709,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -673,7 +719,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
 
     setTimeout(async () => {
-      this.updateProgress(params.id)
+      this.installProgress(params.id)
     }, 1000)
 
     const patch: Operation<
@@ -745,7 +791,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     const patch: Operation[] = params.ids.map(id => {
       setTimeout(async () => {
-        this.updateProgress(id)
+        this.installProgress(id)
       }, 2000)
 
       return {
@@ -1013,7 +1059,57 @@ export class MockApiService extends ApiService {
     return '4120e092-05ab-4de2-9fbd-c3f1f4b1df9e' // no significance, randomly generated
   }
 
-  private async updateProgress(id: string): Promise {
+  private async initProgress(): Promise {
+    const progress = JSON.parse(JSON.stringify(PROGRESS))
+
+    for (let [i, phase] of progress.phases.entries()) {
+      if (
+        !phase.progress ||
+        typeof phase.progress !== 'object' ||
+        !phase.progress.total
+      ) {
+        await pauseFor(2000)
+
+        progress.phases[i].progress = true
+
+        if (
+          progress.overall &&
+          typeof progress.overall === 'object' &&
+          progress.overall.total
+        ) {
+          const step = progress.overall.total / progress.phases.length
+          progress.overall.done += step
+        }
+      } else {
+        const step = phase.progress.total / 4
+
+        while (phase.progress.done < phase.progress.total) {
+          await pauseFor(200)
+
+          phase.progress.done += step
+
+          if (
+            progress.overall &&
+            typeof progress.overall === 'object' &&
+            progress.overall.total
+          ) {
+            const step = progress.overall.total / progress.phases.length / 4
+
+            progress.overall.done += step
+          }
+
+          if (phase.progress.done === phase.progress.total) {
+            await pauseFor(250)
+
+            progress.phases[i].progress = true
+          }
+        }
+      }
+    }
+    return progress
+  }
+
+  private async installProgress(id: string): Promise {
     const progress = JSON.parse(JSON.stringify(PROGRESS))
 
     for (let [i, phase] of progress.phases.entries()) {
@@ -1194,10 +1290,6 @@ export class MockApiService extends ApiService {
   }
 
   private async mockRevision(patch: Operation[]): Promise {
-    if (!this.sequence) {
-      const { sequence } = this.bootstrapper.init()
-      this.sequence = sequence
-    }
     const revision = {
       id: ++this.sequence,
       patch,
diff --git a/web/projects/ui/src/app/services/auth.service.ts b/web/projects/ui/src/app/services/auth.service.ts
index 5d755aa98..9c16d0e26 100644
--- a/web/projects/ui/src/app/services/auth.service.ts
+++ b/web/projects/ui/src/app/services/auth.service.ts
@@ -12,7 +12,7 @@ export enum AuthState {
   providedIn: 'root',
 })
 export class AuthService {
-  private readonly LOGGED_IN_KEY = 'loggedInKey'
+  private readonly LOGGED_IN_KEY = 'loggedIn'
   private readonly authState$ = new ReplaySubject(1)
 
   readonly isVerified$ = this.authState$.pipe(
diff --git a/web/projects/ui/src/app/services/connection.service.ts b/web/projects/ui/src/app/services/connection.service.ts
index a45d5ec4c..7d3328503 100644
--- a/web/projects/ui/src/app/services/connection.service.ts
+++ b/web/projects/ui/src/app/services/connection.service.ts
@@ -1,25 +1,23 @@
-import { Injectable } from '@angular/core'
-import { combineLatest, fromEvent, merge, ReplaySubject } from 'rxjs'
-import { distinctUntilChanged, map, startWith } from 'rxjs/operators'
+import { inject, Injectable } from '@angular/core'
+import { combineLatest, Observable, shareReplay } from 'rxjs'
+import { distinctUntilChanged, map } from 'rxjs/operators'
+import { NetworkService } from 'src/app/services/network.service'
+import { StateService } from 'src/app/services/state.service'
 
 @Injectable({
   providedIn: 'root',
 })
-export class ConnectionService {
-  readonly networkConnected$ = merge(
-    fromEvent(window, 'online'),
-    fromEvent(window, 'offline'),
-  ).pipe(
-    startWith(null),
-    map(() => navigator.onLine),
-    distinctUntilChanged(),
-  )
-  readonly websocketConnected$ = new ReplaySubject(1)
-  readonly connected$ = combineLatest([
-    this.networkConnected$,
-    this.websocketConnected$.pipe(distinctUntilChanged()),
+export class ConnectionService extends Observable {
+  private readonly stream$ = combineLatest([
+    inject(NetworkService),
+    inject(StateService).pipe(map(Boolean)),
   ]).pipe(
     map(([network, websocket]) => network && websocket),
     distinctUntilChanged(),
+    shareReplay(1),
   )
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
 }
diff --git a/web/projects/ui/src/app/services/eos.service.ts b/web/projects/ui/src/app/services/eos.service.ts
index dcd1d0de0..81b97bf11 100644
--- a/web/projects/ui/src/app/services/eos.service.ts
+++ b/web/projects/ui/src/app/services/eos.service.ts
@@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'
 import { Emver } from '@start9labs/shared'
 import { BehaviorSubject, combineLatest } from 'rxjs'
 import { distinctUntilChanged, map } from 'rxjs/operators'
-import { MarketplaceEOS } from 'src/app/services/api/api.types'
+import { OSUpdate } from 'src/app/services/api/api.types'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { PatchDB } from 'patch-db-client'
 import { getServerInfo } from 'src/app/util/get-server-info'
@@ -12,7 +12,7 @@ import { DataModel } from './patch-db/data-model'
   providedIn: 'root',
 })
 export class EOSService {
-  eos?: MarketplaceEOS
+  osUpdate?: OSUpdate
   updateAvailable$ = new BehaviorSubject(false)
 
   readonly updating$ = this.patch.watch$('serverInfo', 'statusInfo').pipe(
@@ -52,9 +52,10 @@ export class EOSService {
   ) {}
 
   async loadEos(): Promise {
-    const { version } = await getServerInfo(this.patch)
-    this.eos = await this.api.getEos()
-    const updateAvailable = this.emver.compare(this.eos.version, version) === 1
+    const { version, id } = await getServerInfo(this.patch)
+    this.osUpdate = await this.api.checkOSUpdate({ serverId: id })
+    const updateAvailable =
+      this.emver.compare(this.osUpdate.version, version) === 1
     this.updateAvailable$.next(updateAvailable)
   }
 }
diff --git a/web/projects/ui/src/app/services/network.service.ts b/web/projects/ui/src/app/services/network.service.ts
new file mode 100644
index 000000000..e1568603d
--- /dev/null
+++ b/web/projects/ui/src/app/services/network.service.ts
@@ -0,0 +1,22 @@
+import { inject, Injectable } from '@angular/core'
+import { WINDOW } from '@ng-web-apis/common'
+import { fromEvent, merge, Observable, shareReplay } from 'rxjs'
+import { distinctUntilChanged, map, startWith } from 'rxjs/operators'
+
+@Injectable({ providedIn: 'root' })
+export class NetworkService extends Observable {
+  private readonly win = inject(WINDOW)
+  private readonly stream$ = merge(
+    fromEvent(this.win, 'online'),
+    fromEvent(this.win, 'offline'),
+  ).pipe(
+    startWith(null),
+    map(() => this.win.navigator.onLine),
+    distinctUntilChanged(),
+    shareReplay(1),
+  )
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
+}
diff --git a/web/projects/ui/src/app/services/patch-data.service.ts b/web/projects/ui/src/app/services/patch-data.service.ts
index 5372a665c..50023c1f1 100644
--- a/web/projects/ui/src/app/services/patch-data.service.ts
+++ b/web/projects/ui/src/app/services/patch-data.service.ts
@@ -1,9 +1,9 @@
 import { Inject, Injectable } from '@angular/core'
 import { ModalController } from '@ionic/angular'
 import { Observable } from 'rxjs'
-import { filter, share, switchMap, take, tap } from 'rxjs/operators'
+import { filter, map, share, switchMap, take, tap } from 'rxjs/operators'
 import { PatchDB } from 'patch-db-client'
-import { DataModel, UIData } from 'src/app/services/patch-db/data-model'
+import { DataModel } from 'src/app/services/patch-db/data-model'
 import { EOSService } from 'src/app/services/eos.service'
 import { OSWelcomePage } from 'src/app/modals/os-welcome/os-welcome.page'
 import { ConfigService } from 'src/app/services/config.service'
@@ -11,21 +11,25 @@ import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { MarketplaceService } from 'src/app/services/marketplace.service'
 import { AbstractMarketplaceService } from '@start9labs/marketplace'
 import { ConnectionService } from 'src/app/services/connection.service'
+import { LocalStorageBootstrap } from './patch-db/local-storage-bootstrap'
 
 // Get data from PatchDb after is starts and act upon it
 @Injectable({
   providedIn: 'root',
 })
-export class PatchDataService extends Observable {
-  private readonly stream$ = this.connectionService.connected$.pipe(
+export class PatchDataService extends Observable {
+  private readonly stream$ = this.connection$.pipe(
     filter(Boolean),
     switchMap(() => this.patch.watch$()),
-    take(1),
-    tap(({ ui }) => {
-      // check for updates to eOS and services
-      this.checkForUpdates()
-      // show eos welcome message
-      this.showEosWelcome(ui.ackWelcome)
+    map((cache, index) => {
+      this.bootstrapper.update(cache)
+
+      if (index === 0) {
+        // check for updates to StartOS and services
+        this.checkForUpdates()
+        // show eos welcome message
+        this.showEosWelcome(cache.ui.ackWelcome)
+      }
     }),
     share(),
   )
@@ -38,7 +42,8 @@ export class PatchDataService extends Observable {
     private readonly embassyApi: ApiService,
     @Inject(AbstractMarketplaceService)
     private readonly marketplaceService: MarketplaceService,
-    private readonly connectionService: ConnectionService,
+    private readonly connection$: ConnectionService,
+    private readonly bootstrapper: LocalStorageBootstrap,
   ) {
     super(subscriber => this.stream$.subscribe(subscriber))
   }
diff --git a/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts b/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
index 2ea5bef02..079def855 100644
--- a/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
+++ b/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
@@ -1,4 +1,4 @@
-import { Bootstrapper, DBCache } from 'patch-db-client'
+import { Dump } from 'patch-db-client'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { Injectable } from '@angular/core'
 import { StorageService } from '../storage.service'
@@ -6,20 +6,18 @@ import { StorageService } from '../storage.service'
 @Injectable({
   providedIn: 'root',
 })
-export class LocalStorageBootstrap implements Bootstrapper {
-  static CONTENT_KEY = 'patch-db-cache'
+export class LocalStorageBootstrap {
+  static CONTENT_KEY = 'patchDB'
 
   constructor(private readonly storage: StorageService) {}
 
-  init(): DBCache {
-    const cache = this.storage.get>(
-      LocalStorageBootstrap.CONTENT_KEY,
-    )
+  init(): Dump {
+    const cache = this.storage.get(LocalStorageBootstrap.CONTENT_KEY)
 
-    return cache || { sequence: 0, data: {} as DataModel }
+    return cache ? { id: 1, value: cache } : { id: 0, value: {} as DataModel }
   }
 
-  update(cache: DBCache): void {
+  update(cache: DataModel): void {
     this.storage.set(LocalStorageBootstrap.CONTENT_KEY, cache)
   }
 }
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts b/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
index 51d29edc8..bb2dcdf57 100644
--- a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
+++ b/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
@@ -1,19 +1,19 @@
 import { InjectionToken, Injector } from '@angular/core'
+import { Revision, Update } from 'patch-db-client'
+import { defer, EMPTY, from, Observable } from 'rxjs'
 import {
   bufferTime,
   catchError,
   filter,
+  startWith,
   switchMap,
   take,
-  tap,
 } from 'rxjs/operators'
-import { Update } from 'patch-db-client'
-import { DataModel } from './data-model'
-import { defer, EMPTY, from, interval, Observable } from 'rxjs'
-import { AuthService } from '../auth.service'
-import { ConnectionService } from '../connection.service'
+import { StateService } from 'src/app/services/state.service'
 import { ApiService } from '../api/embassy-api.service'
-import { ConfigService } from '../config.service'
+import { AuthService } from '../auth.service'
+import { DataModel } from './data-model'
+import { LocalStorageBootstrap } from './local-storage-bootstrap'
 
 export const PATCH_SOURCE = new InjectionToken[]>>(
   '',
@@ -25,33 +25,31 @@ export function sourceFactory(
   // defer() needed to avoid circular dependency with ApiService, since PatchDB is needed there
   return defer(() => {
     const api = injector.get(ApiService)
-    const authService = injector.get(AuthService)
-    const connectionService = injector.get(ConnectionService)
-    const configService = injector.get(ConfigService)
-    const isTor = configService.isTor()
-    const timeout = isTor ? 16000 : 4000
+    const auth = injector.get(AuthService)
+    const state = injector.get(StateService)
+    const bootstrapper = injector.get(LocalStorageBootstrap)
 
-    const websocket$ = api.openPatchWebsocket$().pipe(
-      bufferTime(250),
-      filter(updates => !!updates.length),
-      catchError((_, watch$) => {
-        connectionService.websocketConnected$.next(false)
+    return auth.isVerified$.pipe(
+      switchMap(verified =>
+        verified ? from(api.subscribeToPatchDB({})) : EMPTY,
+      ),
+      switchMap(({ dump, guid }) =>
+        api.openWebsocket$(guid, {}).pipe(
+          bufferTime(250),
+          filter(revisions => !!revisions.length),
+          startWith([dump]),
+        ),
+      ),
+      catchError((_, original$) => {
+        state.retrigger()
 
-        return interval(timeout).pipe(
-          switchMap(() =>
-            from(api.echo({ message: 'ping', timeout })).pipe(
-              catchError(() => EMPTY),
-            ),
-          ),
+        return state.pipe(
+          filter(current => current === 'running'),
           take(1),
-          switchMap(() => watch$),
+          switchMap(() => original$),
         )
       }),
-      tap(() => connectionService.websocketConnected$.next(true)),
-    )
-
-    return authService.isVerified$.pipe(
-      switchMap(verified => (verified ? websocket$ : EMPTY)),
+      startWith([bootstrapper.init()]),
     )
   })
 }
diff --git a/web/projects/ui/src/app/services/patch-monitor.service.ts b/web/projects/ui/src/app/services/patch-monitor.service.ts
index cafb9f0fe..675531dda 100644
--- a/web/projects/ui/src/app/services/patch-monitor.service.ts
+++ b/web/projects/ui/src/app/services/patch-monitor.service.ts
@@ -4,24 +4,19 @@ import { tap } from 'rxjs/operators'
 import { PatchDB } from 'patch-db-client'
 import { AuthService } from 'src/app/services/auth.service'
 import { DataModel } from './patch-db/data-model'
-import { LocalStorageBootstrap } from './patch-db/local-storage-bootstrap'
 
 // Start and stop PatchDb upon verification
 @Injectable({
   providedIn: 'root',
 })
-export class PatchMonitorService extends Observable {
-  // @TODO not happy with Observable
+export class PatchMonitorService extends Observable {
   private readonly stream$ = this.authService.isVerified$.pipe(
-    tap(verified =>
-      verified ? this.patch.start(this.bootstrapper) : this.patch.stop(),
-    ),
+    tap(verified => (verified ? this.patch.start() : this.patch.stop())),
   )
 
   constructor(
     private readonly authService: AuthService,
     private readonly patch: PatchDB,
-    private readonly bootstrapper: LocalStorageBootstrap,
   ) {
     super(subscriber => this.stream$.subscribe(subscriber))
   }
diff --git a/web/projects/ui/src/app/services/state.service.ts b/web/projects/ui/src/app/services/state.service.ts
new file mode 100644
index 000000000..33569a751
--- /dev/null
+++ b/web/projects/ui/src/app/services/state.service.ts
@@ -0,0 +1,136 @@
+import { inject, Injectable } from '@angular/core'
+import { CanActivateFn, IsActiveMatchOptions, Router } from '@angular/router'
+import { ALWAYS_TRUE_HANDLER } from '@taiga-ui/cdk'
+import { TuiAlertService, TuiNotification } from '@taiga-ui/core'
+import {
+  BehaviorSubject,
+  combineLatest,
+  concat,
+  EMPTY,
+  exhaustMap,
+  from,
+  merge,
+  Observable,
+  startWith,
+  Subject,
+  timer,
+} from 'rxjs'
+import {
+  catchError,
+  filter,
+  map,
+  shareReplay,
+  skip,
+  switchMap,
+  take,
+  takeUntil,
+  tap,
+} from 'rxjs/operators'
+import { RR } from 'src/app/services/api/api.types'
+import { ApiService } from 'src/app/services/api/embassy-api.service'
+import { NetworkService } from 'src/app/services/network.service'
+
+const OPTIONS: IsActiveMatchOptions = {
+  paths: 'subset',
+  queryParams: 'exact',
+  fragment: 'ignored',
+  matrixParams: 'ignored',
+}
+
+@Injectable({
+  providedIn: 'root',
+})
+export class StateService extends Observable {
+  private readonly alerts = inject(TuiAlertService)
+  private readonly api = inject(ApiService)
+  private readonly router = inject(Router)
+  private readonly network$ = inject(NetworkService)
+
+  private readonly single$ = new Subject()
+
+  private readonly trigger$ = new BehaviorSubject(undefined)
+  private readonly poll$ = this.trigger$.pipe(
+    switchMap(() =>
+      timer(0, 2000).pipe(
+        switchMap(() =>
+          from(this.api.getState()).pipe(catchError(() => EMPTY)),
+        ),
+        take(1),
+      ),
+    ),
+  )
+
+  private readonly stream$ = merge(this.single$, this.poll$).pipe(
+    tap(state => {
+      switch (state) {
+        case 'initializing':
+          this.router.navigate(['initializing'], { replaceUrl: true })
+          break
+        case 'error':
+          this.router.navigate(['diagnostic'], { replaceUrl: true })
+          break
+        case 'running':
+          if (
+            this.router.isActive('initializing', OPTIONS) ||
+            this.router.isActive('diagnostic', OPTIONS)
+          ) {
+            this.router.navigate([''], { replaceUrl: true })
+          }
+
+          break
+      }
+    }),
+    startWith(null),
+    shareReplay(1),
+  )
+
+  private readonly alert = merge(
+    this.trigger$.pipe(skip(1)),
+    this.network$.pipe(filter(v => !v)),
+  )
+    .pipe(
+      exhaustMap(() =>
+        concat(
+          this.alerts
+            .open('Trying to reach server', {
+              label: 'State unknown',
+              autoClose: false,
+              status: TuiNotification.Error,
+            })
+            .pipe(
+              takeUntil(
+                combineLatest([this.stream$, this.network$]).pipe(
+                  filter(state => state.every(Boolean)),
+                ),
+              ),
+            ),
+          this.alerts.open('Connection restored', {
+            label: 'Server reached',
+            status: TuiNotification.Success,
+          }),
+        ),
+      ),
+    )
+    .subscribe() // @TODO shouldn't this be subscribed in app component with the others? Do we ever need to unsubscribe?
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
+
+  retrigger() {
+    this.trigger$.next()
+  }
+
+  async syncState() {
+    const state = await this.api.getState()
+    this.single$.next(state)
+  }
+}
+
+export function stateNot(state: RR.ServerState[]): CanActivateFn {
+  return () =>
+    inject(StateService).pipe(
+      filter(current => !current || !state.includes(current)),
+      map(ALWAYS_TRUE_HANDLER),
+    )
+}
diff --git a/web/projects/ui/src/app/services/storage.service.ts b/web/projects/ui/src/app/services/storage.service.ts
index ec87864b5..e59eba439 100644
--- a/web/projects/ui/src/app/services/storage.service.ts
+++ b/web/projects/ui/src/app/services/storage.service.ts
@@ -1,7 +1,7 @@
 import { Inject, Injectable } from '@angular/core'
 import { DOCUMENT } from '@angular/common'
 
-const PREFIX = '_embassystorage/_embassykv/'
+const PREFIX = '_startos/'
 
 @Injectable({
   providedIn: 'root',
@@ -15,16 +15,21 @@ export class StorageService {
     return JSON.parse(String(this.storage.getItem(`${PREFIX}${key}`)))
   }
 
-  set(key: string, value: T) {
+  set(key: string, value: any) {
     this.storage.setItem(`${PREFIX}${key}`, JSON.stringify(value))
   }
 
   clear() {
-    Array.from(
-      { length: this.storage.length },
-      (_, i) => this.storage.key(i) || '',
-    )
-      .filter(key => key.startsWith(PREFIX))
-      .forEach(key => this.storage.removeItem(key))
+    this.storage.clear()
+  }
+
+  migrate036() {
+    const oldPrefix = '_embassystorage/_embassykv/'
+    if (!!this.storage.getItem(`${oldPrefix}loggedInKey`)) {
+      const cache = this.storage.getItem(`${oldPrefix}patch-db-cache`)
+      this.clear()
+      this.set('loggedIn', true)
+      this.set('patchDB', cache)
+    }
   }
 }