From 9a01a0df8e2bcaa9b7539fd98a812223b69df141 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Thu, 21 Jul 2022 15:18:44 -0600 Subject: [PATCH] refactor build process (#1675) * add nc-broadcast to view initialization.sh logs * include stderr * refactor build * add frontend/config.json as frontend dependency * fix nc-broadcast * always run all workflows * address dependabot alerts * fix build caching * remove registries.json * more efficient build --- .github/workflows/backend-pr.yaml | 12 +-- .github/workflows/frontend-pr.yaml | 7 +- .github/workflows/product.yaml | 5 + .gitignore | 3 + Makefile | 37 +++++-- backend/Cargo.lock | 157 ++++++++++------------------ backend/Cargo.toml | 3 +- backend/build-dev.sh | 3 + backend/build-portable-dev.sh | 3 + backend/build-portable.sh | 3 + backend/build-prod.sh | 4 + build-cargo-dep.sh | 21 ++++ build/initialization.service | 1 + build/initialization.sh | 1 + build/nc-broadcast.service | 13 +++ build/partitioning.sh | 2 +- build/user-data | 86 --------------- build/user-data-dev | 86 --------------- build/write-image.sh | 25 +++-- check-environment.sh | 7 ++ check-git-hash.sh | 9 ++ frontend/build-config.js | 4 - frontend/config-sample.json | 4 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- frontend/projects/ui/postprocess.ts | 15 --- frontend/registries.json | 14 --- system-images/compat/Cargo.lock | 2 +- 28 files changed, 178 insertions(+), 355 deletions(-) create mode 100755 build-cargo-dep.sh create mode 100644 build/nc-broadcast.service delete mode 100755 build/user-data delete mode 100755 build/user-data-dev create mode 100755 check-environment.sh create mode 100755 check-git-hash.sh delete mode 100644 frontend/projects/ui/postprocess.ts delete mode 100644 frontend/registries.json diff --git a/.github/workflows/backend-pr.yaml b/.github/workflows/backend-pr.yaml index bc21a5e69..7719641f3 100644 --- a/.github/workflows/backend-pr.yaml +++ b/.github/workflows/backend-pr.yaml @@ -3,12 +3,6 @@ name: Backend PR on: workflow_call: workflow_dispatch: - pull_request: - branches: - - master - paths: - - 'backend/**' - - 'libs/**' jobs: libs: @@ -44,7 +38,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + libs/target/ key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }} - name: Build v8 snapshot @@ -89,14 +83,14 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + backend/target/ key: ${{ runner.os }}-cargo-backend-${{ hashFiles('backend/Cargo.lock') }} - name: Build backend run: make backend - name: 'Tar files to preserve file permissions' - run: tar -cvf backend.tar backend/target/aarch64-unknown-linux-gnu/release/embassy* + run: tar -cvf backend.tar ENVIRONMENT.txt GIT_HASH.txt backend/target/aarch64-unknown-linux-gnu/release/embassy* - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/frontend-pr.yaml b/.github/workflows/frontend-pr.yaml index eca149f4b..421699727 100644 --- a/.github/workflows/frontend-pr.yaml +++ b/.github/workflows/frontend-pr.yaml @@ -3,11 +3,6 @@ name: Frontend PR on: workflow_call: workflow_dispatch: - pull_request: - branches: - - master - paths: - - 'frontend/**' jobs: frontend: @@ -38,7 +33,7 @@ jobs: run: make frontends - name: 'Tar files to preserve file permissions' - run: tar -cvf frontend.tar frontend/dist frontend/config.json + run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt frontend/dist frontend/config.json - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/product.yaml b/.github/workflows/product.yaml index b8c750f85..d6796af13 100644 --- a/.github/workflows/product.yaml +++ b/.github/workflows/product.yaml @@ -5,6 +5,11 @@ on: push: branches: - master + - next + pull_request: + branches: + - master + - next jobs: compat: diff --git a/.gitignore b/.gitignore index 94d2a6324..977f6819a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ deploy_web.sh deploy_web.sh secrets.db .vscode/ +/cargo-deps/**/* +ENVIRONMENT.txt +GIT_HASH.txt \ No newline at end of file diff --git a/Makefile b/Makefile index a4e16d73d..6d8d5e2ca 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ +ENVIRONMENT_FILE := $(shell ./check-environment.sh) +GIT_HASH_FILE := $(shell ./check-git-hash.sh) EMBASSY_BINS := backend/target/aarch64-unknown-linux-gnu/release/embassyd backend/target/aarch64-unknown-linux-gnu/release/embassy-init backend/target/aarch64-unknown-linux-gnu/release/embassy-cli backend/target/aarch64-unknown-linux-gnu/release/embassy-sdk EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build) COMPAT_SRC := $(shell find system-images/compat/src) UTILS_SRC := $(shell find system-images/utils/Dockerfile) BACKEND_SRC := $(shell find backend/src) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock -FRONTEND_SRC := $(shell find frontend/projects) $(shell find frontend/assets) +FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell find frontend/assets) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist +FRONTEND_UI_SRC := $(shell find frontend/projects/ui) +FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard) +FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui) PATCH_DB_CLIENT_SRC = $(shell find patch-db/client -not -path patch-db/client/dist) -GIT_REFS := $(shell find .git/refs/heads) -TMP_FILE := $(shell mktemp) .DELETE_ON_ERROR: @@ -27,11 +30,12 @@ clean: rm -rf frontend/dist rm -rf patch-db/client/node_modules rm -rf patch-db/client/dist + sudo rm -rf cargo-deps sdk: cd backend/ && ./install-sdk.sh -eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar +eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) ! test -f eos.img || rm eos.img if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi @@ -57,31 +61,42 @@ snapshots: libs/snapshot-creator/Cargo.toml cd libs/ && ./build-v8-snapshot.sh cd libs/ && ./build-arm-v8-snapshot.sh -$(EMBASSY_BINS): $(BACKEND_SRC) +$(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) cd backend && ./build-prod.sh + touch $(EMBASSY_BINS) frontend/node_modules: frontend/package.json npm --prefix frontend ci -$(EMBASSY_UIS): $(FRONTEND_SRC) frontend/node_modules patch-db/client patch-db/client/dist frontend/config.json - npm --prefix frontend run build:all +frontend/dist/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE) + npm --prefix frontend run build:ui -frontend/config.json: .git/HEAD $(GIT_REFS) +frontend/dist/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE) + npm --prefix frontend run build:setup-wizard + +frontend/dist/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE) + npm --prefix frontend run build:diagnostic-ui + +frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json jq '.useMocks = false' frontend/config-sample.json > frontend/config.json npm --prefix frontend run-script build-config patch-db/client/node_modules: patch-db/client/package.json - npm --prefix patch-db/client install + npm --prefix patch-db/client ci patch-db/client/dist: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules ! test -d patch-db/client/dist || rm -rf patch-db/client/dist + rm -rf frontend/.angular/cache npm --prefix patch-db/client run build # this is a convenience step to build all frontends - it is not referenced elsewhere in this file -frontends: frontend/node_modules frontend/config.json $(EMBASSY_UIS) +frontends: $(EMBASSY_UIS) # this is a convenience step to build the UI -ui: frontend/node_modules frontend/config.json frontend/dist/ui +ui: frontend/dist/ui # this is a convenience step to build the backend backend: $(EMBASSY_BINS) + +cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast: + ./build-cargo-dep.sh nc-broadcast \ No newline at end of file diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0c69bfa3b..f96fac727 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -369,7 +369,7 @@ checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5" dependencies = [ "base64 0.13.0", "bollard-stubs", - "bytes 1.1.0", + "bytes", "futures-core", "futures-util", "hex", @@ -377,13 +377,13 @@ dependencies = [ "hyper", "hyperlocal", "log", - "pin-project-lite 0.2.9", + "pin-project-lite", "serde", "serde_derive", "serde_json", "serde_urlencoded", "thiserror", - "tokio 1.19.2", + "tokio", "tokio-util", "url", "winapi", @@ -423,12 +423,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.1.0" @@ -1115,7 +1109,7 @@ dependencies = [ "libc", "log", "models", - "nix 0.24.1", + "nix 0.24.2", "nom 7.1.1", "num", "num_enum", @@ -1148,8 +1142,7 @@ dependencies = [ "stderrlog", "tar", "thiserror", - "tokio 1.19.2", - "tokio-compat-02", + "tokio", "tokio-stream", "tokio-tar", "tokio-tungstenite", @@ -1284,11 +1277,11 @@ dependencies = [ [[package]] name = "fd-lock-rs" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a24e88458a5abfcd1cb81622511306f1aea43b900ddb9b34ff5ad8857a7685" +checksum = "ef0f547e1d79e058664f2ea7d3a6d82b2ddd5fea4a6650b97b70c38979f34db3" dependencies = [ - "nix 0.19.1", + "nix 0.24.2", ] [[package]] @@ -1484,7 +1477,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.9", + "pin-project-lite", "pin-utils", "slab", ] @@ -1561,7 +1554,7 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ - "bytes 1.1.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -1569,7 +1562,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.19.2", + "tokio", "tokio-util", "tracing", ] @@ -1628,7 +1621,7 @@ version = "0.1.0" dependencies = [ "models", "pin-project", - "tokio 1.19.2", + "tokio", ] [[package]] @@ -1671,7 +1664,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ - "bytes 1.1.0", + "bytes", "fnv", "itoa 1.0.2", ] @@ -1682,9 +1675,9 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.1.0", + "bytes", "http", - "pin-project-lite 0.2.9", + "pin-project-lite", ] [[package]] @@ -1720,7 +1713,7 @@ version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ - "bytes 1.1.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -1730,9 +1723,9 @@ dependencies = [ "httparse", "httpdate", "itoa 1.0.2", - "pin-project-lite 0.2.9", + "pin-project-lite", "socket2", - "tokio 1.19.2", + "tokio", "tower-service", "tracing", "want", @@ -1744,10 +1737,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.1.0", + "bytes", "hyper", "native-tls", - "tokio 1.19.2", + "tokio", "tokio-native-tls", ] @@ -1764,7 +1757,7 @@ dependencies = [ "hyper", "log", "sha-1", - "tokio 1.19.2", + "tokio", "tokio-tungstenite", ] @@ -1778,7 +1771,7 @@ dependencies = [ "hex", "hyper", "pin-project", - "tokio 1.19.2", + "tokio", ] [[package]] @@ -1969,7 +1962,7 @@ dependencies = [ "swc_macros_common", "swc_visit", "swc_visit_macros", - "tokio 1.19.2", + "tokio", "tracing", ] @@ -2309,18 +2302,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "nix" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" -dependencies = [ - "bitflags", - "cc", - "cfg-if 1.0.0", - "libc", -] - [[package]] name = "nix" version = "0.23.1" @@ -2336,9 +2317,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -2679,7 +2660,7 @@ dependencies = [ "serde_cbor 0.11.1", "serde_json", "thiserror", - "tokio 1.19.2", + "tokio", "tracing", "tracing-error 0.1.2", ] @@ -2816,12 +2797,6 @@ dependencies = [ "syn 1.0.96", ] -[[package]] -name = "pin-project-lite" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -3184,7 +3159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64 0.13.0", - "bytes 1.1.0", + "bytes", "cookie", "cookie_store", "encoding_rs", @@ -3202,12 +3177,12 @@ dependencies = [ "mime", "native-tls", "percent-encoding", - "pin-project-lite 0.2.9", + "pin-project-lite", "proc-macro-hack", "serde", "serde_json", "serde_urlencoded", - "tokio 1.19.2", + "tokio", "tokio-native-tls", "tokio-socks", "tokio-util", @@ -3225,7 +3200,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc7d9a4063adcdfbb061aa3011b813cce95d3a8990f9ccb5ea85017916a1b0b" dependencies = [ - "bytes 1.1.0", + "bytes", "cookie", "cookie_store", "reqwest", @@ -3276,7 +3251,7 @@ dependencies = [ "serde_cbor 0.11.2", "serde_json", "thiserror", - "tokio 1.19.2", + "tokio", "url", "yajrc", ] @@ -3555,7 +3530,7 @@ version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ca1daa2506c9d62744fff84d3534192f2e1c70cdf3bed95f298d89156c00b06" dependencies = [ - "bytes 1.1.0", + "bytes", "derive_more", "serde", "v8", @@ -3794,7 +3769,7 @@ dependencies = [ "atoi", "bitflags", "byteorder", - "bytes 1.1.0", + "bytes", "chrono", "crc", "crossbeam-queue", @@ -3860,7 +3835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "874e93a365a598dc3dadb197565952cb143ae4aa716f7bcc933a8d836f6bf89f" dependencies = [ "once_cell", - "tokio 1.19.2", + "tokio", "tokio-rustls", ] @@ -4516,52 +4491,26 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" -[[package]] -name = "tokio" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "num_cpus", - "pin-project-lite 0.1.12", - "slab", -] - [[package]] name = "tokio" version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ - "bytes 1.1.0", + "bytes", "libc", "memchr", "mio", "num_cpus", "once_cell", "parking_lot 0.12.1", - "pin-project-lite 0.2.9", + "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "winapi", ] -[[package]] -name = "tokio-compat-02" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d4237822b7be8fff0a7a27927462fad435dcb6650f95cea9e946bf6bdc7e07" -dependencies = [ - "bytes 0.5.6", - "once_cell", - "pin-project-lite 0.2.9", - "tokio 0.2.25", - "tokio 1.19.2", - "tokio-stream", -] - [[package]] name = "tokio-macros" version = "1.8.0" @@ -4580,7 +4529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.19.2", + "tokio", ] [[package]] @@ -4590,7 +4539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", - "tokio 1.19.2", + "tokio", "webpki", ] @@ -4603,7 +4552,7 @@ dependencies = [ "either", "futures-util", "thiserror", - "tokio 1.19.2", + "tokio", ] [[package]] @@ -4613,8 +4562,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ "futures-core", - "pin-project-lite 0.2.9", - "tokio 1.19.2", + "pin-project-lite", + "tokio", "tokio-util", ] @@ -4627,7 +4576,7 @@ dependencies = [ "futures-core", "libc", "redox_syscall 0.2.13", - "tokio 1.19.2", + "tokio", "tokio-stream", "xattr", ] @@ -4640,7 +4589,7 @@ checksum = "06cda1232a49558c46f8a504d5b93101d42c0bf7f911f12a105ba48168f821ae" dependencies = [ "futures-util", "log", - "tokio 1.19.2", + "tokio", "tungstenite", ] @@ -4650,11 +4599,11 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ - "bytes 1.1.0", + "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.9", - "tokio 1.19.2", + "pin-project-lite", + "tokio", "tracing", ] @@ -4684,7 +4633,7 @@ dependencies = [ "serde_derive", "sha2 0.9.9", "sha3", - "tokio 1.19.2", + "tokio", ] [[package]] @@ -4700,7 +4649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if 1.0.0", - "pin-project-lite 0.2.9", + "pin-project-lite", "tracing-attributes", "tracing-core", ] @@ -4821,7 +4770,7 @@ dependencies = [ "rand 0.8.5", "thiserror", "time 0.3.11", - "tokio 1.19.2", + "tokio", "trust-dns-proto", ] @@ -4846,7 +4795,7 @@ dependencies = [ "smallvec", "thiserror", "tinyvec", - "tokio 1.19.2", + "tokio", "url", ] @@ -4857,7 +4806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a395a2e0fd8aac9b4613767a5b4ba4b2040de1b767fa03ace8c9d6f351d60b2d" dependencies = [ "async-trait", - "bytes 1.1.0", + "bytes", "cfg-if 1.0.0", "enum-as-inner", "env_logger 0.9.0", @@ -4867,7 +4816,7 @@ dependencies = [ "serde", "thiserror", "time 0.3.11", - "tokio 1.19.2", + "tokio", "toml", "trust-dns-client", "trust-dns-proto", @@ -4887,7 +4836,7 @@ checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" dependencies = [ "base64 0.13.0", "byteorder", - "bytes 1.1.0", + "bytes", "http", "httparse", "log", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 836f6b1e0..87d1d2a19 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -67,7 +67,7 @@ divrem = "1.0.0" ed25519 = { version = "1.5.2", features = ["pkcs8", "pem", "alloc"] } ed25519-dalek = { version = "1.0.1", features = ["serde"] } emver = { version = "0.1.6", features = ["serde"] } -fd-lock-rs = "0.1.3" +fd-lock-rs = "0.1.4" futures = "0.3.21" git-version = "0.3.5" helpers = { path = "../libs/helpers" } @@ -129,7 +129,6 @@ stderrlog = "0.5.3" tar = "0.4.38" thiserror = "1.0.31" tokio = { version = "1.19.2", features = ["full"] } -tokio-compat-02 = "0.2.0" tokio-stream = { version = "0.1.9", features = ["io-util", "sync"] } tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" } tokio-tungstenite = "0.17.1" diff --git a/backend/build-dev.sh b/backend/build-dev.sh index dcec8a6a9..479244184 100755 --- a/backend/build-dev.sh +++ b/backend/build-dev.sh @@ -18,4 +18,7 @@ alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry": cd .. rust-arm64-builder sh -c "(cd backend && cargo build)" cd backend + +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/build-portable-dev.sh b/backend/build-portable-dev.sh index e2161e226..f2c792af9 100755 --- a/backend/build-portable-dev.sh +++ b/backend/build-portable-dev.sh @@ -18,3 +18,6 @@ alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features)" cd backend + +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/backend/build-portable.sh b/backend/build-portable.sh index 9f3cc9abf..4c04520bf 100755 --- a/backend/build-portable.sh +++ b/backend/build-portable.sh @@ -18,3 +18,6 @@ alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features)" cd backend + +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/backend/build-prod.sh b/backend/build-prod.sh index 0eca69eed..7fe706ee2 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -30,4 +30,8 @@ else rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS)" fi cd backend + +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo + #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/build-cargo-dep.sh b/build-cargo-dep.sh new file mode 100755 index 000000000..08fad4799 --- /dev/null +++ b/build-cargo-dep.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +shopt -s expand_aliases + +if [ "$0" != "./build-cargo-dep.sh" ]; then + >&2 echo "Must be run from embassy-os directory" + exit 1 +fi + +USE_TTY= +if tty -s; then + USE_TTY="-it" +fi + +mkdir -p cargo-deps +alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64' + +rust-arm64-builder cargo install "$1" --target-dir /home/rust/src +sudo chown -R $USER cargo-deps +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/build/initialization.service b/build/initialization.service index 35c654370..f4514e4bf 100644 --- a/build/initialization.service +++ b/build/initialization.service @@ -8,6 +8,7 @@ Restart=on-failure RestartSec=5s ExecStart=/usr/local/bin/initialization.sh RemainAfterExit=true +StandardOutput=append:/var/log/initialization.log [Install] WantedBy=multi-user.target diff --git a/build/initialization.sh b/build/initialization.sh index 0ccc73833..61486c2da 100755 --- a/build/initialization.sh +++ b/build/initialization.sh @@ -127,6 +127,7 @@ sed -i 's/^/usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u /g' /boot/cmd # making that *sudo docker stats* command fulfil its purpose by displaying all metrics sed -i 's/rootwait quiet.*/rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory quiet/g' /boot/cmdline.txt +systemctl disable nc-broadcast.service systemctl disable initialization.service sudo systemctl restart NetworkManager diff --git a/build/nc-broadcast.service b/build/nc-broadcast.service new file mode 100644 index 000000000..3c6b09eca --- /dev/null +++ b/build/nc-broadcast.service @@ -0,0 +1,13 @@ +[Unit] +Description=Writes initialization logs to network +Requires=initialization.service + +[Service] +Type=oneshot +Restart=on-failure +RestartSec=5s +ExecStart=/usr/local/bin/nc-broadcast --input=/var/log/initialization.log --tee 0.0.0.0:8080 +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/build/partitioning.sh b/build/partitioning.sh index 8efd63d7a..2c86ea6f3 100755 --- a/build/partitioning.sh +++ b/build/partitioning.sh @@ -3,4 +3,4 @@ set -e # Use fdisk to create DOS partition table with 4 primary partitions, set 1 as bootable, write, and quite -(echo o; echo x; echo i; echo "0xcb15ae4d"; echo r; echo n; echo p; echo 1; echo 2048; echo 526335; echo t; echo c; echo n; echo p; echo 2; echo 526336; echo 1050623; echo t; echo 2; echo c; echo n; echo p; echo 3; echo 1050624; echo 16083455; echo n; echo p; echo 16083456; echo 31116287; echo a; echo 1; echo w) | sudo fdisk ${OUTPUT_DEVICE} +(echo o; echo x; echo i; echo "0xcb15ae4d"; echo r; echo n; echo p; echo 1; echo 2048; echo 526335; echo t; echo c; echo n; echo p; echo 2; echo 526336; echo 1050623; echo t; echo 2; echo c; echo n; echo p; echo 3; echo 1050624; echo 16083455; echo n; echo p; echo 16083456; echo 31116287; echo a; echo 1; echo w) | sudo fdisk ${OUTPUT_DEVICE} > /dev/null diff --git a/build/user-data b/build/user-data deleted file mode 100755 index 2d5018b25..000000000 --- a/build/user-data +++ /dev/null @@ -1,86 +0,0 @@ -#cloud-config - -# This is the user-data configuration file for cloud-init. By default this sets -# up an initial user called "ubuntu" with password "ubuntu", which must be -# changed at first login. However, many additional actions can be initiated on -# first boot from this file. The cloud-init documentation has more details: -# -# https://cloudinit.readthedocs.io/ -# -# Please note that the YAML format employed by this file is sensitive to -# differences in whitespace; if you are editing this file in an editor (like -# Notepad) which uses literal tabs, take care to only use spaces for -# indentation. See the following link for more details: -# -# https://en.wikipedia.org/wiki/YAML -# -# Some additional examples are provided in comments below the default -# configuration. - -# On first boot, set the (default) ubuntu user's password to "ubuntu" and -# expire user passwords -#chpasswd: -# expire: true -# list: -# - ubuntu:ubuntu - -# Enable password authentication with the SSH daemon -#ssh_pwauth: true - -## On first boot, use ssh-import-id to give the specific users SSH access to -## the default user -#ssh_import_id: -#- lp:my_launchpad_username -#- gh:my_github_username - -## Add users and groups to the system, and import keys with the ssh-import-id -## utility -#groups: -#- robot: [robot] -#- robotics: [robot] -#- pi -# -#users: -#- default -#- name: robot -# gecos: Mr. Robot -# primary_group: robot -# groups: users -# ssh_import_id: foobar -# lock_passwd: false -# passwd: $5$hkui88$nvZgIle31cNpryjRfO9uArF7DYiBcWEnjqq7L1AQNN3 - -## Update apt database and upgrade packages on first boot -#package_update: true -#package_upgrade: true - -## Install additional packages on first boot -#packages: -#- pwgen -#- pastebinit -#- [libpython2.7, 2.7.3-0ubuntu3.1] - -## Write arbitrary files to the file-system (including binaries!) -#write_files: -#- path: /etc/default/keyboard -# content: | -# # KEYBOARD configuration file -# # Consult the keyboard(5) manual page. -# XKBMODEL="pc105" -# XKBLAYOUT="gb" -# XKBVARIANT="" -# XKBOPTIONS="ctrl: nocaps" -# permissions: '0644' -# owner: root:root -#- encoding: gzip -# path: /usr/bin/hello -# content: !!binary | -# H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= -# owner: root:root -# permissions: '0755' - -## Run arbitrary commands at rc.local like time -#runcmd: -#- [ ls, -l, / ] -#- [ sh, -xc, "echo $(date) ': hello world!'" ] -#- [ wget, "http://ubuntu.com", -O, /run/mydir/index.html ] diff --git a/build/user-data-dev b/build/user-data-dev deleted file mode 100755 index 828f39ee3..000000000 --- a/build/user-data-dev +++ /dev/null @@ -1,86 +0,0 @@ -#cloud-config - -# This is the user-data configuration file for cloud-init. By default this sets -# up an initial user called "ubuntu" with password "ubuntu", which must be -# changed at first login. However, many additional actions can be initiated on -# first boot from this file. The cloud-init documentation has more details: -# -# https://cloudinit.readthedocs.io/ -# -# Please note that the YAML format employed by this file is sensitive to -# differences in whitespace; if you are editing this file in an editor (like -# Notepad) which uses literal tabs, take care to only use spaces for -# indentation. See the following link for more details: -# -# https://en.wikipedia.org/wiki/YAML -# -# Some additional examples are provided in comments below the default -# configuration. - -# On first boot, set the (default) ubuntu user's password to "ubuntu" and -# expire user passwords -chpasswd: - expire: true - list: - - ubuntu:ubuntu - -# Enable password authentication with the SSH daemon -ssh_pwauth: true - -## On first boot, use ssh-import-id to give the specific users SSH access to -## the default user -#ssh_import_id: -#- lp:my_launchpad_username -#- gh:my_github_username - -## Add users and groups to the system, and import keys with the ssh-import-id -## utility -#groups: -#- robot: [robot] -#- robotics: [robot] -#- pi -# -#users: -#- default -#- name: robot -# gecos: Mr. Robot -# primary_group: robot -# groups: users -# ssh_import_id: foobar -# lock_passwd: false -# passwd: $5$hkui88$nvZgIle31cNpryjRfO9uArF7DYiBcWEnjqq7L1AQNN3 - -## Update apt database and upgrade packages on first boot -#package_update: true -#package_upgrade: true - -## Install additional packages on first boot -#packages: -#- pwgen -#- pastebinit -#- [libpython2.7, 2.7.3-0ubuntu3.1] - -## Write arbitrary files to the file-system (including binaries!) -#write_files: -#- path: /etc/default/keyboard -# content: | -# # KEYBOARD configuration file -# # Consult the keyboard(5) manual page. -# XKBMODEL="pc105" -# XKBLAYOUT="gb" -# XKBVARIANT="" -# XKBOPTIONS="ctrl: nocaps" -# permissions: '0644' -# owner: root:root -#- encoding: gzip -# path: /usr/bin/hello -# content: !!binary | -# H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= -# owner: root:root -# permissions: '0755' - -## Run arbitrary commands at rc.local like time -#runcmd: -#- [ ls, -l, / ] -#- [ sh, -xc, "echo $(date) ': hello world!'" ] -#- [ wget, "http://ubuntu.com", -O, /run/mydir/index.html ] diff --git a/build/write-image.sh b/build/write-image.sh index 415d56000..9f67e57b6 100755 --- a/build/write-image.sh +++ b/build/write-image.sh @@ -11,7 +11,7 @@ function partition_for () { } # Write contents of LOOPDEV (Ubuntu image) to sd card and make filesystems, then detach the loop device -echo USING $LOOPDEV TO IMAGE $OUTPUT_DEVICE +echo USING $LOOPDEV TO IMAGE $OUTPUT_DEVICE WITH ENVIRONMENT $ENVIRONMENT sudo dd if=${LOOPDEV}p1 of=`partition_for ${OUTPUT_DEVICE} 1` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress sudo mkfs.vfat -F 32 `partition_for ${OUTPUT_DEVICE} 2` sudo dd if=${LOOPDEV}p2 of=`partition_for ${OUTPUT_DEVICE} 3` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress @@ -29,17 +29,11 @@ sudo e2label `partition_for ${OUTPUT_DEVICE} 4` blue mkdir -p /tmp/eos-mnt sudo mount `partition_for ${OUTPUT_DEVICE} 1` /tmp/eos-mnt -if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then - sudo cp build/user-data-dev /tmp/eos-mnt/user-data -else - sudo cp build/user-data /tmp/eos-mnt/user-data -fi - sudo sed -i 's/PARTUUID=cb15ae4d-02/PARTUUID=cb15ae4d-03/g' /tmp/eos-mnt/cmdline.txt sudo sed -i 's/ init=\/usr\/lib\/raspi-config\/init_resize.sh//g' /tmp/eos-mnt/cmdline.txt -cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp -echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp +cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp > /dev/null +echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null sudo mv /tmp/eos-mnt/config.txt.tmp /tmp/eos-mnt/config.txt sudo touch /tmp/eos-mnt/ssh @@ -51,10 +45,17 @@ sudo umount /tmp/eos-mnt sudo mount `partition_for ${OUTPUT_DEVICE} 3` /tmp/eos-mnt -sudo mkdir /tmp/eos-mnt/media/boot-rw -sudo mkdir /tmp/eos-mnt/embassy-os +sudo mkdir /tmp/eos-mnt/media/boot-rw +sudo mkdir /tmp/eos-mnt/embassy-os +sudo mkdir /tmp/eos-mnt/etc/embassy +sudo cp ENVIRONMENT.txt /tmp/eos-mnt/etc/embassy +sudo cp GIT_HASH.txt /tmp/eos-mnt/etc/embassy sudo cp build/fstab /tmp/eos-mnt/etc/fstab sudo cp build/journald.conf /tmp/eos-mnt/etc/systemd/journald.conf + +# copy over cargo dependencies +sudo cp cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast /tmp/eos-mnt/usr/local/bin + # Enter the backend directory, copy over the built EmbassyOS binaries and systemd services, edit the nginx config, then create the .ssh directory cd backend/ @@ -97,5 +98,7 @@ fi sudo cp ./build/initialization.service /tmp/eos-mnt/etc/systemd/system/initialization.service sudo ln -s /etc/systemd/system/initialization.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/initialization.service +sudo cp ./build/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/nc-broadcast.service +sudo ln -s /etc/systemd/system/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/nc-broadcast.service sudo umount /tmp/eos-mnt diff --git a/check-environment.sh b/check-environment.sh new file mode 100755 index 000000000..9cf7e8ee3 --- /dev/null +++ b/check-environment.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if ! [ -f ./ENVIRONMENT.txt ] || [ "$(cat ./ENVIRONMENT.txt)" != "$ENVIRONMENT" ]; then + echo -n "$ENVIRONMENT" > ./ENVIRONMENT.txt +fi + +echo -n ./ENVIRONMENT.txt \ No newline at end of file diff --git a/check-git-hash.sh b/check-git-hash.sh new file mode 100755 index 000000000..8c286840d --- /dev/null +++ b/check-git-hash.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +GIT_HASH="$(git describe --always --abbrev=40 --dirty=-modified)" + +if ! [ -f ./GIT_HASH.txt ] || [ "$(cat ./GIT_HASH.txt)" != "$GIT_HASH" ]; then + echo -n "$GIT_HASH" > ./GIT_HASH.txt +fi + +echo -n ./GIT_HASH.txt \ No newline at end of file diff --git a/frontend/build-config.js b/frontend/build-config.js index d8aebc49b..8c220f63c 100755 --- a/frontend/build-config.js +++ b/frontend/build-config.js @@ -1,7 +1,6 @@ // @ts-check const fs = require('fs') const childProcess = require('child_process') -const { env } = require('process') const gitHash = String( childProcess.execSync('git describe --always --abbrev=40 --dirty=-modified'), @@ -9,8 +8,5 @@ const gitHash = String( const origConfig = require('./config.json') -const registries = require('./registries.json') - origConfig['gitHash'] = gitHash -origConfig.ui['marketplace'] = registries.prod fs.writeFileSync('./config.json', JSON.stringify(origConfig, null, 2)) diff --git a/frontend/config-sample.json b/frontend/config-sample.json index 1727d1e5c..0622437d0 100644 --- a/frontend/config-sample.json +++ b/frontend/config-sample.json @@ -16,8 +16,8 @@ "skipStartupAlerts": true }, "marketplace": { - "url": "https://beta-registry-0-3.start9labs.com/", - "name": "Embassy Marketplace" + "url": "https://registry.start9.com/", + "name": "Start9 Marketplace" } }, "gitHash": "" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index da651f248..e1124aa5b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "embassy-os", - "version": "0.3.1", + "version": "0.3.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "embassy-os", - "version": "0.3.1", + "version": "0.3.1.1", "dependencies": { "@angular/animations": "^13.3.0", "@angular/common": "^13.3.0", diff --git a/frontend/package.json b/frontend/package.json index 1dfef47e0..659ed9a4f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,7 +13,7 @@ "build:deps": "rm -rf .angular/cache && cd ../patch-db/client && npm ci && npm run build", "build:diagnostic-ui": "ng run diagnostic-ui:build", "build:setup-wizard": "ng run setup-wizard:build", - "build:ui": "ng run ui:build && tsc projects/ui/postprocess.ts && node projects/ui/postprocess.js && git log | head -n1 > dist/ui/git-hash.txt", + "build:ui": "ng run ui:build", "build:all": "npm run build:deps && npm run build:diagnostic-ui && npm run build:setup-wizard && npm run build:ui", "start:diagnostic-ui": "npm run-script build-config && ionic serve --project diagnostic-ui --host 0.0.0.0", "start:setup-wizard": "npm run-script build-config && ionic serve --project setup-wizard --host 0.0.0.0", diff --git a/frontend/projects/ui/postprocess.ts b/frontend/projects/ui/postprocess.ts deleted file mode 100644 index bfd497559..000000000 --- a/frontend/projects/ui/postprocess.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { parse } from 'node-html-parser' -import * as fs from 'fs' - -let index = fs.readFileSync('./dist/ui/index.html').toString('utf-8') - -const root = parse(index) -for (let elem of root.querySelectorAll('link')) { - if (elem.getAttribute('rel') === 'stylesheet') { - const sheet = fs - .readFileSync('./dist/ui/' + elem.getAttribute('href')) - .toString('utf-8') - index = index.replace(elem.toString(), '') - } -} -fs.writeFileSync('./dist/ui/index.html', index) diff --git a/frontend/registries.json b/frontend/registries.json deleted file mode 100644 index 755f63edc..000000000 --- a/frontend/registries.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "prod": { - "url": "https://registry.start9.com", - "name": "Embassy Marketplace" - }, - "beta": { - "url": "https://beta-registry-0-3.start9labs.com", - "name": "Beta Marketplace" - }, - "alpha": { - "url": "https://alpha-registry-0-3.start9labs.com", - "name": "Alpha Marketplace" - } -} diff --git a/system-images/compat/Cargo.lock b/system-images/compat/Cargo.lock index af3c2a9db..ccb5de1f2 100644 --- a/system-images/compat/Cargo.lock +++ b/system-images/compat/Cargo.lock @@ -931,7 +931,7 @@ dependencies = [ [[package]] name = "embassy-os" -version = "0.3.1" +version = "0.3.1-rev.1" dependencies = [ "aes", "async-stream",