refactor build process (#1675)

* add nc-broadcast to view initialization.sh logs

* include stderr

* refactor build

* add frontend/config.json as frontend dependency

* fix nc-broadcast

* always run all workflows

* address dependabot alerts

* fix build caching

* remove registries.json

* more efficient build

.github/workflows/backend-pr.yaml

@@ -3,12 +3,6 @@ name: Backend PR
 on:
   workflow_call:
   workflow_dispatch:
-  pull_request:
-    branches:
-    - master
-    paths:
-    - 'backend/**'
-    - 'libs/**'
 
 jobs:
   libs:
@@ -44,7 +38,7 @@ jobs:
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          libs/target/
         key: ${{ runner.os }}-cargo-libs-${{ matrix.target }}-${{ hashFiles('libs/Cargo.lock') }}
 
     - name: Build v8 snapshot
@@ -89,14 +83,14 @@ jobs:
           ~/.cargo/registry/index/
           ~/.cargo/registry/cache/
           ~/.cargo/git/db/
-          target/
+          backend/target/
         key: ${{ runner.os }}-cargo-backend-${{ hashFiles('backend/Cargo.lock') }}
 
     - name: Build backend
       run: make backend
 
     - name: 'Tar files to preserve file permissions'
-      run: tar -cvf backend.tar backend/target/aarch64-unknown-linux-gnu/release/embassy*
+      run: tar -cvf backend.tar ENVIRONMENT.txt GIT_HASH.txt backend/target/aarch64-unknown-linux-gnu/release/embassy*
 
     - uses: actions/upload-artifact@v3
       with:

.github/workflows/frontend-pr.yaml

@@ -3,11 +3,6 @@ name: Frontend PR
 on:
   workflow_call:
   workflow_dispatch:
-  pull_request:
-    branches:
-    - master
-    paths:
-    - 'frontend/**'
 
 jobs:
   frontend:
@@ -38,7 +33,7 @@ jobs:
       run: make frontends
 
     - name: 'Tar files to preserve file permissions'
-      run: tar -cvf frontend.tar frontend/dist frontend/config.json
+      run: tar -cvf frontend.tar ENVIRONMENT.txt GIT_HASH.txt frontend/dist frontend/config.json
 
     - uses: actions/upload-artifact@v3
       with:

.github/workflows/product.yaml

@@ -5,6 +5,11 @@ on:
   push:
     branches:
     - master
+    - next
+  pull_request:
+    branches:
+    - master
+    - next
 
 jobs:
   compat:

.gitignore

@@ -12,3 +12,6 @@ deploy_web.sh
 deploy_web.sh
 secrets.db
 .vscode/
+/cargo-deps/**/*
+ENVIRONMENT.txt
+GIT_HASH.txt

Makefile

@@ -1,13 +1,16 @@
+ENVIRONMENT_FILE := $(shell ./check-environment.sh)
+GIT_HASH_FILE := $(shell ./check-git-hash.sh)
 EMBASSY_BINS := backend/target/aarch64-unknown-linux-gnu/release/embassyd backend/target/aarch64-unknown-linux-gnu/release/embassy-init backend/target/aarch64-unknown-linux-gnu/release/embassy-cli backend/target/aarch64-unknown-linux-gnu/release/embassy-sdk
 EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui
 EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build)
 COMPAT_SRC := $(shell find system-images/compat/src)
 UTILS_SRC := $(shell find system-images/utils/Dockerfile)
 BACKEND_SRC := $(shell find backend/src) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock
-FRONTEND_SRC := $(shell find frontend/projects) $(shell find frontend/assets)
+FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell find frontend/assets) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist
+FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
+FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
+FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)
 PATCH_DB_CLIENT_SRC = $(shell find patch-db/client -not -path patch-db/client/dist)
-GIT_REFS := $(shell find .git/refs/heads)
-TMP_FILE := $(shell mktemp)
 
 .DELETE_ON_ERROR:
 
@@ -27,11 +30,12 @@ clean:
 	rm -rf frontend/dist
 	rm -rf patch-db/client/node_modules
 	rm -rf patch-db/client/dist
+	sudo rm -rf cargo-deps
 
 sdk: 
 	cd backend/ && ./install-sdk.sh
 
-eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar 
+eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
 	! test -f eos.img || rm eos.img
 	if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi
 
@@ -57,31 +61,42 @@ snapshots: libs/snapshot-creator/Cargo.toml
 	cd libs/  && ./build-v8-snapshot.sh
 	cd libs/  && ./build-arm-v8-snapshot.sh
 
-$(EMBASSY_BINS): $(BACKEND_SRC) 
+$(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
 	cd backend && ./build-prod.sh
+	touch $(EMBASSY_BINS)
 
 frontend/node_modules: frontend/package.json
 	npm --prefix frontend ci
 
-$(EMBASSY_UIS): $(FRONTEND_SRC) frontend/node_modules patch-db/client patch-db/client/dist frontend/config.json
+frontend/dist/ui: $(FRONTEND_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
-	npm --prefix frontend run build:all
+	npm --prefix frontend run build:ui
 
-frontend/config.json: .git/HEAD $(GIT_REFS)
+frontend/dist/setup-wizard: $(FRONTEND_SETUP_WIZARD_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:setup-wizard
+
+frontend/dist/diagnostic-ui: $(FRONTEND_DIAGNOSTIC_UI_SRC) $(FRONTEND_SHARED_SRC) $(ENVIRONMENT_FILE)
+	npm --prefix frontend run build:diagnostic-ui
+
+frontend/config.json: $(GIT_HASH_FILE) frontend/config-sample.json
 	jq '.useMocks = false' frontend/config-sample.json > frontend/config.json
 	npm --prefix frontend run-script build-config
 
 patch-db/client/node_modules: patch-db/client/package.json
-	npm --prefix patch-db/client install
+	npm --prefix patch-db/client ci
 
 patch-db/client/dist: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules
 	! test -d patch-db/client/dist || rm -rf patch-db/client/dist
+	rm -rf frontend/.angular/cache
 	npm --prefix patch-db/client run build
 
 # this is a convenience step to build all frontends - it is not referenced elsewhere in this file
-frontends: frontend/node_modules frontend/config.json $(EMBASSY_UIS) 
+frontends: $(EMBASSY_UIS) 
 
 # this is a convenience step to build the UI
-ui: frontend/node_modules frontend/config.json frontend/dist/ui
+ui: frontend/dist/ui
 
 # this is a convenience step to build the backend
 backend: $(EMBASSY_BINS)
+
+cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast:
+	./build-cargo-dep.sh nc-broadcast

backend/Cargo.lock

@@ -369,7 +369,7 @@ checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5"
 dependencies = [
  "base64 0.13.0",
  "bollard-stubs",
- "bytes 1.1.0",
+ "bytes",
  "futures-core",
  "futures-util",
  "hex",
@@ -377,13 +377,13 @@ dependencies = [
  "hyper",
  "hyperlocal",
  "log",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "serde",
  "serde_derive",
  "serde_json",
  "serde_urlencoded",
  "thiserror",
- "tokio 1.19.2",
+ "tokio",
  "tokio-util",
  "url",
  "winapi",
@@ -423,12 +423,6 @@ version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
-[[package]]
-name = "bytes"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
-
 [[package]]
 name = "bytes"
 version = "1.1.0"
@@ -1115,7 +1109,7 @@ dependencies = [
  "libc",
  "log",
  "models",
- "nix 0.24.1",
+ "nix 0.24.2",
  "nom 7.1.1",
  "num",
  "num_enum",
@@ -1148,8 +1142,7 @@ dependencies = [
  "stderrlog",
  "tar",
  "thiserror",
- "tokio 1.19.2",
+ "tokio",
- "tokio-compat-02",
  "tokio-stream",
  "tokio-tar",
  "tokio-tungstenite",
@@ -1284,11 +1277,11 @@ dependencies = [
 
 [[package]]
 name = "fd-lock-rs"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a24e88458a5abfcd1cb81622511306f1aea43b900ddb9b34ff5ad8857a7685"
+checksum = "ef0f547e1d79e058664f2ea7d3a6d82b2ddd5fea4a6650b97b70c38979f34db3"
 dependencies = [
- "nix 0.19.1",
+ "nix 0.24.2",
 ]
 
 [[package]]
@@ -1484,7 +1477,7 @@ dependencies = [
  "futures-sink",
  "futures-task",
  "memchr",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "pin-utils",
  "slab",
 ]
@@ -1561,7 +1554,7 @@ version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
@@ -1569,7 +1562,7 @@ dependencies = [
  "http",
  "indexmap",
  "slab",
- "tokio 1.19.2",
+ "tokio",
  "tokio-util",
  "tracing",
 ]
@@ -1628,7 +1621,7 @@ version = "0.1.0"
 dependencies = [
  "models",
  "pin-project",
- "tokio 1.19.2",
+ "tokio",
 ]
 
 [[package]]
@@ -1671,7 +1664,7 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "fnv",
  "itoa 1.0.2",
 ]
@@ -1682,9 +1675,9 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "http",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
 ]
 
 [[package]]
@@ -1720,7 +1713,7 @@ version = "0.14.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1730,9 +1723,9 @@ dependencies = [
  "httparse",
  "httpdate",
  "itoa 1.0.2",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "socket2",
- "tokio 1.19.2",
+ "tokio",
  "tower-service",
  "tracing",
  "want",
@@ -1744,10 +1737,10 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "hyper",
  "native-tls",
- "tokio 1.19.2",
+ "tokio",
  "tokio-native-tls",
 ]
 
@@ -1764,7 +1757,7 @@ dependencies = [
  "hyper",
  "log",
  "sha-1",
- "tokio 1.19.2",
+ "tokio",
  "tokio-tungstenite",
 ]
 
@@ -1778,7 +1771,7 @@ dependencies = [
  "hex",
  "hyper",
  "pin-project",
- "tokio 1.19.2",
+ "tokio",
 ]
 
 [[package]]
@@ -1969,7 +1962,7 @@ dependencies = [
  "swc_macros_common",
  "swc_visit",
  "swc_visit_macros",
- "tokio 1.19.2",
+ "tokio",
  "tracing",
 ]
 
@@ -2309,18 +2302,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "nix"
-version = "0.19.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if 1.0.0",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.23.1"
@@ -2336,9 +2317,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.24.1"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9"
+checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
@@ -2679,7 +2660,7 @@ dependencies = [
  "serde_cbor 0.11.1",
  "serde_json",
  "thiserror",
- "tokio 1.19.2",
+ "tokio",
  "tracing",
  "tracing-error 0.1.2",
 ]
@@ -2816,12 +2797,6 @@ dependencies = [
  "syn 1.0.96",
 ]
 
-[[package]]
-name = "pin-project-lite"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777"
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -3184,7 +3159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92"
 dependencies = [
  "base64 0.13.0",
- "bytes 1.1.0",
+ "bytes",
  "cookie",
  "cookie_store",
  "encoding_rs",
@@ -3202,12 +3177,12 @@ dependencies = [
  "mime",
  "native-tls",
  "percent-encoding",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "proc-macro-hack",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "tokio 1.19.2",
+ "tokio",
  "tokio-native-tls",
  "tokio-socks",
  "tokio-util",
@@ -3225,7 +3200,7 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5fc7d9a4063adcdfbb061aa3011b813cce95d3a8990f9ccb5ea85017916a1b0b"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "cookie",
  "cookie_store",
  "reqwest",
@@ -3276,7 +3251,7 @@ dependencies = [
  "serde_cbor 0.11.2",
  "serde_json",
  "thiserror",
- "tokio 1.19.2",
+ "tokio",
  "url",
  "yajrc",
 ]
@@ -3555,7 +3530,7 @@ version = "0.47.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3ca1daa2506c9d62744fff84d3534192f2e1c70cdf3bed95f298d89156c00b06"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "derive_more",
  "serde",
  "v8",
@@ -3794,7 +3769,7 @@ dependencies = [
  "atoi",
  "bitflags",
  "byteorder",
- "bytes 1.1.0",
+ "bytes",
  "chrono",
  "crc",
  "crossbeam-queue",
@@ -3860,7 +3835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "874e93a365a598dc3dadb197565952cb143ae4aa716f7bcc933a8d836f6bf89f"
 dependencies = [
  "once_cell",
- "tokio 1.19.2",
+ "tokio",
  "tokio-rustls",
 ]
 
@@ -4516,52 +4491,26 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
-[[package]]
-name = "tokio"
-version = "0.2.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092"
-dependencies = [
- "bytes 0.5.6",
- "num_cpus",
- "pin-project-lite 0.1.12",
- "slab",
-]
-
 [[package]]
 name = "tokio"
 version = "1.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "libc",
  "memchr",
  "mio",
  "num_cpus",
  "once_cell",
  "parking_lot 0.12.1",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
  "winapi",
 ]
 
-[[package]]
-name = "tokio-compat-02"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7d4237822b7be8fff0a7a27927462fad435dcb6650f95cea9e946bf6bdc7e07"
-dependencies = [
- "bytes 0.5.6",
- "once_cell",
- "pin-project-lite 0.2.9",
- "tokio 0.2.25",
- "tokio 1.19.2",
- "tokio-stream",
-]
-
 [[package]]
 name = "tokio-macros"
 version = "1.8.0"
@@ -4580,7 +4529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
 dependencies = [
  "native-tls",
- "tokio 1.19.2",
+ "tokio",
 ]
 
 [[package]]
@@ -4590,7 +4539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
 dependencies = [
  "rustls",
- "tokio 1.19.2",
+ "tokio",
  "webpki",
 ]
 
@@ -4603,7 +4552,7 @@ dependencies = [
  "either",
  "futures-util",
  "thiserror",
- "tokio 1.19.2",
+ "tokio",
 ]
 
 [[package]]
@@ -4613,8 +4562,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9"
 dependencies = [
  "futures-core",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
- "tokio 1.19.2",
+ "tokio",
  "tokio-util",
 ]
 
@@ -4627,7 +4576,7 @@ dependencies = [
  "futures-core",
  "libc",
  "redox_syscall 0.2.13",
- "tokio 1.19.2",
+ "tokio",
  "tokio-stream",
  "xattr",
 ]
@@ -4640,7 +4589,7 @@ checksum = "06cda1232a49558c46f8a504d5b93101d42c0bf7f911f12a105ba48168f821ae"
 dependencies = [
  "futures-util",
  "log",
- "tokio 1.19.2",
+ "tokio",
  "tungstenite",
 ]
 
@@ -4650,11 +4599,11 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45"
 dependencies = [
- "bytes 1.1.0",
+ "bytes",
  "futures-core",
  "futures-sink",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
- "tokio 1.19.2",
+ "tokio",
  "tracing",
 ]
 
@@ -4684,7 +4633,7 @@ dependencies = [
  "serde_derive",
  "sha2 0.9.9",
  "sha3",
- "tokio 1.19.2",
+ "tokio",
 ]
 
 [[package]]
@@ -4700,7 +4649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160"
 dependencies = [
  "cfg-if 1.0.0",
- "pin-project-lite 0.2.9",
+ "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
 ]
@@ -4821,7 +4770,7 @@ dependencies = [
  "rand 0.8.5",
  "thiserror",
  "time 0.3.11",
- "tokio 1.19.2",
+ "tokio",
  "trust-dns-proto",
 ]
 
@@ -4846,7 +4795,7 @@ dependencies = [
  "smallvec",
  "thiserror",
  "tinyvec",
- "tokio 1.19.2",
+ "tokio",
  "url",
 ]
 
@@ -4857,7 +4806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a395a2e0fd8aac9b4613767a5b4ba4b2040de1b767fa03ace8c9d6f351d60b2d"
 dependencies = [
  "async-trait",
- "bytes 1.1.0",
+ "bytes",
  "cfg-if 1.0.0",
  "enum-as-inner",
  "env_logger 0.9.0",
@@ -4867,7 +4816,7 @@ dependencies = [
  "serde",
  "thiserror",
  "time 0.3.11",
- "tokio 1.19.2",
+ "tokio",
  "toml",
  "trust-dns-client",
  "trust-dns-proto",
@@ -4887,7 +4836,7 @@ checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5"
 dependencies = [
  "base64 0.13.0",
  "byteorder",
- "bytes 1.1.0",
+ "bytes",
  "http",
  "httparse",
  "log",

backend/Cargo.toml

@@ -67,7 +67,7 @@ divrem = "1.0.0"
 ed25519 = { version = "1.5.2", features = ["pkcs8", "pem", "alloc"] }
 ed25519-dalek = { version = "1.0.1", features = ["serde"] }
 emver = { version = "0.1.6", features = ["serde"] }
-fd-lock-rs = "0.1.3"
+fd-lock-rs = "0.1.4"
 futures = "0.3.21"
 git-version = "0.3.5"
 helpers = { path = "../libs/helpers" }
@@ -129,7 +129,6 @@ stderrlog = "0.5.3"
 tar = "0.4.38"
 thiserror = "1.0.31"
 tokio = { version = "1.19.2", features = ["full"] }
-tokio-compat-02 = "0.2.0"
 tokio-stream = { version = "0.1.9", features = ["io-util", "sync"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
 tokio-tungstenite = "0.17.1"

backend/build-dev.sh

@@ -18,4 +18,7 @@ alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":
 cd ..
 rust-arm64-builder sh -c "(cd backend && cargo build)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo
 #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

backend/build-portable-dev.sh

@@ -18,3 +18,6 @@ alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/
 cd ..
 rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo

backend/build-portable.sh

@@ -18,3 +18,6 @@ alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/
 cd ..
 rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features)"
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo

backend/build-prod.sh

@@ -30,4 +30,8 @@ else
 	rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS)"
 fi
 cd backend
+
+sudo chown -R $USER target
+sudo chown -R $USER ~/.cargo
+
 #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd

build-cargo-dep.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+shopt -s expand_aliases
+
+if [ "$0" != "./build-cargo-dep.sh" ]; then
+	>&2 echo "Must be run from embassy-os directory"
+	exit 1
+fi
+
+USE_TTY=
+if tty -s; then
+	USE_TTY="-it"
+fi
+
+mkdir -p cargo-deps
+alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64'
+
+rust-arm64-builder cargo install "$1" --target-dir /home/rust/src
+sudo chown -R $USER cargo-deps
+sudo chown -R $USER ~/.cargo

build/initialization.service

@@ -8,6 +8,7 @@ Restart=on-failure
 RestartSec=5s
 ExecStart=/usr/local/bin/initialization.sh
 RemainAfterExit=true
+StandardOutput=append:/var/log/initialization.log
 
 [Install]
 WantedBy=multi-user.target

build/initialization.sh

@@ -127,6 +127,7 @@ sed -i 's/^/usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u /g' /boot/cmd
 # making that *sudo docker stats* command fulfil its purpose by displaying all metrics
 sed -i 's/rootwait quiet.*/rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory quiet/g' /boot/cmdline.txt
 
+systemctl disable nc-broadcast.service
 systemctl disable initialization.service
 sudo systemctl restart NetworkManager
 

build/nc-broadcast.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Writes initialization logs to network
+Requires=initialization.service
+
+[Service]
+Type=oneshot
+Restart=on-failure
+RestartSec=5s
+ExecStart=/usr/local/bin/nc-broadcast --input=/var/log/initialization.log --tee 0.0.0.0:8080
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

build/partitioning.sh

@@ -3,4 +3,4 @@
 set -e
 
 # Use fdisk to create DOS partition table with 4 primary partitions, set 1 as bootable, write, and quite
-(echo o; echo x; echo i; echo "0xcb15ae4d"; echo r; echo n; echo p; echo 1; echo 2048; echo 526335; echo t; echo c; echo n; echo p; echo 2; echo 526336; echo 1050623; echo t; echo 2; echo c; echo n; echo p; echo 3; echo 1050624; echo 16083455; echo n; echo p; echo 16083456; echo 31116287; echo a; echo 1; echo w) | sudo fdisk ${OUTPUT_DEVICE}
+(echo o; echo x; echo i; echo "0xcb15ae4d"; echo r; echo n; echo p; echo 1; echo 2048; echo 526335; echo t; echo c; echo n; echo p; echo 2; echo 526336; echo 1050623; echo t; echo 2; echo c; echo n; echo p; echo 3; echo 1050624; echo 16083455; echo n; echo p; echo 16083456; echo 31116287; echo a; echo 1; echo w) | sudo fdisk ${OUTPUT_DEVICE} > /dev/null

build/user-data

@@ -1,86 +0,0 @@
-#cloud-config
-
-# This is the user-data configuration file for cloud-init. By default this sets
-# up an initial user called "ubuntu" with password "ubuntu", which must be
-# changed at first login. However, many additional actions can be initiated on
-# first boot from this file. The cloud-init documentation has more details:
-#
-# https://cloudinit.readthedocs.io/
-#
-# Please note that the YAML format employed by this file is sensitive to
-# differences in whitespace; if you are editing this file in an editor (like
-# Notepad) which uses literal tabs, take care to only use spaces for
-# indentation. See the following link for more details:
-#
-# https://en.wikipedia.org/wiki/YAML
-#
-# Some additional examples are provided in comments below the default
-# configuration.
-
-# On first boot, set the (default) ubuntu user's password to "ubuntu" and
-# expire user passwords
-#chpasswd:
-#  expire: true
-#  list:
-#  - ubuntu:ubuntu
-
-# Enable password authentication with the SSH daemon
-#ssh_pwauth: true
-
-## On first boot, use ssh-import-id to give the specific users SSH access to
-## the default user
-#ssh_import_id:
-#- lp:my_launchpad_username
-#- gh:my_github_username
-
-## Add users and groups to the system, and import keys with the ssh-import-id
-## utility
-#groups:
-#- robot: [robot]
-#- robotics: [robot]
-#- pi
-#
-#users:
-#- default
-#- name: robot
-#  gecos: Mr. Robot
-#  primary_group: robot
-#  groups: users
-#  ssh_import_id: foobar
-#  lock_passwd: false
-#  passwd: $5$hkui88$nvZgIle31cNpryjRfO9uArF7DYiBcWEnjqq7L1AQNN3
-
-## Update apt database and upgrade packages on first boot
-#package_update: true
-#package_upgrade: true
-
-## Install additional packages on first boot
-#packages:
-#- pwgen
-#- pastebinit
-#- [libpython2.7, 2.7.3-0ubuntu3.1]
-
-## Write arbitrary files to the file-system (including binaries!)
-#write_files:
-#- path: /etc/default/keyboard
-#  content: |
-#    # KEYBOARD configuration file
-#    # Consult the keyboard(5) manual page.
-#    XKBMODEL="pc105"
-#    XKBLAYOUT="gb"
-#    XKBVARIANT=""
-#    XKBOPTIONS="ctrl: nocaps"
-#  permissions: '0644'
-#  owner: root:root
-#- encoding: gzip
-#  path: /usr/bin/hello
-#  content: !!binary |
-#    H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA=
-#  owner: root:root
-#  permissions: '0755'
-
-## Run arbitrary commands at rc.local like time
-#runcmd:
-#- [ ls, -l, / ]
-#- [ sh, -xc, "echo $(date) ': hello world!'" ]
-#- [ wget, "http://ubuntu.com", -O, /run/mydir/index.html ]

build/user-data-dev

@@ -1,86 +0,0 @@
-#cloud-config
-
-# This is the user-data configuration file for cloud-init. By default this sets
-# up an initial user called "ubuntu" with password "ubuntu", which must be
-# changed at first login. However, many additional actions can be initiated on
-# first boot from this file. The cloud-init documentation has more details:
-#
-# https://cloudinit.readthedocs.io/
-#
-# Please note that the YAML format employed by this file is sensitive to
-# differences in whitespace; if you are editing this file in an editor (like
-# Notepad) which uses literal tabs, take care to only use spaces for
-# indentation. See the following link for more details:
-#
-# https://en.wikipedia.org/wiki/YAML
-#
-# Some additional examples are provided in comments below the default
-# configuration.
-
-# On first boot, set the (default) ubuntu user's password to "ubuntu" and
-# expire user passwords
-chpasswd:
-  expire: true
-  list:
-  - ubuntu:ubuntu
-
-# Enable password authentication with the SSH daemon
-ssh_pwauth: true
-
-## On first boot, use ssh-import-id to give the specific users SSH access to
-## the default user
-#ssh_import_id:
-#- lp:my_launchpad_username
-#- gh:my_github_username
-
-## Add users and groups to the system, and import keys with the ssh-import-id
-## utility
-#groups:
-#- robot: [robot]
-#- robotics: [robot]
-#- pi
-#
-#users:
-#- default
-#- name: robot
-#  gecos: Mr. Robot
-#  primary_group: robot
-#  groups: users
-#  ssh_import_id: foobar
-#  lock_passwd: false
-#  passwd: $5$hkui88$nvZgIle31cNpryjRfO9uArF7DYiBcWEnjqq7L1AQNN3
-
-## Update apt database and upgrade packages on first boot
-#package_update: true
-#package_upgrade: true
-
-## Install additional packages on first boot
-#packages:
-#- pwgen
-#- pastebinit
-#- [libpython2.7, 2.7.3-0ubuntu3.1]
-
-## Write arbitrary files to the file-system (including binaries!)
-#write_files:
-#- path: /etc/default/keyboard
-#  content: |
-#    # KEYBOARD configuration file
-#    # Consult the keyboard(5) manual page.
-#    XKBMODEL="pc105"
-#    XKBLAYOUT="gb"
-#    XKBVARIANT=""
-#    XKBOPTIONS="ctrl: nocaps"
-#  permissions: '0644'
-#  owner: root:root
-#- encoding: gzip
-#  path: /usr/bin/hello
-#  content: !!binary |
-#    H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA=
-#  owner: root:root
-#  permissions: '0755'
-
-## Run arbitrary commands at rc.local like time
-#runcmd:
-#- [ ls, -l, / ]
-#- [ sh, -xc, "echo $(date) ': hello world!'" ]
-#- [ wget, "http://ubuntu.com", -O, /run/mydir/index.html ]

build/write-image.sh

@@ -11,7 +11,7 @@ function partition_for () {
 }
 
 # Write contents of LOOPDEV (Ubuntu image) to sd card and make filesystems, then detach the loop device
-echo USING $LOOPDEV TO IMAGE $OUTPUT_DEVICE
+echo USING $LOOPDEV TO IMAGE $OUTPUT_DEVICE WITH ENVIRONMENT $ENVIRONMENT
 sudo dd if=${LOOPDEV}p1 of=`partition_for ${OUTPUT_DEVICE} 1` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress
 sudo mkfs.vfat -F 32 `partition_for ${OUTPUT_DEVICE} 2`
 sudo dd if=${LOOPDEV}p2 of=`partition_for ${OUTPUT_DEVICE} 3` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress
@@ -29,17 +29,11 @@ sudo e2label `partition_for ${OUTPUT_DEVICE} 4` blue
 mkdir -p /tmp/eos-mnt
 sudo mount `partition_for ${OUTPUT_DEVICE} 1` /tmp/eos-mnt
 
-if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then
-	sudo cp build/user-data-dev /tmp/eos-mnt/user-data
-else
-	sudo cp build/user-data /tmp/eos-mnt/user-data
-fi
-
 sudo sed -i 's/PARTUUID=cb15ae4d-02/PARTUUID=cb15ae4d-03/g' /tmp/eos-mnt/cmdline.txt
 sudo sed -i 's/ init=\/usr\/lib\/raspi-config\/init_resize.sh//g' /tmp/eos-mnt/cmdline.txt
 
-cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp
+cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp > /dev/null
-echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp
+echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null
 sudo mv /tmp/eos-mnt/config.txt.tmp /tmp/eos-mnt/config.txt
 sudo touch /tmp/eos-mnt/ssh
 
@@ -53,8 +47,15 @@ sudo mount `partition_for ${OUTPUT_DEVICE} 3` /tmp/eos-mnt
 
 sudo mkdir /tmp/eos-mnt/media/boot-rw
 sudo mkdir /tmp/eos-mnt/embassy-os
+sudo mkdir /tmp/eos-mnt/etc/embassy
+sudo cp ENVIRONMENT.txt /tmp/eos-mnt/etc/embassy
+sudo cp GIT_HASH.txt /tmp/eos-mnt/etc/embassy
 sudo cp build/fstab /tmp/eos-mnt/etc/fstab
 sudo cp build/journald.conf /tmp/eos-mnt/etc/systemd/journald.conf
+
+# copy over cargo dependencies
+sudo cp cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast /tmp/eos-mnt/usr/local/bin
+
 # Enter the backend directory, copy over the built EmbassyOS binaries and systemd services, edit the nginx config, then create the .ssh directory
 cd backend/
 
@@ -97,5 +98,7 @@ fi
 
 sudo cp ./build/initialization.service /tmp/eos-mnt/etc/systemd/system/initialization.service
 sudo ln -s /etc/systemd/system/initialization.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/initialization.service
+sudo cp ./build/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/nc-broadcast.service
+sudo ln -s /etc/systemd/system/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/nc-broadcast.service
 
 sudo umount /tmp/eos-mnt

check-environment.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if ! [ -f ./ENVIRONMENT.txt ] || [ "$(cat ./ENVIRONMENT.txt)" != "$ENVIRONMENT" ]; then
+    echo -n "$ENVIRONMENT" > ./ENVIRONMENT.txt
+fi
+
+echo -n ./ENVIRONMENT.txt

check-git-hash.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+GIT_HASH="$(git describe --always --abbrev=40 --dirty=-modified)"
+
+if ! [ -f ./GIT_HASH.txt ] || [ "$(cat ./GIT_HASH.txt)" != "$GIT_HASH" ]; then
+    echo -n "$GIT_HASH" > ./GIT_HASH.txt
+fi
+
+echo -n ./GIT_HASH.txt

frontend/build-config.js

@@ -1,7 +1,6 @@
 // @ts-check
 const fs = require('fs')
 const childProcess = require('child_process')
-const { env } = require('process')
 
 const gitHash = String(
   childProcess.execSync('git describe --always --abbrev=40 --dirty=-modified'),
@@ -9,8 +8,5 @@ const gitHash = String(
 
 const origConfig = require('./config.json')
 
-const registries = require('./registries.json')
-
 origConfig['gitHash'] = gitHash
-origConfig.ui['marketplace'] = registries.prod
 fs.writeFileSync('./config.json', JSON.stringify(origConfig, null, 2))

frontend/config-sample.json

@@ -16,8 +16,8 @@
       "skipStartupAlerts": true
     },
     "marketplace": {
-      "url": "https://beta-registry-0-3.start9labs.com/",
+      "url": "https://registry.start9.com/",
-      "name": "Embassy Marketplace"
+      "name": "Start9 Marketplace"
     }
   },
   "gitHash": ""

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "embassy-os",
-  "version": "0.3.1",
+  "version": "0.3.1.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "embassy-os",
-      "version": "0.3.1",
+      "version": "0.3.1.1",
       "dependencies": {
         "@angular/animations": "^13.3.0",
         "@angular/common": "^13.3.0",

frontend/package.json

@@ -13,7 +13,7 @@
     "build:deps": "rm -rf .angular/cache && cd ../patch-db/client && npm ci && npm run build",
     "build:diagnostic-ui": "ng run diagnostic-ui:build",
     "build:setup-wizard": "ng run setup-wizard:build",
-    "build:ui": "ng run ui:build && tsc projects/ui/postprocess.ts && node projects/ui/postprocess.js && git log | head -n1 > dist/ui/git-hash.txt",
+    "build:ui": "ng run ui:build",
     "build:all": "npm run build:deps && npm run build:diagnostic-ui && npm run build:setup-wizard && npm run build:ui",
     "start:diagnostic-ui": "npm run-script build-config && ionic serve --project diagnostic-ui --host 0.0.0.0",
     "start:setup-wizard": "npm run-script build-config && ionic serve --project setup-wizard --host 0.0.0.0",

frontend/projects/ui/postprocess.ts

@@ -1,15 +0,0 @@
-import { parse } from 'node-html-parser'
-import * as fs from 'fs'
-
-let index = fs.readFileSync('./dist/ui/index.html').toString('utf-8')
-
-const root = parse(index)
-for (let elem of root.querySelectorAll('link')) {
-  if (elem.getAttribute('rel') === 'stylesheet') {
-    const sheet = fs
-      .readFileSync('./dist/ui/' + elem.getAttribute('href'))
-      .toString('utf-8')
-    index = index.replace(elem.toString(), '<style>' + sheet + '</style>')
-  }
-}
-fs.writeFileSync('./dist/ui/index.html', index)

frontend/registries.json

@@ -1,14 +0,0 @@
-{
-  "prod": {
-    "url": "https://registry.start9.com",
-    "name": "Embassy Marketplace"
-  },
-  "beta": {
-    "url": "https://beta-registry-0-3.start9labs.com",
-    "name": "Beta Marketplace"
-  },
-  "alpha": {
-    "url": "https://alpha-registry-0-3.start9labs.com",
-    "name": "Alpha Marketplace"
-  }
-}

system-images/compat/Cargo.lock

@@ -931,7 +931,7 @@ dependencies = [
 
 [[package]]
 name = "embassy-os"
-version = "0.3.1"
+version = "0.3.1-rev.1"
 dependencies = [
  "aes",
  "async-stream",