fix live-build resolv.conf (#3035)

* fix live-build resolv.conf

* improved debuggability

Makefile

@@ -30,8 +30,10 @@ ALL_TARGETS := $(STARTD_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE
 		echo cargo-deps/aarch64-unknown-linux-musl/release/pi-beep; \
 	fi) \
 	$(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then \
-		echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; \
 		echo cargo-deps/$(ARCH)-unknown-linux-musl/release/flamegraph; \
+	fi') \
+	$(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)console($$|-) ]]; then \
+		echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; \
 	fi')
 REBUILD_TYPES = 1
 
@@ -139,9 +141,11 @@ install: $(ALL_TARGETS)
 	$(call ln,/usr/bin/startbox,$(DESTDIR)/usr/bin/start-cli)
 	if [ "$(PLATFORM)" = "raspberrypi" ]; then $(call cp,cargo-deps/aarch64-unknown-linux-musl/release/pi-beep,$(DESTDIR)/usr/bin/pi-beep); fi
 	if /bin/bash -c '[[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]'; then \
-		$(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console,$(DESTDIR)/usr/bin/tokio-console); \
 		$(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/flamegraph,$(DESTDIR)/usr/bin/flamegraph); \
 	fi
+	if /bin/bash -c '[[ "${ENVIRONMENT}" =~ (^|-)console($$|-) ]]'; then \
+		$(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console,$(DESTDIR)/usr/bin/tokio-console); \
+	fi
 	$(call cp,cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs,$(DESTDIR)/usr/bin/startos-backup-fs)
 	$(call ln,/usr/bin/startos-backup-fs,$(DESTDIR)/usr/sbin/mount.backup-fs)
 	
@@ -264,7 +268,7 @@ container-runtime/dist/node_modules/.package-lock.json container-runtime/dist/pa
 container-runtime/rootfs.$(ARCH).squashfs: container-runtime/debian.$(ARCH).squashfs container-runtime/container-runtime.service container-runtime/update-image.sh container-runtime/deb-install.sh container-runtime/dist/index.js container-runtime/dist/node_modules/.package-lock.json core/target/$(ARCH)-unknown-linux-musl/release/containerbox
 	ARCH=$(ARCH) REQUIRES=linux ./build/os-compat/run-compat.sh ./container-runtime/update-image.sh
 
-build/lib/depends build/lib/conflicts: build/dpkg-deps/*
+build/lib/depends build/lib/conflicts: $(ENVIRONMENT_FILE) build/dpkg-deps/*
 	build/dpkg-deps/generate.sh
 
 $(FIRMWARE_ROMS): build/lib/firmware.json download-firmware.sh $(PLATFORM_FILE)

build/lib/motd

@@ -1,34 +1,123 @@
 #!/bin/sh
-printf "\n"
-printf "Welcome to\n"
-cat << "ASCII"
 
-           ███████        
-         █    █    █      
-       █     █ █     █    
-      █     █   █     █   
-      █    █     █    █   
-       █  █       █  █    
-         █         █      
-           ███████        
+parse_essential_db_info() {
+    DB_DUMP="/tmp/startos_db.json"
 
-    _____     __ ___  __  __
-   (_  |  /\ |__) |  /  \(_
-   __) | /  \|  \ |  \__/__)
-ASCII
-printf "                            v$(cat /usr/lib/startos/VERSION.txt)\n\n"
-printf "   %s (%s %s)\n" "$(uname -o)" "$(uname -r)" "$(uname -m)"
-printf "   Git Hash: $(cat /usr/lib/startos/GIT_HASH.txt)"
-if [ -n "$(cat /usr/lib/startos/ENVIRONMENT.txt)" ]; then
-    printf " ~ $(cat /usr/lib/startos/ENVIRONMENT.txt)\n"
-else
-    printf "\n"
+    if command -v start-cli >/dev/null 2>&1; then
+        start-cli db dump > "$DB_DUMP" 2>/dev/null || return 1
+    else
+        return 1
+    fi
+
+    if command -v jq >/dev/null 2>&1 && [ -f "$DB_DUMP" ]; then
+        HOSTNAME=$(jq -r '.value.serverInfo.hostname // "unknown"' "$DB_DUMP" 2>/dev/null)
+        VERSION=$(jq -r '.value.serverInfo.version // "unknown"' "$DB_DUMP" 2>/dev/null)
+        RAM_BYTES=$(jq -r '.value.serverInfo.ram // 0' "$DB_DUMP" 2>/dev/null)
+        WAN_IP=$(jq -r '.value.serverInfo.network.gateways[].ipInfo.wanIp // "unknown"' "$DB_DUMP" 2>/dev/null | head -1)
+        NTP_SYNCED=$(jq -r '.value.serverInfo.ntpSynced // false' "$DB_DUMP" 2>/dev/null)
+
+        if [ "$RAM_BYTES" != "0" ] && [ "$RAM_BYTES" != "null" ]; then
+            RAM_GB=$(echo "scale=1; $RAM_BYTES / 1073741824" | bc 2>/dev/null || echo "unknown")
+        else
+            RAM_GB="unknown"
+        fi
+
+        RUNNING_SERVICES=$(jq -r '[.value.packageData[] | select(.status.main == "running")] | length' "$DB_DUMP" 2>/dev/null)
+        TOTAL_SERVICES=$(jq -r '.value.packageData | length' "$DB_DUMP" 2>/dev/null)
+
+        rm -f "$DB_DUMP"
+        return 0
+    else
+        rm -f "$DB_DUMP" 2>/dev/null
+        return 1
+    fi
+}
+
+DB_INFO_AVAILABLE=0
+if parse_essential_db_info; then
+    DB_INFO_AVAILABLE=1
 fi
 
-printf "\n"
-printf " * Documentation:  https://docs.start9.com\n"
-printf " * Management:     https://%s.local\n" "$(hostname)"
-printf " * Support:        https://start9.com/contact\n"
-printf " * Source Code:    https://github.com/Start9Labs/start-os\n"
-printf " * License:        MIT\n"
-printf "\n"
+if [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$VERSION" != "unknown" ]; then
+    version_display="v$VERSION"
+else
+    version_display="v$(cat /usr/lib/startos/VERSION.txt 2>/dev/null || echo 'unknown')"
+fi
+
+printf "\n\033[1;37m    ▄▄▀▀▀▀▀▄▄\033[0m\n"
+printf "\033[1;37m  ▄▀    ▄    ▀▄      ▄▄▄▄▄  ▄▄▄▄▄▄▄    ▄      ▄▄▄▄▄   ▄▄▄▄▄▄▄ \033[1;31m▄██████▄ ▄██████\033[0m\n"
+printf "\033[1;37m █     █ █     █    █          █      █ █     █    ▀▄    █    \033[1;31m██    ██ ██     \033[0m\n"
+printf "\033[1;37m█     █   █     █   ▀▄▄▄▄      █     █   █    █ ▄▄▄▀     █    \033[1;31m██    ██ ▀█████▄\033[0m\n"
+printf "\033[1;37m█    █     █    █        █     █    █     █   █  ▀▄      █    \033[1;31m██    ██      ██\033[0m\n"
+printf "\033[1;37m █  █       █  █    ▄▄▄▄▄▀     █   █       █  █    ▀▄    █    \033[1;31m▀██████▀ ██████▀\033[0m\n"
+printf "\033[1;37m   █         █\033[0m\n"
+printf "\033[1;37m     ▀▀▄▄▄▀▀                                                  $version_display\033[0m\n\n"
+
+uptime_str=$(uptime | awk -F'up ' '{print $2}' | awk -F',' '{print $1}' | sed 's/^ *//')
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$RAM_GB" != "unknown" ]; then
+    memory_used=$(free -m | awk 'NR==2{printf "%.0fMB", $3}')
+    memory_display="$memory_used / ${RAM_GB}GB"
+else
+    memory_display=$(free -m | awk 'NR==2{printf "%.0fMB / %.0fMB", $3, $2}')
+fi
+
+root_usage=$(df -h / | awk 'NR==2{printf "%s (%s free)", $5, $4}')
+
+if [ -d "/media/startos/data/package-data" ]; then
+    data_usage=$(df -h /media/startos/data/package-data | awk 'NR==2{printf "%s (%s free)", $5, $4}')
+else
+    data_usage="N/A"
+fi
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ]; then
+    services_text="$RUNNING_SERVICES/$TOTAL_SERVICES running"
+else
+    services_text="Unknown"
+fi
+
+local_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}' | head -1)
+if [ -z "$local_ip" ]; then local_ip="N/A"; fi
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$WAN_IP" != "unknown" ]; then
+    wan_ip="$WAN_IP"
+else
+    wan_ip="N/A"
+fi
+
+printf "    \033[1;37m┌─ SYSTEM STATUS ───────────────────────────────────────────────────┐\033[0m\n"
+printf "    \033[1;37m│\033[0m %-8s  \033[0;33m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Uptime:" "$uptime_str" "Memory:" "$memory_display"
+printf "    \033[1;37m│\033[0m %-8s  \033[0;33m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Root:" "$root_usage" "Data:" "$data_usage"
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ]; then
+    if [ "$RUNNING_SERVICES" -eq "$TOTAL_SERVICES" ] && [ "$TOTAL_SERVICES" -gt 0 ]; then
+        printf "    \033[1;37m│\033[0m %-8s \033[0;32m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Services:" "$services_text" "WAN:" "$wan_ip"
+    elif [ "$RUNNING_SERVICES" -gt 0 ]; then
+        printf "    \033[1;37m│\033[0m %-8s \033[0;33m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Services:" "$services_text" "WAN:" "$wan_ip"
+    else
+        printf "    \033[1;37m│\033[0m %-8s \033[0;31m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Services:" "$services_text" "WAN:" "$wan_ip"
+    fi
+else
+    printf "    \033[1;37m│\033[0m %-8s \033[0;37m%-22s\033[0m %-8s \033[0;33m%-23s\033[0m \033[1;37m│\033[0m\n" "Services:" "$services_text" "WAN:" "$wan_ip"
+fi
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$NTP_SYNCED" = "true" ]; then
+    printf "    \033[1;37m│\033[0m %-8s  \033[0;33m%-22s\033[0m %-8s \033[0;32m%-23s\033[0m \033[1;37m│\033[0m\n" "Local:" "$local_ip" "NTP:" "Synced"
+elif [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$NTP_SYNCED" = "false" ]; then
+    printf "    \033[1;37m│\033[0m %-8s  \033[0;33m%-22s\033[0m %-8s \033[0;31m%-23s\033[0m \033[1;37m│\033[0m\n" "Local:" "$local_ip" "NTP:" "Not Synced"
+else
+    printf "    \033[1;37m│\033[0m %-8s  \033[0;33m%-22s\033[0m %-8s \033[0;37m%-23s\033[0m \033[1;37m│\033[0m\n" "Local:" "$local_ip" "NTP:" "Unknown"
+fi
+
+printf "    \033[1;37m└───────────────────────────────────────────────────────────────────┘\033[0m"
+
+if [ "$DB_INFO_AVAILABLE" -eq 1 ] && [ "$HOSTNAME" != "unknown" ]; then
+    web_url="https://$HOSTNAME.local"
+else
+    web_url="https://$(hostname).local"
+fi
+printf "\n    \033[1;37m┌──────────────────────────────────────────────────── QUICK ACCESS ─┐\033[0m\n"
+printf "    \033[1;37m│\033[0m Web Interface: \033[0;36m%-50s\033[0m \033[1;37m│\033[0m\n" "$web_url"
+printf "    \033[1;37m│\033[0m Documentation: \033[0;36m%-50s\033[0m \033[1;37m│\033[0m\n" "https://staging.docs.start9.com"
+printf "    \033[1;37m│\033[0m Support:       \033[0;36m%-50s\033[0m \033[1;37m│\033[0m\n" "https://start9.com/contact"
+printf "    \033[1;37m└───────────────────────────────────────────────────────────────────┘\033[0m\n\n"

container-runtime/deb-install.sh

@@ -2,6 +2,7 @@
 
 set -e
 
+
 mkdir -p /run/systemd/resolve
 echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf
 
@@ -13,6 +14,7 @@ source ~/.bashrc
 nvm install 22
 ln -s $(which node) /usr/bin/node
 
+sed -i '/\(^\|#\)DNSStubListener=/c\DNSStubListener=no' /etc/systemd/resolved.conf
 sed -i '/\(^\|#\)Storage=/c\Storage=persistent' /etc/systemd/journald.conf
 sed -i '/\(^\|#\)Compress=/c\Compress=yes' /etc/systemd/journald.conf
 sed -i '/\(^\|#\)SystemMaxUse=/c\SystemMaxUse=1G' /etc/systemd/journald.conf
@@ -20,4 +22,7 @@ sed -i '/\(^\|#\)ForwardToSyslog=/c\ForwardToSyslog=no' /etc/systemd/journald.co
 
 systemctl enable container-runtime.service
 
-rm -rf /run/systemd
+rm -rf /run/systemd
+
+rm /etc/resolv.conf
+echo "nameserver 10.0.3.1" > /etc/resolv.conf

container-runtime/package-lock.json

@@ -38,7 +38,7 @@
     },
     "../sdk/dist": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.36",
+      "version": "0.4.0-beta.41",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

container-runtime/update-image.sh

@@ -40,7 +40,7 @@ sudo chown 0:0 tmp/combined/lib/systemd/system/container-runtime.service
 sudo cp container-runtime-failure.service tmp/combined/lib/systemd/system/container-runtime-failure.service
 sudo chown 0:0 tmp/combined/lib/systemd/system/container-runtime-failure.service
 sudo cp ../core/target/$ARCH-unknown-linux-musl/release/containerbox tmp/combined/usr/bin/start-container
-echo -e '#!/bin/bash\nexec start-container $@' | sudo tee tmp/combined/usr/bin/start-cli # TODO: remove
+echo -e '#!/bin/bash\nexec start-container "$@"' | sudo tee tmp/combined/usr/bin/start-cli # TODO: remove
 sudo chmod +x tmp/combined/usr/bin/start-cli
 sudo chown 0:0 tmp/combined/usr/bin/start-container
 echo container-runtime | sha256sum | head -c 32 | cat - <(echo) | sudo tee tmp/combined/etc/machine-id

core/build-cli.sh

@@ -5,50 +5,60 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 set -ea
 shopt -s expand_aliases
 
-if [ -z "$ARCH" ]; then
-	ARCH=$(uname -m)
+if [ -z "${ARCH:-}" ]; then
+  ARCH=$(uname -m)
 fi
 if [ "$ARCH" = "arm64" ]; then
   ARCH="aarch64"
 fi
 
-if [ -z "$KERNEL_NAME" ]; then
-	KERNEL_NAME=$(uname -s)
+if [ -z "${KERNEL_NAME:-}" ]; then
+  KERNEL_NAME=$(uname -s)
 fi
 
-if [ -z "$TARGET" ]; then
-	if [ "$KERNEL_NAME" = "Linux" ]; then
-		TARGET="$ARCH-unknown-linux-musl"
-	elif [ "$KERNEL_NAME" = "Darwin" ]; then
-		TARGET="$ARCH-apple-darwin"
-	else
-		>&2 echo "unknown kernel $KERNEL_NAME"
-		exit 1
-	fi
+if [ -z "${TARGET:-}" ]; then
+  if [ "$KERNEL_NAME" = "Linux" ]; then
+    TARGET="$ARCH-unknown-linux-musl"
+  elif [ "$KERNEL_NAME" = "Darwin" ]; then
+    TARGET="$ARCH-apple-darwin"
+  else
+    >&2 echo "unknown kernel $KERNEL_NAME"
+    exit 1
+  fi
 fi
 
 USE_TTY=
 if tty -s; then
-	USE_TTY="-it"
+  USE_TTY="-it"
 fi
 
 cd ..
-FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
-RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
-	RUSTFLAGS="--cfg tokio_unstable"
+# Ensure GIT_HASH.txt exists if not created by higher-level build steps
+if [ ! -f GIT_HASH.txt ] && command -v git >/dev/null 2>&1; then
+  git rev-parse HEAD > GIT_HASH.txt || true
 fi
 
-if which zig > /dev/null && [ "$ENFORCE_USE_DOCKER" != 1 ]; do
-	echo "FEATURES=\"$FEATURES\""
-	echo "RUSTFLAGS=\"$RUSTFLAGS\""
-	RUSTFLAGS=$RUSTFLAGS sh -c "cd core && cargo zigbuild --release --no-default-features --features cli,$FEATURES --locked --bin start-cli --target=$TARGET"
-else
-	alias 'rust-zig-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/cargo-zigbuild'
-	RUSTFLAGS=$RUSTFLAGS rust-zig-builder sh -c "cd core && cargo zigbuild --release --no-default-features --features cli,$FEATURES --locked --bin start-cli --target=$TARGET"
+FEATURES="$(echo "${ENVIRONMENT:-}" | sed 's/-/,/g')"
+FEATURE_ARGS="cli"
+if [ -n "$FEATURES" ]; then
+  FEATURE_ARGS="$FEATURE_ARGS,$FEATURES"
+fi
 
-	if [ "$(ls -nd core/target/$TARGET/release/start-cli | awk '{ print $3 }')" != "$UID" ]; then
-		rust-zig-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo"
-	fi
+RUSTFLAGS=""
+if [[ "${ENVIRONMENT:-}" =~ (^|-)console($|-) ]]; then
+  RUSTFLAGS="--cfg tokio_unstable"
+fi
+
+if command -v zig >/dev/null 2>&1 && [ "${ENFORCE_USE_DOCKER:-0}" != "1" ]; then
+  echo "FEATURES=\"$FEATURES\""
+  echo "RUSTFLAGS=\"$RUSTFLAGS\""
+  RUSTFLAGS=$RUSTFLAGS sh -c "cd core && cargo zigbuild --release --no-default-features --features $FEATURE_ARGS --locked --bin start-cli --target=$TARGET"
+else
+  alias 'rust-zig-builder'='docker run '"$USE_TTY"' --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/cargo-zigbuild'
+  RUSTFLAGS=$RUSTFLAGS rust-zig-builder sh -c "cd core && cargo zigbuild --release --no-default-features --features $FEATURE_ARGS --locked --bin start-cli --target=$TARGET"
+
+  if [ "$(ls -nd "core/target/$TARGET/release/start-cli" | awk '{ print $3 }')" != "$UID" ]; then
+    rust-zig-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo"
+  fi
 fi

core/build-containerbox.sh

@@ -22,7 +22,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/build-registrybox.sh

@@ -22,7 +22,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/build-startbox.sh

@@ -27,7 +27,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/build-ts.sh

@@ -22,7 +22,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/build-tunnelbox.sh

@@ -22,7 +22,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/models/Cargo.toml

@@ -13,7 +13,6 @@ color-eyre = "0.6.2"
 ed25519-dalek = { version = "2.0.0", features = ["serde"] }
 gpt = "4.1.0"
 lazy_static = "1.4"
-lettre = { version = "0.11", default-features = false }
 mbrman = "0.6.0"
 exver = { version = "0.2.0", git = "https://github.com/Start9Labs/exver-rs.git", features = [
   "serde",
@@ -36,5 +35,6 @@ ts-rs = "9"
 thiserror = "2.0"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1.39"
+typeid = "1"
 yasi = { version = "0.1.6", features = ["serde", "ts-rs"] }
 zbus = "5"

core/models/src/data_url.rs

@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::path::Path;
+use std::str::FromStr;
 
 use base64::Engine;
 use color_eyre::eyre::eyre;
@@ -14,28 +15,26 @@ use crate::{mime, Error, ErrorKind, ResultExt};
 #[derive(Clone, TS)]
 #[ts(type = "string")]
 pub struct DataUrl<'a> {
-    mime: InternedString,
-    data: Cow<'a, [u8]>,
+    pub mime: InternedString,
+    pub data: Cow<'a, [u8]>,
 }
 impl<'a> DataUrl<'a> {
     pub const DEFAULT_MIME: &'static str = "application/octet-stream";
     pub const MAX_SIZE: u64 = 100 * 1024;
 
-    // data:{mime};base64,{data}
-    pub fn to_string(&self) -> String {
+    fn to_string(&self) -> String {
         use std::fmt::Write;
-        let mut res = String::with_capacity(self.data_url_len_without_mime() + self.mime.len());
-        let _ = write!(res, "data:{};base64,", self.mime);
-        base64::engine::general_purpose::STANDARD.encode_string(&self.data, &mut res);
+        let mut res = String::with_capacity(self.len());
+        write!(&mut res, "{self}").unwrap();
         res
     }
 
-    fn data_url_len_without_mime(&self) -> usize {
+    fn len_without_mime(&self) -> usize {
         5 + 8 + (4 * self.data.len() / 3) + 3
     }
 
-    pub fn data_url_len(&self) -> usize {
-        self.data_url_len_without_mime() + self.mime.len()
+    pub fn len(&self) -> usize {
+        self.len_without_mime() + self.mime.len()
     }
 
     pub fn from_slice(mime: &str, data: &'a [u8]) -> Self {
@@ -44,6 +43,10 @@ impl<'a> DataUrl<'a> {
             data: Cow::Borrowed(data),
         }
     }
+
+    pub fn canonical_ext(&self) -> Option<&'static str> {
+        mime::unmime(&self.mime)
+    }
 }
 impl DataUrl<'static> {
     pub async fn from_reader(
@@ -109,12 +112,57 @@ impl DataUrl<'static> {
     }
 }
 
+impl<'a> std::fmt::Display for DataUrl<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "data:{};base64,{}",
+            self.mime,
+            base64::display::Base64Display::new(
+                &*self.data,
+                &base64::engine::general_purpose::STANDARD
+            )
+        )
+    }
+}
 impl<'a> std::fmt::Debug for DataUrl<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(&self.to_string())
+        std::fmt::Display::fmt(self, f)
     }
 }
 
+#[derive(Debug)]
+pub struct DataUrlParseError;
+impl std::fmt::Display for DataUrlParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "invalid base64 url")
+    }
+}
+impl std::error::Error for DataUrlParseError {}
+impl From<DataUrlParseError> for Error {
+    fn from(e: DataUrlParseError) -> Self {
+        Error::new(e, ErrorKind::ParseUrl)
+    }
+}
+
+impl FromStr for DataUrl<'static> {
+    type Err = DataUrlParseError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        s.strip_prefix("data:")
+            .and_then(|v| v.split_once(";base64,"))
+            .and_then(|(mime, data)| {
+                Some(DataUrl {
+                    mime: InternedString::intern(mime),
+                    data: Cow::Owned(
+                        base64::engine::general_purpose::STANDARD
+                            .decode(data)
+                            .ok()?,
+                    ),
+                })
+            })
+            .ok_or(DataUrlParseError)
+    }
+}
 impl<'de> Deserialize<'de> for DataUrl<'static> {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
@@ -130,21 +178,9 @@ impl<'de> Deserialize<'de> for DataUrl<'static> {
             where
                 E: serde::de::Error,
             {
-                v.strip_prefix("data:")
-                    .and_then(|v| v.split_once(";base64,"))
-                    .and_then(|(mime, data)| {
-                        Some(DataUrl {
-                            mime: InternedString::intern(mime),
-                            data: Cow::Owned(
-                                base64::engine::general_purpose::STANDARD
-                                    .decode(data)
-                                    .ok()?,
-                            ),
-                        })
-                    })
-                    .ok_or_else(|| {
-                        E::invalid_value(serde::de::Unexpected::Str(v), &"a valid base64 data url")
-                    })
+                v.parse().map_err(|_| {
+                    E::invalid_value(serde::de::Unexpected::Str(v), &"a valid base64 data url")
+                })
             }
         }
         deserializer.deserialize_any(Visitor)
@@ -168,6 +204,6 @@ fn doesnt_reallocate() {
             mime: InternedString::intern("png"),
             data: Cow::Borrowed(&random[..i]),
         };
-        assert_eq!(icon.to_string().capacity(), icon.data_url_len());
+        assert_eq!(icon.to_string().capacity(), icon.len());
     }
 }

core/models/src/errors.rs

@@ -94,7 +94,6 @@ pub enum ErrorKind {
     DBus = 75,
     InstallFailed = 76,
     UpdateFailed = 77,
-    Smtp = 78,
 }
 impl ErrorKind {
     pub fn as_str(&self) -> &'static str {
@@ -177,7 +176,6 @@ impl ErrorKind {
             DBus => "DBus Error",
             InstallFailed => "Install Failed",
             UpdateFailed => "Update Failed",
-            Smtp => "SMTP Error",
         }
     }
 }
@@ -190,6 +188,7 @@ impl Display for ErrorKind {
 #[derive(Debug)]
 pub struct Error {
     pub source: color_eyre::eyre::Error,
+    pub debug: Option<color_eyre::eyre::Error>,
     pub kind: ErrorKind,
     pub revision: Option<Revision>,
     pub task: Option<JoinHandle<()>>,
@@ -201,9 +200,15 @@ impl Display for Error {
     }
 }
 impl Error {
-    pub fn new<E: Into<color_eyre::eyre::Error>>(source: E, kind: ErrorKind) -> Self {
+    pub fn new<E: Into<color_eyre::eyre::Error> + std::fmt::Debug + 'static>(
+        source: E,
+        kind: ErrorKind,
+    ) -> Self {
+        let debug = (typeid::of::<E>() == typeid::of::<color_eyre::eyre::Error>())
+            .then(|| eyre!("{source:?}"));
         Error {
             source: source.into(),
+            debug,
             kind,
             revision: None,
             task: None,
@@ -211,11 +216,8 @@ impl Error {
     }
     pub fn clone_output(&self) -> Self {
         Error {
-            source: ErrorData {
-                details: format!("{}", self.source),
-                debug: format!("{:?}", self.source),
-            }
-            .into(),
+            source: eyre!("{}", self.source),
+            debug: self.debug.as_ref().map(|e| eyre!("{e}")),
             kind: self.kind,
             revision: self.revision.clone(),
             task: None,
@@ -372,21 +374,6 @@ impl From<patch_db::value::Error> for Error {
         }
     }
 }
-impl From<lettre::error::Error> for Error {
-    fn from(e: lettre::error::Error) -> Self {
-        Error::new(e, ErrorKind::Smtp)
-    }
-}
-impl From<lettre::transport::smtp::Error> for Error {
-    fn from(e: lettre::transport::smtp::Error) -> Self {
-        Error::new(e, ErrorKind::Smtp)
-    }
-}
-impl From<lettre::address::AddressError> for Error {
-    fn from(e: lettre::address::AddressError) -> Self {
-        Error::new(e, ErrorKind::Smtp)
-    }
-}
 
 #[derive(Clone, Deserialize, Serialize)]
 pub struct ErrorData {
@@ -556,25 +543,24 @@ where
 impl<T, E> ResultExt<T, E> for Result<T, E>
 where
     color_eyre::eyre::Error: From<E>,
+    E: std::fmt::Debug + 'static,
 {
     fn with_kind(self, kind: ErrorKind) -> Result<T, Error> {
-        self.map_err(|e| Error {
-            source: e.into(),
-            kind,
-            revision: None,
-            task: None,
-        })
+        self.map_err(|e| Error::new(e, kind))
     }
 
     fn with_ctx<F: FnOnce(&E) -> (ErrorKind, D), D: Display>(self, f: F) -> Result<T, Error> {
         self.map_err(|e| {
             let (kind, ctx) = f(&e);
+            let debug = (typeid::of::<E>() == typeid::of::<color_eyre::eyre::Error>())
+                .then(|| eyre!("{ctx}: {e:?}"));
             let source = color_eyre::eyre::Error::from(e);
-            let ctx = format!("{}: {}", ctx, source);
-            let source = source.wrap_err(ctx);
+            let with_ctx = format!("{ctx}: {source}");
+            let source = source.wrap_err(with_ctx);
             Error {
                 kind,
                 source,
+                debug,
                 revision: None,
                 task: None,
             }
@@ -595,25 +581,24 @@ where
 }
 impl<T> ResultExt<T, Error> for Result<T, Error> {
     fn with_kind(self, kind: ErrorKind) -> Result<T, Error> {
-        self.map_err(|e| Error {
-            source: e.source,
-            kind,
-            revision: e.revision,
-            task: e.task,
-        })
+        self.map_err(|e| Error { kind, ..e })
     }
 
     fn with_ctx<F: FnOnce(&Error) -> (ErrorKind, D), D: Display>(self, f: F) -> Result<T, Error> {
         self.map_err(|e| {
             let (kind, ctx) = f(&e);
             let source = e.source;
-            let ctx = format!("{}: {}", ctx, source);
-            let source = source.wrap_err(ctx);
+            let with_ctx = format!("{ctx}: {source}");
+            let source = source.wrap_err(with_ctx);
+            let debug = e.debug.map(|e| {
+                let with_ctx = format!("{ctx}: {e}");
+                e.wrap_err(with_ctx)
+            });
             Error {
                 kind,
                 source,
-                revision: e.revision,
-                task: e.task,
+                debug,
+                ..e
             }
         })
     }

core/models/src/id/gateway.rs

@@ -14,12 +14,14 @@ impl GatewayId {
         &*self.0
     }
 }
-impl<T> From<T> for GatewayId
-where
-    T: Into<InternedString>,
-{
-    fn from(value: T) -> Self {
-        Self(value.into())
+impl From<InternedString> for GatewayId {
+    fn from(value: InternedString) -> Self {
+        Self(value)
+    }
+}
+impl From<GatewayId> for InternedString {
+    fn from(value: GatewayId) -> Self {
+        value.0
     }
 }
 impl FromStr for GatewayId {

core/run-tests.sh

@@ -22,7 +22,7 @@ cd ..
 FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')"
 RUSTFLAGS=""
 
-if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then
+if [[ "${ENVIRONMENT}" =~ (^|-)console($|-) ]]; then
 	RUSTFLAGS="--cfg tokio_unstable"
 fi
 

core/startos/Cargo.toml

@@ -14,7 +14,7 @@ keywords = [
 name = "start-os"
 readme = "README.md"
 repository = "https://github.com/Start9Labs/start-os"
-version = "0.4.0-alpha.10" # VERSION_BUMP
+version = "0.4.0-alpha.11" # VERSION_BUMP
 license = "MIT"
 
 [lib]
@@ -48,13 +48,14 @@ cli-registry = []
 cli-startd = []
 cli-tunnel = []
 default = ["cli", "startd", "registry", "cli-container", "tunnel"]
-dev = []
+dev = ["backtrace-on-stack-overflow"]
 docker = []
 registry = []
-startd = []
+startd = ["mail-send"]
 test = []
 tunnel = []
-unstable = ["console-subscriber", "tokio/tracing"]
+console = ["console-subscriber", "tokio/tracing"]
+unstable = ["backtrace-on-stack-overflow"]
 
 [dependencies]
 arti-client = { version = "0.33", features = [
@@ -81,26 +82,26 @@ async-stream = "0.3.5"
 async-trait = "0.1.74"
 axum = { version = "0.8.4", features = ["ws"] }
 barrage = "0.2.3"
-backhand = "0.23.0"
+backhand = "0.21.0"
+backtrace-on-stack-overflow = { version = "0.3.0", optional = true }
 base32 = "0.5.0"
 base64 = "0.22.1"
 base64ct = "1.6.0"
 basic-cookies = "0.1.4"
-bech32 = "0.11.0"
 blake3 = { version = "1.5.0", features = ["mmap", "rayon"] }
 bytes = "1"
 chrono = { version = "0.4.31", features = ["serde"] }
 clap = { version = "4.4.12", features = ["string"] }
 color-eyre = "0.6.2"
-console = "0.16.0"
+console = "0.15.7"
 console-subscriber = { version = "0.4.1", optional = true }
 const_format = "0.2.34"
 cookie = "0.18.0"
-cookie_store = "0.22.0"
+cookie_store = "0.21.0"
 der = { version = "0.7.9", features = ["derive", "pem"] }
 digest = "0.10.7"
 divrem = "1.0.0"
-dns-lookup = "3.0.0"
+dns-lookup = "2.1.0"
 ed25519 = { version = "2.2.3", features = ["pkcs8", "pem", "alloc"] }
 ed25519-dalek = { version = "2.2.0", features = [
   "serde",
@@ -142,11 +143,10 @@ imbl = { version = "6", features = ["serde", "small-chunks"] }
 imbl-value = { version = "0.4.3", features = ["ts-rs"] }
 include_dir = { version = "0.7.3", features = ["metadata"] }
 indexmap = { version = "2.0.2", features = ["serde"] }
-indicatif = { version = "0.18.0", features = ["tokio"] }
+indicatif = { version = "0.17.7", features = ["tokio"] }
 inotify = "0.11.0"
 integer-encoding = { version = "4.0.0", features = ["tokio_async"] }
 ipnet = { version = "2.8.0", features = ["serde"] }
-iroh = { version = "0.91.2", features = ["discovery-pkarr-dht"] }
 isocountry = "0.3.2"
 itertools = "0.14.0"
 jaq-core = "0.10.1"
@@ -155,16 +155,7 @@ josekit = "0.10.3"
 jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
 lazy_async_pool = "0.3.3"
 lazy_format = "2.0"
-lazy_static = "1.5.0"
-lettre = { version = "0.11.18", default-features = false, features = [
-  "smtp-transport",
-  "pool",
-  "hostname",
-  "builder",
-  "tokio1-rustls",
-  "rustls-platform-verifier",
-  "aws-lc-rs",
-] }
+lazy_static = "1.4.0"
 libc = "0.2.149"
 log = "0.4.20"
 mio = "1"
@@ -197,23 +188,23 @@ pkcs8 = { version = "0.10.2", features = ["std"] }
 prettytable-rs = "0.10.0"
 procfs = { version = "0.17.0", optional = true }
 proptest = "1.3.1"
-proptest-derive = "0.6.0"
+proptest-derive = "0.5.0"
 pty-process = { version = "0.5.1", optional = true }
 qrcode = "0.14.1"
 rand = "0.9.2"
 regex = "1.10.2"
 reqwest = { version = "0.12.4", features = ["stream", "json", "socks"] }
-reqwest_cookie_store = "0.9.0"
+reqwest_cookie_store = "0.8.0"
 rpassword = "7.2.0"
 rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "master" }
-rust-argon2 = "3.0.0"
+rust-argon2 = "2.0.0"
 rustyline-async = "0.4.1"
 safelog = { version = "0.4.8", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
 semver = { version = "1.0.20", features = ["serde"] }
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_cbor = { package = "ciborium", version = "0.2.1" }
 serde_json = "1.0"
-serde_toml = { package = "toml", version = "0.9.5" }
+serde_toml = { package = "toml", version = "0.8.2" }
 serde_urlencoded = "0.7"
 serde_with = { version = "3.4.0", features = ["macros", "json"] }
 serde_yaml = { package = "serde_yml", version = "0.0.12" }
@@ -238,7 +229,7 @@ tokio = { version = "1.38.1", features = ["full"] }
 tokio-rustls = "0.26.0"
 tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }
 tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
-tokio-tungstenite = { version = "0.27.0", features = ["native-tls", "url"] }
+tokio-tungstenite = { version = "0.26.2", features = ["native-tls", "url"] }
 tokio-util = { version = "0.7.9", features = ["io"] }
 tor-cell = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
 tor-hscrypto = { version = "0.33", features = [
@@ -270,6 +261,7 @@ urlencoding = "2.1.3"
 uuid = { version = "1.4.1", features = ["v4"] }
 zbus = "5.1.1"
 zeroize = "1.6.0"
+mail-send = { git = "https://github.com/dr-bonez/mail-send.git", branch = "main", optional = true }
 rustls = "0.23.20"
 rustls-pki-types = { version = "1.10.1", features = ["alloc"] }
 

core/startos/src/action.rs

@@ -4,7 +4,7 @@ use clap::{CommandFactory, FromArgMatches, Parser};
 pub use models::ActionId;
 use models::{PackageId, ReplayId};
 use qrcode::QrCode;
-use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 use ts_rs::TS;
@@ -14,7 +14,7 @@ use crate::db::model::package::TaskSeverity;
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
 use crate::util::serde::{
-    HandlerExtSerde, StdinDeserializable, WithIoFormat, display_serializable,
+    display_serializable, HandlerExtSerde, StdinDeserializable, WithIoFormat,
 };
 
 pub fn action_api<C: Context>() -> ParentHandler<C> {
@@ -52,6 +52,7 @@ pub fn action_api<C: Context>() -> ParentHandler<C> {
 #[ts(export)]
 #[serde(rename_all = "camelCase")]
 pub struct ActionInput {
+    #[serde(default)]
     pub event_id: Guid,
     #[ts(type = "Record<string, unknown>")]
     pub spec: Value,

core/startos/src/bins/startd.rs

@@ -132,8 +132,6 @@ async fn inner_main(
     .await?;
     rpc_ctx.shutdown().await?;
 
-    tracing::info!("RPC Context is dropped");
-
     Ok(shutdown)
 }
 

core/startos/src/context/init.rs

@@ -6,10 +6,10 @@ use tokio::sync::broadcast::Sender;
 use tokio::sync::watch;
 use tracing::instrument;
 
-use crate::Error;
 use crate::context::config::ServerConfig;
 use crate::progress::FullProgressTracker;
 use crate::rpc_continuations::RpcContinuations;
+use crate::Error;
 
 pub struct InitContextSeed {
     pub config: ServerConfig,
@@ -25,10 +25,12 @@ impl InitContext {
     #[instrument(skip_all)]
     pub async fn init(cfg: &ServerConfig) -> Result<Self, Error> {
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let mut progress = FullProgressTracker::new();
+        progress.enable_logging(true);
         Ok(Self(Arc::new(InitContextSeed {
             config: cfg.clone(),
             error: watch::channel(None).0,
-            progress: FullProgressTracker::new(),
+            progress,
             shutdown,
             rpc_continuations: RpcContinuations::new(),
         })))

core/startos/src/context/rpc.rs

@@ -76,6 +76,11 @@ pub struct RpcContextSeed {
     pub start_time: Instant,
     pub crons: SyncMutex<BTreeMap<Guid, NonDetachingJoinHandle<()>>>,
 }
+impl Drop for RpcContextSeed {
+    fn drop(&mut self) {
+        tracing::info!("RpcContext is dropped");
+    }
+}
 
 pub struct Hardware {
     pub devices: Vec<LshwDevice>,
@@ -269,7 +274,7 @@ impl RpcContext {
         self.crons.mutate(|c| std::mem::take(c));
         self.services.shutdown_all().await?;
         self.is_closed.store(true, Ordering::SeqCst);
-        tracing::info!("RPC Context is shutdown");
+        tracing::info!("RpcContext is shutdown");
         Ok(())
     }
 

core/startos/src/context/setup.rs

@@ -10,15 +10,14 @@ use josekit::jwk::Jwk;
 use patch_db::PatchDb;
 use rpc_toolkit::Context;
 use serde::{Deserialize, Serialize};
-use tokio::sync::OnceCell;
 use tokio::sync::broadcast::Sender;
+use tokio::sync::OnceCell;
 use tracing::instrument;
 use ts_rs::TS;
 
-use crate::MAIN_DATA;
 use crate::account::AccountInfo;
-use crate::context::RpcContext;
 use crate::context::config::ServerConfig;
+use crate::context::RpcContext;
 use crate::disk::OsPartitionInfo;
 use crate::hostname::Hostname;
 use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter};
@@ -28,6 +27,7 @@ use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations};
 use crate::setup::SetupProgress;
 use crate::shutdown::Shutdown;
 use crate::util::net::WebSocketExt;
+use crate::MAIN_DATA;
 
 lazy_static::lazy_static! {
     pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
@@ -86,6 +86,8 @@ impl SetupContext {
         config: &ServerConfig,
     ) -> Result<Self, Error> {
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
+        let mut progress = FullProgressTracker::new();
+        progress.enable_logging(true);
         Ok(Self(Arc::new(SetupContextSeed {
             webserver: webserver.acceptor_setter(),
             config: config.clone(),
@@ -96,7 +98,7 @@ impl SetupContext {
                 )
             })?,
             disable_encryption: config.disable_encryption.unwrap_or(false),
-            progress: FullProgressTracker::new(),
+            progress,
             task: OnceCell::new(),
             result: OnceCell::new(),
             disk_guid: OnceCell::new(),

core/startos/src/db/model/public.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
 
 use chrono::{DateTime, Utc};
@@ -19,8 +19,8 @@ use crate::account::AccountInfo;
 use crate::db::model::package::AllPackageData;
 use crate::net::acme::AcmeProvider;
 use crate::net::forward::START9_BRIDGE_IFACE;
-use crate::net::host::Host;
 use crate::net::host::binding::{AddSslOptions, BindInfo, BindOptions, NetInfo};
+use crate::net::host::Host;
 use crate::net::utils::ipv6_is_local;
 use crate::net::vhost::AlpnInfo;
 use crate::prelude::*;
@@ -202,8 +202,10 @@ pub struct NetworkInfo {
 #[model = "Model<Self>"]
 #[ts(export)]
 pub struct DnsSettings {
-    pub dhcp_servers: Vec<SocketAddr>,
-    pub static_servers: Option<Vec<SocketAddr>>,
+    #[ts(type = "string[]")]
+    pub dhcp_servers: VecDeque<SocketAddr>,
+    #[ts(type = "string[] | null")]
+    pub static_servers: Option<VecDeque<SocketAddr>>,
 }
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)]
@@ -211,6 +213,7 @@ pub struct DnsSettings {
 #[model = "Model<Self>"]
 #[ts(export)]
 pub struct NetworkInterfaceInfo {
+    pub name: Option<InternedString>,
     pub public: Option<bool>,
     pub secure: Option<bool>,
     pub ip_info: Option<IpInfo>,
@@ -218,8 +221,9 @@ pub struct NetworkInterfaceInfo {
 impl NetworkInterfaceInfo {
     pub fn loopback() -> (&'static GatewayId, &'static Self) {
         lazy_static! {
-            static ref LO: GatewayId = GatewayId::from("lo");
+            static ref LO: GatewayId = GatewayId::from(InternedString::intern("lo"));
             static ref LOOPBACK: NetworkInterfaceInfo = NetworkInterfaceInfo {
+                name: Some(InternedString::from_static("Loopback")),
                 public: Some(false),
                 secure: Some(true),
                 ip_info: Some(IpInfo {
@@ -248,8 +252,10 @@ impl NetworkInterfaceInfo {
     }
     pub fn lxc_bridge() -> (&'static GatewayId, &'static Self) {
         lazy_static! {
-            static ref LXCBR0: GatewayId = GatewayId::from(START9_BRIDGE_IFACE);
+            static ref LXCBR0: GatewayId =
+                GatewayId::from(InternedString::intern(START9_BRIDGE_IFACE));
             static ref LXC_BRIDGE: NetworkInterfaceInfo = NetworkInterfaceInfo {
+                name: Some(InternedString::from_static("LXC Bridge Interface")),
                 public: Some(false),
                 secure: Some(true),
                 ip_info: Some(IpInfo {

core/startos/src/dependencies.rs

@@ -1,13 +1,14 @@
 use std::collections::BTreeMap;
+use std::path::Path;
 
 use imbl_value::InternedString;
 use models::PackageId;
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 
-use crate::Error;
 use crate::prelude::*;
 use crate::util::PathOrUrl;
+use crate::Error;
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)]
 #[model = "Model<Self>"]
@@ -24,14 +25,57 @@ impl Map for Dependencies {
     }
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "camelCase")]
 #[model = "Model<Self>"]
-#[ts(export)]
 pub struct DepInfo {
     pub description: Option<String>,
     pub optional: bool,
-    pub s9pk: Option<PathOrUrl>,
+    #[serde(flatten)]
+    pub metadata: Option<MetadataSrc>,
+}
+impl TS for DepInfo {
+    type WithoutGenerics = Self;
+    fn decl() -> String {
+        format!("type {} = {}", Self::name(), Self::inline())
+    }
+    fn decl_concrete() -> String {
+        Self::decl()
+    }
+    fn name() -> String {
+        "DepInfo".into()
+    }
+    fn inline() -> String {
+        "{ description: string | null, optional: boolean } & MetadataSrc".into()
+    }
+    fn inline_flattened() -> String {
+        Self::inline()
+    }
+    fn visit_dependencies(v: &mut impl ts_rs::TypeVisitor)
+    where
+        Self: 'static,
+    {
+        v.visit::<MetadataSrc>()
+    }
+    fn output_path() -> Option<&'static std::path::Path> {
+        Some(Path::new("DepInfo.ts"))
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum MetadataSrc {
+    Metadata(Metadata),
+    S9pk(Option<PathOrUrl>), // backwards compatibility
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct Metadata {
+    pub title: InternedString,
+    pub icon: PathOrUrl,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]

core/startos/src/developer/mod.rs

@@ -31,7 +31,7 @@ pub async fn write_developer_key(
         secret_key: secret.to_bytes(),
         public_key: Some(PublicKeyBytes(VerifyingKey::from(secret).to_bytes())),
     };
-    let mut file = create_file_mod(path, 0o046).await?;
+    let mut file = create_file_mod(path, 0o640).await?;
     file.write_all(
         keypair_bytes
             .to_pkcs8_pem(base64ct::LineEnding::default())

core/startos/src/disk/mount/guard.rs

@@ -10,8 +10,8 @@ use tracing::instrument;
 
 use super::filesystem::{FileSystem, MountType, ReadOnly, ReadWrite};
 use super::util::unmount;
-use crate::Error;
 use crate::util::{Invoke, Never};
+use crate::Error;
 
 pub const TMP_MOUNTPOINT: &'static str = "/media/startos/tmp";
 
@@ -74,7 +74,7 @@ impl MountGuard {
     }
     pub async fn unmount(mut self, delete_mountpoint: bool) -> Result<(), Error> {
         if self.mounted {
-            unmount(&self.mountpoint, false).await?;
+            unmount(&self.mountpoint, !cfg!(feature = "unstable")).await?;
             if delete_mountpoint {
                 match tokio::fs::remove_dir(&self.mountpoint).await {
                     Err(e) if e.raw_os_error() == Some(39) => Ok(()), // directory not empty

core/startos/src/init.rs

@@ -247,7 +247,8 @@ pub async fn init(
     Command::new("killall")
         .arg("journalctl")
         .invoke(crate::ErrorKind::Journald)
-        .await?;
+        .await
+        .log_err();
     mount_logs.complete();
     tokio::io::copy(
         &mut open_file("/run/startos/init.log").await?,
@@ -495,14 +496,7 @@ pub async fn init_progress(ctx: InitContext) -> Result<InitProgressRes, Error> {
                         }
                     );
 
-                    if let Err(e) = ws
-                        .close_result(res.map(|_| "complete").map_err(|e| {
-                            tracing::error!("error in init progress websocket: {e}");
-                            tracing::debug!("{e:?}");
-                            e
-                        }))
-                        .await
-                    {
+                    if let Err(e) = ws.close_result(res.map(|_| "complete")).await {
                         tracing::error!("error closing init progress websocket: {e}");
                         tracing::debug!("{e:?}");
                     }

core/startos/src/logs.rs

@@ -15,7 +15,7 @@ use itertools::Itertools;
 use models::{FromStrParser, PackageId};
 use rpc_toolkit::yajrc::RpcError;
 use rpc_toolkit::{
-    CallRemote, Context, Empty, HandlerArgs, HandlerExt, HandlerFor, ParentHandler, from_fn_async,
+    from_fn_async, CallRemote, Context, Empty, HandlerArgs, HandlerExt, HandlerFor, ParentHandler,
 };
 use serde::de::{self, DeserializeOwned};
 use serde::{Deserialize, Serialize};
@@ -30,9 +30,9 @@ use crate::error::ResultExt;
 use crate::lxc::ContainerId;
 use crate::prelude::*;
 use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations};
-use crate::util::Invoke;
 use crate::util::net::WebSocketExt;
 use crate::util::serde::Reversible;
+use crate::util::Invoke;
 
 #[pin_project::pin_project]
 pub struct LogStream {
@@ -551,8 +551,8 @@ pub async fn journalctl(
     let deserialized_entries = String::from_utf8(cmd.invoke(ErrorKind::Journald).await?)?
         .lines()
         .map(serde_json::from_str::<JournalctlEntry>)
-        .collect::<Result<Vec<_>, _>>()
-        .with_kind(ErrorKind::Deserialization)?;
+        .filter_map(|e| e.ok())
+        .collect::<Vec<_>>();
 
     if follow {
         let mut follow_cmd = gen_journalctl_command(&id);
@@ -573,11 +573,8 @@ pub async fn journalctl(
 
         let follow_deserialized_entries = journalctl_entries
             .map_err(|e| Error::new(e, crate::ErrorKind::Journald))
-            .and_then(|s| {
-                futures::future::ready(
-                    serde_json::from_str::<JournalctlEntry>(&s)
-                        .with_kind(crate::ErrorKind::Deserialization),
-                )
+            .try_filter_map(|s| {
+                futures::future::ready(Ok(serde_json::from_str::<JournalctlEntry>(&s).ok()))
             });
 
         let entries = futures::stream::iter(deserialized_entries)

core/startos/src/main.rs

@@ -1,3 +1,7 @@
 fn main() {
+    #[cfg(feature = "backtrace-on-stack-overflow")]
+    unsafe {
+        backtrace_on_stack_overflow::enable()
+    };
     startos::bins::startbox()
 }

core/startos/src/middleware/auth.rs

@@ -43,7 +43,7 @@ pub trait AuthContext: SignatureAuthContext {
     const LOCAL_AUTH_COOKIE_OWNERSHIP: &str;
     fn init_auth_cookie() -> impl Future<Output = Result<(), Error>> + Send {
         async {
-            let mut file = create_file_mod(Self::LOCAL_AUTH_COOKIE_PATH, 0o046).await?;
+            let mut file = create_file_mod(Self::LOCAL_AUTH_COOKIE_PATH, 0o640).await?;
             file.write_all(BASE64.encode(random::<[u8; 32]>()).as_bytes())
                 .await?;
             file.sync_all().await?;

core/startos/src/net/dns.rs

@@ -1,43 +1,46 @@
 use std::borrow::Borrow;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
+use std::str::FromStr;
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
 use clap::Parser;
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
-use futures::{FutureExt, StreamExt, TryStreamExt};
+use futures::{FutureExt, StreamExt};
 use helpers::NonDetachingJoinHandle;
 use hickory_client::client::Client;
-use hickory_client::proto::DnsHandle;
 use hickory_client::proto::runtime::TokioRuntimeProvider;
 use hickory_client::proto::tcp::TcpClientStream;
 use hickory_client::proto::udp::UdpClientStream;
-use hickory_client::proto::xfer::{DnsExchangeBackground, DnsRequestOptions};
-use hickory_server::ServerFuture;
+use hickory_client::proto::xfer::DnsRequestOptions;
+use hickory_client::proto::DnsHandle;
 use hickory_server::authority::MessageResponseBuilder;
 use hickory_server::proto::op::{Header, ResponseCode};
 use hickory_server::proto::rr::{Name, Record, RecordType};
 use hickory_server::server::{Request, RequestHandler, ResponseHandler, ResponseInfo};
+use hickory_server::ServerFuture;
 use imbl::OrdMap;
 use imbl_value::InternedString;
 use itertools::Itertools;
 use models::{GatewayId, OptionExt, PackageId};
+use patch_db::json_ptr::JsonPointer;
 use rpc_toolkit::{
-    Context, HandlerArgs, HandlerExt, ParentHandler, from_fn_async, from_fn_blocking,
+    from_fn_async, from_fn_blocking, Context, HandlerArgs, HandlerExt, ParentHandler,
 };
 use serde::{Deserialize, Serialize};
 use tokio::net::{TcpListener, UdpSocket};
 use tracing::instrument;
 
 use crate::context::RpcContext;
-use crate::db::model::Database;
 use crate::db::model::public::NetworkInterfaceInfo;
+use crate::db::model::Database;
 use crate::net::gateway::NetworkInterfaceWatcher;
 use crate::prelude::*;
+use crate::util::actor::background::BackgroundJobQueue;
 use crate::util::io::file_string_stream;
-use crate::util::serde::{HandlerExtSerde, display_serializable};
+use crate::util::serde::{display_serializable, HandlerExtSerde};
 use crate::util::sync::{SyncRwLock, Watch};
 
 pub fn dns_api<C: Context>() -> ParentHandler<C> {
@@ -161,97 +164,146 @@ impl DnsClient {
         Self {
             client: client.clone(),
             _thread: tokio::spawn(async move {
-                loop {
-                    if let Err::<(), Error>(e) = async {
-                        let mut stream = file_string_stream("/run/systemd/resolve/resolv.conf")
-                            .filter_map(|a| futures::future::ready(a.transpose()))
-                            .boxed();
-                        let mut conf: String = stream
-                            .next()
-                            .await
-                            .or_not_found("/run/systemd/resolve/resolv.conf")??;
-                        let mut prev_nameservers = Vec::new();
-                        let mut bg = BTreeMap::<SocketAddr, BoxFuture<_>>::new();
-                        loop {
-                            let nameservers = conf
-                                .lines()
-                                .map(|l| l.trim())
-                                .filter_map(|l| l.strip_prefix("nameserver "))
-                                .skip(2)
-                                .map(|n| {
-                                    n.parse::<SocketAddr>()
-                                        .or_else(|_| n.parse::<IpAddr>().map(|a| (a, 53).into()))
-                                })
-                                .collect::<Result<Vec<_>, _>>()?;
-                            let static_nameservers = db
-                                .mutate(|db| {
-                                    let dns = db
-                                        .as_public_mut()
-                                        .as_server_info_mut()
-                                        .as_network_mut()
-                                        .as_dns_mut();
-                                    dns.as_dhcp_servers_mut().ser(&nameservers)?;
-                                    dns.as_static_servers().de()
-                                })
-                                .await
-                                .result?;
-                            let nameservers = static_nameservers.unwrap_or(nameservers);
-                            if nameservers != prev_nameservers {
-                                let mut existing: BTreeMap<_, _> =
-                                    client.peek(|c| c.iter().cloned().collect());
-                                let mut new = Vec::with_capacity(nameservers.len());
-                                for addr in &nameservers {
-                                    if let Some(existing) = existing.remove(addr) {
-                                        new.push((*addr, existing));
-                                    } else {
-                                        let client = if let Ok((client, bg_thread)) =
-                                            Client::connect(
-                                                UdpClientStream::builder(
-                                                    *addr,
-                                                    TokioRuntimeProvider::new(),
-                                                )
-                                                .build(),
-                                            )
-                                            .await
+                let (bg, mut runner) = BackgroundJobQueue::new();
+                runner
+                    .run_while(async move {
+                        let dhcp_ns_db = db.clone();
+                        bg.add_job(async move {
+                            loop {
+                                if let Err(e) = async {
+                                    let mut stream =
+                                        file_string_stream("/run/systemd/resolve/resolv.conf")
+                                            .filter_map(|a| futures::future::ready(a.transpose()))
+                                            .boxed();
+                                    while let Some(conf) = stream.next().await {
+                                        let conf: String = conf?;
+                                        let mut nameservers = conf
+                                            .lines()
+                                            .map(|l| l.trim())
+                                            .filter_map(|l| l.strip_prefix("nameserver "))
+                                            .map(|n| {
+                                                n.parse::<SocketAddr>().or_else(|_| {
+                                                    n.parse::<IpAddr>().map(|a| (a, 53).into())
+                                                })
+                                            })
+                                            .collect::<Result<VecDeque<_>, _>>()?;
+                                        if nameservers
+                                            .front()
+                                            .map_or(false, |addr| addr.ip().is_loopback())
                                         {
-                                            bg.insert(*addr, bg_thread.boxed());
-                                            client
-                                        } else {
-                                            let (stream, sender) = TcpClientStream::new(
-                                                *addr,
-                                                None,
-                                                Some(Duration::from_secs(30)),
-                                                TokioRuntimeProvider::new(),
-                                            );
-                                            let (client, bg_thread) =
-                                                Client::new(stream, sender, None)
-                                                    .await
-                                                    .with_kind(ErrorKind::Network)?;
-                                            bg.insert(*addr, bg_thread.boxed());
-                                            client
-                                        };
-                                        new.push((*addr, client));
+                                            nameservers.pop_front();
+                                        }
+                                        if nameservers.front().map_or(false, |addr| {
+                                            addr.ip() == IpAddr::from([1, 1, 1, 1])
+                                        }) {
+                                            nameservers.pop_front();
+                                        }
+                                        dhcp_ns_db
+                                            .mutate(|db| {
+                                                let dns = db
+                                                    .as_public_mut()
+                                                    .as_server_info_mut()
+                                                    .as_network_mut()
+                                                    .as_dns_mut();
+                                                dns.as_dhcp_servers_mut().ser(&nameservers)
+                                            })
+                                            .await
+                                            .result?
                                     }
+                                    Ok::<_, Error>(())
+                                }
+                                .await
+                                {
+                                    tracing::error!("{e}");
+                                    tracing::debug!("{e:?}");
+                                    tokio::time::sleep(Duration::from_secs(1)).await;
                                 }
-                                bg.retain(|n, _| nameservers.iter().any(|a| a == n));
-                                prev_nameservers = nameservers;
-                                client.replace(new);
                             }
-                            tokio::select! {
-                                c = stream.next() => conf = c.or_not_found("/run/systemd/resolve/resolv.conf")??,
-                                _ = futures::future::join(
-                                    futures::future::join_all(bg.values_mut()),
-                                    futures::future::pending::<()>(),
-                                ) => (),
+                        });
+                        loop {
+                            if let Err::<(), Error>(e) = async {
+                                let mut static_changed = db
+                                    .subscribe(
+                                        "/public/serverInfo/network/dns/staticServers"
+                                            .parse::<JsonPointer>()
+                                            .with_kind(ErrorKind::Database)?,
+                                    )
+                                    .await;
+                                let mut prev_nameservers = VecDeque::new();
+                                let mut bg = BTreeMap::<SocketAddr, BoxFuture<_>>::new();
+                                loop {
+                                    let dns = db
+                                        .peek()
+                                        .await
+                                        .into_public()
+                                        .into_server_info()
+                                        .into_network()
+                                        .into_dns();
+                                    let nameservers = dns
+                                        .as_static_servers()
+                                        .transpose_ref()
+                                        .unwrap_or_else(|| dns.as_dhcp_servers())
+                                        .de()?;
+                                    if nameservers != prev_nameservers {
+                                        let mut existing: BTreeMap<_, _> =
+                                            client.peek(|c| c.iter().cloned().collect());
+                                        let mut new = Vec::with_capacity(nameservers.len());
+                                        for addr in &nameservers {
+                                            if let Some(existing) = existing.remove(addr) {
+                                                new.push((*addr, existing));
+                                            } else {
+                                                let client = if let Ok((client, bg_thread)) =
+                                                    Client::connect(
+                                                        UdpClientStream::builder(
+                                                            *addr,
+                                                            TokioRuntimeProvider::new(),
+                                                        )
+                                                        .build(),
+                                                    )
+                                                    .await
+                                                {
+                                                    bg.insert(*addr, bg_thread.boxed());
+                                                    client
+                                                } else {
+                                                    let (stream, sender) = TcpClientStream::new(
+                                                        *addr,
+                                                        None,
+                                                        Some(Duration::from_secs(30)),
+                                                        TokioRuntimeProvider::new(),
+                                                    );
+                                                    let (client, bg_thread) =
+                                                        Client::new(stream, sender, None)
+                                                            .await
+                                                            .with_kind(ErrorKind::Network)?;
+                                                    bg.insert(*addr, bg_thread.boxed());
+                                                    client
+                                                };
+                                                new.push((*addr, client));
+                                            }
+                                        }
+                                        bg.retain(|n, _| nameservers.iter().any(|a| a == n));
+                                        prev_nameservers = nameservers;
+                                        client.replace(new);
+                                    }
+                                    futures::future::select(
+                                        static_changed.recv().boxed(),
+                                        futures::future::join(
+                                            futures::future::join_all(bg.values_mut()),
+                                            futures::future::pending::<()>(),
+                                        ),
+                                    )
+                                    .await;
+                                }
                             }
+                            .await
+                            {
+                                tracing::error!("{e}");
+                                tracing::debug!("{e:?}");
+                            }
+                            tokio::time::sleep(Duration::from_secs(1)).await;
                         }
-                    }
-                    .await
-                    {
-                        tracing::error!("{e}");
-                        tracing::debug!("{e:?}");
-                    }
-                }
+                    })
+                    .await;
             })
             .into(),
         }
@@ -269,6 +321,12 @@ impl DnsClient {
     }
 }
 
+lazy_static::lazy_static! {
+    static ref LOCALHOST: Name = Name::from_ascii("localhost").unwrap();
+    static ref STARTOS: Name = Name::from_ascii("startos").unwrap();
+    static ref EMBASSY: Name = Name::from_ascii("embassy").unwrap();
+}
+
 struct Resolver {
     client: DnsClient,
     net_iface: Watch<OrdMap<GatewayId, NetworkInterfaceInfo>>,
@@ -276,9 +334,12 @@ struct Resolver {
 }
 impl Resolver {
     fn resolve(&self, name: &Name, src: IpAddr) -> Option<Vec<IpAddr>> {
+        if name.zone_of(&*LOCALHOST) {
+            return Some(vec![Ipv4Addr::LOCALHOST.into(), Ipv6Addr::LOCALHOST.into()]);
+        }
         self.resolve.peek(|r| {
             if r.private_domains
-                .get(&*name.to_lowercase().to_ascii())
+                .get(&*name.to_lowercase().to_utf8().trim_end_matches('.'))
                 .map_or(false, |d| d.strong_count() > 0)
             {
                 if let Some(res) = self.net_iface.peek(|i| {
@@ -295,36 +356,30 @@ impl Resolver {
                     return Some(res);
                 }
             }
-            match name.iter().next_back() {
-                Some(b"embassy") | Some(b"startos") => {
-                    if let Some(pkg) = name.iter().rev().skip(1).next() {
-                        if let Some(ip) = r.services.get(&Some(
-                            std::str::from_utf8(pkg)
-                                .unwrap_or_default()
-                                .parse()
-                                .unwrap_or_default(),
-                        )) {
-                            Some(
-                                ip.iter()
-                                    .filter(|(_, rc)| rc.strong_count() > 0)
-                                    .map(|(ip, _)| (*ip).into())
-                                    .collect(),
-                            )
-                        } else {
-                            None
-                        }
-                    } else if let Some(ip) = r.services.get(&None) {
-                        Some(
-                            ip.iter()
-                                .filter(|(_, rc)| rc.strong_count() > 0)
-                                .map(|(ip, _)| (*ip).into())
-                                .collect(),
-                        )
-                    } else {
-                        None
-                    }
+            if STARTOS.zone_of(name) || EMBASSY.zone_of(name) {
+                let Ok(pkg) = name
+                    .trim_to(2)
+                    .iter()
+                    .next()
+                    .map(std::str::from_utf8)
+                    .transpose()
+                    .map_err(|_| ())
+                    .and_then(|s| s.map(PackageId::from_str).transpose().map_err(|_| ()))
+                else {
+                    return None;
+                };
+                if let Some(ip) = r.services.get(&pkg) {
+                    Some(
+                        ip.iter()
+                            .filter(|(_, rc)| rc.strong_count() > 0)
+                            .map(|(ip, _)| (*ip).into())
+                            .collect(),
+                    )
+                } else {
+                    None
                 }
-                _ => None,
+            } else {
+                None
             }
         })
     }
@@ -343,13 +398,19 @@ impl RequestHandler for Resolver {
             if let Some(ip) = self.resolve(query.name().borrow(), req.src.ip()) {
                 match query.query_type() {
                     RecordType::A => {
+                        let mut header = Header::response_from_request(request.header());
+                        header.set_recursion_available(true);
                         response_handle
                             .send_response(
                                 MessageResponseBuilder::from_message_request(&*request).build(
-                                    Header::response_from_request(request.header()),
+                                    header,
                                     &ip.into_iter()
                                         .filter_map(|a| {
-                                            if let IpAddr::V4(a) = a { Some(a) } else { None }
+                                            if let IpAddr::V4(a) = a {
+                                                Some(a)
+                                            } else {
+                                                None
+                                            }
                                         })
                                         .map(|ip| {
                                             Record::from_rdata(
@@ -367,13 +428,19 @@ impl RequestHandler for Resolver {
                             .await
                     }
                     RecordType::AAAA => {
+                        let mut header = Header::response_from_request(request.header());
+                        header.set_recursion_available(true);
                         response_handle
                             .send_response(
                                 MessageResponseBuilder::from_message_request(&*request).build(
-                                    Header::response_from_request(request.header()),
+                                    header,
                                     &ip.into_iter()
                                         .filter_map(|a| {
-                                            if let IpAddr::V6(a) = a { Some(a) } else { None }
+                                            if let IpAddr::V6(a) = a {
+                                                Some(a)
+                                            } else {
+                                                None
+                                            }
                                         })
                                         .map(|ip| {
                                             Record::from_rdata(
@@ -391,11 +458,12 @@ impl RequestHandler for Resolver {
                             .await
                     }
                     _ => {
-                        let res = Header::response_from_request(request.header());
+                        let mut header = Header::response_from_request(request.header());
+                        header.set_recursion_available(true);
                         response_handle
                             .send_response(
                                 MessageResponseBuilder::from_message_request(&*request).build(
-                                    res.into(),
+                                    header.into(),
                                     [],
                                     [],
                                     [],
@@ -407,16 +475,22 @@ impl RequestHandler for Resolver {
                 }
             } else {
                 let query = query.original().clone();
-                let mut streams = self.client.lookup(query, DnsRequestOptions::default());
+                let mut opt = DnsRequestOptions::default();
+                opt.recursion_desired = request.recursion_desired();
+                let mut streams = self.client.lookup(query, opt);
                 let mut err = None;
                 for stream in streams.iter_mut() {
                     match tokio::time::timeout(Duration::from_secs(5), stream.next()).await {
                         Ok(Some(Err(e))) => err = Some(e),
                         Ok(Some(Ok(msg))) => {
+                            let mut header = msg.header().clone();
+                            header.set_id(request.id());
+                            header.set_checking_disabled(request.checking_disabled());
+                            header.set_recursion_available(true);
                             return response_handle
                                 .send_response(
                                     MessageResponseBuilder::from_message_request(&*request).build(
-                                        Header::response_from_request(request.header()),
+                                        header,
                                         msg.answers(),
                                         msg.name_servers(),
                                         &msg.soa().map(|s| s.to_owned().into_record_of_rdata()),
@@ -432,12 +506,13 @@ impl RequestHandler for Resolver {
                     tracing::error!("{e}");
                     tracing::debug!("{e:?}");
                 }
-                let mut res = Header::response_from_request(request.header());
-                res.set_response_code(ResponseCode::ServFail);
+                let mut header = Header::response_from_request(request.header());
+                header.set_recursion_available(true);
+                header.set_response_code(ResponseCode::ServFail);
                 response_handle
                     .send_response(
                         MessageResponseBuilder::from_message_request(&*request).build(
-                            res,
+                            header,
                             [],
                             [],
                             [],
@@ -453,12 +528,13 @@ impl RequestHandler for Resolver {
             Err(e) => {
                 tracing::error!("{}", e);
                 tracing::debug!("{:?}", e);
-                let mut res = Header::response_from_request(request.header());
-                res.set_response_code(ResponseCode::ServFail);
+                let mut header = Header::response_from_request(request.header());
+                header.set_recursion_available(true);
+                header.set_response_code(ResponseCode::ServFail);
                 response_handle
                     .send_response(
                         MessageResponseBuilder::from_message_request(&*request).build(
-                            res,
+                            header,
                             [],
                             [],
                             [],
@@ -466,7 +542,7 @@ impl RequestHandler for Resolver {
                         ),
                     )
                     .await
-                    .unwrap_or(res.into())
+                    .unwrap_or(header.into())
             }
         }
     }

core/startos/src/net/forward.rs

@@ -7,16 +7,19 @@ use helpers::NonDetachingJoinHandle;
 use id_pool::IdPool;
 use imbl::OrdMap;
 use models::GatewayId;
+use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tokio::process::Command;
 use tokio::sync::mpsc;
 
+use crate::context::{CliContext, RpcContext};
 use crate::db::model::public::NetworkInterfaceInfo;
-use crate::net::gateway::{DynInterfaceFilter, InterfaceFilter};
+use crate::net::gateway::{DynInterfaceFilter, InterfaceFilter, SecureFilter};
 use crate::net::utils::ipv6_is_link_local;
 use crate::prelude::*;
-use crate::util::Invoke;
+use crate::util::serde::{display_serializable, HandlerExtSerde};
 use crate::util::sync::Watch;
+use crate::util::Invoke;
 
 pub const START9_BRIDGE_IFACE: &str = "lxcbr0";
 pub const FIRST_DYNAMIC_PRIVATE_PORT: u16 = 49152;
@@ -42,6 +45,42 @@ impl AvailablePorts {
     }
 }
 
+pub fn forward_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new().subcommand(
+        "dump-table",
+        from_fn_async(
+            |ctx: RpcContext| async move { ctx.net_controller.forward.dump_table().await },
+        )
+        .with_display_serializable()
+        .with_custom_display_fn(|HandlerArgs { params, .. }, res| {
+            use prettytable::*;
+
+            if let Some(format) = params.format {
+                return display_serializable(format, res);
+            }
+
+            let mut table = Table::new();
+            table.add_row(row![bc => "FROM", "TO", "FILTER / GATEWAY"]);
+
+            for (external, target) in res.0 {
+                table.add_row(row![external, target.target, target.filter]);
+                for (source, gateway) in target.gateways {
+                    table.add_row(row![
+                        format!("{}:{}", source, external),
+                        target.target,
+                        gateway
+                    ]);
+                }
+            }
+
+            table.print_tty(false)?;
+
+            Ok(())
+        })
+        .with_call_remote::<CliContext>(),
+    )
+}
+
 struct ForwardRequest {
     external: u16,
     target: SocketAddr,
@@ -49,6 +88,7 @@ struct ForwardRequest {
     rc: Weak<()>,
 }
 
+#[derive(Clone)]
 struct ForwardEntry {
     external: u16,
     target: SocketAddr,
@@ -96,7 +136,7 @@ impl ForwardEntry {
         let mut keep = BTreeSet::<SocketAddr>::new();
         for (iface, info) in ip_info
             .iter()
-            .chain([NetworkInterfaceInfo::loopback()])
+            // .chain([NetworkInterfaceInfo::loopback()])
             .filter(|(id, info)| filter_ref.filter(*id, *info))
         {
             if let Some(ip_info) = &info.ip_info {
@@ -155,10 +195,9 @@ impl ForwardEntry {
             *self = Self::new(external, target, rc);
             self.update(ip_info, Some(filter)).await?;
         } else {
-            if self.prev_filter != filter {
-                self.update(ip_info, Some(filter)).await?;
-            }
             self.rc = rc;
+            self.update(ip_info, Some(filter).filter(|f| f != &self.prev_filter))
+                .await?;
         }
         Ok(())
     }
@@ -174,7 +213,7 @@ impl Drop for ForwardEntry {
     }
 }
 
-#[derive(Default)]
+#[derive(Default, Clone)]
 struct ForwardState {
     state: BTreeMap<u16, ForwardEntry>,
 }
@@ -197,7 +236,7 @@ impl ForwardState {
         for entry in self.state.values_mut() {
             entry.update(ip_info, None).await?;
         }
-        self.state.retain(|_, fwd| !fwd.forwards.is_empty());
+        self.state.retain(|_, fwd| fwd.rc.strong_count() > 0);
         Ok(())
     }
 }
@@ -209,28 +248,68 @@ fn err_has_exited<T>(_: T) -> Error {
     )
 }
 
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ForwardTable(pub BTreeMap<u16, ForwardTarget>);
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct ForwardTarget {
+    pub target: SocketAddr,
+    pub filter: String,
+    pub gateways: BTreeMap<SocketAddr, GatewayId>,
+}
+impl From<&ForwardState> for ForwardTable {
+    fn from(value: &ForwardState) -> Self {
+        Self(
+            value
+                .state
+                .iter()
+                .map(|(external, entry)| {
+                    (
+                        *external,
+                        ForwardTarget {
+                            target: entry.target,
+                            filter: format!("{:?}", entry.prev_filter),
+                            gateways: entry.forwards.clone(),
+                        },
+                    )
+                })
+                .collect(),
+        )
+    }
+}
+
+enum ForwardCommand {
+    Forward(ForwardRequest, oneshot::Sender<Result<(), Error>>),
+    Sync(oneshot::Sender<Result<(), Error>>),
+    DumpTable(oneshot::Sender<ForwardTable>),
+}
+
+#[test]
+fn test() {
+    assert_ne!(
+        false.into_dyn(),
+        SecureFilter { secure: false }.into_dyn().into_dyn()
+    );
+}
+
 pub struct PortForwardController {
-    req: mpsc::UnboundedSender<(Option<ForwardRequest>, oneshot::Sender<Result<(), Error>>)>,
+    req: mpsc::UnboundedSender<ForwardCommand>,
     _thread: NonDetachingJoinHandle<()>,
 }
 impl PortForwardController {
     pub fn new(mut ip_info: Watch<OrdMap<GatewayId, NetworkInterfaceInfo>>) -> Self {
-        let (req_send, mut req_recv) = mpsc::unbounded_channel::<(
-            Option<ForwardRequest>,
-            oneshot::Sender<Result<(), Error>>,
-        )>();
+        let (req_send, mut req_recv) = mpsc::unbounded_channel::<ForwardCommand>();
         let thread = NonDetachingJoinHandle::from(tokio::spawn(async move {
             let mut state = ForwardState::default();
             let mut interfaces = ip_info.read_and_mark_seen();
             loop {
                 tokio::select! {
                     msg = req_recv.recv() => {
-                        if let Some((msg, re)) = msg {
-                            if let Some(req) = msg {
-                                re.send(state.handle_request(req, &interfaces).await).ok();
-                            } else {
-                                re.send(state.sync(&interfaces).await).ok();
-                            }
+                        if let Some(cmd) = msg {
+                            match cmd {
+                                ForwardCommand::Forward(req, re) => re.send(state.handle_request(req, &interfaces).await).ok(),
+                                ForwardCommand::Sync(re) => re.send(state.sync(&interfaces).await).ok(),
+                                ForwardCommand::DumpTable(re) => re.send((&state).into()).ok(),
+                            };
                         } else {
                             break;
                         }
@@ -250,19 +329,19 @@ impl PortForwardController {
     pub async fn add(
         &self,
         external: u16,
-        filter: impl InterfaceFilter,
+        filter: DynInterfaceFilter,
         target: SocketAddr,
     ) -> Result<Arc<()>, Error> {
         let rc = Arc::new(());
         let (send, recv) = oneshot::channel();
         self.req
-            .send((
-                Some(ForwardRequest {
+            .send(ForwardCommand::Forward(
+                ForwardRequest {
                     external,
                     target,
-                    filter: filter.into_dyn(),
+                    filter,
                     rc: Arc::downgrade(&rc),
-                }),
+                },
                 send,
             ))
             .map_err(err_has_exited)?;
@@ -271,13 +350,25 @@ impl PortForwardController {
     }
     pub async fn gc(&self) -> Result<(), Error> {
         let (send, recv) = oneshot::channel();
-        self.req.send((None, send)).map_err(err_has_exited)?;
+        self.req
+            .send(ForwardCommand::Sync(send))
+            .map_err(err_has_exited)?;
 
         recv.await.map_err(err_has_exited)?
     }
+    pub async fn dump_table(&self) -> Result<ForwardTable, Error> {
+        let (req, res) = oneshot::channel();
+        self.req
+            .send(ForwardCommand::DumpTable(req))
+            .map_err(err_has_exited)?;
+        res.await.map_err(err_has_exited)
+    }
 }
 
 async fn forward(interface: &str, source: SocketAddr, target: SocketAddr) -> Result<(), Error> {
+    if source.is_ipv6() {
+        return Ok(()); // TODO: socat? ip6tables?
+    }
     Command::new("/usr/lib/startos/scripts/forward-port")
         .env("iiface", interface)
         .env("oiface", START9_BRIDGE_IFACE)
@@ -291,6 +382,9 @@ async fn forward(interface: &str, source: SocketAddr, target: SocketAddr) -> Res
 }
 
 async fn unforward(interface: &str, source: SocketAddr, target: SocketAddr) -> Result<(), Error> {
+    if source.is_ipv6() {
+        return Ok(()); // TODO: socat? ip6tables?
+    }
     Command::new("/usr/lib/startos/scripts/forward-port")
         .env("UNDO", "1")
         .env("iiface", interface)

core/startos/src/net/gateway.rs

@@ -16,33 +16,33 @@ use itertools::Itertools;
 use models::GatewayId;
 use nix::net::if_::if_nametoindex;
 use patch_db::json_ptr::JsonPointer;
-use rpc_toolkit::{Context, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tokio::io::{AsyncBufReadExt, BufReader};
 use tokio::net::{TcpListener, TcpStream};
 use tokio::process::Command;
+use tokio::sync::oneshot;
 use ts_rs::TS;
 use zbus::proxy::{PropertyChanged, PropertyStream, SignalStream};
 use zbus::zvariant::{
-    DICT_ENTRY_SIG_END_STR, DeserializeDict, Dict, OwnedObjectPath, OwnedValue, Type as ZType,
-    Value as ZValue,
+    DeserializeDict, Dict, OwnedObjectPath, OwnedValue, Type as ZType, Value as ZValue,
 };
-use zbus::{Connection, proxy};
+use zbus::{proxy, Connection};
 
 use crate::context::{CliContext, RpcContext};
-use crate::db::model::Database;
 use crate::db::model::public::{IpInfo, NetworkInterfaceInfo, NetworkInterfaceType};
+use crate::db::model::Database;
 use crate::net::forward::START9_BRIDGE_IFACE;
 use crate::net::gateway::device::DeviceProxy;
 use crate::net::utils::ipv6_is_link_local;
 use crate::net::web_server::Accept;
 use crate::prelude::*;
-use crate::util::Invoke;
 use crate::util::collections::OrdMapIterMut;
 use crate::util::future::Until;
 use crate::util::io::open_file;
-use crate::util::serde::{HandlerExtSerde, display_serializable};
+use crate::util::serde::{display_serializable, HandlerExtSerde};
 use crate::util::sync::{SyncMutex, Watch};
+use crate::util::Invoke;
 
 pub fn gateway_api<C: Context>() -> ParentHandler<C> {
     ParentHandler::new()
@@ -130,64 +130,61 @@ async fn list_interfaces(
 #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)]
 #[ts(export)]
 struct NetworkInterfaceSetPublicParams {
-    interface: GatewayId,
+    gateway: GatewayId,
     public: Option<bool>,
 }
 
 async fn set_public(
     ctx: RpcContext,
-    NetworkInterfaceSetPublicParams { interface, public }: NetworkInterfaceSetPublicParams,
+    NetworkInterfaceSetPublicParams { gateway, public }: NetworkInterfaceSetPublicParams,
 ) -> Result<(), Error> {
     ctx.net_controller
         .net_iface
-        .set_public(&interface, Some(public.unwrap_or(true)))
+        .set_public(&gateway, Some(public.unwrap_or(true)))
         .await
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)]
 #[ts(export)]
-struct UnsetInboundParams {
-    interface: GatewayId,
+struct UnsetPublicParams {
+    gateway: GatewayId,
 }
 
 async fn unset_public(
     ctx: RpcContext,
-    UnsetInboundParams { interface }: UnsetInboundParams,
+    UnsetPublicParams { gateway }: UnsetPublicParams,
 ) -> Result<(), Error> {
     ctx.net_controller
         .net_iface
-        .set_public(&interface, None)
+        .set_public(&gateway, None)
         .await
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)]
 #[ts(export)]
-struct ForgetInterfaceParams {
-    interface: GatewayId,
+struct ForgetGatewayParams {
+    gateway: GatewayId,
 }
 
 async fn forget_iface(
     ctx: RpcContext,
-    ForgetInterfaceParams { interface }: ForgetInterfaceParams,
+    ForgetGatewayParams { gateway }: ForgetGatewayParams,
 ) -> Result<(), Error> {
-    ctx.net_controller.net_iface.forget(&interface).await
+    ctx.net_controller.net_iface.forget(&gateway).await
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)]
 #[ts(export)]
-struct RenameInterfaceParams {
-    interface: GatewayId,
-    name: String,
+struct RenameGatewayParams {
+    id: GatewayId,
+    name: InternedString,
 }
 
 async fn set_name(
     ctx: RpcContext,
-    RenameInterfaceParams { interface, name }: RenameInterfaceParams,
+    RenameGatewayParams { id, name }: RenameGatewayParams,
 ) -> Result<(), Error> {
-    ctx.net_controller
-        .net_iface
-        .set_name(&interface, &name)
-        .await
+    ctx.net_controller.net_iface.set_name(&id, name).await
 }
 
 #[proxy(
@@ -247,12 +244,17 @@ mod active_connection {
     default_service = "org.freedesktop.NetworkManager"
 )]
 trait ConnectionSettings {
+    fn get_settings(&self) -> Result<HashMap<String, HashMap<String, OwnedValue>>, Error>;
+
     fn update2(
         &self,
-        settings: HashMap<String, HashMap<String, ZValue<'_>>>,
+        settings: HashMap<String, HashMap<String, OwnedValue>>,
         flags: u32,
         args: HashMap<String, ZValue<'_>>,
-    ) -> Result<(), Error>;
+    ) -> Result<HashMap<String, OwnedValue>, Error>;
+
+    #[zbus(signal)]
+    fn updated(&self) -> Result<(), Error>;
 }
 
 #[proxy(
@@ -556,6 +558,12 @@ async fn watch_ip(
                 let active_connection_proxy =
                     active_connection::ActiveConnectionProxy::new(&connection, dac).await?;
 
+                let settings_proxy = ConnectionSettingsProxy::new(
+                    &connection,
+                    active_connection_proxy.connection().await?,
+                )
+                .await?;
+
                 let mut until = Until::new()
                     .with_stream(
                         active_connection_proxy
@@ -569,7 +577,8 @@ async fn watch_ip(
                             .receive_dhcp4_config_changed()
                             .await
                             .stub(),
-                    );
+                    )
+                    .with_stream(settings_proxy.receive_updated().await?.into_inner().stub());
 
                 loop {
                     until
@@ -628,12 +637,10 @@ async fn watch_ip(
                                         let lan_ip = [
                                             Some(ip4_proxy.gateway().await?)
                                                 .filter(|g| !g.is_empty())
-                                                .map(|g| g.parse::<IpAddr>())
-                                                .transpose()?,
+                                                .and_then(|g| g.parse::<IpAddr>().log_err()),
                                             Some(ip6_proxy.gateway().await?)
                                                 .filter(|g| !g.is_empty())
-                                                .map(|g| g.parse::<IpAddr>())
-                                                .transpose()?,
+                                                .and_then(|g| g.parse::<IpAddr>().log_err()),
                                         ]
                                         .into_iter()
                                         .filter_map(|a| a)
@@ -650,9 +657,11 @@ async fn watch_ip(
                                             }
                                             if let Some(dns) = dhcp.domain_name_servers {
                                                 dns_servers.extend(
-                                                    dns.split(",")
-                                                        .map(|s| s.trim().parse::<IpAddr>())
-                                                        .collect::<Result<Vec<_>, _>>()?,
+                                                    dns.split_ascii_whitespace()
+                                                        .filter_map(|s| {
+                                                            s.parse::<IpAddr>().log_err()
+                                                        })
+                                                        .collect::<Vec<_>>(),
                                                 );
                                             }
                                         }
@@ -676,7 +685,7 @@ async fn watch_ip(
                                         } else {
                                             None
                                         };
-                                        let ip_info = Some(IpInfo {
+                                        let mut ip_info = IpInfo {
                                             name: name.clone(),
                                             scope_id,
                                             device_type,
@@ -685,22 +694,33 @@ async fn watch_ip(
                                             wan_ip,
                                             ntp_servers,
                                             dns_servers,
-                                        });
+                                        };
 
                                         write_to.send_if_modified(
                                             |m: &mut OrdMap<GatewayId, NetworkInterfaceInfo>| {
-                                                let (public, secure) = m
+                                                let (name, public, secure, prev_wan_ip) = m
                                                     .get(&iface)
-                                                    .map_or((None, None), |i| (i.public, i.secure));
+                                                    .map_or((None, None, None, None), |i| {
+                                                        (
+                                                            i.name.clone(),
+                                                            i.public,
+                                                            i.secure,
+                                                            i.ip_info
+                                                                .as_ref()
+                                                                .and_then(|i| i.wan_ip),
+                                                        )
+                                                    });
+                                                ip_info.wan_ip = ip_info.wan_ip.or(prev_wan_ip);
                                                 m.insert(
                                                     iface.clone(),
                                                     NetworkInterfaceInfo {
+                                                        name,
                                                         public,
                                                         secure,
-                                                        ip_info: ip_info.clone(),
+                                                        ip_info: Some(ip_info.clone()),
                                                     },
                                                 )
-                                                .filter(|old| &old.ip_info == &ip_info)
+                                                .filter(|old| &old.ip_info == &Some(ip_info))
                                                 .is_none()
                                             },
                                         );
@@ -765,7 +785,13 @@ impl NetworkInterfaceWatcher {
         watch_activated: impl IntoIterator<Item = GatewayId>,
     ) -> Self {
         let ip_info = Watch::new(OrdMap::new());
-        let activated = Watch::new(watch_activated.into_iter().map(|k| (k, false)).collect());
+        let activated = Watch::new(
+            watch_activated
+                .into_iter()
+                .chain([NetworkInterfaceInfo::lxc_bridge().0.clone()])
+                .map(|k| (k, false))
+                .collect(),
+        );
         Self {
             activated: activated.clone(),
             ip_info: ip_info.clone(),
@@ -818,9 +844,11 @@ impl NetworkInterfaceWatcher {
             Ok(())
         })?;
         let ip_info = self.ip_info.clone_unseen();
+        let activated = self.activated.clone_unseen();
         Ok(NetworkInterfaceListener {
             _arc: arc,
             ip_info,
+            activated,
             listeners: ListenerMap::new(port),
         })
     }
@@ -924,11 +952,12 @@ impl NetworkInterfaceController {
         Ok(())
     }
     pub fn new(db: TypedPatchDb<Database>) -> Self {
+        let (seeded_send, seeded) = oneshot::channel();
         let watcher = NetworkInterfaceWatcher::new(
             {
                 let db = db.clone();
                 async move {
-                    match db
+                    let info = match db
                         .peek()
                         .await
                         .as_public()
@@ -948,21 +977,26 @@ impl NetworkInterfaceController {
                             tracing::debug!("{e:?}");
                             OrdMap::new()
                         }
-                    }
+                    };
+                    let _ = seeded_send.send(info.clone());
+                    info
                 }
             },
-            [START9_BRIDGE_IFACE.into()],
+            [InternedString::from_static(START9_BRIDGE_IFACE).into()],
         );
-        let mut ip_info = watcher.subscribe();
+        let mut ip_info_watch = watcher.subscribe();
+        ip_info_watch.mark_seen();
         Self {
             db: db.clone(),
             watcher,
             _sync: tokio::spawn(async move {
                 let res: Result<(), Error> = async {
+                    let mut ip_info = seeded.await.ok();
                     loop {
                         if let Err(e) = async {
-                            let ip_info = ip_info.read();
-                            Self::sync(&db, &ip_info).boxed().await?;
+                            if let Some(ip_info) = ip_info {
+                                Self::sync(&db, &ip_info).boxed().await?;
+                            }
 
                             Ok::<_, Error>(())
                         }
@@ -972,7 +1006,8 @@ impl NetworkInterfaceController {
                             tracing::debug!("{e:?}");
                         }
 
-                        let _ = ip_info.changed().await;
+                        let _ = ip_info_watch.changed().await;
+                        ip_info = Some(ip_info_watch.read());
                     }
                 }
                 .await;
@@ -1092,66 +1127,33 @@ impl NetworkInterfaceController {
         Ok(())
     }
 
-    pub async fn set_name(&self, interface: &GatewayId, name: &str) -> Result<(), Error> {
-        let (dump, mut sub) = self
+    pub async fn set_name(&self, interface: &GatewayId, name: InternedString) -> Result<(), Error> {
+        let mut sub = self
             .db
-            .dump_and_sub(
+            .subscribe(
                 "/public/serverInfo/network/gateways"
                     .parse::<JsonPointer<_, _>>()
                     .with_kind(ErrorKind::Database)?
                     .join_end(interface.as_str())
-                    .join_end("ipInfo")
                     .join_end("name"),
             )
             .await;
-        let change = dump.value.as_str().or_not_found(interface)? != name;
-
-        if !change {
-            return Ok(());
+        let changed = self.watcher.ip_info.send_if_modified(|i| {
+            i.get_mut(interface)
+                .map(|i| {
+                    if i.name.as_ref() != Some(&name) {
+                        i.name = Some(name);
+                        true
+                    } else {
+                        false
+                    }
+                })
+                .unwrap_or(false)
+        });
+        if changed {
+            sub.recv().await;
         }
 
-        let connection = Connection::system().await?;
-
-        let netman_proxy = NetworkManagerProxy::new(&connection).await?;
-
-        let device = Some(
-            netman_proxy
-                .get_device_by_ip_iface(interface.as_str())
-                .await?,
-        )
-        .filter(|o| &**o != "/")
-        .or_not_found(lazy_format!("{interface} in NetworkManager"))?;
-
-        let device_proxy = DeviceProxy::new(&connection, device).await?;
-
-        let dac = Some(device_proxy.active_connection().await?)
-            .filter(|o| &**o != "/")
-            .or_not_found(lazy_format!("ActiveConnection for {interface}"))?;
-
-        let dac_proxy = active_connection::ActiveConnectionProxy::new(&connection, dac).await?;
-
-        let settings = Some(dac_proxy.connection().await?)
-            .filter(|o| &**o != "/")
-            .or_not_found(lazy_format!("ConnectionSettings for {interface}"))?;
-
-        let settings_proxy = ConnectionSettingsProxy::new(&connection, settings).await?;
-
-        settings_proxy
-            .update2(
-                [(
-                    "connection".into(),
-                    [("id".into(), zbus::zvariant::Value::Str(name.into()))]
-                        .into_iter()
-                        .collect(),
-                )]
-                .into_iter()
-                .collect(),
-                0x1,
-                HashMap::new(),
-            )
-            .await?;
-
-        sub.recv().await;
         Ok(())
     }
 }
@@ -1236,7 +1238,7 @@ pub struct PublicFilter {
 }
 impl InterfaceFilter for PublicFilter {
     fn filter(&self, _: &GatewayId, info: &NetworkInterfaceInfo) -> bool {
-        self.public || !info.public()
+        self.public == info.public()
     }
 }
 
@@ -1327,6 +1329,9 @@ impl InterfaceFilter for DynInterfaceFilter {
     fn as_any(&self) -> &dyn Any {
         self.0.as_any()
     }
+    fn into_dyn(self) -> DynInterfaceFilter {
+        self
+    }
 }
 impl DynInterfaceFilter {
     fn new<T: InterfaceFilter>(value: T) -> Self {
@@ -1363,15 +1368,14 @@ impl ListenerMap {
     fn update(
         &mut self,
         ip_info: &OrdMap<GatewayId, NetworkInterfaceInfo>,
+        lxc_bridge: bool,
         filter: &impl InterfaceFilter,
     ) -> Result<(), Error> {
         let mut keep = BTreeSet::<SocketAddr>::new();
         for (_, info) in ip_info
             .iter()
-            .chain([
-                NetworkInterfaceInfo::loopback(),
-                NetworkInterfaceInfo::lxc_bridge(),
-            ])
+            .chain([NetworkInterfaceInfo::loopback()])
+            .chain(Some(NetworkInterfaceInfo::lxc_bridge()).filter(|_| lxc_bridge))
             .filter(|(id, info)| filter.filter(*id, *info))
         {
             if let Some(ip_info) = &info.ip_info {
@@ -1459,6 +1463,7 @@ pub fn lookup_info_by_addr(
 
 pub struct NetworkInterfaceListener {
     pub ip_info: Watch<OrdMap<GatewayId, NetworkInterfaceInfo>>,
+    activated: Watch<BTreeMap<GatewayId, bool>>,
     listeners: ListenerMap,
     _arc: Arc<()>,
 }
@@ -1474,21 +1479,29 @@ impl NetworkInterfaceListener {
         filter: &impl InterfaceFilter,
     ) -> Poll<Result<Accepted, Error>> {
         while self.ip_info.poll_changed(cx).is_ready()
+            || self.activated.poll_changed(cx).is_ready()
             || !DynInterfaceFilterT::eq(&self.listeners.prev_filter, filter.as_any())
         {
+            let lxc_bridge = self.activated.peek(|a| {
+                a.get(NetworkInterfaceInfo::lxc_bridge().0)
+                    .copied()
+                    .unwrap_or_default()
+            });
             self.ip_info
-                .peek_and_mark_seen(|ip_info| self.listeners.update(ip_info, filter))?;
+                .peek_and_mark_seen(|ip_info| self.listeners.update(ip_info, lxc_bridge, filter))?;
         }
         self.listeners.poll_accept(cx)
     }
 
     pub(super) fn new(
         mut ip_info: Watch<OrdMap<GatewayId, NetworkInterfaceInfo>>,
+        activated: Watch<BTreeMap<GatewayId, bool>>,
         port: u16,
     ) -> Self {
         ip_info.mark_unseen();
         Self {
             ip_info,
+            activated,
             listeners: ListenerMap::new(port),
             _arc: Arc::new(()),
         }
@@ -1532,11 +1545,15 @@ pub struct SelfContainedNetworkInterfaceListener {
 impl SelfContainedNetworkInterfaceListener {
     pub fn bind(port: u16) -> Self {
         let ip_info = Watch::new(OrdMap::new());
-        let _watch_thread =
-            tokio::spawn(watcher(ip_info.clone(), Watch::new(BTreeMap::new()))).into();
+        let activated = Watch::new(
+            [(NetworkInterfaceInfo::lxc_bridge().0.clone(), false)]
+                .into_iter()
+                .collect(),
+        );
+        let _watch_thread = tokio::spawn(watcher(ip_info.clone(), activated.clone())).into();
         Self {
             _watch_thread,
-            listener: NetworkInterfaceListener::new(ip_info, port),
+            listener: NetworkInterfaceListener::new(ip_info, activated, port),
         }
     }
 }

core/startos/src/net/host/address.rs

@@ -4,17 +4,17 @@ use std::net::Ipv4Addr;
 use clap::Parser;
 use imbl_value::InternedString;
 use models::GatewayId;
-use rpc_toolkit::{Context, Empty, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 
 use crate::context::{CliContext, RpcContext};
 use crate::db::model::DatabaseModel;
 use crate::net::acme::AcmeProvider;
-use crate::net::host::{HostApiKind, all_hosts};
+use crate::net::host::{all_hosts, HostApiKind};
 use crate::net::tor::OnionAddress;
 use crate::prelude::*;
-use crate::util::serde::{HandlerExtSerde, display_serializable};
+use crate::util::serde::{display_serializable, HandlerExtSerde};
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -27,6 +27,7 @@ pub enum HostAddress {
     Domain {
         address: InternedString,
         public: Option<PublicDomainConfig>,
+        private: bool,
     },
 }
 
@@ -70,11 +71,14 @@ fn handle_duplicates(db: &mut DatabaseModel) -> Result<(), Error> {
         for onion in host.as_onions().de()? {
             check_onion(&mut onions, onion)?;
         }
-        for domain in host.as_public_domains().keys()? {
-            check_domain(&mut domains, domain)?;
+        let public = host.as_public_domains().keys()?;
+        for domain in &public {
+            check_domain(&mut domains, domain.clone())?;
         }
         for domain in host.as_private_domains().de()? {
-            check_domain(&mut domains, domain)?;
+            if !public.contains(&domain) {
+                check_domain(&mut domains, domain)?;
+            }
         }
     }
     for host in not_in_use {
@@ -88,18 +92,21 @@ fn handle_duplicates(db: &mut DatabaseModel) -> Result<(), Error> {
         for onion in host.as_onions().de()? {
             check_onion(&mut onions, onion)?;
         }
-        for domain in host.as_public_domains().keys()? {
-            check_domain(&mut domains, domain)?;
+        let public = host.as_public_domains().keys()?;
+        for domain in &public {
+            check_domain(&mut domains, domain.clone())?;
         }
         for domain in host.as_private_domains().de()? {
-            check_domain(&mut domains, domain)?;
+            if !public.contains(&domain) {
+                check_domain(&mut domains, domain)?;
+            }
         }
     }
     Ok(())
 }
 
-pub fn address_api<C: Context, Kind: HostApiKind>()
--> ParentHandler<C, Kind::Params, Kind::InheritedParams> {
+pub fn address_api<C: Context, Kind: HostApiKind>(
+) -> ParentHandler<C, Kind::Params, Kind::InheritedParams> {
     ParentHandler::<C, Kind::Params, Kind::InheritedParams>::new()
         .subcommand(
             "domain",
@@ -198,16 +205,21 @@ pub fn address_api<C: Context, Kind: HostApiKind>()
                             HostAddress::Domain {
                                 address,
                                 public: Some(PublicDomainConfig { gateway, acme }),
+                                private,
                             } => {
                                 table.add_row(row![
                                     address,
-                                    &format!("YES ({gateway})"),
+                                    &format!(
+                                        "{} ({gateway})",
+                                        if *private { "YES" } else { "ONLY" }
+                                    ),
                                     acme.as_ref().map(|a| a.0.as_str()).unwrap_or("NONE")
                                 ]);
                             }
                             HostAddress::Domain {
                                 address,
                                 public: None,
+                                ..
                             } => {
                                 table.add_row(row![address, &format!("NO"), "N/A"]);
                             }

core/startos/src/net/host/binding.rs

@@ -135,11 +135,12 @@ impl BindInfo {
 }
 impl InterfaceFilter for NetInfo {
     fn filter(&self, id: &GatewayId, info: &NetworkInterfaceInfo) -> bool {
-        if info.public() {
-            self.public_enabled.contains(id)
-        } else {
-            !self.private_disabled.contains(id)
-        }
+        info.ip_info.is_some()
+            && if info.public() {
+                self.public_enabled.contains(id)
+            } else {
+                !self.private_disabled.contains(id)
+            }
     }
 }
 

core/startos/src/net/host/mod.rs

@@ -6,15 +6,15 @@ use clap::Parser;
 use imbl_value::InternedString;
 use itertools::Itertools;
 use models::{HostId, PackageId};
-use rpc_toolkit::{Context, Empty, HandlerExt, OrEmpty, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, OrEmpty, ParentHandler};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 
 use crate::context::RpcContext;
 use crate::db::model::DatabaseModel;
 use crate::net::forward::AvailablePorts;
-use crate::net::host::address::{HostAddress, PublicDomainConfig, address_api};
-use crate::net::host::binding::{BindInfo, BindOptions, binding};
+use crate::net::host::address::{address_api, HostAddress, PublicDomainConfig};
+use crate::net::host::binding::{binding, BindInfo, BindOptions};
 use crate::net::service_interface::HostnameInfo;
 use crate::net::tor::OnionAddress;
 use crate::prelude::*;
@@ -56,14 +56,17 @@ impl Host {
                     .map(|(address, config)| HostAddress::Domain {
                         address: address.clone(),
                         public: Some(config.clone()),
+                        private: self.private_domains.contains(address),
                     }),
             )
             .chain(
                 self.private_domains
                     .iter()
+                    .filter(|a| !self.public_domains.contains_key(*a))
                     .map(|address| HostAddress::Domain {
                         address: address.clone(),
                         public: None,
+                        private: true,
                     }),
             )
     }

core/startos/src/net/iroh.rs

@@ -1,585 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::net::SocketAddr;
-use std::str::FromStr;
-use std::sync::{Arc, Weak};
-
-use clap::Parser;
-use color_eyre::eyre::eyre;
-use futures::{FutureExt, StreamExt};
-use helpers::NonDetachingJoinHandle;
-use imbl_value::InternedString;
-use iroh::{Endpoint, NodeId, SecretKey};
-use itertools::Itertools;
-use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
-use serde::{Deserialize, Serialize};
-use tokio::net::TcpStream;
-
-use crate::context::{CliContext, RpcContext};
-use crate::prelude::*;
-use crate::util::actor::background::BackgroundJobQueue;
-use crate::util::io::ReadWriter;
-use crate::util::serde::{
-    deserialize_from_str, display_serializable, serialize_display, HandlerExtSerde, Pem,
-    PemEncoding, WithIoFormat,
-};
-use crate::util::sync::{SyncMutex, SyncRwLock, Watch};
-
-const HRP: bech32::Hrp = bech32::Hrp::parse_unchecked("iroh");
-
-#[derive(Debug, Clone, Copy)]
-pub struct IrohAddress(pub NodeId);
-impl std::fmt::Display for IrohAddress {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        bech32::encode_lower_to_fmt::<bech32::Bech32m, _>(f, HRP, self.0.as_bytes())
-            .map_err(|_| std::fmt::Error)?;
-        write!(f, ".p2p.start9.to")
-    }
-}
-impl FromStr for IrohAddress {
-    type Err = Error;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        if let Some(b32) = s.strip_suffix(".p2p.start9.to") {
-            let (hrp, data) = bech32::decode(b32).with_kind(ErrorKind::ParseNetAddress)?;
-            ensure_code!(
-                hrp == HRP,
-                ErrorKind::ParseNetAddress,
-                "not an iroh address"
-            );
-            Ok(Self(
-                NodeId::from_bytes(&*<Box<[u8; 32]>>::try_from(data).map_err(|_| {
-                    Error::new(eyre!("invalid length"), ErrorKind::ParseNetAddress)
-                })?)
-                .with_kind(ErrorKind::ParseNetAddress)?,
-            ))
-        } else {
-            Err(Error::new(
-                eyre!("Invalid iroh address"),
-                ErrorKind::ParseNetAddress,
-            ))
-        }
-    }
-}
-impl Serialize for IrohAddress {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        serialize_display(self, serializer)
-    }
-}
-impl<'de> Deserialize<'de> for IrohAddress {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        deserialize_from_str(deserializer)
-    }
-}
-impl PartialEq for IrohAddress {
-    fn eq(&self, other: &Self) -> bool {
-        self.0.as_ref() == other.0.as_ref()
-    }
-}
-impl Eq for IrohAddress {}
-impl PartialOrd for IrohAddress {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        self.0.as_ref().partial_cmp(other.0.as_ref())
-    }
-}
-impl Ord for IrohAddress {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        self.0.as_ref().cmp(other.0.as_ref())
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct IrohSecretKey(pub SecretKey);
-impl IrohSecretKey {
-    pub fn iroh_address(&self) -> IrohAddress {
-        IrohAddress(self.0.public())
-    }
-    pub fn generate() -> Self {
-        Self(SecretKey::generate(
-            &mut ssh_key::rand_core::OsRng::default(),
-        ))
-    }
-}
-impl PemEncoding for IrohSecretKey {
-    fn from_pem<E: serde::de::Error>(pem: &str) -> Result<Self, E> {
-        ed25519_dalek::SigningKey::from_pem(pem)
-            .map(From::from)
-            .map(Self)
-    }
-    fn to_pem<E: serde::ser::Error>(&self) -> Result<String, E> {
-        self.0.secret().to_pem()
-    }
-}
-
-#[derive(Default, Debug, Deserialize, Serialize)]
-pub struct IrohKeyStore(BTreeMap<IrohAddress, Pem<IrohSecretKey>>);
-impl Map for IrohKeyStore {
-    type Key = IrohAddress;
-    type Value = Pem<IrohSecretKey>;
-    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
-        Self::key_string(key)
-    }
-    fn key_string(key: &Self::Key) -> Result<imbl_value::InternedString, Error> {
-        Ok(InternedString::from_display(key))
-    }
-}
-impl IrohKeyStore {
-    pub fn new() -> Self {
-        Self::default()
-    }
-    pub fn insert(&mut self, key: IrohSecretKey) {
-        self.0.insert(key.iroh_address(), Pem::new(key));
-    }
-}
-impl Model<IrohKeyStore> {
-    pub fn new_key(&mut self) -> Result<IrohSecretKey, Error> {
-        let key = IrohSecretKey::generate();
-        self.insert(&key.iroh_address(), &Pem::new(key))?;
-        Ok(key)
-    }
-    pub fn insert_key(&mut self, key: &IrohSecretKey) -> Result<(), Error> {
-        self.insert(&key.iroh_address(), Pem::new_ref(key))
-    }
-    pub fn get_key(&self, address: &IrohAddress) -> Result<IrohSecretKey, Error> {
-        self.as_idx(address)
-            .or_not_found(lazy_format!("private key for {address}"))?
-            .de()
-            .map(|k| k.0)
-    }
-}
-
-pub fn iroh_api<C: Context>() -> ParentHandler<C> {
-    ParentHandler::new()
-        .subcommand(
-            "list-services",
-            from_fn_async(list_services)
-                .with_display_serializable()
-                .with_custom_display_fn(|handle, result| display_services(handle.params, result))
-                .with_about("Display the status of running iroh services")
-                .with_call_remote::<CliContext>(),
-        )
-        .subcommand(
-            "key",
-            key::<C>().with_about("Manage the iroh service key store"),
-        )
-}
-
-pub fn key<C: Context>() -> ParentHandler<C> {
-    ParentHandler::new()
-        .subcommand(
-            "generate",
-            from_fn_async(generate_key)
-                .with_about("Generate an iroh service key and add it to the key store")
-                .with_call_remote::<CliContext>(),
-        )
-        .subcommand(
-            "add",
-            from_fn_async(add_key)
-                .with_about("Add an iroh service key to the key store")
-                .with_call_remote::<CliContext>(),
-        )
-        .subcommand(
-            "list",
-            from_fn_async(list_keys)
-                .with_custom_display_fn(|_, res| {
-                    for addr in res {
-                        println!("{addr}");
-                    }
-                    Ok(())
-                })
-                .with_about("List iroh services with keys in the key store")
-                .with_call_remote::<CliContext>(),
-        )
-}
-
-pub async fn generate_key(ctx: RpcContext) -> Result<IrohAddress, Error> {
-    ctx.db
-        .mutate(|db| {
-            Ok(db
-                .as_private_mut()
-                .as_key_store_mut()
-                .as_iroh_mut()
-                .new_key()?
-                .iroh_address())
-        })
-        .await
-        .result
-}
-
-#[derive(Deserialize, Serialize, Parser)]
-pub struct AddKeyParams {
-    pub key: Pem<IrohSecretKey>,
-}
-
-pub async fn add_key(
-    ctx: RpcContext,
-    AddKeyParams { key }: AddKeyParams,
-) -> Result<IrohAddress, Error> {
-    ctx.db
-        .mutate(|db| {
-            db.as_private_mut()
-                .as_key_store_mut()
-                .as_iroh_mut()
-                .insert_key(&key.0)
-        })
-        .await
-        .result?;
-    Ok(key.iroh_address())
-}
-
-pub async fn list_keys(ctx: RpcContext) -> Result<BTreeSet<IrohAddress>, Error> {
-    ctx.db
-        .peek()
-        .await
-        .into_private()
-        .into_key_store()
-        .into_iroh()
-        .keys()
-}
-
-pub fn display_services(
-    params: WithIoFormat<Empty>,
-    services: BTreeMap<IrohAddress, IrohServiceInfo>,
-) -> Result<(), Error> {
-    use prettytable::*;
-
-    if let Some(format) = params.format {
-        return display_serializable(format, services);
-    }
-
-    let mut table = Table::new();
-    table.add_row(row![bc => "ADDRESS", "BINDINGS"]);
-    for (service, info) in services {
-        let row = row![
-            &service.to_string(),
-            &info
-                .bindings
-                .into_iter()
-                .map(|((subdomain, port), addr)| lazy_format!("{subdomain}:{port} -> {addr}"))
-                .join("; ")
-        ];
-        table.add_row(row);
-    }
-    table.print_tty(false)?;
-    Ok(())
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct IrohServiceInfo {
-    pub bindings: BTreeMap<(InternedString, u16), SocketAddr>,
-}
-
-pub async fn list_services(
-    ctx: RpcContext,
-    _: Empty,
-) -> Result<BTreeMap<IrohAddress, IrohServiceInfo>, Error> {
-    ctx.net_controller.iroh.list_services().await
-}
-
-#[derive(Clone)]
-pub struct IrohController(Arc<IrohControllerInner>);
-struct IrohControllerInner {
-    // client: Endpoint,
-    services: SyncMutex<BTreeMap<IrohAddress, IrohService>>,
-}
-impl IrohController {
-    pub fn new() -> Result<Self, Error> {
-        Ok(Self(Arc::new(IrohControllerInner {
-            services: SyncMutex::new(BTreeMap::new()),
-        })))
-    }
-
-    pub fn service(&self, key: IrohSecretKey) -> Result<IrohService, Error> {
-        self.0.services.mutate(|s| {
-            use std::collections::btree_map::Entry;
-            let addr = key.iroh_address();
-            match s.entry(addr) {
-                Entry::Occupied(e) => Ok(e.get().clone()),
-                Entry::Vacant(e) => Ok(e
-                    .insert(IrohService::launch(self.0.client.clone(), key)?)
-                    .clone()),
-            }
-        })
-    }
-
-    pub async fn gc(&self, addr: Option<IrohAddress>) -> Result<(), Error> {
-        if let Some(addr) = addr {
-            if let Some(s) = self.0.services.mutate(|s| {
-                let rm = if let Some(s) = s.get(&addr) {
-                    !s.gc()
-                } else {
-                    false
-                };
-                if rm {
-                    s.remove(&addr)
-                } else {
-                    None
-                }
-            }) {
-                s.shutdown().await
-            } else {
-                Ok(())
-            }
-        } else {
-            for s in self.0.services.mutate(|s| {
-                let mut rm = Vec::new();
-                s.retain(|_, s| {
-                    if s.gc() {
-                        true
-                    } else {
-                        rm.push(s.clone());
-                        false
-                    }
-                });
-                rm
-            }) {
-                s.shutdown().await?;
-            }
-            Ok(())
-        }
-    }
-
-    pub async fn list_services(&self) -> Result<BTreeMap<IrohAddress, IrohServiceInfo>, Error> {
-        Ok(self
-            .0
-            .services
-            .peek(|s| s.iter().map(|(a, s)| (a.clone(), s.info())).collect()))
-    }
-
-    pub async fn connect_iroh(
-        &self,
-        addr: &IrohAddress,
-        port: u16,
-    ) -> Result<Box<dyn ReadWriter + Unpin + Send + Sync + 'static>, Error> {
-        if let Some(target) = self.0.services.peek(|s| {
-            s.get(addr).and_then(|s| {
-                s.0.bindings.peek(|b| {
-                    b.get(&port).and_then(|b| {
-                        b.iter()
-                            .find(|(_, rc)| rc.strong_count() > 0)
-                            .map(|(a, _)| *a)
-                    })
-                })
-            })
-        }) {
-            Ok(Box::new(
-                TcpStream::connect(target)
-                    .await
-                    .with_kind(ErrorKind::Network)?,
-            ))
-        } else {
-            todo!()
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct IrohService(Arc<IrohServiceData>);
-struct IrohServiceData {
-    service: Endpoint,
-    bindings: Arc<SyncRwLock<BTreeMap<(InternedString, u16), BTreeMap<SocketAddr, Weak<()>>>>>,
-    _thread: NonDetachingJoinHandle<()>,
-}
-impl IrohService {
-    fn launch(
-        mut client: Watch<(usize, IrohClient<TokioRustlsRuntime>)>,
-        key: IrohSecretKey,
-    ) -> Result<Self, Error> {
-        let service = Arc::new(SyncMutex::new(None));
-        let bindings = Arc::new(SyncRwLock::new(BTreeMap::<
-            u16,
-            BTreeMap<SocketAddr, Weak<()>>,
-        >::new()));
-        Ok(Self(Arc::new(IrohServiceData {
-            service: service.clone(),
-            bindings: bindings.clone(),
-            _thread: tokio::spawn(async move {
-                let (bg, mut runner) = BackgroundJobQueue::new();
-                runner
-                    .run_while(async {
-                        loop {
-                            if let Err(e) = async {
-                                client.wait_for(|(_,c)| c.bootstrap_status().ready_for_traffic()).await;
-                                let epoch = client.peek(|(e, c)| {
-                                    ensure_code!(c.bootstrap_status().ready_for_traffic(), ErrorKind::Iroh, "client recycled");
-                                    Ok::<_, Error>(*e)
-                                })?;
-                                let (new_service, stream) = client.peek(|(_, c)| {
-                                    c.launch_onion_service_with_hsid(
-                                        IrohServiceConfigBuilder::default()
-                                            .nickname(
-                                                key.iroh_address()
-                                                    .to_string()
-                                                    .trim_end_matches(".onion")
-                                                    .parse::<HsNickname>()
-                                                    .with_kind(ErrorKind::Iroh)?,
-                                            )
-                                            .build()
-                                            .with_kind(ErrorKind::Iroh)?,
-                                        key.clone().0,
-                                    )
-                                    .with_kind(ErrorKind::Iroh)
-                                })?;
-                                let mut status_stream = new_service.status_events();
-                                bg.add_job(async move {
-                                    while let Some(status) = status_stream.next().await {
-                                        // TODO: health daemon?
-                                    }
-                                });
-                                service.replace(Some(new_service));
-                                let mut stream = tor_hsservice::handle_rend_requests(stream);
-                                while let Some(req) = tokio::select! {
-                                    req = stream.next() => req,
-                                    _ = client.wait_for(|(e, _)| *e != epoch) => None
-                                } {
-                                    bg.add_job({
-                                        let bg = bg.clone();
-                                        let bindings = bindings.clone();
-                                        async move {
-                                            if let Err(e) = async {
-                                                let IncomingStreamRequest::Begin(begin) =
-                                                    req.request()
-                                                else {
-                                                    return req
-                                                        .reject(tor_cell::relaycell::msg::End::new_with_reason(
-                                                            tor_cell::relaycell::msg::EndReason::DONE,
-                                                        ))
-                                                        .await
-                                                        .with_kind(ErrorKind::Iroh);
-                                                };
-                                                let Some(target) = bindings.peek(|b| {
-                                                    b.get(&begin.port()).and_then(|a| {
-                                                        a.iter()
-                                                            .find(|(_, rc)| rc.strong_count() > 0)
-                                                            .map(|(addr, _)| *addr)
-                                                    })
-                                                }) else {
-                                                    return req
-                                                        .reject(tor_cell::relaycell::msg::End::new_with_reason(
-                                                            tor_cell::relaycell::msg::EndReason::DONE,
-                                                        ))
-                                                        .await
-                                                        .with_kind(ErrorKind::Iroh);
-                                                };
-                                                bg.add_job(async move {
-                                                    if let Err(e) = async {
-                                                        let mut outgoing =
-                                                            TcpStream::connect(target)
-                                                                .await
-                                                                .with_kind(ErrorKind::Network)?;
-                                                        let mut incoming = req
-                                                            .accept(Connected::new_empty())
-                                                            .await
-                                                            .with_kind(ErrorKind::Iroh)?;
-                                                        if let Err(e) =
-                                                            tokio::io::copy_bidirectional(
-                                                                &mut outgoing,
-                                                                &mut incoming,
-                                                            )
-                                                            .await
-                                                        {
-                                                            tracing::error!("Iroh Stream Error: {e}");
-                                                            tracing::debug!("{e:?}");
-                                                        }
-
-                                                        Ok::<_, Error>(())
-                                                    }
-                                                    .await
-                                                    {
-                                                        tracing::trace!("Iroh Stream Error: {e}");
-                                                        tracing::trace!("{e:?}");
-                                                    }
-                                                });
-                                                Ok::<_, Error>(())
-                                            }
-                                            .await
-                                            {
-                                                tracing::trace!("Iroh Request Error: {e}");
-                                                tracing::trace!("{e:?}");
-                                            }
-                                        }
-                                    });
-                                }
-                                Ok::<_, Error>(())
-                            }
-                            .await
-                            {
-                                tracing::error!("Iroh Client Error: {e}");
-                                tracing::debug!("{e:?}");
-                            }
-                        }
-                    })
-                    .await
-            })
-            .into(),
-        })))
-    }
-
-    pub fn proxy_all<Rcs: FromIterator<Arc<()>>>(
-        &self,
-        bindings: impl IntoIterator<Item = (InternedString, u16, SocketAddr)>,
-    ) -> Rcs {
-        self.0.bindings.mutate(|b| {
-            bindings
-                .into_iter()
-                .map(|(subdomain, port, target)| {
-                    let entry = b
-                        .entry((subdomain, port))
-                        .or_default()
-                        .entry(target)
-                        .or_default();
-                    if let Some(rc) = entry.upgrade() {
-                        rc
-                    } else {
-                        let rc = Arc::new(());
-                        *entry = Arc::downgrade(&rc);
-                        rc
-                    }
-                })
-                .collect()
-        })
-    }
-
-    pub fn gc(&self) -> bool {
-        self.0.bindings.mutate(|b| {
-            b.retain(|_, targets| {
-                targets.retain(|_, rc| rc.strong_count() > 0);
-                !targets.is_empty()
-            });
-            !b.is_empty()
-        })
-    }
-
-    pub async fn shutdown(self) -> Result<(), Error> {
-        self.0.service.replace(None);
-        self.0._thread.abort();
-        Ok(())
-    }
-
-    pub fn state(&self) -> IrohServiceState {
-        self.0
-            .service
-            .peek(|s| s.as_ref().map(|s| s.status().state().into()))
-            .unwrap_or(IrohServiceState::Bootstrapping)
-    }
-
-    pub fn info(&self) -> IrohServiceInfo {
-        IrohServiceInfo {
-            state: self.state(),
-            bindings: self.0.bindings.peek(|b| {
-                b.iter()
-                    .filter_map(|(port, b)| {
-                        b.iter()
-                            .find(|(_, rc)| rc.strong_count() > 0)
-                            .map(|(addr, _)| (*port, *addr))
-                    })
-                    .collect()
-            }),
-        }
-    }
-}

core/startos/src/net/keys.rs

@@ -2,17 +2,14 @@ use serde::{Deserialize, Serialize};
 
 use crate::account::AccountInfo;
 use crate::net::acme::AcmeCertStore;
-use crate::net::iroh::IrohKeyStore;
 use crate::net::ssl::CertStore;
-use crate::net::tor::OnionKeyStore;
+use crate::net::tor::OnionStore;
 use crate::prelude::*;
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]
 #[model = "Model<Self>"]
 pub struct KeyStore {
-    pub onion: OnionKeyStore,
-    #[serde(default)]
-    pub iroh: IrohKeyStore,
+    pub onion: OnionStore,
     pub local_certs: CertStore,
     #[serde(default)]
     pub acme: AcmeCertStore,
@@ -20,8 +17,7 @@ pub struct KeyStore {
 impl KeyStore {
     pub fn new(account: &AccountInfo) -> Result<Self, Error> {
         let mut res = Self {
-            onion: OnionKeyStore::new(),
-            iroh: IrohKeyStore::new(),
+            onion: OnionStore::new(),
             local_certs: CertStore::new(account)?,
             acme: AcmeCertStore::new(),
         };

core/startos/src/net/mod.rs

@@ -5,7 +5,6 @@ pub mod dns;
 pub mod forward;
 pub mod gateway;
 pub mod host;
-pub mod iroh;
 pub mod keys;
 pub mod mdns;
 pub mod net_controller;
@@ -34,6 +33,10 @@ pub fn net_api<C: Context>() -> ParentHandler<C> {
             "dns",
             dns::dns_api::<C>().with_about("Manage and query DNS"),
         )
+        .subcommand(
+            "forward",
+            forward::forward_api::<C>().with_about("Manage port forwards"),
+        )
         .subcommand(
             "gateway",
             gateway::gateway_api::<C>().with_about("View and edit gateway configurations"),

core/startos/src/net/net_controller.rs

@@ -11,7 +11,7 @@ use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 use tracing::instrument;
 
-use crate::db::model::public::NetworkInterfaceInfo;
+use crate::db::model::public::{NetworkInterfaceInfo, NetworkInterfaceType};
 use crate::db::model::Database;
 use crate::error::ErrorCollection;
 use crate::hostname::Hostname;
@@ -24,8 +24,7 @@ use crate::net::gateway::{
 use crate::net::host::address::HostAddress;
 use crate::net::host::binding::{AddSslOptions, BindId, BindOptions};
 use crate::net::host::{host_for, Host, Hosts};
-use crate::net::iroh::IrohController;
-use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname};
+use crate::net::service_interface::{GatewayInfo, HostnameInfo, IpHostname, OnionHostname};
 use crate::net::socks::SocksController;
 use crate::net::tor::{OnionAddress, TorController, TorSecretKey};
 use crate::net::utils::ipv6_is_local;
@@ -38,7 +37,6 @@ use crate::HOST_IP;
 pub struct NetController {
     pub(crate) db: TypedPatchDb<Database>,
     pub(super) tor: TorController,
-    pub(super) iroh: IrohController,
     pub(super) vhost: VHostController,
     pub(crate) net_iface: Arc<NetworkInterfaceController>,
     pub(super) dns: DnsController,
@@ -56,12 +54,10 @@ impl NetController {
     ) -> Result<Self, Error> {
         let net_iface = Arc::new(NetworkInterfaceController::new(db.clone()));
         let tor = TorController::new()?;
-        let iroh = IrohController::new()?;
         let socks = SocksController::new(socks_listen, tor.clone())?;
         Ok(Self {
             db: db.clone(),
             tor,
-            iroh,
             vhost: VHostController::new(db.clone(), net_iface.clone()),
             dns: DnsController::init(db, &net_iface.watcher).await?,
             forward: PortForwardController::new(net_iface.watcher.subscribe()),
@@ -299,7 +295,11 @@ impl NetServiceData {
                                     ); // TODO: wrap onion ssl stream directly in tor ctrl
                                 }
                             }
-                            HostAddress::Domain { address, public } => {
+                            HostAddress::Domain {
+                                address,
+                                public,
+                                private,
+                            } => {
                                 if hostnames.insert(address.clone()) {
                                     let address = Some(address.clone());
                                     if ssl.preferred_external_port == 443 {
@@ -325,10 +325,19 @@ impl NetServiceData {
                                                 TargetInfo {
                                                     filter: AndFilter(
                                                         bind.net.clone(),
-                                                        OrFilter(
-                                                            IdFilter(public.gateway.clone()),
-                                                            PublicFilter { public: false },
-                                                        ),
+                                                        if private {
+                                                            OrFilter(
+                                                                IdFilter(public.gateway.clone()),
+                                                                PublicFilter { public: false },
+                                                            )
+                                                            .into_dyn()
+                                                        } else {
+                                                            AndFilter(
+                                                                IdFilter(public.gateway.clone()),
+                                                                PublicFilter { public: true },
+                                                            )
+                                                            .into_dyn()
+                                                        },
                                                     )
                                                     .into_dyn(),
                                                     acme: public.acme.clone(),
@@ -358,7 +367,16 @@ impl NetServiceData {
                                                 TargetInfo {
                                                     filter: AndFilter(
                                                         bind.net.clone(),
-                                                        IdFilter(public.gateway.clone()),
+                                                        if private {
+                                                            OrFilter(
+                                                                IdFilter(public.gateway.clone()),
+                                                                PublicFilter { public: false },
+                                                            )
+                                                            .into_dyn()
+                                                        } else {
+                                                            IdFilter(public.gateway.clone())
+                                                                .into_dyn()
+                                                        },
                                                     )
                                                     .into_dyn(),
                                                     acme: public.acme.clone(),
@@ -370,7 +388,11 @@ impl NetServiceData {
                                             vhosts.insert(
                                                 (address.clone(), external),
                                                 TargetInfo {
-                                                    filter: bind.net.clone().into_dyn(),
+                                                    filter: AndFilter(
+                                                        bind.net.clone(),
+                                                        PublicFilter { public: false },
+                                                    )
+                                                    .into_dyn(),
                                                     acme: None,
                                                     addr,
                                                     connect_ssl: connect_ssl.clone(),
@@ -405,66 +427,91 @@ impl NetServiceData {
                 }
                 let mut bind_hostname_info: Vec<HostnameInfo> =
                     hostname_info.remove(port).unwrap_or_default();
-                for (interface, info) in net_ifaces
+                for (gateway_id, info) in net_ifaces
                     .iter()
                     .filter(|(id, info)| bind.net.filter(id, info))
                 {
-                    if !info.public() {
+                    let gateway = GatewayInfo {
+                        id: gateway_id.clone(),
+                        name: info
+                            .name
+                            .clone()
+                            .or_else(|| info.ip_info.as_ref().map(|i| i.name.clone()))
+                            .unwrap_or_else(|| gateway_id.clone().into()),
+                        public: info.public(),
+                    };
+                    let port = bind.net.assigned_port.filter(|_| {
+                        bind.options.secure.map_or(false, |s| {
+                            !(s.ssl && bind.options.add_ssl.is_some()) || info.secure()
+                        })
+                    });
+                    if !info.public()
+                        && info.ip_info.as_ref().map_or(false, |i| {
+                            i.device_type != Some(NetworkInterfaceType::Wireguard)
+                        })
+                    {
                         bind_hostname_info.push(HostnameInfo::Ip {
-                            gateway_id: interface.clone(),
+                            gateway: gateway.clone(),
                             public: false,
                             hostname: IpHostname::Local {
                                 value: InternedString::from_display(&{
                                     let hostname = &hostname;
                                     lazy_format!("{hostname}.local")
                                 }),
-                                port: bind.net.assigned_port,
+                                port,
                                 ssl_port: bind.net.assigned_ssl_port,
                             },
                         });
                     }
                     for address in host.addresses() {
                         if let HostAddress::Domain {
-                            address, public, ..
+                            address,
+                            public,
+                            private,
                         } = address
                         {
-                            if bind
-                                .options
-                                .add_ssl
-                                .as_ref()
-                                .map_or(false, |ssl| ssl.preferred_external_port == 443)
-                            {
-                                bind_hostname_info.push(HostnameInfo::Ip {
-                                    gateway_id: interface.clone(),
-                                    public: public.is_some(),
-                                    hostname: IpHostname::Domain {
-                                        value: address.clone(),
-                                        port: None,
-                                        ssl_port: Some(443),
-                                    },
-                                });
-                            } else {
-                                bind_hostname_info.push(HostnameInfo::Ip {
-                                    gateway_id: interface.clone(),
-                                    public: public.is_some(),
-                                    hostname: IpHostname::Domain {
-                                        value: address.clone(),
-                                        port: bind.net.assigned_port,
-                                        ssl_port: bind.net.assigned_ssl_port,
-                                    },
-                                });
+                            let private = private && !info.public();
+                            let public =
+                                public.as_ref().map_or(false, |p| &p.gateway == gateway_id);
+                            if public || private {
+                                if bind
+                                    .options
+                                    .add_ssl
+                                    .as_ref()
+                                    .map_or(false, |ssl| ssl.preferred_external_port == 443)
+                                {
+                                    bind_hostname_info.push(HostnameInfo::Ip {
+                                        gateway: gateway.clone(),
+                                        public,
+                                        hostname: IpHostname::Domain {
+                                            value: address.clone(),
+                                            port: None,
+                                            ssl_port: Some(443),
+                                        },
+                                    });
+                                } else {
+                                    bind_hostname_info.push(HostnameInfo::Ip {
+                                        gateway: gateway.clone(),
+                                        public,
+                                        hostname: IpHostname::Domain {
+                                            value: address.clone(),
+                                            port,
+                                            ssl_port: bind.net.assigned_ssl_port,
+                                        },
+                                    });
+                                }
                             }
                         }
                     }
                     if let Some(ip_info) = &info.ip_info {
                         let public = info.public();
-                        if let Some(wan_ip) = ip_info.wan_ip.filter(|_| public) {
+                        if let Some(wan_ip) = ip_info.wan_ip {
                             bind_hostname_info.push(HostnameInfo::Ip {
-                                gateway_id: interface.clone(),
-                                public,
+                                gateway: gateway.clone(),
+                                public: true,
                                 hostname: IpHostname::Ipv4 {
                                     value: wan_ip,
-                                    port: bind.net.assigned_port,
+                                    port,
                                     ssl_port: bind.net.assigned_ssl_port,
                                 },
                             });
@@ -474,11 +521,11 @@ impl NetServiceData {
                                 IpNet::V4(net) => {
                                     if !public {
                                         bind_hostname_info.push(HostnameInfo::Ip {
-                                            gateway_id: interface.clone(),
+                                            gateway: gateway.clone(),
                                             public,
                                             hostname: IpHostname::Ipv4 {
                                                 value: net.addr(),
-                                                port: bind.net.assigned_port,
+                                                port,
                                                 ssl_port: bind.net.assigned_ssl_port,
                                             },
                                         });
@@ -486,12 +533,12 @@ impl NetServiceData {
                                 }
                                 IpNet::V6(net) => {
                                     bind_hostname_info.push(HostnameInfo::Ip {
-                                        gateway_id: interface.clone(),
+                                        gateway: gateway.clone(),
                                         public: public && !ipv6_is_local(net.addr()),
                                         hostname: IpHostname::Ipv6 {
                                             value: net.addr(),
                                             scope_id: ip_info.scope_id,
-                                            port: bind.net.assigned_port,
+                                            port,
                                             ssl_port: bind.net.assigned_ssl_port,
                                         },
                                     });

core/startos/src/net/service_interface.rs

@@ -12,8 +12,7 @@ use ts_rs::TS;
 #[serde(tag = "kind")]
 pub enum HostnameInfo {
     Ip {
-        #[ts(type = "string")]
-        gateway_id: GatewayId,
+        gateway: GatewayInfo,
         public: bool,
         hostname: IpHostname,
     },
@@ -30,6 +29,15 @@ impl HostnameInfo {
     }
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub struct GatewayInfo {
+    pub id: GatewayId,
+    pub name: InternedString,
+    pub public: bool,
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize, TS)]
 #[ts(export)]
 #[serde(rename_all = "camelCase")]

core/startos/src/net/socks.rs

@@ -1,5 +1,6 @@
 use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
 use std::sync::Arc;
+use std::time::Duration;
 
 use helpers::NonDetachingJoinHandle;
 use socks5_impl::protocol::{Address, Reply};
@@ -22,15 +23,21 @@ pub struct SocksController {
 }
 impl SocksController {
     pub fn new(listen: SocketAddr, tor: TorController) -> Result<Self, Error> {
-        let auth: AuthAdaptor<()> = Arc::new(NoAuth);
-        let listener = TcpListener::from_std(
-            mio::net::TcpListener::bind(listen)
-                .with_kind(ErrorKind::Network)?
-                .into(),
-        )
-        .with_kind(ErrorKind::Network)?;
         Ok(Self {
             _thread: tokio::spawn(async move {
+                let auth: AuthAdaptor<()> = Arc::new(NoAuth);
+                let listener;
+                loop {
+                    if let Some(l) = TcpListener::bind(listen)
+                        .await
+                        .with_kind(ErrorKind::Network)
+                        .log_err()
+                    {
+                        listener = l;
+                        break;
+                    }
+                    tokio::time::sleep(Duration::from_secs(1)).await;
+                }
                 let (bg, mut runner) = BackgroundJobQueue::new();
                 runner
                     .run_while(async {
@@ -149,8 +156,8 @@ impl SocksController {
                                         }
                                         .await
                                         {
-                                            tracing::error!("SOCKS5 Stream Error: {e}");
-                                            tracing::debug!("{e:?}");
+                                            tracing::trace!("SOCKS5 Stream Error: {e}");
+                                            tracing::trace!("{e:?}");
                                         }
                                     });
                                 }

core/startos/src/net/tor.rs

@@ -6,7 +6,7 @@ use std::sync::{Arc, Weak};
 use std::time::{Duration, Instant};
 
 use arti_client::config::onion_service::OnionServiceConfigBuilder;
-use arti_client::{TorClient, TorClientConfig};
+use arti_client::{DataStream, TorClient, TorClientConfig};
 use base64::Engine;
 use clap::Parser;
 use color_eyre::eyre::eyre;
@@ -62,7 +62,7 @@ impl FromStr for OnionAddress {
                 Cow::Owned(format!("{s}.onion"))
             }
             .parse::<HsId>()
-            .with_kind(ErrorKind::ParseNetAddress)?,
+            .with_kind(ErrorKind::Tor)?,
         ))
     }
 }
@@ -165,8 +165,8 @@ impl<'de> Deserialize<'de> for TorSecretKey {
 }
 
 #[derive(Default, Deserialize, Serialize)]
-pub struct OnionKeyStore(BTreeMap<OnionAddress, TorSecretKey>);
-impl Map for OnionKeyStore {
+pub struct OnionStore(BTreeMap<OnionAddress, TorSecretKey>);
+impl Map for OnionStore {
     type Key = OnionAddress;
     type Value = TorSecretKey;
     fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
@@ -176,7 +176,7 @@ impl Map for OnionKeyStore {
         Ok(InternedString::from_display(key))
     }
 }
-impl OnionKeyStore {
+impl OnionStore {
     pub fn new() -> Self {
         Self::default()
     }
@@ -184,7 +184,7 @@ impl OnionKeyStore {
         self.0.insert(key.onion_address(), key);
     }
 }
-impl Model<OnionKeyStore> {
+impl Model<OnionStore> {
     pub fn new_key(&mut self) -> Result<TorSecretKey, Error> {
         let key = TorSecretKey::generate();
         self.insert(&key.onion_address(), &key)?;
@@ -199,7 +199,7 @@ impl Model<OnionKeyStore> {
             .de()
     }
 }
-impl std::fmt::Debug for OnionKeyStore {
+impl std::fmt::Debug for OnionStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         struct OnionStoreMap<'a>(&'a BTreeMap<OnionAddress, TorSecretKey>);
         impl<'a> std::fmt::Debug for OnionStoreMap<'a> {
@@ -227,7 +227,7 @@ pub fn tor_api<C: Context>() -> ParentHandler<C> {
             from_fn_async(list_services)
                 .with_display_serializable()
                 .with_custom_display_fn(|handle, result| display_services(handle.params, result))
-                .with_about("Show the status of running onion services")
+                .with_about("Display Tor V3 Onion Addresses")
                 .with_call_remote::<CliContext>(),
         )
         .subcommand(
@@ -417,6 +417,7 @@ impl TorController {
             0,
             TorClient::with_runtime(TokioRustlsRuntime::current()?)
                 .config(config.build().with_kind(ErrorKind::Tor)?)
+                .local_resource_timeout(Duration::from_secs(0))
                 .create_unbootstrapped()?,
         ));
         let reset = Arc::new(Notify::new());
@@ -424,10 +425,10 @@ impl TorController {
         let bootstrapper_client = client.clone();
         let bootstrapper = tokio::spawn(async move {
             loop {
+                let (epoch, client): (usize, _) = bootstrapper_client.read();
                 if let Err(e) = Until::new()
                     .with_async_fn(|| bootstrapper_reset.notified().map(Ok))
                     .run(async {
-                        let (epoch, client): (usize, _) = bootstrapper_client.read();
                         let mut events = client.bootstrap_events();
                         let bootstrap_fut =
                             client.bootstrap().map(|res| res.with_kind(ErrorKind::Tor));
@@ -560,7 +561,7 @@ impl TorController {
                                 }
                                 .await
                                 {
-                                    tracing::error!("Tor Client Creation Error: {e}");
+                                    tracing::error!("Tor Client Health Error: {e}");
                                     tracing::debug!("{e:?}");
                                 }
                             }
@@ -569,21 +570,7 @@ impl TorController {
                                 HEALTH_CHECK_FAILURE_ALLOWANCE
                             );
                         }
-                        if let Err::<(), Error>(e) = async {
-                            tokio::time::sleep(RETRY_COOLDOWN).await;
-                            bootstrapper_client.send((
-                                epoch.wrapping_add(1),
-                                TorClient::with_runtime(TokioRustlsRuntime::current()?)
-                                    .config(config.build().with_kind(ErrorKind::Tor)?)
-                                    .create_unbootstrapped()?,
-                            ));
-                            Ok(())
-                        }
-                        .await
-                        {
-                            tracing::error!("Tor Client Creation Error: {e}");
-                            tracing::debug!("{e:?}");
-                        }
+
                         Ok(())
                     })
                     .await
@@ -591,6 +578,24 @@ impl TorController {
                     tracing::error!("Tor Bootstrapper Error: {e}");
                     tracing::debug!("{e:?}");
                 }
+                if let Err::<(), Error>(e) = async {
+                    tokio::time::sleep(RETRY_COOLDOWN).await;
+                    bootstrapper_client.send((
+                        epoch.wrapping_add(1),
+                        TorClient::with_runtime(TokioRustlsRuntime::current()?)
+                            .config(config.build().with_kind(ErrorKind::Tor)?)
+                            .local_resource_timeout(Duration::from_secs(0))
+                            .create_unbootstrapped_async()
+                            .await?,
+                    ));
+                    tracing::debug!("TorClient recycled");
+                    Ok(())
+                }
+                .await
+                {
+                    tracing::error!("Tor Client Creation Error: {e}");
+                    tracing::debug!("{e:?}");
+                }
             }
         })
         .into();
@@ -729,14 +734,15 @@ impl OnionService {
                             if let Err(e) = async {
                                 client.wait_for(|(_,c)| c.bootstrap_status().ready_for_traffic()).await;
                                 let epoch = client.peek(|(e, c)| {
-                                    ensure_code!(c.bootstrap_status().ready_for_traffic(), ErrorKind::Tor, "client recycled");
+                                    ensure_code!(c.bootstrap_status().ready_for_traffic(), ErrorKind::Tor, "TorClient recycled");
                                     Ok::<_, Error>(*e)
                                 })?;
+                                let addr = key.onion_address();
                                 let (new_service, stream) = client.peek(|(_, c)| {
                                     c.launch_onion_service_with_hsid(
                                         OnionServiceConfigBuilder::default()
                                             .nickname(
-                                                key.onion_address()
+                                                addr
                                                     .to_string()
                                                     .trim_end_matches(".onion")
                                                     .parse::<HsNickname>()
@@ -749,8 +755,20 @@ impl OnionService {
                                     .with_kind(ErrorKind::Tor)
                                 })?;
                                 let mut status_stream = new_service.status_events();
+                                let mut status = new_service.status();
+                                if status.state().is_fully_reachable() {
+                                    tracing::debug!("{addr} is fully reachable");
+                                } else {
+                                    tracing::debug!("{addr} is not fully reachable");
+                                }
                                 bg.add_job(async move {
-                                    while let Some(status) = status_stream.next().await {
+                                    while let Some(new_status) = status_stream.next().await {
+                                        if status.state().is_fully_reachable() && !new_status.state().is_fully_reachable() {
+                                            tracing::debug!("{addr} is no longer fully reachable");
+                                        } else if !status.state().is_fully_reachable() && new_status.state().is_fully_reachable() {
+                                            tracing::debug!("{addr} is now fully reachable");
+                                        }
+                                        status = new_status;
                                         // TODO: health daemon?
                                     }
                                 });
@@ -806,8 +824,8 @@ impl OnionService {
                                                             )
                                                             .await
                                                         {
-                                                            tracing::error!("Tor Stream Error: {e}");
-                                                            tracing::debug!("{e:?}");
+                                                            tracing::trace!("Tor Stream Error: {e}");
+                                                            tracing::trace!("{e:?}");
                                                         }
 
                                                         Ok::<_, Error>(())

core/startos/src/net/tunnel.rs

@@ -1,16 +1,17 @@
 use clap::Parser;
 use imbl_value::InternedString;
 use models::GatewayId;
-use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use patch_db::json_ptr::JsonPointer;
+use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tokio::process::Command;
 use ts_rs::TS;
 
 use crate::context::{CliContext, RpcContext};
-use crate::db::model::public::NetworkInterfaceType;
+use crate::db::model::public::{NetworkInterfaceInfo, NetworkInterfaceType};
 use crate::prelude::*;
+use crate::util::io::{write_file_atomic, TmpDir};
 use crate::util::Invoke;
-use crate::util::io::{TmpDir, write_file_atomic};
 
 pub fn tunnel_api<C: Context>() -> ParentHandler<C> {
     ParentHandler::new()
@@ -32,7 +33,6 @@ pub fn tunnel_api<C: Context>() -> ParentHandler<C> {
 #[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)]
 #[ts(export)]
 pub struct AddTunnelParams {
-    #[ts(type = "string")]
     name: InternedString,
     config: String,
     public: bool,
@@ -46,26 +46,46 @@ pub async fn add_tunnel(
         public,
     }: AddTunnelParams,
 ) -> Result<GatewayId, Error> {
-    let existing = ctx
-        .db
-        .peek()
-        .await
-        .into_public()
-        .into_server_info()
-        .into_network()
-        .into_gateways()
-        .keys()?;
-    let mut iface = GatewayId::from("wg0");
-    for id in 1.. {
-        if !existing.contains(&iface) {
-            break;
+    let ifaces = ctx.net_controller.net_iface.watcher.subscribe();
+    let mut iface = GatewayId::from(InternedString::intern("wg0"));
+    if !ifaces.send_if_modified(|i| {
+        for id in 1..256 {
+            if !i.contains_key(&iface) {
+                i.insert(
+                    iface.clone(),
+                    NetworkInterfaceInfo {
+                        name: Some(name),
+                        public: Some(public),
+                        secure: None,
+                        ip_info: None,
+                    },
+                );
+                return true;
+            }
+            iface = InternedString::from_display(&lazy_format!("wg{id}")).into();
         }
-        iface = InternedString::from_display(&lazy_format!("wg{id}")).into();
+        false
+    }) {
+        return Err(Error::new(
+            eyre!("too many wireguard interfaces"),
+            ErrorKind::InvalidRequest,
+        ));
     }
+
+    let mut sub = ctx
+        .db
+        .subscribe(
+            "/public/serverInfo/network/gateways"
+                .parse::<JsonPointer>()
+                .with_kind(ErrorKind::Database)?
+                .join_end(iface.as_str())
+                .join_end("ipInfo"),
+        )
+        .await;
+
     let tmpdir = TmpDir::new().await?;
     let conf = tmpdir.join(&iface).with_extension("conf");
     write_file_atomic(&conf, &config).await?;
-    let mut ifaces = ctx.net_controller.net_iface.watcher.subscribe();
     Command::new("nmcli")
         .arg("connection")
         .arg("import")
@@ -77,14 +97,7 @@ pub async fn add_tunnel(
         .await?;
     tmpdir.delete().await?;
 
-    ifaces.wait_for(|ifaces| ifaces.contains_key(&iface)).await;
-
-    ctx.net_controller
-        .net_iface
-        .set_public(&iface, Some(public))
-        .await?;
-
-    ctx.net_controller.net_iface.set_name(&iface, &name).await?;
+    sub.recv().await;
 
     Ok(iface)
 }

core/startos/src/net/web_server.rs

@@ -258,6 +258,7 @@ impl<A: Accept + Send + Sync + 'static> WebServer<A> {
                         .await
                         {
                             err = Some(e);
+                            tokio::time::sleep(Duration::from_millis(100)).await;
                         } else {
                             break;
                         }

core/startos/src/notifications.rs

@@ -3,13 +3,13 @@ use std::fmt;
 use std::str::FromStr;
 
 use chrono::{DateTime, Utc};
-use clap::Parser;
 use clap::builder::ValueParserFactory;
+use clap::Parser;
 use color_eyre::eyre::eyre;
 use helpers::const_true;
 use imbl_value::InternedString;
 use models::{FromStrParser, PackageId};
-use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 use ts_rs::TS;
@@ -155,6 +155,16 @@ pub async fn remove(
             for id in ids {
                 n.remove(&id)?;
             }
+            let mut unread = 0;
+            for (_, n) in n.as_entries()? {
+                if !n.as_seen().de()? {
+                    unread += 1;
+                }
+            }
+            db.as_public_mut()
+                .as_server_info_mut()
+                .as_unread_notification_count_mut()
+                .ser(&unread)?;
             Ok(())
         })
         .await
@@ -179,6 +189,16 @@ pub async fn remove_before(
             for id in n.keys()?.range(..before) {
                 n.remove(&id)?;
             }
+            let mut unread = 0;
+            for (_, n) in n.as_entries()? {
+                if !n.as_seen().de()? {
+                    unread += 1;
+                }
+            }
+            db.as_public_mut()
+                .as_server_info_mut()
+                .as_unread_notification_count_mut()
+                .ser(&unread)?;
             Ok(())
         })
         .await

core/startos/src/s9pk/v2/compat.rs

@@ -8,7 +8,7 @@ use models::{ImageId, VolumeId};
 use tokio::io::{AsyncRead, AsyncSeek, AsyncWriteExt};
 use tokio::process::Command;
 
-use crate::dependencies::{DepInfo, Dependencies};
+use crate::dependencies::{DepInfo, Dependencies, MetadataSrc};
 use crate::prelude::*;
 use crate::s9pk::manifest::{DeviceFilter, Manifest};
 use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
@@ -16,10 +16,10 @@ use crate::s9pk::merkle_archive::source::TmpSource;
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure};
 use crate::s9pk::v1::reader::S9pkReader;
-use crate::s9pk::v2::pack::{CONTAINER_TOOL, ImageSource, PackSource};
+use crate::s9pk::v2::pack::{ImageSource, PackSource, CONTAINER_TOOL};
 use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
+use crate::util::io::{create_file, TmpDir};
 use crate::util::Invoke;
-use crate::util::io::{TmpDir, create_file};
 
 pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01];
 
@@ -225,7 +225,7 @@ impl TryFrom<ManifestV1> for Manifest {
                             DepInfo {
                                 description: value.description,
                                 optional: !value.requirement.required(),
-                                s9pk: None,
+                                metadata: None,
                             },
                         )
                     })

core/startos/src/s9pk/v2/pack.rs

@@ -3,10 +3,10 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use clap::Parser;
-use futures::future::{BoxFuture, ready};
+use futures::future::{ready, BoxFuture};
 use futures::{FutureExt, TryStreamExt};
 use imbl_value::InternedString;
-use models::{ImageId, PackageId, VersionString};
+use models::{DataUrl, ImageId, PackageId, VersionString};
 use serde::{Deserialize, Serialize};
 use tokio::process::Command;
 use tokio::sync::OnceCell;
@@ -15,23 +15,23 @@ use tracing::{debug, warn};
 use ts_rs::TS;
 
 use crate::context::CliContext;
-use crate::dependencies::DependencyMetadata;
+use crate::dependencies::{DependencyMetadata, MetadataSrc};
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
-use crate::s9pk::S9pk;
 use crate::s9pk::git_hash::GitHash;
 use crate::s9pk::manifest::Manifest;
 use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
 use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::{
-    ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource, into_dyn_read,
+    into_dyn_read, ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource,
 };
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::v2::SIG_CONTEXT;
-use crate::util::io::{TmpDir, create_file, open_file};
+use crate::s9pk::S9pk;
+use crate::util::io::{create_file, open_file, TmpDir};
 use crate::util::serde::IoFormat;
-use crate::util::{Invoke, PathOrUrl, new_guid};
+use crate::util::{new_guid, Invoke, PathOrUrl};
 
 #[cfg(not(feature = "docker"))]
 pub const CONTAINER_TOOL: &str = "podman";
@@ -369,12 +369,10 @@ impl ImageSource {
                 workdir,
                 ..
             } => {
-                vec![
-                    workdir
-                        .as_deref()
-                        .unwrap_or(Path::new("."))
-                        .join(dockerfile.as_deref().unwrap_or(Path::new("Dockerfile"))),
-                ]
+                vec![workdir
+                    .as_deref()
+                    .unwrap_or(Path::new("."))
+                    .join(dockerfile.as_deref().unwrap_or(Path::new("Dockerfile")))]
             }
             Self::DockerTag(_) => Vec::new(),
         }
@@ -699,53 +697,77 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
 
     let mut to_insert = Vec::new();
     for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
-        if let Some(s9pk) = dependency.s9pk.take() {
-            let s9pk = match s9pk {
-                PathOrUrl::Path(path) => {
-                    S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
-                        .await?
-                        .into_dyn()
-                }
-                PathOrUrl::Url(url) => {
-                    if url.scheme() == "http" || url.scheme() == "https" {
-                        S9pk::deserialize(
-                            &Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
-                            None,
-                        )
-                        .await?
-                        .into_dyn()
-                    } else {
-                        return Err(Error::new(
-                            eyre!("unknown scheme: {}", url.scheme()),
-                            ErrorKind::InvalidRequest,
-                        ));
+        if let Some((title, icon)) = match dependency.metadata.take() {
+            Some(MetadataSrc::Metadata(metadata)) => {
+                let icon = match metadata.icon {
+                    PathOrUrl::Path(path) => DataUrl::from_path(path).await?,
+                    PathOrUrl::Url(url) => {
+                        if url.scheme() == "http" || url.scheme() == "https" {
+                            DataUrl::from_response(ctx.client.get(url).send().await?).await?
+                        } else if url.scheme() == "data" {
+                            url.as_str().parse()?
+                        } else {
+                            return Err(Error::new(
+                                eyre!("unknown scheme: {}", url.scheme()),
+                                ErrorKind::InvalidRequest,
+                            ));
+                        }
                     }
-                }
-            };
+                };
+                Some((metadata.title, icon))
+            }
+            Some(MetadataSrc::S9pk(Some(s9pk))) => {
+                let s9pk = match s9pk {
+                    PathOrUrl::Path(path) => {
+                        S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
+                            .await?
+                            .into_dyn()
+                    }
+                    PathOrUrl::Url(url) => {
+                        if url.scheme() == "http" || url.scheme() == "https" {
+                            S9pk::deserialize(
+                                &Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
+                                None,
+                            )
+                            .await?
+                            .into_dyn()
+                        } else {
+                            return Err(Error::new(
+                                eyre!("unknown scheme: {}", url.scheme()),
+                                ErrorKind::InvalidRequest,
+                            ));
+                        }
+                    }
+                };
+                Some((
+                    s9pk.as_manifest().title.clone(),
+                    s9pk.icon_data_url().await?,
+                ))
+            }
+            Some(MetadataSrc::S9pk(None)) | None => {
+                warn!("no metadata specified for {id}, leaving metadata empty");
+                None
+            }
+        } {
             let dep_path = Path::new("dependencies").join(id);
             to_insert.push((
                 dep_path.join("metadata.json"),
                 Entry::file(TmpSource::new(
                     tmp_dir.clone(),
                     PackSource::Buffered(
-                        IoFormat::Json
-                            .to_vec(&DependencyMetadata {
-                                title: s9pk.as_manifest().title.clone(),
-                            })?
-                            .into(),
+                        IoFormat::Json.to_vec(&DependencyMetadata { title })?.into(),
                     ),
                 )),
             ));
-            let icon = s9pk.icon().await?;
             to_insert.push((
-                dep_path.join(&*icon.0),
+                dep_path
+                    .join("icon")
+                    .with_extension(icon.canonical_ext().unwrap_or("ico")),
                 Entry::file(TmpSource::new(
                     tmp_dir.clone(),
-                    PackSource::Buffered(icon.1.expect_file()?.to_vec(icon.1.hash()).await?.into()),
+                    PackSource::Buffered(icon.data.into_owned().into()),
                 )),
             ));
-        } else {
-            warn!("no s9pk specified for {id}, leaving metadata empty");
         }
     }
     for (path, source) in to_insert {
@@ -797,8 +819,17 @@ pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<P
     let mut ingredients = vec![js_path, params.icon().await?, params.license().await?];
 
     for (_, dependency) in manifest.dependencies.0 {
-        if let Some(PathOrUrl::Path(p)) = dependency.s9pk {
-            ingredients.push(p);
+        match dependency.metadata {
+            Some(MetadataSrc::Metadata(crate::dependencies::Metadata {
+                icon: PathOrUrl::Path(icon),
+                ..
+            })) => {
+                ingredients.push(icon);
+            }
+            Some(MetadataSrc::S9pk(Some(PathOrUrl::Path(s9pk)))) => {
+                ingredients.push(s9pk);
+            }
+            _ => (),
         }
     }
 

core/startos/src/service/effects/callbacks.rs

@@ -6,7 +6,7 @@ use std::time::{Duration, SystemTime};
 use clap::Parser;
 use futures::future::join_all;
 use helpers::NonDetachingJoinHandle;
-use imbl::{Vector, vector};
+use imbl::{vector, Vector};
 use imbl_value::InternedString;
 use models::{HostId, PackageId, ServiceInterfaceId};
 use serde::{Deserialize, Serialize};
@@ -264,7 +264,6 @@ impl CallbackHandler {
         }
     }
     pub async fn call(mut self, args: Vector<Value>) -> Result<(), Error> {
-        crate::dbg!(eyre!("callback fired: {}", self.handle.is_active()));
         if let Some(seed) = self.seed.upgrade() {
             seed.persistent_container
                 .callback(self.handle.take(), args)

core/startos/src/service/mod.rs

@@ -15,12 +15,12 @@ use futures::future::BoxFuture;
 use futures::stream::FusedStream;
 use futures::{FutureExt, SinkExt, StreamExt, TryStreamExt};
 use helpers::NonDetachingJoinHandle;
-use imbl_value::{InternedString, json};
+use imbl_value::{json, InternedString};
 use itertools::Itertools;
 use models::{ActionId, HostId, ImageId, PackageId};
 use nix::sys::signal::Signal;
 use persistent_container::{PersistentContainer, Subcontainer};
-use rpc_toolkit::{CallRemoteHandler, Empty, HandlerArgs, HandlerFor, from_fn_async};
+use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor};
 use serde::{Deserialize, Serialize};
 use service_actor::ServiceActor;
 use start_stop::StartStop;
@@ -47,11 +47,11 @@ use crate::service::action::update_tasks;
 use crate::service::rpc::{ExitParams, InitKind};
 use crate::service::service_map::InstallProgressHandles;
 use crate::service::uninstall::cleanup;
-use crate::util::Never;
 use crate::util::actor::concurrent::ConcurrentActor;
-use crate::util::io::{AsyncReadStream, TermSize, create_file, delete_file};
+use crate::util::io::{create_file, delete_file, AsyncReadStream, TermSize};
 use crate::util::net::WebSocketExt;
 use crate::util::serde::Pem;
+use crate::util::Never;
 use crate::volume::data_dir;
 use crate::{CAP_1_KiB, DATA_DIR};
 

core/startos/src/service/uninstall.rs

@@ -4,7 +4,7 @@ use models::PackageId;
 
 use crate::context::RpcContext;
 use crate::prelude::*;
-use crate::volume::data_dir;
+use crate::volume::PKG_VOLUME_DIR;
 use crate::{DATA_DIR, PACKAGE_DATA};
 
 pub async fn cleanup(ctx: &RpcContext, id: &PackageId, soft: bool) -> Result<(), Error> {
@@ -45,11 +45,11 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, soft: bool) -> Result<(),
         {
             let state = pde.state_info.expect_removing()?;
             if !soft {
-                for volume_id in &state.manifest.volumes {
-                    let path = data_dir(DATA_DIR, &state.manifest.id, volume_id);
-                    if tokio::fs::metadata(&path).await.is_ok() {
-                        tokio::fs::remove_dir_all(&path).await?;
-                    }
+                let path = Path::new(DATA_DIR)
+                    .join(PKG_VOLUME_DIR)
+                    .join(&state.manifest.id);
+                if tokio::fs::metadata(&path).await.is_ok() {
+                    tokio::fs::remove_dir_all(&path).await?;
                 }
                 let logs_dir = Path::new(PACKAGE_DATA)
                     .join("logs")

core/startos/src/system.rs

@@ -9,7 +9,7 @@ use color_eyre::eyre::eyre;
 use futures::{FutureExt, TryStreamExt};
 use imbl::vector;
 use imbl_value::InternedString;
-use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
+use rpc_toolkit::{Context, Empty, HandlerExt, ParentHandler, from_fn_async};
 use rustls::RootCertStore;
 use rustls_pki_types::CertificateDer;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -24,12 +24,12 @@ use crate::logs::{LogSource, LogsParams, SYSTEM_UNIT};
 use crate::prelude::*;
 use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations};
 use crate::shutdown::Shutdown;
-use crate::util::cpupower::{get_available_governors, set_governor, Governor};
+use crate::util::Invoke;
+use crate::util::cpupower::{Governor, get_available_governors, set_governor};
 use crate::util::io::open_file;
 use crate::util::net::WebSocketExt;
-use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
+use crate::util::serde::{HandlerExtSerde, WithIoFormat, display_serializable};
 use crate::util::sync::Watch;
-use crate::util::Invoke;
 use crate::{MAIN_DATA, PACKAGE_DATA};
 
 pub fn experimental<C: Context>() -> ParentHandler<C> {
@@ -1024,7 +1024,7 @@ pub struct TestSmtpParams {
     #[arg(long)]
     pub login: String,
     #[arg(long)]
-    pub password: String,
+    pub password: Option<String>,
 }
 pub async fn test_smtp(
     _: RpcContext,
@@ -1037,23 +1037,74 @@ pub async fn test_smtp(
         password,
     }: TestSmtpParams,
 ) -> Result<(), Error> {
-    use lettre::message::header::ContentType;
-    use lettre::transport::smtp::authentication::Credentials;
-    use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
+    #[cfg(feature = "mail-send")]
+    {
+        use mail_send::SmtpClientBuilder;
+        use mail_send::mail_builder::{self, MessageBuilder};
+        use rustls_pki_types::pem::PemObject;
 
-    AsyncSmtpTransport::<Tokio1Executor>::relay(&server)?
-        .credentials(Credentials::new(login, password))
-        .build()
-        .send(
-            Message::builder()
-                .from(from.parse()?)
-                .to(to.parse()?)
-                .subject("StartOS Test Email")
-                .header(ContentType::TEXT_PLAIN)
-                .body("This is a test email sent from your StartOS Server".to_owned())?,
-        )
-        .await?;
-    Ok(())
+        let Some(pass_val) = password else {
+            return Err(Error::new(
+                eyre!("mail-send requires a password"),
+                ErrorKind::InvalidRequest,
+            ));
+        };
+
+        let mut root_cert_store = RootCertStore::empty();
+        let pem = tokio::fs::read("/etc/ssl/certs/ca-certificates.crt").await?;
+        for cert in CertificateDer::pem_slice_iter(&pem) {
+            root_cert_store.add_parsable_certificates([cert.with_kind(ErrorKind::OpenSsl)?]);
+        }
+
+        let cfg = Arc::new(
+            rustls::ClientConfig::builder_with_provider(Arc::new(
+                rustls::crypto::ring::default_provider(),
+            ))
+            .with_safe_default_protocol_versions()?
+            .with_root_certificates(root_cert_store)
+            .with_no_client_auth(),
+        );
+        let client = SmtpClientBuilder::new_with_tls_config(server, port, cfg)
+            .implicit_tls(false)
+            .credentials((login.split("@").next().unwrap().to_owned(), pass_val));
+
+        fn parse_address<'a>(addr: &'a str) -> mail_builder::headers::address::Address<'a> {
+            if addr.find("<").map_or(false, |start| {
+                addr.find(">").map_or(false, |end| start < end)
+            }) {
+                addr.split_once("<")
+                    .map(|(name, addr)| (name.trim(), addr.strip_suffix(">").unwrap_or(addr)))
+                    .unwrap()
+                    .into()
+            } else {
+                addr.into()
+            }
+        }
+
+        let message = MessageBuilder::new()
+            .from(parse_address(&from))
+            .to(parse_address(&to))
+            .subject("StartOS Test Email")
+            .text_body("This is a test email sent from your StartOS Server");
+        client
+            .connect()
+            .await
+            .map_err(|e| {
+                Error::new(
+                    eyre!("mail-send connection error: {:?}", e),
+                    ErrorKind::Unknown,
+                )
+            })?
+            .send(message)
+            .await
+            .map_err(|e| Error::new(eyre!("mail-send send error: {:?}", e), ErrorKind::Unknown))?;
+        Ok(())
+    }
+    #[cfg(not(feature = "mail-send"))]
+    Err(Error::new(
+        eyre!("test-smtp requires mail-send feature to be enabled"),
+        ErrorKind::InvalidRequest,
+    ))
 }
 
 #[tokio::test]

core/startos/src/tunnel/api.rs

@@ -2,7 +2,7 @@ use std::net::Ipv4Addr;
 
 use clap::Parser;
 use ipnet::Ipv4Net;
-use rpc_toolkit::{Context, Empty, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 
 use crate::context::CliContext;
@@ -22,7 +22,7 @@ pub fn tunnel_api<C: Context>() -> ParentHandler<C> {
             subnet_api::<C>().with_about("Add, remove, or modify subnets"),
         )
     // .subcommand(
-    //     "forward",
+    //     "port-forward",
     //     ParentHandler::<C>::new()
     //         .subcommand(
     //             "add",
@@ -77,19 +77,19 @@ pub fn subnet_api<C: Context>() -> ParentHandler<C, SubnetParams> {
     //         .with_call_remote::<CliContext>(),
     // )
     // .subcommand(
-    //     "add-client",
-    //     from_fn_async(add_client)
+    //     "add-device",
+    //     from_fn_async(add_device)
     //         .with_metadata("sync_db", Value::Bool(true))
     //         .no_display()
-    //         .with_about("Add a client to a subnet")
+    //         .with_about("Add a device to a subnet")
     //         .with_call_remote::<CliContext>(),
     // )
     // .subcommand(
-    //     "remove-client",
-    //     from_fn_async(remove_client)
+    //     "remove-device",
+    //     from_fn_async(remove_device)
     //         .with_metadata("sync_db", Value::Bool(true))
     //         .no_display()
-    //         .with_about("Remove a client from a subnet")
+    //         .with_about("Remove a device from a subnet")
     //         .with_call_remote::<CliContext>(),
     // )
 }

core/startos/src/util/collections/eq_set.rs

@@ -173,18 +173,18 @@ impl<T: Eq> EqSet<T> {
 
     /// Retains only the elements specified by the predicate.
     ///
-    /// In other words, remove all pairs `(k, v)` for which `f(&k, &mut v)` returns `false`.
-    /// The elements are visited in ascending value order.
+    /// In other words, remove all elements `x` for which `f(&x)` returns `false`.
+    /// The elements are visited in order.
     ///
     /// # Examples
     ///
     /// ```
     /// use startos::util::collections::EqSet;
     ///
-    /// let mut set: EqSet<i32, i32> = (0..8).set(|x| (x, x*10)).collect();
+    /// let mut set: EqSet<i32> = (0..8).collect();
     /// // Keep only the elements with even-numbered values.
-    /// set.retain(|&k, _| k % 2 == 0);
-    /// assert!(set.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
+    /// set.retain(|x| *x % 2 == 0);
+    /// assert!(set.into_iter().eq(vec![0, 2, 4, 6]));
     /// ```
     #[inline]
     pub fn retain<F>(&mut self, f: F)
@@ -210,9 +210,9 @@ impl<T: Eq> EqSet<T> {
     /// a.insert("c"); // Note: "c" also present in b.
     ///
     /// let mut b = EqSet::new();
-    /// b.insert(3, "c"); // Note: "c" also present in a.
-    /// b.insert(4, "d");
-    /// b.insert(5, "e");
+    /// b.insert("c"); // Note: "c" also present in a.
+    /// b.insert("d");
+    /// b.insert("e");
     ///
     /// a.append(&mut b);
     ///
@@ -246,7 +246,7 @@ impl<T: Eq> EqSet<T> {
     // /// ```
     // /// use startos::util::collections::EqSet;
     // ///
-    // /// let mut set: EqSet<i32, i32> = (0..8).set(|x| (x, x)).collect();
+    // /// let mut set: EqSet<(i32, i32)> = (0..8).map(|x| (x, x)).collect();
     // /// let evens: EqSet<_, _> = set.extract_if(|k, _v| k % 2 == 0).collect();
     // /// let odds = set;
     // /// assert_eq!(evens.values().copied().collect::<Vec<_>>(), [0, 2, 4, 6]);
@@ -366,7 +366,7 @@ impl<T: Eq, const N: usize> From<[T; N]> for EqSet<T> {
     /// use startos::util::collections::EqSet;
     ///
     /// let set1 = EqSet::from([(1, 2), (3, 4)]);
-    /// let set2: EqSet<_, _> = [(1, 2), (3, 4)].into();
+    /// let set2: EqSet<_> = [(1, 2), (3, 4)].into();
     /// assert_eq!(set1, set2);
     /// ```
     fn from(arr: [T; N]) -> Self {

core/startos/src/util/future.rs

@@ -1,7 +1,7 @@
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-use futures::future::{BoxFuture, FusedFuture, abortable, pending};
+use futures::future::{abortable, pending, BoxFuture, FusedFuture};
 use futures::stream::{AbortHandle, Abortable, BoxStream};
 use futures::{Future, FutureExt, Stream, StreamExt};
 use tokio::sync::watch;

core/startos/src/util/io.rs

@@ -912,7 +912,7 @@ impl AsRef<Path> for TmpDir {
 }
 impl Drop for TmpDir {
     fn drop(&mut self) {
-        if self.path.exists() {
+        if self.path != PathBuf::new() && self.path.exists() {
             let path = std::mem::take(&mut self.path);
             tokio::spawn(async move {
                 tokio::fs::remove_dir_all(&path).await.log_err();
@@ -1575,7 +1575,7 @@ pub fn file_string_stream(
             loop {
                 match stream.watches().add(
                     &path,
-                    WatchMask::MODIFY | WatchMask::MOVE_SELF | WatchMask::MOVED_TO | WatchMask::DELETE_SELF,
+                    WatchMask::MODIFY | WatchMask::CLOSE_WRITE | WatchMask::MOVE_SELF | WatchMask::MOVED_TO | WatchMask::DELETE_SELF,
                 ) {
                     Ok(_) => break,
                     Err(e) if e.kind() == std::io::ErrorKind::NotFound => {

core/startos/src/util/logger.rs

@@ -59,7 +59,7 @@ impl StartOSLogger {
     fn base_subscriber(logfile: LogFile) -> impl Subscriber {
         use tracing_error::ErrorLayer;
         use tracing_subscriber::prelude::*;
-        use tracing_subscriber::{EnvFilter, fmt};
+        use tracing_subscriber::{fmt, EnvFilter};
 
         let filter_layer = || {
             EnvFilter::builder()
@@ -80,10 +80,8 @@ impl StartOSLogger {
 
         let sub = tracing_subscriber::registry();
 
-        #[cfg(feature = "unstable")]
+        #[cfg(feature = "console-subscriber")]
         let sub = sub.with(console_subscriber::spawn());
-        #[cfg(not(feature = "unstable"))]
-        let sub = sub.with(filter_layer());
 
         let sub = sub.with(fmt_layer).with(ErrorLayer::default());
 

core/startos/src/util/mod.rs

@@ -14,10 +14,10 @@ use ::serde::{Deserialize, Serialize};
 use async_trait::async_trait;
 use color_eyre::eyre::{self, eyre};
 use fd_lock_rs::FdLock;
-use futures::FutureExt;
 use futures::future::BoxFuture;
-pub use helpers::NonDetachingJoinHandle;
+use futures::FutureExt;
 use helpers::canonicalize;
+pub use helpers::NonDetachingJoinHandle;
 use imbl_value::InternedString;
 use lazy_static::lazy_static;
 pub use models::VersionString;
@@ -25,7 +25,7 @@ use pin_project::pin_project;
 use sha2::Digest;
 use tokio::fs::File;
 use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
-use tokio::sync::{Mutex, OwnedMutexGuard, RwLock, oneshot};
+use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
 use tracing::instrument;
 use ts_rs::TS;
 use url::Url;
@@ -197,17 +197,17 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
     }
     #[instrument(skip_all)]
     async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
-        let cmd_str = self
-            .cmd
-            .as_std()
-            .get_program()
-            .to_string_lossy()
-            .into_owned();
         self.cmd.kill_on_drop(true);
         if self.input.is_some() {
             self.cmd.stdin(Stdio::piped());
         }
         if self.pipe.is_empty() {
+            let cmd_str = self
+                .cmd
+                .as_std()
+                .get_program()
+                .to_string_lossy()
+                .into_owned();
             if self.capture {
                 self.cmd.stdout(Stdio::piped());
                 self.cmd.stderr(Stdio::piped());
@@ -256,6 +256,7 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
                 .take()
                 .map(|i| Box::new(i) as Box<dyn AsyncRead + Unpin + Send>);
             for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() {
+                let cmd_str = cmd.as_std().get_program().to_string_lossy().into_owned();
                 let last = idx == len - 1;
                 if self.capture || !last {
                     cmd.stdout(Stdio::piped());

core/startos/src/version/mod.rs

@@ -5,14 +5,14 @@ use std::panic::{RefUnwindSafe, UnwindSafe};
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
 use futures::{Future, FutureExt};
-use imbl_value::{InternedString, to_value};
+use imbl_value::{to_value, InternedString};
 use patch_db::json_ptr::ROOT;
 
-use crate::Error;
 use crate::context::RpcContext;
 use crate::db::model::Database;
 use crate::prelude::*;
 use crate::progress::PhaseProgressTrackerHandle;
+use crate::Error;
 
 mod v0_3_5;
 mod v0_3_5_1;
@@ -50,8 +50,9 @@ mod v0_4_0_alpha_8;
 mod v0_4_0_alpha_9;
 
 mod v0_4_0_alpha_10;
+mod v0_4_0_alpha_11;
 
-pub type Current = v0_4_0_alpha_10::Version; // VERSION_BUMP
+pub type Current = v0_4_0_alpha_11::Version; // VERSION_BUMP
 
 impl Current {
     #[instrument(skip(self, db))]
@@ -164,7 +165,8 @@ enum Version {
     V0_4_0_alpha_7(Wrapper<v0_4_0_alpha_7::Version>),
     V0_4_0_alpha_8(Wrapper<v0_4_0_alpha_8::Version>),
     V0_4_0_alpha_9(Wrapper<v0_4_0_alpha_9::Version>),
-    V0_4_0_alpha_10(Wrapper<v0_4_0_alpha_10::Version>), // VERSION_BUMP
+    V0_4_0_alpha_10(Wrapper<v0_4_0_alpha_10::Version>),
+    V0_4_0_alpha_11(Wrapper<v0_4_0_alpha_11::Version>), // VERSION_BUMP
     Other(exver::Version),
 }
 
@@ -217,7 +219,8 @@ impl Version {
             Self::V0_4_0_alpha_7(v) => DynVersion(Box::new(v.0)),
             Self::V0_4_0_alpha_8(v) => DynVersion(Box::new(v.0)),
             Self::V0_4_0_alpha_9(v) => DynVersion(Box::new(v.0)),
-            Self::V0_4_0_alpha_10(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
+            Self::V0_4_0_alpha_10(v) => DynVersion(Box::new(v.0)),
+            Self::V0_4_0_alpha_11(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
             Self::Other(v) => {
                 return Err(Error::new(
                     eyre!("unknown version {v}"),
@@ -262,7 +265,8 @@ impl Version {
             Version::V0_4_0_alpha_7(Wrapper(x)) => x.semver(),
             Version::V0_4_0_alpha_8(Wrapper(x)) => x.semver(),
             Version::V0_4_0_alpha_9(Wrapper(x)) => x.semver(),
-            Version::V0_4_0_alpha_10(Wrapper(x)) => x.semver(), // VERSION_BUMP
+            Version::V0_4_0_alpha_10(Wrapper(x)) => x.semver(),
+            Version::V0_4_0_alpha_11(Wrapper(x)) => x.semver(), // VERSION_BUMP
             Version::Other(x) => x.clone(),
         }
     }

core/startos/src/version/update_details/v0_4_0.md

@@ -16,25 +16,7 @@ v0.4.0 is a complete rewrite of StartOS, almost nothing survived. After nearly s
 
 ## Changelog
 
-- [Improve user interface](#user-interface)
-- [Add translations](#translations)
-- [Switch to lxc-based container runtime](#lxc)
-- [Update s9pk archive format](#s9pk-archive-format)
-- [Improve Actions](#actions)
-- [Use squashfs images for OS updates](#squashfs-updates)
-- [Introduce Typescript package API and SDK](#typescript-sdk)
-- [Remove Postgresql](#remove-postgressql)
-- [Enable sending emails via SMTP](#smtp)
-- [Support SSH password auth](#ssh-password-auth)
-- [Allow managing Tor addresses](#tor-addresses)
-- [Implement detailed progress reporting](#progress-reporting)
-- [Improve registry protocol](#registry-protocol)
-- [Replace unique .local URLs with unique ports](#lan-port-forwarding)
-- [Use start-fs Fuse module for improved backups](#improved-backups)
-- [Switch to Exver for versioning](#exver)
-- [Add clearnet hosting](#clearnet)
-
-### User interface
+### Improved User interface
 
 We re-wrote the StartOS UI to be more performant, more intuitive, and better looking on both mobile and desktop. Enjoy.
 
@@ -42,31 +24,31 @@ We re-wrote the StartOS UI to be more performant, more intuitive, and better loo
 
 StartOS v0.4.0 supports multiple languages and also makes it easy to add more later on.
 
-### LXC
+### LXC Container Runtime
 
-Replacing both Docker and Podman, StartOS v0.4.0 uses a nested container paradigm based on LXC for the outer container and linux namespaces for sub containers. This architecture naturally support multi container setups.
+Neither Docker nor Podman offer the reliability and flexibility needed for StartOS. Instead, v0.4.0 uses a nested container paradigm based on LXC for the outer container and Linux namespaces for sub containers. This architecture naturally supports multi container setups.
 
-### S9PK archive format
+### New S9PK archive format
 
 The S9PK archive format has been overhauled to allow for signature verification of partial downloads, and allow direct mounting of container images without unpacking the s9pk.
 
-### Actions
+### Improved Actions
 
-Actions take arbitrary form input and return arbitrary responses, thus satisfying the needs of both Config and Properties, which have been removed. The new actions API gives packages developers the ability to break up Config and Properties into smaller, more specific formats, or to exclude them entirely without polluting the UI. Improved form design and new input types round out the actions experience.
+Actions take arbitrary form input and return arbitrary responses, thus satisfying the needs of both "Config" and "Properties", which have now been removed. The new actions API gives package developers the ability to break up Config and Properties into smaller, more specific formats, or to exclude them entirely without polluting the UI. Improved form design and new input types round out the new actions experience.
 
-### Squashfs updates
+### Squashfs Images for OS Updates
 
-StartOS now uses squashfs images to represent OS updates. This allows for better update verification, and improved reliability over rsync updates.
+StartOS now uses squashfs images instead of rsync for OS updates. This allows for better update verification and improved reliability.
 
-### Typescript SDK
+### Typescript Package API and SDK
 
-Package developers can now take advantage of StartOS APIs using the new start-sdk, available in Typescript. A barebones StartOS package (s9pk) can be produced in minutes with minimal knowledge or skill. More advanced developers can use the SDK to create highly customized user experiences with their service.
+Package developers can now take advantage of StartOS APIs using the new start-sdk, available in Typescript. A barebones StartOS package (s9pk) can be produced in minutes with minimal knowledge or skill. More advanced developers can use the SDK to create highly customized user experiences for their service.
 
-### Remove PostgresSQL
+### Removed PostgresSQL
 
 StartOS itself has miniscule data persistence needs. PostgresSQL was overkill and has been removed in favor of lightweight PatchDB.
 
-### SMTP
+### Sending Emails via SMTP
 
 You can now add your Gmail, SES, or other SMTP credentials to StartOS in order to send deliver email notifications from StartOS and from installed services that support SMTP.
 
@@ -74,17 +56,17 @@ You can now add your Gmail, SES, or other SMTP credentials to StartOS in order t
 
 You can now SSH into your server using your master password. SSH public key authentication is still supported as well.
 
-### Tor addresses
+### Tor Address Management
 
 StartOS v0.4.0 supports adding and removing Tor addresses for StartOS and all service interfaces. You can even provide your own private key instead of using one auto-generated by StartOS. This has the added benefit of permitting vanity addresses.
 
-### Progress reporting
+### Progress Reporting
 
 A new progress reporting API enabled package developers to create unique phases and provide real-time progress reporting for actions such as installing, updating, or backing up a service.
 
-### Registry protocol
+### Registry Protocol
 
-The new registry protocol bifurcates package indexing (listing/validating) and package hosting (downloading). Registries are now simple indexes of packages that reference binaries hosted in arbitrary locations, locally or externally. For example, when someone visits the Start9 Registry, the currated list of packages comes from Start9. But when someone installs a listed service, the package binary is being downloaded from Github. The registry also valides the binary. This makes it much easier to host a custom registry, since it is just a currated list of services tat reference package binaries hosted on Github or elsewhere.
+The new registry protocol bifurcates package indexing (listing/validating) and package hosting (downloading). Registries are now simple indexes of packages that reference binaries hosted in arbitrary locations, locally or externally. For example, when someone visits the Start9 Registry, the curated list of packages comes from Start9. But when someone installs a listed service, the package binary is being downloaded from Github. The registry also validates the binary. This makes it much easier to host a custom registry, since it is just a curated list of services tat reference package binaries hosted on Github or elsewhere.
 
 ### LAN port forwarding
 
@@ -96,12 +78,34 @@ The new start-fs fuse module unifies file system expectations for various platfo
 
 ### Exver
 
-StartOS now uses Extended Versioning (Exver), which consists of three parts: (1) a Semver-compliant upstream version, (2) a Semver-compliant wrapper version, and (3) an optional "flavor" prefix. Flavors can be thought of as alternative implementations of services, where a user would only want one or the other installed, and data can feasibly be migrating between the two. Another common characteristic of flavors is that they satisfy the same API requirement of dependents, though this is not strictly necessary. A valid Exver looks something like this: `#knots:28.0.:1.0-beta.1`. This would translate to "the first beta release of StartOS wrapper version 1.0 of Bitcoin Knots version 27.0".
+StartOS now uses Extended Versioning (Exver), which consists of three parts: (1) a Semver-compliant upstream version, (2) a Semver-compliant wrapper version, and (3) an optional "flavor" prefix. Flavors can be thought of as alternative implementations of services, where a user would only want one or the other installed, and data can feasibly be migrating between the two. Another common characteristic of flavors is that they satisfy the same API requirement of dependents, though this is not strictly necessary. A valid Exver looks something like this: `#knots:29.0:1.0-beta.1`. This would translate to "the first beta release of StartOS wrapper version 1.0 of Bitcoin Knots version 29.0".
 
-### Clearnet
+### ACME
 
-It is now possible (and easy) to expose service interfaces to the public Internet on standard domains. There are two options, both of which are easy to accomplish:
+StartOS now supports using ACME protocol to automatically obtain SSL/TLS certificates from widely trusted certificate authorities, such as Let's Encrypt, for your public domains. This means people visiting your public websites and APIs will not need to download and trust your server's Root CA.
 
-1. Open ports on your router. This option is free and supported by all routers. The drawback is that your home IP address is revealed to anyone accessing an exposed interface. For example, hosting a blog in this way would reveal your home IP address, and therefore your approximate location, to readers.
+### Gateways
 
-2. Use a Wireguard VPN to proxy web traffic. This option requires provisioning a $5-$10/month VPS and running a one-line script. The result is the successful obfuscation of the users home IP address.
+Gateways connect your server to the Internet. They process outbound traffic, and under certain conditions, they also permit inbound traffic. For example, your router is a gateway. It is now possible add gateways to StartOS, such as StartTunnel, in order to more granularly control how your installed services are exposed to the Internet.
+
+### Static DNS Servers
+
+By default, StartOS uses the DNS servers it receives via DHCP from its gateway(s). It is now possible to override these DNS servers with custom, static ones.
+
+### Internal DNS Server
+
+StartOS runs its own DNS server and automatically adds records for your private domains. You can update your router or other gateway to use StartOS DNS server in order to resolve these domains locally.
+
+### Private Domains
+
+A private domain is like to your server's .local, except it also works for VPN connectivity, and it can be _anything_. It can be a real domain you control, a made up domain, or even a domain controlled by someone else.
+
+Similar to your local domain, private domains can only be accessed when connected to the same LAN as your server, either physically or via VPN, and they require trusting your server's Root CA.
+
+### Public Domains (Clearnet)
+
+It is now easy to expose service interfaces to the public Internet on a public domain you control. There are two options, both of which are easy to accomplish:
+
+1. Open ports on your router. This option is free and supported by all routers. The drawback is that your home IP address is revealed to anyone accessing an exposed interface.
+
+2. Use a Wireguard reverse tunnel, such as [StartTunnel](#start-tunnel) to proxy web traffic. This option requires renting a $5-$10/month VPS and installing StartTunnel (or similar). The result is a new gateway, a virtual router in the cloud, that you can use to expose service interfaces instead of your real router, thereby hiding your IP address from visitors.

core/startos/src/version/v0_3_6_alpha_10.rs

@@ -6,7 +6,7 @@ use models::GatewayId;
 use serde::{Deserialize, Serialize};
 
 use super::v0_3_5::V0_3_0_COMPAT;
-use super::{VersionT, v0_3_6_alpha_9};
+use super::{v0_3_6_alpha_9, VersionT};
 use crate::net::host::address::PublicDomainConfig;
 use crate::net::tor::OnionAddress;
 use crate::prelude::*;
@@ -75,7 +75,7 @@ impl VersionT for Version {
                             domains.insert(
                                 address.clone(),
                                 PublicDomainConfig {
-                                    gateway: GatewayId::from("lo"),
+                                    gateway: GatewayId::from(InternedString::intern("lo")),
                                     acme: None,
                                 },
                             );

core/startos/src/version/v0_4_0_alpha_10.rs

@@ -67,6 +67,7 @@ impl VersionT for Version {
                     private.insert(domain.clone());
                 }
             }
+            host["hostnameInfo"] = json!({});
             host["publicDomains"] = to_value(&public)?;
             host["privateDomains"] = to_value(&private)?;
             Ok::<_, Error>(())
@@ -96,7 +97,7 @@ impl VersionT for Version {
         }
         fix_host(&mut db["public"]["serverInfo"]["network"]["host"])?;
         let network = &mut db["public"]["serverInfo"]["network"];
-        network["gateways"] = network["networkInterfaces"].clone();
+        network["gateways"] = json!({});
         network["dns"] = json!({
             "dhcpServers": [],
         });

core/startos/src/version/v0_4_0_alpha_11.rs

@@ -0,0 +1,37 @@
+use exver::{PreReleaseSegment, VersionRange};
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{v0_4_0_alpha_10, VersionT};
+use crate::prelude::*;
+
+lazy_static::lazy_static! {
+    static ref V0_4_0_alpha_11: exver::Version = exver::Version::new(
+        [0, 4, 0],
+        [PreReleaseSegment::String("alpha".into()), 11.into()]
+    );
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_4_0_alpha_10::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_4_0_alpha_11.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    #[instrument]
+    fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
+        Ok(Value::Null)
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}

image-recipe/build.sh

@@ -61,7 +61,7 @@ PLATFORM_CONFIG_EXTRAS=()
 if [ "${IB_TARGET_PLATFORM}" = "raspberrypi" ]; then
 	PLATFORM_CONFIG_EXTRAS+=( --firmware-binary false )
 	PLATFORM_CONFIG_EXTRAS+=( --firmware-chroot false )
-	PLATFORM_CONFIG_EXTRAS+=( --linux-packages linux-image-6.12.20+rpt )
+	PLATFORM_CONFIG_EXTRAS+=( --linux-packages linux-image-6.12.47+rpt )
 	PLATFORM_CONFIG_EXTRAS+=( --linux-flavours "rpi-v8 rpi-2712" )
 elif [ "${IB_TARGET_PLATFORM}" = "rockchip64" ]; then
 	PLATFORM_CONFIG_EXTRAS+=( --linux-flavours rockchip64 )
@@ -204,8 +204,8 @@ if [ "${IB_TARGET_PLATFORM}" = "raspberrypi" ]; then
 		echo "Configuring raspi kernel '\$v'"
     	extract-ikconfig "/usr/lib/modules/\$v/kernel/kernel/configs.ko.xz" > /boot/config-\$v
 	done
-	mkinitramfs -c gzip -o /boot/initramfs8 6.12.25-v8+
-	mkinitramfs -c gzip -o /boot/initramfs_2712 6.12.25-v8-16k+
+	mkinitramfs -c gzip -o /boot/initramfs8 6.12.47-v8+
+	mkinitramfs -c gzip -o /boot/initramfs_2712 6.12.47-v8-16k+
 fi
 
 useradd --shell /bin/bash -G startos -m start9
@@ -231,7 +231,8 @@ lb chroot
 lb installer
 lb binary_chroot
 lb chroot_prep install all mode-apt-install-binary mode-archives-chroot
-ln -sf /run/systemd/resolve/stub-resolv.conf chroot/chroot/etc/resolv.conf
+echo "nameserver 127.0.0.1" > chroot/chroot/etc/resolv.conf
+echo "nameserver 1.1.1.1" >> chroot/chroot/etc/resolv.conf # Cloudflare DNS Fallback
 lb binary_rootfs
 
 cp $prep_results_dir/binary/live/filesystem.squashfs $RESULTS_DIR/$IMAGE_BASENAME.squashfs

sdk/base/lib/osBindings/DepInfo.ts

@@ -1,8 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { PathOrUrl } from "./PathOrUrl"
+import type { MetadataSrc } from "./MetadataSrc"
 
 export type DepInfo = {
   description: string | null
   optional: boolean
-  s9pk: PathOrUrl | null
-}
+} & MetadataSrc

sdk/base/lib/osBindings/DnsSettings.ts

@@ -1,6 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
 export type DnsSettings = {
-  dhcpServers: Array<string>
-  staticServers: Array<string> | null
+  dhcpServers: string[]
+  staticServers: string[] | null
 }

sdk/base/lib/osBindings/ForgetGatewayParams.ts

@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { GatewayId } from "./GatewayId"
 
-export type UnsetInboundParams = { interface: GatewayId }
+export type ForgetGatewayParams = { gateway: GatewayId }

sdk/base/lib/osBindings/GatewayInfo.ts

@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { GatewayId } from "./GatewayId"
 
-export type RenameInterfaceParams = { interface: GatewayId; name: string }
+export type GatewayInfo = { id: GatewayId; name: string; public: boolean }

sdk/base/lib/osBindings/HostnameInfo.ts

@@ -1,7 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { GatewayInfo } from "./GatewayInfo"
 import type { IpHostname } from "./IpHostname"
 import type { OnionHostname } from "./OnionHostname"
 
 export type HostnameInfo =
-  | { kind: "ip"; gatewayId: string; public: boolean; hostname: IpHostname }
+  | { kind: "ip"; gateway: GatewayInfo; public: boolean; hostname: IpHostname }
   | { kind: "onion"; hostname: OnionHostname }

sdk/base/lib/osBindings/Metadata.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { PathOrUrl } from "./PathOrUrl"
+
+export type Metadata = { title: string; icon: PathOrUrl }

sdk/base/lib/osBindings/MetadataSrc.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Metadata } from "./Metadata"
+import type { PathOrUrl } from "./PathOrUrl"
+
+export type MetadataSrc = { metadata: Metadata } | { s9pk: PathOrUrl | null }

sdk/base/lib/osBindings/NetworkInterfaceInfo.ts

@@ -2,6 +2,7 @@
 import type { IpInfo } from "./IpInfo"
 
 export type NetworkInterfaceInfo = {
+  name: string | null
   public: boolean | null
   secure: boolean | null
   ipInfo: IpInfo | null

sdk/base/lib/osBindings/NetworkInterfaceSetPublicParams.ts

@@ -2,6 +2,6 @@
 import type { GatewayId } from "./GatewayId"
 
 export type NetworkInterfaceSetPublicParams = {
-  interface: GatewayId
+  gateway: GatewayId
   public: boolean | null
 }

sdk/base/lib/osBindings/RenameGatewayParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { GatewayId } from "./GatewayId"
+
+export type RenameGatewayParams = { id: GatewayId; name: string }

sdk/base/lib/osBindings/UnsetPublicParams.ts

@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { GatewayId } from "./GatewayId"
 
-export type ForgetInterfaceParams = { interface: GatewayId }
+export type UnsetPublicParams = { gateway: GatewayId }

sdk/base/lib/osBindings/index.ts

@@ -76,10 +76,11 @@ export { EventId } from "./EventId"
 export { ExportActionParams } from "./ExportActionParams"
 export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams"
 export { FileType } from "./FileType"
-export { ForgetInterfaceParams } from "./ForgetInterfaceParams"
+export { ForgetGatewayParams } from "./ForgetGatewayParams"
 export { FullIndex } from "./FullIndex"
 export { FullProgress } from "./FullProgress"
 export { GatewayId } from "./GatewayId"
+export { GatewayInfo } from "./GatewayInfo"
 export { GetActionInputParams } from "./GetActionInputParams"
 export { GetContainerIpParams } from "./GetContainerIpParams"
 export { GetHostInfoParams } from "./GetHostInfoParams"
@@ -128,6 +129,8 @@ export { Manifest } from "./Manifest"
 export { MaybeUtf8String } from "./MaybeUtf8String"
 export { MebiBytes } from "./MebiBytes"
 export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment"
+export { MetadataSrc } from "./MetadataSrc"
+export { Metadata } from "./Metadata"
 export { MetricsCpu } from "./MetricsCpu"
 export { MetricsDisk } from "./MetricsDisk"
 export { MetricsGeneral } from "./MetricsGeneral"
@@ -172,7 +175,7 @@ export { RemovePackageFromCategoryParams } from "./RemovePackageFromCategoryPara
 export { RemovePackageParams } from "./RemovePackageParams"
 export { RemoveTunnelParams } from "./RemoveTunnelParams"
 export { RemoveVersionParams } from "./RemoveVersionParams"
-export { RenameInterfaceParams } from "./RenameInterfaceParams"
+export { RenameGatewayParams } from "./RenameGatewayParams"
 export { ReplayId } from "./ReplayId"
 export { RequestCommitment } from "./RequestCommitment"
 export { RunActionParams } from "./RunActionParams"
@@ -208,7 +211,7 @@ export { TaskSeverity } from "./TaskSeverity"
 export { TaskTrigger } from "./TaskTrigger"
 export { Task } from "./Task"
 export { TestSmtpParams } from "./TestSmtpParams"
-export { UnsetInboundParams } from "./UnsetInboundParams"
+export { UnsetPublicParams } from "./UnsetPublicParams"
 export { UpdatingState } from "./UpdatingState"
 export { VerifyCifsParams } from "./VerifyCifsParams"
 export { VersionSignerParams } from "./VersionSignerParams"

sdk/base/lib/util/getServiceInterface.ts

@@ -21,23 +21,81 @@ type FilterKinds = "onion" | "local" | "domain" | "ip" | "ipv4" | "ipv6"
 export type Filter = {
   visibility?: "public" | "private"
   kind?: FilterKinds | FilterKinds[]
+  predicate?: (h: HostnameInfo) => boolean
   exclude?: Filter
 }
 
+type VisibilityFilter<V extends "public" | "private"> = V extends "public"
+  ? (HostnameInfo & { public: true }) | VisibilityFilter<Exclude<V, "public">>
+  : V extends "private"
+    ?
+        | (HostnameInfo & { public: false })
+        | VisibilityFilter<Exclude<V, "private">>
+    : never
+type KindFilter<K extends FilterKinds> = K extends "onion"
+  ? (HostnameInfo & { kind: "onion" }) | KindFilter<Exclude<K, "onion">>
+  : K extends "local"
+    ?
+        | (HostnameInfo & { kind: "ip"; hostname: { kind: "local" } })
+        | KindFilter<Exclude<K, "local">>
+    : K extends "domain"
+      ?
+          | (HostnameInfo & { kind: "ip"; hostname: { kind: "domain" } })
+          | KindFilter<Exclude<K, "domain">>
+      : K extends "ipv4"
+        ?
+            | (HostnameInfo & { kind: "ip"; hostname: { kind: "ipv4" } })
+            | KindFilter<Exclude<K, "ipv4">>
+        : K extends "ipv6"
+          ?
+              | (HostnameInfo & { kind: "ip"; hostname: { kind: "ipv6" } })
+              | KindFilter<Exclude<K, "ipv6">>
+          : K extends "ip"
+            ? KindFilter<Exclude<K, "ip"> | "ipv4" | "ipv6">
+            : never
+
+type FilterReturnTy<F extends Filter> = F extends {
+  visibility: infer V extends "public" | "private"
+}
+  ? VisibilityFilter<V> & FilterReturnTy<Omit<F, "visibility">>
+  : F extends {
+        kind: (infer K extends FilterKinds) | (infer K extends FilterKinds)[]
+      }
+    ? KindFilter<K> & FilterReturnTy<Omit<F, "kind">>
+    : F extends {
+          predicate: (h: HostnameInfo) => h is infer H extends HostnameInfo
+        }
+      ? H & FilterReturnTy<Omit<F, "predicate">>
+      : F extends { exclude: infer E extends Filter } // MUST BE LAST
+        ? HostnameInfo extends FilterReturnTy<E>
+          ? HostnameInfo
+          : Exclude<HostnameInfo, FilterReturnTy<E>>
+        : HostnameInfo
+
 type Formats = "hostname-info" | "urlstring" | "url"
-type FormatReturnTy<Format extends Formats> = Format extends "hostname-info"
-  ? HostnameInfo
+type FormatReturnTy<
+  F extends Filter,
+  Format extends Formats,
+> = Format extends "hostname-info"
+  ? FilterReturnTy<F> | FormatReturnTy<F, Exclude<Format, "hostname-info">>
   : Format extends "url"
-    ? URL
-    : UrlString
+    ? URL | FormatReturnTy<F, Exclude<Format, "url">>
+    : Format extends "urlstring"
+      ? UrlString | FormatReturnTy<F, Exclude<Format, "urlstring">>
+      : never
 
 export type Filled = {
   hostnames: HostnameInfo[]
 
-  filter: <Format extends Formats = "urlstring">(
-    filter: Filter,
+  toUrls: (h: HostnameInfo) => {
+    url: UrlString | null
+    sslUrl: UrlString | null
+  }
+
+  filter: <F extends Filter, Format extends Formats = "urlstring">(
+    filter: F,
     format?: Format,
-  ) => FormatReturnTy<Format>[]
+  ) => FormatReturnTy<F, Format>[]
 
   publicHostnames: HostnameInfo[]
   onionHostnames: HostnameInfo[]
@@ -83,8 +141,8 @@ const negate =
 const unique = <A>(values: A[]) => Array.from(new Set(values))
 export const addressHostToUrl = (
   { scheme, sslScheme, username, suffix }: AddressInfo,
-  host: HostnameInfo,
-): UrlString[] => {
+  hostname: HostnameInfo,
+): { url: UrlString | null; sslUrl: UrlString | null } => {
   const res = []
   const fmt = (scheme: string | null, host: HostnameInfo, port: number) => {
     const excludePort =
@@ -109,14 +167,16 @@ export const addressHostToUrl = (
       username ? `${username}@` : ""
     }${hostname}${excludePort ? "" : `:${port}`}${suffix}`
   }
-  if (host.hostname.sslPort !== null) {
-    res.push(fmt(sslScheme, host, host.hostname.sslPort))
+  let url = null
+  if (hostname.hostname.sslPort !== null) {
+    url = fmt(sslScheme, hostname, hostname.hostname.sslPort)
   }
-  if (host.hostname.port !== null) {
-    res.push(fmt(scheme, host, host.hostname.port))
+  let sslUrl = null
+  if (hostname.hostname.port !== null) {
+    sslUrl = fmt(scheme, hostname, hostname.hostname.port)
   }
 
-  return res
+  return { url, sslUrl }
 }
 
 function filterRec(
@@ -124,6 +184,10 @@ function filterRec(
   filter: Filter,
   invert: boolean,
 ): HostnameInfo[] {
+  if (filter.predicate) {
+    const pred = filter.predicate
+    hostnames = hostnames.filter((h) => invert !== pred(h))
+  }
   if (filter.visibility === "public")
     hostnames = hostnames.filter(
       (h) => invert !== (h.kind === "onion" || h.public),
@@ -164,19 +228,28 @@ export const filledAddress = (
   host: Host,
   addressInfo: AddressInfo,
 ): FilledAddressInfo => {
-  const toUrl = addressHostToUrl.bind(null, addressInfo)
+  const toUrls = addressHostToUrl.bind(null, addressInfo)
+  const toUrlArray = (h: HostnameInfo) => {
+    const u = toUrls(h)
+    return [u.url, u.sslUrl].filter((u) => u !== null)
+  }
   const hostnames = host.hostnameInfo[addressInfo.internalPort] ?? []
 
   return {
     ...addressInfo,
     hostnames,
-    filter: <T extends Formats = "urlstring">(filter: Filter, format?: T) => {
-      const res = filterRec(hostnames, filter, false)
-      if (format === "hostname-info") return res as FormatReturnTy<T>[]
-      const urls = res.flatMap(toUrl)
-      if (format === "url")
-        return urls.map((u) => new URL(u)) as FormatReturnTy<T>[]
-      return urls as FormatReturnTy<T>[]
+    toUrls,
+    filter: <F extends Filter, Format extends Formats = "urlstring">(
+      filter: F,
+      format?: Format,
+    ) => {
+      const filtered = filterRec(hostnames, filter, false)
+      let res: FormatReturnTy<F, Format>[] = filtered as any
+      if (format === "hostname-info") return res
+      const urls = filtered.flatMap(toUrlArray)
+      if (format === "url") res = urls.map((u) => new URL(u)) as any
+      else res = urls as any
+      return res
     },
     get publicHostnames() {
       return hostnames.filter((h) => h.kind === "onion" || h.public)
@@ -215,28 +288,28 @@ export const filledAddress = (
       )
     },
     get urls() {
-      return this.hostnames.flatMap(toUrl)
+      return this.hostnames.flatMap(toUrlArray)
     },
     get publicUrls() {
-      return this.publicHostnames.flatMap(toUrl)
+      return this.publicHostnames.flatMap(toUrlArray)
     },
     get onionUrls() {
-      return this.onionHostnames.flatMap(toUrl)
+      return this.onionHostnames.flatMap(toUrlArray)
     },
     get localUrls() {
-      return this.localHostnames.flatMap(toUrl)
+      return this.localHostnames.flatMap(toUrlArray)
     },
     get ipUrls() {
-      return this.ipHostnames.flatMap(toUrl)
+      return this.ipHostnames.flatMap(toUrlArray)
     },
     get ipv4Urls() {
-      return this.ipv4Hostnames.flatMap(toUrl)
+      return this.ipv4Hostnames.flatMap(toUrlArray)
     },
     get ipv6Urls() {
-      return this.ipv6Hostnames.flatMap(toUrl)
+      return this.ipv6Hostnames.flatMap(toUrlArray)
     },
     get nonIpUrls() {
-      return this.nonIpHostnames.flatMap(toUrl)
+      return this.nonIpHostnames.flatMap(toUrlArray)
     },
   }
 }

sdk/package/lib/StartSdk.ts

@@ -61,7 +61,7 @@ import {
 } from "../../base/lib/inits"
 import { DropGenerator } from "../../base/lib/util/Drop"
 
-export const OSVersion = testTypeVersion("0.4.0-alpha.10")
+export const OSVersion = testTypeVersion("0.4.0-alpha.11")
 
 // prettier-ignore
 type AnyNeverCond<T extends any[], Then, Else> = 

sdk/package/lib/mainFn/Daemons.ts

@@ -11,7 +11,7 @@ import * as CP from "node:child_process"
 
 export { Daemon } from "./Daemon"
 export { CommandController } from "./CommandController"
-import { HealthDaemon } from "./HealthDaemon"
+import { EXIT_SUCCESS, HealthDaemon } from "./HealthDaemon"
 import { Daemon } from "./Daemon"
 import { CommandController } from "./CommandController"
 import { HealthCheck } from "../health/HealthCheck"
@@ -91,6 +91,10 @@ type NewDaemonParams<
   subcontainer: C
 }
 
+type OptionalParamSync<T> = T | (() => T | null)
+type OptionalParamAsync<T> = () => Promise<T | null>
+type OptionalParam<T> = OptionalParamSync<T> | OptionalParamAsync<T>
+
 type AddDaemonParams<
   Manifest extends T.SDKManifest,
   Ids extends string,
@@ -160,7 +164,6 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
     readonly started:
       | ((onTerm: () => PromiseLike<void>) => PromiseLike<null>)
       | null,
-    readonly daemons: Promise<Daemon<Manifest>>[],
     readonly ids: Ids[],
     readonly healthDaemons: HealthDaemon<Manifest>[],
   ) {}
@@ -189,9 +192,37 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
       options.started,
       [],
       [],
-      [],
     )
   }
+
+  private addDaemonImpl<Id extends string>(
+    id: Id,
+    daemon: Promise<
+      Daemon<Manifest, SubContainer<Manifest, T.Effects> | null>
+    > | null,
+    requires: Ids[],
+    ready: Ready | typeof EXIT_SUCCESS,
+  ) {
+    const healthDaemon = new HealthDaemon(
+      daemon,
+      requires
+        .map((x) => this.ids.indexOf(x))
+        .filter((x) => x >= 0)
+        .map((id) => this.healthDaemons[id]),
+      id,
+      ready,
+      this.effects,
+    )
+    const ids = [...this.ids, id] as (Ids | Id)[]
+    const healthDaemons = [...this.healthDaemons, healthDaemon]
+    return new Daemons<Manifest, Ids | Id>(
+      this.effects,
+      this.started,
+      ids,
+      healthDaemons,
+    )
+  }
+
   /**
    * Returns the complete list of daemons, including the one defined here
    * @param id
@@ -205,36 +236,42 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
       ErrorDuplicateId<Id> extends Id ? never :
       Id extends Ids ? ErrorDuplicateId<Id> :
       Id,
-    options: AddDaemonParams<Manifest, Ids, Id, C>,
+    options: OptionalParamSync<AddDaemonParams<Manifest, Ids, Id, C>>,
+  ): Daemons<Manifest, Ids | Id>
+  addDaemon<Id extends string, C extends SubContainer<Manifest> | null>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: OptionalParamAsync<AddDaemonParams<Manifest, Ids, Id, C>>,
+  ): Promise<Daemons<Manifest, Ids | Id>>
+  addDaemon<Id extends string, C extends SubContainer<Manifest> | null>(
+    id: Id,
+    options: OptionalParam<AddDaemonParams<Manifest, Ids, Id, C>>,
   ) {
-    const daemon =
-      "daemon" in options
-        ? Promise.resolve(options.daemon)
-        : Daemon.of<Manifest>()<C>(
-            this.effects,
-            options.subcontainer,
-            options.exec,
-          )
-    const healthDaemon = new HealthDaemon(
-      daemon,
-      options.requires
-        .map((x) => this.ids.indexOf(x))
-        .filter((x) => x >= 0)
-        .map((id) => this.healthDaemons[id]),
-      id,
-      options.ready,
-      this.effects,
-    )
-    const daemons = [...this.daemons, daemon]
-    const ids = [...this.ids, id] as (Ids | Id)[]
-    const healthDaemons = [...this.healthDaemons, healthDaemon]
-    return new Daemons<Manifest, Ids | Id>(
-      this.effects,
-      this.started,
-      daemons,
-      ids,
-      healthDaemons,
-    )
+    const prev = this
+    const res = (options: AddDaemonParams<Manifest, Ids, Id, C> | null) => {
+      if (!options) return prev
+      const daemon =
+        "daemon" in options
+          ? Promise.resolve(options.daemon)
+          : Daemon.of<Manifest>()<C>(
+              this.effects,
+              options.subcontainer,
+              options.exec,
+            )
+      return prev.addDaemonImpl(id, daemon, options.requires, options.ready)
+    }
+    if (options instanceof Function) {
+      const opts = options()
+      if (opts instanceof Promise) {
+        return opts.then(res)
+      }
+      return res(opts)
+    }
+    return res(options)
   }
 
   /**
@@ -245,40 +282,45 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
    * @returns a new Daemons object
    */
   addOneshot<Id extends string, C extends SubContainer<Manifest> | null>(
-    id: "" extends Id
-      ? never
-      : ErrorDuplicateId<Id> extends Id
-        ? never
-        : Id extends Ids
-          ? ErrorDuplicateId<Id>
-          : Id,
-    options: AddOneshotParams<Manifest, Ids, Id, C>,
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: OptionalParamSync<AddOneshotParams<Manifest, Ids, Id, C>>,
+  ): Daemons<Manifest, Ids | Id>
+  addOneshot<Id extends string, C extends SubContainer<Manifest> | null>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: OptionalParamAsync<AddOneshotParams<Manifest, Ids, Id, C>>,
+  ): Promise<Daemons<Manifest, Ids | Id>>
+  addOneshot<Id extends string, C extends SubContainer<Manifest> | null>(
+    id: Id,
+    options: OptionalParam<AddOneshotParams<Manifest, Ids, Id, C>>,
   ) {
-    const daemon = Oneshot.of<Manifest>()<C>(
-      this.effects,
-      options.subcontainer,
-      options.exec,
-    )
-    const healthDaemon = new HealthDaemon<Manifest>(
-      daemon,
-      options.requires
-        .map((x) => this.ids.indexOf(x))
-        .filter((x) => x >= 0)
-        .map((id) => this.healthDaemons[id]),
-      id,
-      "EXIT_SUCCESS",
-      this.effects,
-    )
-    const daemons = [...this.daemons, daemon]
-    const ids = [...this.ids, id] as (Ids | Id)[]
-    const healthDaemons = [...this.healthDaemons, healthDaemon]
-    return new Daemons<Manifest, Ids | Id>(
-      this.effects,
-      this.started,
-      daemons,
-      ids,
-      healthDaemons,
-    )
+    const prev = this
+    const res = (options: AddOneshotParams<Manifest, Ids, Id, C> | null) => {
+      if (!options) return prev
+      const daemon = Oneshot.of<Manifest>()<C>(
+        this.effects,
+        options.subcontainer,
+        options.exec,
+      )
+      return prev.addDaemonImpl(id, daemon, options.requires, EXIT_SUCCESS)
+    }
+    if (options instanceof Function) {
+      const opts = options()
+      if (opts instanceof Promise) {
+        return opts.then(res)
+      }
+      return res(opts)
+    }
+    return res(options)
   }
 
   /**
@@ -288,35 +330,40 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
    * @returns a new Daemons object
    */
   addHealthCheck<Id extends string>(
-    id: "" extends Id
-      ? never
-      : ErrorDuplicateId<Id> extends Id
-        ? never
-        : Id extends Ids
-          ? ErrorDuplicateId<Id>
-          : Id,
-    options: AddHealthCheckParams<Ids, Id>,
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: OptionalParamSync<AddHealthCheckParams<Ids, Id>>,
+  ): Daemons<Manifest, Ids | Id>
+  addHealthCheck<Id extends string>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: OptionalParamAsync<AddHealthCheckParams<Ids, Id>>,
+  ): Promise<Daemons<Manifest, Ids | Id>>
+  addHealthCheck<Id extends string>(
+    id: Id,
+    options: OptionalParam<AddHealthCheckParams<Ids, Id>>,
   ) {
-    const healthDaemon = new HealthDaemon<Manifest>(
-      null,
-      options.requires
-        .map((x) => this.ids.indexOf(x))
-        .filter((x) => x >= 0)
-        .map((id) => this.healthDaemons[id]),
-      id,
-      options.ready,
-      this.effects,
-    )
-    const daemons = [...this.daemons]
-    const ids = [...this.ids, id] as (Ids | Id)[]
-    const healthDaemons = [...this.healthDaemons, healthDaemon]
-    return new Daemons<Manifest, Ids | Id>(
-      this.effects,
-      this.started,
-      daemons,
-      ids,
-      healthDaemons,
-    )
+    const prev = this
+    const res = (options: AddHealthCheckParams<Ids, Id> | null) => {
+      if (!options) return prev
+      return prev.addDaemonImpl(id, null, options.requires, options.ready)
+    }
+    if (options instanceof Function) {
+      const opts = options()
+      if (opts instanceof Promise) {
+        return opts.then(res)
+      }
+      return res(opts)
+    }
+    return res(options)
   }
 
   /**
@@ -353,7 +400,6 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
     const daemons = await new Daemons<Manifest, Ids>(
       this.effects,
       this.started,
-      [...this.daemons, daemon],
       this.ids,
       [...this.healthDaemons, healthDaemon],
     ).build()

sdk/package/lib/util/fileHelper.ts

@@ -269,7 +269,7 @@ export class FileHelper<A> {
     eq: (left: B | null | undefined, right: B | null) => boolean,
     abort?: AbortSignal,
   ) {
-    let res
+    let prev: { value: B | null } | null = null
     while (effects.isInContext && !abort?.aborted) {
       if (await exists(this.path)) {
         const ctrl = new AbortController()
@@ -287,8 +287,10 @@ export class FileHelper<A> {
             }
           })
           .catch((e) => console.error(asError(e)))
-        if (!eq(res, newRes)) yield newRes
-        res = newRes
+        if (!prev || !eq(prev.value, newRes)) {
+          yield newRes
+        }
+        prev = { value: newRes }
         await listen
       } else {
         yield null

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.36",
+  "version": "0.4.0-beta.41",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.36",
+      "version": "0.4.0-beta.41",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.36",
+  "version": "0.4.0-beta.41",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "startos-ui",
-  "version": "0.4.0-alpha.10",
+  "version": "0.4.0-alpha.11",
   "author": "Start9 Labs, Inc",
   "homepage": "https://start9.com/",
   "license": "MIT",
@@ -46,18 +46,18 @@
     "@noble/hashes": "^1.4.0",
     "@start9labs/argon2": "^0.3.0",
     "@start9labs/start-sdk": "file:../sdk/baseDist",
-    "@taiga-ui/addon-charts": "4.51.0",
-    "@taiga-ui/addon-commerce": "4.51.0",
-    "@taiga-ui/addon-mobile": "4.51.0",
-    "@taiga-ui/addon-table": "4.51.0",
-    "@taiga-ui/cdk": "4.51.0",
-    "@taiga-ui/core": "4.51.0",
+    "@taiga-ui/addon-charts": "4.52.0",
+    "@taiga-ui/addon-commerce": "4.52.0",
+    "@taiga-ui/addon-mobile": "4.52.0",
+    "@taiga-ui/addon-table": "4.52.0",
+    "@taiga-ui/cdk": "4.52.0",
+    "@taiga-ui/core": "4.52.0",
     "@taiga-ui/dompurify": "4.1.11",
-    "@taiga-ui/event-plugins": "4.6.0",
-    "@taiga-ui/experimental": "4.51.0",
-    "@taiga-ui/icons": "4.51.0",
-    "@taiga-ui/kit": "4.51.0",
-    "@taiga-ui/layout": "4.51.0",
+    "@taiga-ui/event-plugins": "4.7.0",
+    "@taiga-ui/experimental": "4.52.0",
+    "@taiga-ui/icons": "4.52.0",
+    "@taiga-ui/kit": "4.52.0",
+    "@taiga-ui/layout": "4.52.0",
     "@taiga-ui/polymorpheus": "4.9.0",
     "ansi-to-html": "^0.7.2",
     "base64-js": "^1.5.1",

web/projects/install-wizard/src/app/app.component.html

@@ -16,7 +16,10 @@
             Back
           </button>
         }
-        <h1>{{ selected ? 'Install Type' : 'Select Disk' }}</h1>
+        <h1>{{ selected ? 'Install Type' : 'StartOS Install' }}</h1>
+        @if (!selected) {
+          <h2>Select Disk</h2>
+        }
         <div [style.color]="'var(--tui-text-negative)'">{{ error }}</div>
       </header>
       <div class="pages">

web/projects/install-wizard/src/app/app.component.scss

@@ -34,6 +34,9 @@ main {
   text-align: center;
   padding-top: 0.25rem;
   margin-bottom: -2rem;
+  h2 {
+    margin-top: 0;
+  }
 }
 
 .back {

web/projects/install-wizard/src/app/app.component.ts

@@ -50,10 +50,7 @@ export class AppComponent {
 
   private async reboot() {
     this.dialogs
-      .open(
-        'Remove the USB stick and reboot your device to begin using your new Start9 server',
-        SUCCESS,
-      )
+      .open('1. Remove the USB stick<br />2. Click "Reboot" below', SUCCESS)
       .subscribe({
         complete: async () => {
           const loader = this.loader.open().subscribe()
@@ -62,7 +59,7 @@ export class AppComponent {
             await this.api.reboot()
             this.dialogs
               .open(
-                'Please wait for StartOS to restart, then refresh this page',
+                'Please wait 1-2 minutes, then refresh this page to access the StartOS setup wizard.',
                 {
                   label: 'Rebooting',
                   size: 's',

web/projects/install-wizard/src/app/app.module.ts

@@ -1,4 +1,8 @@
-import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
+import {
+  provideHttpClient,
+  withFetch,
+  withInterceptorsFromDi,
+} from '@angular/common/http'
 import { NgModule } from '@angular/core'
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations'
 import {
@@ -50,7 +54,7 @@ const {
       provide: RELATIVE_URL,
       useValue: `/${api.url}/${api.version}`,
     },
-    provideHttpClient(withInterceptorsFromDi()),
+    provideHttpClient(withInterceptorsFromDi(), withFetch()),
   ],
   bootstrap: [AppComponent],
 })

web/projects/install-wizard/src/app/app.utils.ts

@@ -3,7 +3,7 @@ import { TuiDialogOptions } from '@taiga-ui/core'
 import { TuiConfirmData } from '@taiga-ui/kit'
 
 export const SUCCESS: Partial<TuiDialogOptions<any>> = {
-  label: 'Install Success',
+  label: 'Install Success!',
   closeable: false,
   size: 's',
   data: { button: 'Reboot' },

web/projects/marketplace/src/components/menu/menu.component.html

@@ -7,7 +7,7 @@
       [url]="registry?.url || ''"
     />
     <h1 [tuiSkeleton]="!registry">
-      {{ registry?.info?.name || 'Unnamed Registry' }}
+      {{ registry?.info?.name || 'Unnamed registry' }}
     </h1>
     <!-- change registry modal -->
     <ng-content select="[slot=desktop]"></ng-content>
@@ -62,12 +62,8 @@
               <div>
                 <!-- link to store for brochure -->
                 <ng-content select="[slot=store-mobile]" />
-                <a
-                  target="_blank"
-                  rel="noreferrer"
-                  href="https://docs.start9.com/latest/packaging-guide/"
-                >
-                  <span>Package a service</span>
+                <a docsLink path="/packaging-guide">
+                  <span>{{ 'Package a service' | i18n }}</span>
                   <tui-icon tuiAppearance="icon" icon="@tui.external-link" />
                 </a>
               </div>
@@ -90,12 +86,8 @@
       <div>
         <!-- link to store for brochure -->
         <ng-content select="[slot=store]" />
-        <a
-          target="_blank"
-          rel="noreferrer"
-          href="https://docs.start9.com/latest/packaging-guide/"
-        >
-          <span>Package a service</span>
+        <a docsLink path="/packaging-guide">
+          <span>{{ 'Package a service' | i18n }}</span>
           <tui-icon tuiAppearance="icon" icon="@tui.external-link" />
         </a>
       </div>

web/projects/marketplace/src/components/menu/menu.component.module.ts

@@ -1,6 +1,10 @@
 import { CommonModule } from '@angular/common'
 import { NgModule } from '@angular/core'
-import { SharedPipesModule } from '@start9labs/shared'
+import {
+  DocsLinkDirective,
+  i18nPipe,
+  SharedPipesModule,
+} from '@start9labs/shared'
 import { TuiLet } from '@taiga-ui/cdk'
 import {
   TuiAppearance,
@@ -31,6 +35,8 @@ import { MenuComponent } from './menu.component'
     TuiSkeleton,
     TuiDrawer,
     TuiPopup,
+    i18nPipe,
+    DocsLinkDirective,
   ],
   declarations: [MenuComponent],
   exports: [MenuComponent],

web/projects/marketplace/src/pages/show/about.component.ts

@@ -6,7 +6,7 @@ import {
   output,
 } from '@angular/core'
 import { MarketplacePkgBase } from '../../types'
-import { CopyService } from '@start9labs/shared'
+import { CopyService, i18nPipe } from '@start9labs/shared'
 import { DatePipe } from '@angular/common'
 import { MarketplaceItemComponent } from './item.component'
 
@@ -44,15 +44,15 @@ import { MarketplaceItemComponent } from './item.component'
             <marketplace-item
               (click)="copyService.copy(gitHash)"
               [data]="gitHash"
-              label="Git Hash"
+              label="Git hash"
               icon="@tui.copy"
               class="item-copy"
             />
           } @else {
             <div class="item-padding">
               <label tuiTitle>
-                <span tuiSubtitle>Git Hash</span>
-                Unknown
+                <span tuiSubtitle>{{ 'Git hash' | i18n }}</span>
+                {{ 'Unknown' | i18n }}
               </label>
             </div>
           }
@@ -128,7 +128,7 @@ import { MarketplaceItemComponent } from './item.component'
     }
   `,
   changeDetection: ChangeDetectionStrategy.OnPush,
-  imports: [MarketplaceItemComponent, DatePipe],
+  imports: [MarketplaceItemComponent, DatePipe, i18nPipe],
 })
 export class MarketplaceAboutComponent {
   readonly copyService = inject(CopyService)

web/projects/marketplace/src/pages/show/dependencies/dependencies.component.ts

@@ -8,13 +8,14 @@ import {
 } from '@angular/core'
 import { MarketplacePkgBase } from '../../../types'
 import { MarketplaceDepItemComponent } from './dependency-item.component'
+import { i18nPipe } from '@start9labs/shared'
 
 @Component({
   selector: 'marketplace-dependencies',
   template: `
     <div class="background-border shadow-color-light box-shadow-lg">
       <div class="dependencies-container">
-        <h2 class="additional-detail-title">Dependencies</h2>
+        <h2 class="additional-detail-title">{{ 'Dependencies' | i18n }}</h2>
         <div class="dependencies-list">
           @for (dep of pkg.dependencyMetadata | keyvalue; track $index) {
             <marketplace-dep-item
@@ -48,7 +49,7 @@ import { MarketplaceDepItemComponent } from './dependency-item.component'
     }
   `,
   changeDetection: ChangeDetectionStrategy.OnPush,
-  imports: [CommonModule, MarketplaceDepItemComponent],
+  imports: [CommonModule, MarketplaceDepItemComponent, i18nPipe],
 })
 export class MarketplaceDependenciesComponent {
   @Input({ required: true })

web/projects/marketplace/src/pages/show/dependencies/dependency-item.component.ts

@@ -1,7 +1,7 @@
 import { KeyValue } from '@angular/common'
 import { ChangeDetectionStrategy, Component, Input } from '@angular/core'
 import { RouterModule } from '@angular/router'
-import { ExverPipesModule } from '@start9labs/shared'
+import { ExverPipesModule, i18nPipe } from '@start9labs/shared'
 import { T } from '@start9labs/start-sdk'
 import { TuiAvatar, TuiLineClamp } from '@taiga-ui/kit'
 import { MarketplacePkgBase } from '../../../types'
@@ -20,9 +20,9 @@ import { MarketplacePkgBase } from '../../../types'
             </span>
             <p>
               @if (dep.value.optional) {
-                <span>(optional)</span>
+                <span>({{ 'Optional' | i18n }})</span>
               } @else {
-                <span>(required)</span>
+                <span>({{ 'Required' | i18n }})</span>
               }
             </p>
           </div>
@@ -49,10 +49,11 @@ import { MarketplacePkgBase } from '../../../types'
       filter: drop-shadow(0 10px 8px rgb(0 0 0 / 0.04))
         drop-shadow(0 4px 3px rgb(0 0 0 / 0.1));
 
-      &:hover {
-        background-color: rgb(63 63 70 / 0.7);
-        cursor: pointer;
-      }
+      // @TODO re-engage when button can link to root with search QP
+      // &:hover {
+      //   background-color: rgb(63 63 70 / 0.7);
+      //   cursor: pointer;
+      // }
     }
 
     .title {
@@ -88,7 +89,7 @@ import { MarketplacePkgBase } from '../../../types'
     }
   `,
   changeDetection: ChangeDetectionStrategy.OnPush,
-  imports: [RouterModule, TuiAvatar, ExverPipesModule, TuiLineClamp],
+  imports: [RouterModule, TuiAvatar, ExverPipesModule, TuiLineClamp, i18nPipe],
 })
 export class MarketplaceDepItemComponent {
   @Input({ required: true })

web/projects/marketplace/src/pages/show/flavors.component.ts

@@ -1,6 +1,6 @@
 import { ChangeDetectionStrategy, Component, Input } from '@angular/core'
 import { RouterLink } from '@angular/router'
-import { SharedPipesModule } from '@start9labs/shared'
+import { i18nPipe, SharedPipesModule } from '@start9labs/shared'
 import { TuiTitle } from '@taiga-ui/core'
 import { TuiAvatar } from '@taiga-ui/kit'
 import { TuiCell } from '@taiga-ui/layout'
@@ -11,7 +11,9 @@ import { MarketplacePkg } from '../../types'
   template: `
     <div class="background-border box-shadow-lg shadow-color-light">
       <div class="box-container">
-        <h2 class="additional-detail-title">Alternative Implementations</h2>
+        <h2 class="additional-detail-title">
+          {{ 'Alternative Implementations' | i18n }}
+        </h2>
         @for (pkg of pkgs; track $index) {
           <a
             tuiCell
@@ -42,7 +44,14 @@ import { MarketplacePkg } from '../../types'
     }
   `,
   changeDetection: ChangeDetectionStrategy.OnPush,
-  imports: [RouterLink, TuiCell, TuiTitle, SharedPipesModule, TuiAvatar],
+  imports: [
+    RouterLink,
+    TuiCell,
+    TuiTitle,
+    SharedPipesModule,
+    TuiAvatar,
+    i18nPipe,
+  ],
 })
 export class MarketplaceFlavorsComponent {
   @Input()

web/projects/marketplace/src/pages/show/item.component.ts

@@ -1,4 +1,5 @@
 import { ChangeDetectionStrategy, Component, Input } from '@angular/core'
+import { i18nKey } from '@start9labs/shared'
 import { TuiIcon, TuiTitle } from '@taiga-ui/core'
 import { TuiFade } from '@taiga-ui/kit'
 
@@ -6,7 +7,7 @@ import { TuiFade } from '@taiga-ui/kit'
   selector: 'marketplace-item',
   template: `
     <label tuiTitle>
-      <span tuiSubtitle>{{ label }}</span>
+      <span tuiSubtitle>{{ label || '' }}</span>
       <span tuiFade>{{ data }}</span>
     </label>
     <tui-icon [icon]="icon" />
@@ -38,7 +39,7 @@ import { TuiFade } from '@taiga-ui/kit'
 })
 export class MarketplaceItemComponent {
   @Input({ required: true })
-  label!: string
+  label!: i18nKey | null
 
   @Input({ required: true })
   icon!: string

web/projects/marketplace/src/pages/show/link.component.ts

@@ -1,5 +1,6 @@
 import { ChangeDetectionStrategy, Component, Input } from '@angular/core'
 import { MarketplaceItemComponent } from './item.component'
+import { i18nKey } from '@start9labs/shared'
 
 @Component({
   selector: 'marketplace-link',
@@ -13,7 +14,7 @@ import { MarketplaceItemComponent } from './item.component'
 })
 export class MarketplaceLinkComponent {
   @Input({ required: true })
-  label!: string
+  label!: i18nKey
 
   @Input({ required: true })
   icon!: string

web/projects/marketplace/src/pages/show/links.component.ts

@@ -5,7 +5,7 @@ import {
   input,
 } from '@angular/core'
 import { ActivatedRoute } from '@angular/router'
-import { CopyService } from '@start9labs/shared'
+import { CopyService, i18nPipe } from '@start9labs/shared'
 import { MarketplacePkgBase } from '../../types'
 import { MarketplaceLinkComponent } from './link.component'
 
@@ -18,13 +18,13 @@ import { MarketplaceLinkComponent } from './link.component'
         <div class="detail-container">
           <marketplace-link
             [url]="pkg().upstreamRepo"
-            label="Upstream Service"
+            label="Upstream service"
             icon="@tui.external-link"
             class="item-pointer"
           />
           <marketplace-link
             [url]="pkg().wrapperRepo"
-            label="StartOS Package"
+            label="StartOS package"
             icon="@tui.external-link"
             class="item-pointer"
           />
@@ -34,7 +34,7 @@ import { MarketplaceLinkComponent } from './link.component'
 
     <div class="background-border shadow-color-light box-shadow-lg">
       <div class="box-container">
-        <h2 class="additional-detail-title">Links</h2>
+        <h2 class="additional-detail-title">{{ 'Links' | i18n }}</h2>
         <div class="detail-container">
           <marketplace-link
             [url]="pkg().marketingSite"
@@ -117,7 +117,7 @@ import { MarketplaceLinkComponent } from './link.component'
     }
   `,
   changeDetection: ChangeDetectionStrategy.OnPush,
-  imports: [MarketplaceLinkComponent],
+  imports: [MarketplaceLinkComponent, i18nPipe],
 })
 export class MarketplaceLinksComponent {
   readonly copyService = inject(CopyService)

web/projects/marketplace/src/pages/show/versions.component.ts

@@ -18,12 +18,12 @@ import { MarketplaceItemComponent } from './item.component'
   template: `
     <div class="background-border shadow-color-light box-shadow-lg">
       <div class="box-container">
-        <h2 class="additional-detail-title">Versions</h2>
+        <h2 class="additional-detail-title">{{ 'Versions' | i18n }}</h2>
         <marketplace-item
           (click)="promptSelectVersion(versionSelect)"
-          data="Select another version"
+          [data]="'Select another version' | i18n"
           icon="@tui.chevron-right"
-          label=""
+          [label]="null"
           class="select"
         />
         <ng-template