misc fixes for alpha.16 (#3091)

* port misc fixes from feature/nvidia

* switch back to official tor proxy on 9050

* refactor OpenUI

* fix typo

* fixes, plus getServiceManifest

* fix EffectCreator, bump to beta.47

* fixes

build/lib/scripts/chroot-and-upgrade

@@ -63,7 +63,7 @@ mount --bind /proc /media/startos/next/proc
 mount --bind /boot /media/startos/next/boot
 mount --bind /media/startos/root /media/startos/next/media/startos/root
 
-if mountpoint /sys/firmware/efi/efivars 2> /dev/null; then
+if mountpoint /sys/firmware/efi/efivars 2>&1 > /dev/null; then
   mount --bind /sys/firmware/efi/efivars /media/startos/next/sys/firmware/efi/efivars
 fi
 
@@ -75,7 +75,7 @@ else
     CHROOT_RES=$?
 fi
 
-if mountpoint /media/startos/next/sys/firmware/efi/efivars 2> /dev/null; then
+if mountpoint /media/startos/next/sys/firmware/efi/efivars 2>&1 > /dev/null; then
   umount /media/startos/next/sys/firmware/efi/efivars 
 fi
 

build/lib/scripts/forward-port

@@ -35,16 +35,20 @@ if [ "$UNDO" = 1 ]; then
     exit $err
 fi
 
+# DNAT: rewrite destination for incoming packets (external traffic)
 iptables -t nat -A ${NAME}_PREROUTING -d "$sip" -p tcp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 iptables -t nat -A ${NAME}_PREROUTING -d "$sip" -p udp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
+
+# DNAT: rewrite destination for locally-originated packets (hairpin from host itself)
 iptables -t nat -A ${NAME}_OUTPUT -d "$sip" -p tcp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 iptables -t nat -A ${NAME}_OUTPUT -d "$sip" -p udp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
 
-iptables -t nat -A ${NAME}_PREROUTING -s "$dip/$dprefix" -d "$sip" -p tcp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
+# MASQUERADE: rewrite source for all forwarded traffic to the destination
-iptables -t nat -A ${NAME}_PREROUTING -s "$dip/$dprefix" -d "$sip" -p udp --dport "$sport" -j DNAT --to-destination "$dip:$dport"
+# This ensures responses are routed back through the host regardless of source IP
-iptables -t nat -A ${NAME}_POSTROUTING -s "$dip/$dprefix" -d "$dip" -p tcp --dport "$dport" -j MASQUERADE
+iptables -t nat -A ${NAME}_POSTROUTING -d "$dip" -p tcp --dport "$dport" -j MASQUERADE
-iptables -t nat -A ${NAME}_POSTROUTING -s "$dip/$dprefix" -d "$dip" -p udp --dport "$dport" -j MASQUERADE
+iptables -t nat -A ${NAME}_POSTROUTING -d "$dip" -p udp --dport "$dport" -j MASQUERADE
 
+# Allow new connections to be forwarded to the destination
 iptables -A ${NAME}_FORWARD -d $dip -p tcp --dport $dport -m state --state NEW -j ACCEPT
 iptables -A ${NAME}_FORWARD -d $dip -p udp --dport $dport -m state --state NEW -j ACCEPT
 

build/lib/scripts/upgrade

@@ -50,12 +50,12 @@ mount --bind /proc /media/startos/next/proc
 mount --bind /boot /media/startos/next/boot
 mount --bind /media/startos/root /media/startos/next/media/startos/root
 
-if mountpoint /boot/efi 2> /dev/null; then
+if mountpoint /boot/efi 2>&1 > /dev/null; then
     mkdir -p /media/startos/next/boot/efi
     mount --bind /boot/efi /media/startos/next/boot/efi
 fi
 
-if mountpoint /sys/firmware/efi/efivars 2> /dev/null; then
+if mountpoint /sys/firmware/efi/efivars 2>&1 > /dev/null; then
     mount --bind /sys/firmware/efi/efivars /media/startos/next/sys/firmware/efi/efivars
 fi
 

container-runtime/package-lock.json

@@ -38,7 +38,7 @@
     },
     "../sdk/dist": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.46",
+      "version": "0.4.0-beta.47",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

container-runtime/src/Adapters/EffectCreator.ts

@@ -178,6 +178,13 @@ export function makeEffects(context: EffectContext): Effects {
         T.Effects["getInstalledPackages"]
       >
     },
+    getServiceManifest(
+      ...[options]: Parameters<T.Effects["getServiceManifest"]>
+    ) {
+      return rpcRound("get-service-manifest", options) as ReturnType<
+        T.Effects["getServiceManifest"]
+      >
+    },
     subcontainer: {
       createFs(options: { imageId: string; name: string }) {
         return rpcRound("subcontainer.create-fs", options) as ReturnType<

container-runtime/update-image-local.sh

@@ -15,4 +15,7 @@ case $ARCH in
         DOCKER_PLATFORM=linux/arm64;;
 esac
 
-docker run --rm $USE_TTY --platform=$DOCKER_PLATFORM -eARCH --privileged -v "$(pwd):/root/start-os" start9/build-env /root/start-os/container-runtime/update-image.sh
+docker run --rm $USE_TTY --platform=$DOCKER_PLATFORM -eARCH --privileged -v "$(pwd):/root/start-os" start9/build-env /root/start-os/container-runtime/update-image.sh
+if [ "$(ls -nd "rootfs.${ARCH}.squashfs" | awk '{ print $3 }')" != "$UID" ]; then
+  docker run --rm $USE_TTY -v "$(pwd):/root/start-os" start9/build-env chown -R $UID:$UID /root/start-os/container-runtime
+fi

core/src/auth.rs

@@ -184,7 +184,11 @@ async fn cli_login<C: SessionAuthContext>(
 where
     CliContext: CallRemote<C>,
 {
-    let password = rpassword::prompt_password("Password: ")?;
+    let password = if let Ok(password) = std::env::var("PASSWORD") {
+        password
+    } else {
+        rpassword::prompt_password("Password: ")?
+    };
 
     ctx.call_remote::<C>(
         &parent_method.into_iter().chain(method).join("."),

core/src/control.rs

@@ -56,8 +56,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
                 .as_idx_mut(&id)
                 .or_not_found(&id)?
                 .as_status_info_mut()
-                .as_desired_mut()
+                .restart()
-                .map_mutate(|s| Ok(s.restart()))
         })
         .await
         .result?;

core/src/net/gateway.rs

@@ -703,22 +703,22 @@ async fn watch_ip(
                                             .into_iter()
                                             .map(IpNet::try_from)
                                             .try_collect()?;
-                                        let tables = ip4_proxy.route_data().await?.into_iter().filter_map(|d|d.table).collect::<Vec<_>>();
+                                        // let tables = ip4_proxy.route_data().await?.into_iter().filter_map(|d|d.table).collect::<Vec<_>>();
-                                        if !tables.is_empty() {
+                                        // if !tables.is_empty() {
-                                            let rules = String::from_utf8(Command::new("ip").arg("rule").arg("list").invoke(ErrorKind::Network).await?)?;
+                                        //     let rules = String::from_utf8(Command::new("ip").arg("rule").arg("list").invoke(ErrorKind::Network).await?)?;
-                                            for table in tables {
+                                        //     for table in tables {
-                                                for subnet in subnets.iter().filter(|s| s.addr().is_ipv4()) {
+                                        //         for subnet in subnets.iter().filter(|s| s.addr().is_ipv4()) {
-                                                    let subnet_string = subnet.trunc().to_string();
+                                        //             let subnet_string = subnet.trunc().to_string();
-                                                    let rule = ["from", &subnet_string, "lookup", &table.to_string()];
+                                        //             let rule = ["from", &subnet_string, "lookup", &table.to_string()];
-                                                    if !rules.contains(&rule.join(" ")) {
+                                        //             if !rules.contains(&rule.join(" ")) {
-                                                        if rules.contains(&rule[..2].join(" ")) {
+                                        //                 if rules.contains(&rule[..2].join(" ")) {
-                                                            Command::new("ip").arg("rule").arg("del").args(&rule[..2]).invoke(ErrorKind::Network).await?;
+                                        //                     Command::new("ip").arg("rule").arg("del").args(&rule[..2]).invoke(ErrorKind::Network).await?;
-                                                        }
+                                        //                 }
-                                                        Command::new("ip").arg("rule").arg("add").args(rule).invoke(ErrorKind::Network).await?;
+                                        //                 Command::new("ip").arg("rule").arg("add").args(rule).invoke(ErrorKind::Network).await?;
-                                                    }
+                                        //             }
-                                                }
+                                        //         }
-                                            }
+                                        //     }
-                                        }
+                                        // }
                                         let wan_ip = if !subnets.is_empty()
                                             && !matches!(
                                                 device_type,

core/src/net/socks.rs

@@ -15,7 +15,7 @@ use crate::util::future::NonDetachingJoinHandle;
 
 pub const DEFAULT_SOCKS_LISTEN: SocketAddr = SocketAddr::V4(SocketAddrV4::new(
     Ipv4Addr::new(HOST_IP[0], HOST_IP[1], HOST_IP[2], HOST_IP[3]),
-    9050,
+    1080,
 ));
 
 pub struct SocksController {

core/src/net/tor/ctor.rs

@@ -43,7 +43,6 @@ const STARTING_HEALTH_TIMEOUT: u64 = 120; // 2min
 
 const TOR_CONTROL: SocketAddr =
     SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 1, 1), 9051));
-const TOR_SOCKS: SocketAddr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 1, 1), 9050));
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct OnionAddress(OnionAddressV3);
@@ -402,10 +401,15 @@ fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), C
 #[derive(Clone)]
 pub struct TorController(Arc<TorControl>);
 impl TorController {
+    const TOR_SOCKS: &[SocketAddr] = &[
+        SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 9050)),
+        SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(10, 0, 3, 1), 9050)),
+    ];
+
     pub fn new() -> Result<Self, Error> {
         Ok(TorController(Arc::new(TorControl::new(
             TOR_CONTROL,
-            TOR_SOCKS,
+            Self::TOR_SOCKS,
         ))))
     }
 
@@ -508,7 +512,7 @@ impl TorController {
             }
             Ok(Box::new(tcp_stream))
         } else {
-            let mut stream = TcpStream::connect(TOR_SOCKS)
+            let mut stream = TcpStream::connect(Self::TOR_SOCKS[0])
                 .await
                 .with_kind(ErrorKind::Tor)?;
             if let Err(e) = socket2::SockRef::from(&stream).set_keepalive(true) {
@@ -595,7 +599,7 @@ enum TorCommand {
 #[instrument(skip_all)]
 async fn torctl(
     tor_control: SocketAddr,
-    tor_socks: SocketAddr,
+    tor_socks: &[SocketAddr],
     recv: &mut mpsc::UnboundedReceiver<TorCommand>,
     services: &mut Watch<
         BTreeMap<
@@ -641,10 +645,21 @@ async fn torctl(
             tokio::fs::remove_dir_all("/var/lib/tor").await?;
             wipe_state.store(false, std::sync::atomic::Ordering::SeqCst);
         }
-        write_file_atomic(
+
-            "/etc/tor/torrc",
+        write_file_atomic("/etc/tor/torrc", {
-            format!("SocksPort {TOR_SOCKS}\nControlPort {TOR_CONTROL}\nCookieAuthentication 1\n"),
+            use std::fmt::Write;
-        )
+            let mut conf = String::new();
+
+            for tor_socks in tor_socks {
+                writeln!(&mut conf, "SocksPort {tor_socks}").unwrap();
+            }
+            writeln!(
+                &mut conf,
+                "ControlPort {tor_control}\nCookieAuthentication 1"
+            )
+            .unwrap();
+            conf
+        })
         .await?;
         tokio::fs::create_dir_all("/var/lib/tor").await?;
         Command::new("chown")
@@ -976,7 +991,10 @@ struct TorControl {
     >,
 }
 impl TorControl {
-    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+    pub fn new(
+        tor_control: SocketAddr,
+        tor_socks: impl AsRef<[SocketAddr]> + Send + 'static,
+    ) -> Self {
         let (send, mut recv) = mpsc::unbounded_channel();
         let services = Watch::new(BTreeMap::new());
         let mut thread_services = services.clone();
@@ -987,7 +1005,7 @@ impl TorControl {
                 loop {
                     if let Err(e) = torctl(
                         tor_control,
-                        tor_socks,
+                        tor_socks.as_ref(),
                         &mut recv,
                         &mut thread_services,
                         &wipe_state,

core/src/service/effects/callbacks.rs

@@ -36,6 +36,7 @@ struct ServiceCallbackMap {
     >,
     get_status: BTreeMap<PackageId, Vec<CallbackHandler>>,
     get_container_ip: BTreeMap<PackageId, Vec<CallbackHandler>>,
+    get_service_manifest: BTreeMap<PackageId, Vec<CallbackHandler>>,
 }
 
 impl ServiceCallbacks {
@@ -68,6 +69,10 @@ impl ServiceCallbacks {
                 v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
                 !v.is_empty()
             });
+            this.get_service_manifest.retain(|_, v| {
+                v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
+                !v.is_empty()
+            });
         })
     }
 
@@ -250,6 +255,25 @@ impl ServiceCallbacks {
                 .filter(|cb| !cb.0.is_empty())
         })
     }
+
+    pub(super) fn add_get_service_manifest(&self, package_id: PackageId, handler: CallbackHandler) {
+        self.mutate(|this| {
+            this.get_service_manifest
+                .entry(package_id)
+                .or_default()
+                .push(handler)
+        })
+    }
+
+    #[must_use]
+    pub fn get_service_manifest(&self, package_id: &PackageId) -> Option<CallbackHandlers> {
+        self.mutate(|this| {
+            this.get_service_manifest
+                .remove(package_id)
+                .map(CallbackHandlers)
+                .filter(|cb| !cb.0.is_empty())
+        })
+    }
 }
 
 pub struct CallbackHandler {

core/src/service/effects/control.rs

@@ -36,8 +36,7 @@ pub async fn restart(context: EffectContext) -> Result<(), Error> {
                 .as_idx_mut(id)
                 .or_not_found(id)?
                 .as_status_info_mut()
-                .as_desired_mut()
+                .restart()
-                .map_mutate(|s| Ok(s.restart()))
         })
         .await
         .result?;

core/src/service/effects/dependency.rs

@@ -14,7 +14,10 @@ use crate::disk::mount::filesystem::bind::{Bind, FileType};
 use crate::disk::mount::filesystem::idmapped::{IdMap, IdMapped};
 use crate::disk::mount::filesystem::{FileSystem, MountType};
 use crate::disk::mount::util::{is_mountpoint, unmount};
+use crate::s9pk::manifest::Manifest;
+use crate::service::effects::callbacks::CallbackHandler;
 use crate::service::effects::prelude::*;
+use crate::service::rpc::CallbackId;
 use crate::status::health_check::NamedHealthCheckResult;
 use crate::util::{FromStrParser, VersionString};
 use crate::volume::data_dir;
@@ -367,3 +370,45 @@ pub async fn check_dependencies(
     }
     Ok(results)
 }
+
+#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetServiceManifestParams {
+    pub package_id: PackageId,
+    #[ts(optional)]
+    #[arg(skip)]
+    pub callback: Option<CallbackId>,
+}
+
+pub async fn get_service_manifest(
+    context: EffectContext,
+    GetServiceManifestParams {
+        package_id,
+        callback,
+    }: GetServiceManifestParams,
+) -> Result<Manifest, Error> {
+    let context = context.deref()?;
+
+    if let Some(callback) = callback {
+        let callback = callback.register(&context.seed.persistent_container);
+        context
+            .seed
+            .ctx
+            .callbacks
+            .add_get_service_manifest(package_id.clone(), CallbackHandler::new(&context, callback));
+    }
+
+    let db = context.seed.ctx.db.peek().await;
+
+    let manifest = db
+        .as_public()
+        .as_package_data()
+        .as_idx(&package_id)
+        .or_not_found(&package_id)?
+        .as_state_info()
+        .as_manifest(ManifestPreference::New)
+        .de()?;
+
+    Ok(manifest)
+}

core/src/service/effects/mod.rs

@@ -88,6 +88,10 @@ pub fn handler<C: Context>() -> ParentHandler<C> {
             "get-installed-packages",
             from_fn_async(dependency::get_installed_packages).no_cli(),
         )
+        .subcommand(
+            "get-service-manifest",
+            from_fn_async(dependency::get_service_manifest).no_cli(),
+        )
         // health
         .subcommand("set-health", from_fn_async(health::set_health).no_cli())
         // subcontainer

core/src/service/mod.rs

@@ -575,6 +575,17 @@ impl Service {
             .await
             .result?;
 
+        // Trigger manifest callbacks after successful installation
+        let manifest = service.seed.persistent_container.s9pk.as_manifest();
+        if let Some(callbacks) = ctx.callbacks.get_service_manifest(&manifest.id) {
+            let manifest_value =
+                serde_json::to_value(manifest).with_kind(ErrorKind::Serialization)?;
+            callbacks
+                .call(imbl::vector![manifest_value.into()])
+                .await
+                .log_err();
+        }
+
         Ok(service)
     }
 

core/src/service/uninstall.rs

@@ -1,5 +1,7 @@
 use std::path::Path;
 
+use imbl::vector;
+
 use crate::context::RpcContext;
 use crate::db::model::package::{InstalledState, InstallingInfo, InstallingState, PackageState};
 use crate::prelude::*;
@@ -65,6 +67,11 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, soft: bool) -> Result<(),
                     ));
                 }
             };
+            // Trigger manifest callbacks with null to indicate uninstall
+            if let Some(callbacks) = ctx.callbacks.get_service_manifest(&manifest.id) {
+                callbacks.call(vector![Value::Null]).await.log_err();
+            }
+
             if !soft {
                 let path = Path::new(DATA_DIR).join(PKG_VOLUME_DIR).join(&manifest.id);
                 if tokio::fs::metadata(&path).await.is_ok() {

core/src/status/mod.rs

@@ -53,6 +53,11 @@ impl Model<StatusInfo> {
         self.as_started_mut().ser(&None)?;
         Ok(())
     }
+    pub fn restart(&mut self) -> Result<(), Error> {
+        self.as_desired_mut().map_mutate(|s| Ok(s.restart()))?;
+        self.as_health_mut().ser(&Default::default())?;
+        Ok(())
+    }
     pub fn init(&mut self) -> Result<(), Error> {
         self.as_started_mut().ser(&None)?;
         self.as_desired_mut().map_mutate(|s| {

sdk/base/lib/Effects.ts

@@ -15,6 +15,7 @@ import {
   CreateTaskParams,
   MountParams,
   StatusInfo,
+  Manifest,
 } from "./osBindings"
 import {
   PackageId,
@@ -83,6 +84,11 @@ export type Effects = {
   mount(options: MountParams): Promise<string>
   /** Returns a list of the ids of all installed packages */
   getInstalledPackages(): Promise<string[]>
+  /** Returns the manifest of a service */
+  getServiceManifest(options: {
+    packageId: PackageId
+    callback?: () => void
+  }): Promise<Manifest>
 
   // health
   /** sets the result of a health check */

sdk/base/lib/osBindings/GetServiceManifestParams.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { CallbackId } from "./CallbackId"
+import type { PackageId } from "./PackageId"
+
+export type GetServiceManifestParams = {
+  packageId: PackageId
+  callback?: CallbackId
+}

sdk/base/lib/osBindings/index.ts

@@ -91,6 +91,7 @@ export { GetPackageParams } from "./GetPackageParams"
 export { GetPackageResponseFull } from "./GetPackageResponseFull"
 export { GetPackageResponse } from "./GetPackageResponse"
 export { GetServiceInterfaceParams } from "./GetServiceInterfaceParams"
+export { GetServiceManifestParams } from "./GetServiceManifestParams"
 export { GetServicePortForwardParams } from "./GetServicePortForwardParams"
 export { GetSslCertificateParams } from "./GetSslCertificateParams"
 export { GetSslKeyParams } from "./GetSslKeyParams"

sdk/base/lib/test/startosTypeValidation.test.ts

@@ -13,6 +13,7 @@ import {
   RunActionParams,
   SetDataVersionParams,
   SetMainStatus,
+  GetServiceManifestParams,
 } from ".././osBindings"
 import { CreateSubcontainerFsParams } from ".././osBindings"
 import { DestroySubcontainerFsParams } from ".././osBindings"
@@ -64,7 +65,6 @@ describe("startosTypeValidation ", () => {
         destroyFs: {} as DestroySubcontainerFsParams,
       },
       clearBindings: {} as ClearBindingsParams,
-      getInstalledPackages: undefined,
       bind: {} as BindParams,
       getHostInfo: {} as WithCallback<GetHostInfoParams>,
       restart: undefined,
@@ -76,6 +76,8 @@ describe("startosTypeValidation ", () => {
       getSslKey: {} as GetSslKeyParams,
       getServiceInterface: {} as WithCallback<GetServiceInterfaceParams>,
       setDependencies: {} as SetDependenciesParams,
+      getInstalledPackages: undefined,
+      getServiceManifest: {} as WithCallback<GetServiceManifestParams>,
       getSystemSmtp: {} as WithCallback<GetSystemSmtpParams>,
       getContainerIp: {} as WithCallback<GetContainerIpParams>,
       getOsIp: undefined,

sdk/base/lib/types/ManifestTypes.ts

@@ -153,10 +153,21 @@ export type SDKManifest = {
 
 // this is hacky but idk a more elegant way
 type ArchOptions = {
-  0: ["x86_64", "aarch64"]
+  0: ["x86_64", "aarch64", "riscv64"]
-  1: ["aarch64", "x86_64"]
+  1: ["aarch64", "x86_64", "riscv64"]
-  2: ["x86_64"]
+  2: ["x86_64", "riscv64", "aarch64"]
-  3: ["aarch64"]
+  3: ["aarch64", "riscv64", "x86_64"]
+  4: ["riscv64", "x86_64", "aarch64"]
+  5: ["riscv64", "aarch64", "x86_64"]
+  6: ["x86_64", "aarch64"]
+  7: ["aarch64", "x86_64"]
+  8: ["x86_64", "riscv64"]
+  9: ["aarch64", "riscv64"]
+  10: ["riscv64", "aarch64"]
+  11: ["riscv64", "x86_64"]
+  12: ["x86_64"]
+  13: ["aarch64"]
+  14: ["riscv64"]
 }
 export type SDKImageInputSpec = {
   [A in keyof ArchOptions]: {

sdk/base/lib/util/getServiceInterface.ts

@@ -5,6 +5,7 @@ import { Effects } from "../Effects"
 import { DropGenerator, DropPromise } from "./Drop"
 import { IpAddress, IPV6_LINK_LOCAL } from "./ip"
 import { deepEqual } from "./deepEqual"
+import { once } from "./once"
 
 export type UrlString = string
 export type HostId = string
@@ -255,6 +256,21 @@ export const filledAddress = (
   function filledAddressFromHostnames<F extends Filter>(
     hostnames: HostnameInfo[],
   ): Filled<F> & AddressInfo {
+    const getNonLocal = once(() =>
+      filledAddressFromHostnames<typeof nonLocalFilter & F>(
+        filterRec(hostnames, nonLocalFilter, false),
+      ),
+    )
+    const getPublic = once(() =>
+      filledAddressFromHostnames<typeof publicFilter & F>(
+        filterRec(hostnames, publicFilter, false),
+      ),
+    )
+    const getOnion = once(() =>
+      filledAddressFromHostnames<typeof onionFilter & F>(
+        filterRec(hostnames, onionFilter, false),
+      ),
+    )
     return {
       ...addressInfo,
       hostnames,
@@ -273,19 +289,13 @@ export const filledAddress = (
         )
       },
       get nonLocal(): Filled<typeof nonLocalFilter & F> {
-        return filledAddressFromHostnames<typeof nonLocalFilter & F>(
+        return getNonLocal()
-          filterRec(hostnames, nonLocalFilter, false),
-        )
       },
       get public(): Filled<typeof publicFilter & F> {
-        return filledAddressFromHostnames<typeof publicFilter & F>(
+        return getPublic()
-          filterRec(hostnames, publicFilter, false),
-        )
       },
       get onion(): Filled<typeof onionFilter & F> {
-        return filledAddressFromHostnames<typeof onionFilter & F>(
+        return getOnion()
-          filterRec(hostnames, onionFilter, false),
-        )
       },
     }
   }

sdk/base/lib/util/index.ts

@@ -4,7 +4,11 @@ export { getDefaultString } from "./getDefaultString"
 export * from "./ip"
 
 /// Not being used, but known to be browser compatible
-export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
+export {
+  GetServiceInterface,
+  getServiceInterface,
+  filledAddress,
+} from "./getServiceInterface"
 export { getServiceInterfaces } from "./getServiceInterfaces"
 export { once } from "./once"
 export { asError } from "./asError"

sdk/package/lib/StartSdk.ts

@@ -46,7 +46,7 @@ import {
   CheckDependencies,
   checkDependencies,
 } from "../../base/lib/dependencies/dependencies"
-import { GetSslCertificate } from "./util"
+import { GetSslCertificate, getServiceManifest } from "./util"
 import { getDataVersion, setDataVersion } from "./version"
 import { MaybeFn } from "../../base/lib/actions/setupActions"
 import { GetInput } from "../../base/lib/actions/setupActions"
@@ -107,6 +107,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
       | "getContainerIp"
       | "getDataVersion"
       | "setDataVersion"
+      | "getServiceManifest"
 
     // prettier-ignore
     type StartSdkEffectWrapper = {
@@ -441,11 +442,12 @@ export class StartSdk<Manifest extends T.SDKManifest> {
       ) => new ServiceInterfaceBuilder({ ...options, effects }),
       getSystemSmtp: <E extends Effects>(effects: E) =>
         new GetSystemSmtp(effects),
-      getSslCerificate: <E extends Effects>(
+      getSslCertificate: <E extends Effects>(
         effects: E,
         hostnames: string[],
         algorithm?: T.Algorithm,
       ) => new GetSslCertificate(effects, hostnames, algorithm),
+      getServiceManifest,
       healthCheck: {
         checkPortListening,
         checkWebUrl,

sdk/package/lib/util/GetServiceManifest.ts

@@ -0,0 +1,152 @@
+import { Effects } from "../../../base/lib/Effects"
+import { Manifest, PackageId } from "../../../base/lib/osBindings"
+import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
+import { deepEqual } from "../../../base/lib/util/deepEqual"
+
+export class GetServiceManifest<Mapped = Manifest> {
+  constructor(
+    readonly effects: Effects,
+    readonly packageId: PackageId,
+    readonly map: (manifest: Manifest | null) => Mapped,
+    readonly eq: (a: Mapped, b: Mapped) => boolean,
+  ) {}
+
+  /**
+   * Returns the manifest of a service. Reruns the context from which it has been called if the underlying value changes
+   */
+  async const() {
+    let abort = new AbortController()
+    const watch = this.watch(abort.signal)
+    const res = await watch.next()
+    if (this.effects.constRetry) {
+      watch.next().then(() => {
+        abort.abort()
+        this.effects.constRetry && this.effects.constRetry()
+      })
+    }
+    return res.value
+  }
+  /**
+   * Returns the manifest of a service. Does nothing if it changes
+   */
+  async once() {
+    const manifest = await this.effects.getServiceManifest({
+      packageId: this.packageId,
+    })
+    return this.map(manifest)
+  }
+
+  private async *watchGen(abort?: AbortSignal) {
+    let prev = null as { value: Mapped } | null
+    const resolveCell = { resolve: () => {} }
+    this.effects.onLeaveContext(() => {
+      resolveCell.resolve()
+    })
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+        resolveCell.resolve = resolve
+      })
+      const next = this.map(
+        await this.effects.getServiceManifest({
+          packageId: this.packageId,
+          callback: () => callback(),
+        }),
+      )
+      if (!prev || !this.eq(prev.value, next)) {
+        prev = { value: next }
+        yield next
+      }
+      await waitForNext
+    }
+    return new Promise<never>((_, rej) => rej(new Error("aborted")))
+  }
+
+  /**
+   * Watches the manifest of a service. Returns an async iterator that yields whenever the value changes
+   */
+  watch(abort?: AbortSignal): AsyncGenerator<Mapped, never, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
+  /**
+   * Watches the manifest of a service. Takes a custom callback function to run whenever it changes
+   */
+  onChange(
+    callback: (
+      value: Mapped | null,
+      error?: Error,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
+  ) {
+    ;(async () => {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
+        try {
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
+        } catch (e) {
+          console.error(
+            "callback function threw an error @ GetServiceManifest.onChange",
+            e,
+          )
+        }
+      }
+    })()
+      .catch((e) => callback(null, e))
+      .catch((e) =>
+        console.error(
+          "callback function threw an error @ GetServiceManifest.onChange",
+          e,
+        ),
+      )
+  }
+
+  /**
+   * Watches the manifest of a service. Returns when the predicate is true
+   */
+  waitFor(pred: (value: Mapped) => boolean): Promise<Mapped> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        throw new Error("context left before predicate passed")
+      }),
+      () => ctrl.abort(),
+    )
+  }
+}
+
+export function getServiceManifest(
+  effects: Effects,
+  packageId: PackageId,
+): GetServiceManifest<Manifest>
+export function getServiceManifest<Mapped>(
+  effects: Effects,
+  packageId: PackageId,
+  map: (manifest: Manifest | null) => Mapped,
+  eq?: (a: Mapped, b: Mapped) => boolean,
+): GetServiceManifest<Mapped>
+export function getServiceManifest<Mapped>(
+  effects: Effects,
+  packageId: PackageId,
+  map?: (manifest: Manifest | null) => Mapped,
+  eq?: (a: Mapped, b: Mapped) => boolean,
+): GetServiceManifest<Mapped> {
+  return new GetServiceManifest(
+    effects,
+    packageId,
+    map ?? ((a) => a as Mapped),
+    eq ?? ((a, b) => deepEqual(a, b)),
+  )
+}

sdk/package/lib/util/index.ts

@@ -1,5 +1,6 @@
 export * from "../../../base/lib/util"
 export { GetSslCertificate } from "./GetSslCertificate"
+export { GetServiceManifest, getServiceManifest } from "./GetServiceManifest"
 
 export { Drop } from "../../../base/lib/util/Drop"
 export { Volume, Volumes } from "./Volume"

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.46",
+  "version": "0.4.0-beta.47",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.46",
+      "version": "0.4.0-beta.47",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.46",
+  "version": "0.4.0-beta.47",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",

web/projects/shared/src/types/workspace-config.ts

@@ -1,3 +1,12 @@
+export type AccessType =
+  | 'tor'
+  | 'mdns'
+  | 'localhost'
+  | 'ipv4'
+  | 'ipv6'
+  | 'domain'
+  | 'wan-ipv4'
+
 export type WorkspaceConfig = {
   gitHash: string
   useMocks: boolean
@@ -8,7 +17,7 @@ export type WorkspaceConfig = {
       version: string
     }
     mocks: {
-      maskAs: 'tor' | 'local' | 'localhost' | 'ipv4' | 'ipv6' | 'clearnet'
+      maskAs: AccessType
       maskAsHttps: boolean
       skipStartupAlerts: boolean
     }

web/projects/ui/src/app/routes/portal/components/interfaces/interface.service.ts

@@ -206,119 +206,67 @@ export class InterfaceService {
 
   /** ${scheme}://${username}@${host}:${externalPort}${suffix} */
   launchableAddress(ui: T.ServiceInterface, host: T.Host): string {
-    const hostnameInfos = this.hostnameInfo(ui, host)
+    const addresses = utils.filledAddress(host, ui.addressInfo)
 
-    if (!hostnameInfos.length) return ''
+    if (!addresses.hostnames.length) return ''
 
-    const addressInfo = ui.addressInfo
+    const publicDomains = addresses.filter({
-    const username = addressInfo.username ? addressInfo.username + '@' : ''
+      kind: 'domain',
-    const suffix = addressInfo.suffix || ''
+      visibility: 'public',
-    const url = new URL(`https://${username}placeholder${suffix}`)
+    })
-    const use = (hostname: {
+    const tor = addresses.filter({ kind: 'onion' })
-      value: string
+    const wanIp = addresses.filter({ kind: 'ipv4', visibility: 'public' })
-      port: number | null
+    const bestPublic = [publicDomains, tor, wanIp].flatMap(h =>
-      sslPort: number | null
+      h.format('urlstring'),
-    }) => {
+    )[0]
-      url.hostname = hostname.value
+    const privateDomains = addresses.filter({
-      const useSsl =
+      kind: 'domain',
-        hostname.port && hostname.sslPort
+      visibility: 'private',
-          ? this.config.isHttps()
+    })
-          : !!hostname.sslPort
+    const mdns = addresses.filter({ kind: 'mdns' })
-      url.protocol = useSsl
+    const bestPrivate = [privateDomains, mdns].flatMap(h =>
-        ? `${addressInfo.sslScheme || 'https'}:`
+      h.format('urlstring'),
-        : `${addressInfo.scheme || 'http'}:`
+    )[0]
-      const port = useSsl ? hostname.sslPort : hostname.port
+
-      const omitPort = useSsl
+    let matching
-        ? ui.addressInfo.sslScheme === 'https' && port === 443
+    let onLan = false
-        : ui.addressInfo.scheme === 'http' && port === 80
+    switch (this.config.accessType) {
-      if (!omitPort && port) url.port = String(port)
+      case 'ipv4':
-    }
+        matching = addresses.nonLocal
-    const useFirst = (
+          .filter({
-      hostnames: (
+            kind: 'ipv4',
-        | {
+            predicate: h => h.hostname.value === this.config.hostname,
-            value: string
+          })
-            port: number | null
+          .format('urlstring')[0]
-            sslPort: number | null
+        onLan = true
-          }
+        break
-        | undefined
+      case 'ipv6':
-      )[],
+        matching = addresses.nonLocal
-    ) => {
+          .filter({
-      const first = hostnames.find(h => h)
+            kind: 'ipv6',
-      if (first) {
+            predicate: h => h.hostname.value === this.config.hostname,
-        use(first)
+          })
-      }
+          .format('urlstring')[0]
-      return !!first
+        break
+      case 'localhost':
+        matching = addresses
+          .filter({ kind: 'localhost' })
+          .format('urlstring')[0]
+        onLan = true
+        break
+      case 'tor':
+        matching = tor.format('urlstring')[0]
+        break
+      case 'mdns':
+        matching = mdns.format('urlstring')[0]
+        onLan = true
+        break
     }
 
-    const ipHostnames = hostnameInfos
+    if (matching) return matching
-      .filter(h => h.kind === 'ip')
+    if (onLan && bestPrivate) return bestPrivate
-      .map(h => h.hostname) as T.IpHostname[]
+    if (bestPublic) return bestPublic
-    const domainHostname = ipHostnames
+    return ''
-      .filter(h => h.kind === 'domain')
-      .map(h => h as T.IpHostname & { kind: 'domain' })
-      .map(h => ({
-        value: h.value,
-        sslPort: h.sslPort,
-        port: h.port,
-      }))[0]
-    const wanIpHostname = hostnameInfos
-      .filter(h => h.kind === 'ip' && h.public && h.hostname.kind !== 'domain')
-      .map(h => h.hostname as Exclude<T.IpHostname, { kind: 'domain' }>)
-      .map(h => ({
-        value: h.value,
-        sslPort: h.sslPort,
-        port: h.port,
-      }))[0]
-    const onionHostname = hostnameInfos
-      .filter(h => h.kind === 'onion')
-      .map(h => h as T.HostnameInfo & { kind: 'onion' })
-      .map(h => ({
-        value: h.hostname.value,
-        sslPort: h.hostname.sslPort,
-        port: h.hostname.port,
-      }))[0]
-    const localHostname = ipHostnames
-      .filter(h => h.kind === 'local')
-      .map(h => h as T.IpHostname & { kind: 'local' })
-      .map(h => ({ value: h.value, sslPort: h.sslPort, port: h.port }))[0]
-
-    if (this.config.isClearnet()) {
-      if (
-        !useFirst([domainHostname, wanIpHostname, onionHostname, localHostname])
-      ) {
-        return ''
-      }
-    } else if (this.config.isTor()) {
-      if (
-        !useFirst([onionHostname, domainHostname, wanIpHostname, localHostname])
-      ) {
-        return ''
-      }
-    } else if (this.config.isIpv6()) {
-      const ipv6Hostname = ipHostnames.find(h => h.kind === 'ipv6') as {
-        kind: 'ipv6'
-        value: string
-        scopeId: number
-        port: number | null
-        sslPort: number | null
-      }
-
-      if (!useFirst([ipv6Hostname, localHostname])) {
-        return ''
-      }
-    } else {
-      // ipv4 or .local or localhost
-
-      if (!localHostname) return ''
-
-      use({
-        value: this.config.hostname,
-        port: localHostname.port,
-        sslPort: localHostname.sslPort,
-      })
-    }
-
-    return url.href
   }
 
   private hostnameInfo(
@@ -330,7 +278,7 @@ export class InterfaceService {
     return (
       hostnameInfo?.filter(
         h =>
-          this.config.isLocalhost() ||
+          this.config.accessType === 'localhost' ||
           !(
             h.kind === 'ip' &&
             ((h.hostname.kind === 'ipv6' &&

web/projects/ui/src/app/routes/portal/routes/sideload/sideload.component.ts

@@ -92,7 +92,7 @@ import { MarketplacePkgSideload, validateS9pk } from './sideload.utils'
   ],
 })
 export default class SideloadComponent {
-  readonly isTor = inject(ConfigService).isTor()
+  readonly isTor = inject(ConfigService).accessType === 'tor'
 
   file: File | null = null
   readonly package = signal<MarketplacePkgSideload | null>(null)

web/projects/ui/src/app/routes/portal/routes/system/routes/general/general.component.ts

@@ -245,7 +245,7 @@ export default class SystemGeneralComponent {
   private readonly errorService = inject(ErrorService)
   private readonly patch = inject<PatchDB<DataModel>>(PatchDB)
   private readonly api = inject(ApiService)
-  private readonly isTor = inject(ConfigService).isTor()
+  private readonly isTor = inject(ConfigService).accessType === 'tor'
   private readonly dialog = inject(DialogService)
   private readonly i18n = inject(i18nPipe)
   private readonly injector = inject(INJECTOR)

web/projects/ui/src/app/routes/portal/routes/system/routes/general/wipe.component.ts

@@ -31,6 +31,6 @@ import { i18nPipe } from '@start9labs/shared'
   imports: [TuiLabel, FormsModule, TuiCheckbox, i18nPipe],
 })
 export class SystemWipeComponent {
-  readonly isTor = inject(ConfigService).isTor()
+  readonly isTor = inject(ConfigService).accessType === 'tor'
   readonly component = inject(SystemGeneralComponent)
 }

web/projects/ui/src/app/services/api/api.fixures.ts

@@ -385,7 +385,7 @@ export namespace Mock {
             docsUrl: 'https://bitcoin.org',
             releaseNotes: 'Even better support for Bitcoin and wallets!',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: BTC_ICON,
             sourceVersion: null,
@@ -420,7 +420,7 @@ export namespace Mock {
             docsUrl: 'https://bitcoinknots.org',
             releaseNotes: 'Even better support for Bitcoin and wallets!',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: BTC_ICON,
             sourceVersion: null,
@@ -465,7 +465,7 @@ export namespace Mock {
             docsUrl: 'https://bitcoin.org',
             releaseNotes: 'Even better support for Bitcoin and wallets!',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: BTC_ICON,
             sourceVersion: null,
@@ -500,7 +500,7 @@ export namespace Mock {
             docsUrl: 'https://bitcoinknots.org',
             releaseNotes: 'Even better support for Bitcoin and wallets!',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: BTC_ICON,
             sourceVersion: null,
@@ -547,7 +547,7 @@ export namespace Mock {
             docsUrl: 'https://lightning.engineering/',
             releaseNotes: 'Upstream release to 0.17.5',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: LND_ICON,
             sourceVersion: null,
@@ -595,7 +595,7 @@ export namespace Mock {
             docsUrl: 'https://lightning.engineering/',
             releaseNotes: 'Upstream release to 0.17.4',
             osVersion: '0.3.6',
-            sdkVersion: '0.4.0-beta.46',
+            sdkVersion: '0.4.0-beta.47',
             gitHash: 'fakehash',
             icon: LND_ICON,
             sourceVersion: null,
@@ -647,7 +647,7 @@ export namespace Mock {
           docsUrl: 'https://bitcoin.org',
           releaseNotes: 'Even better support for Bitcoin and wallets!',
           osVersion: '0.3.6',
-          sdkVersion: '0.4.0-beta.46',
+          sdkVersion: '0.4.0-beta.47',
           gitHash: 'fakehash',
           icon: BTC_ICON,
           sourceVersion: null,
@@ -682,7 +682,7 @@ export namespace Mock {
           docsUrl: 'https://bitcoinknots.org',
           releaseNotes: 'Even better support for Bitcoin and wallets!',
           osVersion: '0.3.6',
-          sdkVersion: '0.4.0-beta.46',
+          sdkVersion: '0.4.0-beta.47',
           gitHash: 'fakehash',
           icon: BTC_ICON,
           sourceVersion: null,
@@ -727,7 +727,7 @@ export namespace Mock {
           docsUrl: 'https://lightning.engineering/',
           releaseNotes: 'Upstream release and minor fixes.',
           osVersion: '0.3.6',
-          sdkVersion: '0.4.0-beta.46',
+          sdkVersion: '0.4.0-beta.47',
           gitHash: 'fakehash',
           icon: LND_ICON,
           sourceVersion: null,
@@ -775,7 +775,7 @@ export namespace Mock {
           marketingSite: '',
           releaseNotes: 'Upstream release and minor fixes.',
           osVersion: '0.3.6',
-          sdkVersion: '0.4.0-beta.46',
+          sdkVersion: '0.4.0-beta.47',
           gitHash: 'fakehash',
           icon: PROXY_ICON,
           sourceVersion: null,

web/projects/ui/src/app/services/config.service.ts

@@ -1,5 +1,5 @@
 import { Inject, Injectable, DOCUMENT } from '@angular/core'
-import { WorkspaceConfig } from '@start9labs/shared'
+import { AccessType, WorkspaceConfig } from '@start9labs/shared'
 import { T, utils } from '@start9labs/start-sdk'
 
 const {
@@ -29,53 +29,29 @@ export class ConfigService {
   supportsWebSockets = !!window.WebSocket
   defaultRegistry = defaultRegistry
 
-  isTor(): boolean {
+  private getAccessType = utils.once(() => {
-    return useMocks ? mocks.maskAs === 'tor' : this.hostname.endsWith('.onion')
+    if (useMocks) return mocks.maskAs
-  }
+    if (this.hostname === 'localhost') return 'localhost'
-
+    if (this.hostname.endsWith('.onion')) return 'tor'
-  isLocalhost(): boolean {
+    if (this.hostname.endsWith('.local')) return 'mdns'
-    return useMocks
+    let ip = null
-      ? mocks.maskAs === 'localhost'
+    try {
-      : this.hostname === 'localhost' || this.hostname === '127.0.0.1'
+      ip = utils.IpAddress.parse(this.hostname.replace(/[\[\]]/g, ''))
+    } catch {}
+    if (ip) {
+      if (utils.IPV4_LOOPBACK.contains(ip) || utils.IPV6_LOOPBACK.contains(ip))
+        return 'localhost'
+      if (ip.isIpv4()) return ip.isPublic() ? 'wan-ipv4' : 'ipv4'
+      return 'ipv6'
+    }
+    return 'domain'
+  })
+  get accessType(): AccessType {
+    return this.getAccessType()
   }
 
   isLanHttp(): boolean {
-    return !this.isTor() && !this.isLocalhost() && !this.isHttps()
+    return !this.isHttps() && !['localhost', 'tor'].includes(this.accessType)
-  }
-
-  private isLocal(): boolean {
-    return useMocks
-      ? mocks.maskAs === 'local'
-      : this.hostname.endsWith('.local')
-  }
-
-  private isLanIpv4(): boolean {
-    return useMocks
-      ? mocks.maskAs === 'ipv4'
-      : new RegExp(utils.Patterns.ipv4.regex).test(this.hostname) &&
-          (this.hostname.startsWith('192.168.') ||
-            this.hostname.startsWith('10.') ||
-            (this.hostname.startsWith('172.') &&
-              !![this.hostname.split('.').map(Number)[1] || NaN].filter(
-                n => n >= 16 && n < 32,
-              ).length))
-  }
-
-  isIpv6(): boolean {
-    return useMocks
-      ? mocks.maskAs === 'ipv6'
-      : new RegExp(utils.Patterns.ipv6.regex).test(this.hostname)
-  }
-
-  isClearnet(): boolean {
-    return useMocks
-      ? mocks.maskAs === 'clearnet'
-      : this.isHttps() &&
-          !this.isTor() &&
-          !this.isLocal() &&
-          !this.isLocalhost() &&
-          !this.isLanIpv4() &&
-          !this.isIpv6()
   }
 
   isHttps(): boolean {