Bugfix/ssl proxy to ssl (#2956)

* fix registry rm command

* fix bind with addSsl on ssl proto

* fix bind with addSsl on ssl proto

* Add pre-release version migrations

* fix os build

* add mime to package deps

* update lockfile

* more ssl fixes

* add waitFor

* improve restart lockup

* beta.26

* fix dependency health check logic

* handle missing health check

* fix port forwards

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

Makefile

@@ -222,7 +222,11 @@ upload-ota: results/$(BASENAME).squashfs
 container-runtime/debian.$(ARCH).squashfs: ./container-runtime/download-base-image.sh
 	ARCH=$(ARCH) ./container-runtime/download-base-image.sh
 
-container-runtime/node_modules/.package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist/package.json
+container-runtime/package-lock.json: sdk/dist/package.json
+	npm --prefix container-runtime i
+	touch container-runtime/package-lock.json
+
+container-runtime/node_modules/.package-lock.json: container-runtime/package-lock.json
 	npm --prefix container-runtime ci
 	touch container-runtime/node_modules/.package-lock.json
 
@@ -277,7 +281,11 @@ core/target/$(ARCH)-unknown-linux-musl/release/containerbox: $(CORE_SRC) $(ENVIR
 	ARCH=$(ARCH) ./core/build-containerbox.sh
 	touch core/target/$(ARCH)-unknown-linux-musl/release/containerbox
 
-web/node_modules/.package-lock.json: web/package.json sdk/baseDist/package.json
+web/package-lock.json: web/package.json sdk/baseDist/package.json
+	npm --prefix web i
+	touch web/package-lock.json
+
+web/node_modules/.package-lock.json: web/package-lock.json
 	npm --prefix web ci
 	touch web/node_modules/.package-lock.json
 

build/lib/scripts/forward-port

@@ -1,18 +1,26 @@
 #!/bin/bash
 
-iptables -F
-iptables -t nat -F
+if [ -z "$iiface" ] || [ -z "$oiface" ] || [ -z "$sip" ] || [ -z "$dip" ] || [ -z "$sport" ] || [ -z "$dport" ]; then
+    >&2 echo 'missing required env var'
+    exit 1
+fi
 
-iptables -t nat -A POSTROUTING -o $iiface -j MASQUERADE
-iptables -t nat -A PREROUTING -i $iiface -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A PREROUTING -i $iiface -p udp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A PREROUTING -i $oiface -s 10.0.3.0/24 -d $sip -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A PREROUTING -i $oiface -s 10.0.3.0/24 -d $sip -p udp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A POSTROUTING -o $oiface -s 10.0.3.0/24 -d $dip/32 -p tcp --dport $dport -j SNAT --to-source $sip:$sport
-iptables -t nat -A POSTROUTING -o $oiface -s 10.0.3.0/24 -d $dip/32 -p udp --dport $dport -j SNAT --to-source $sip:$sport
+kind="-A"
+
+if [ "$UNDO" = 1 ]; then
+    kind="-D"
+fi
+
+iptables -t nat "$kind" POSTROUTING -o $iiface -j MASQUERADE
+iptables -t nat "$kind" PREROUTING -i $iiface -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" PREROUTING -i $iiface -p udp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" PREROUTING -i $oiface -s $dip/24 -d $sip -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" PREROUTING -i $oiface -s $dip/24 -d $sip -p udp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" POSTROUTING -o $oiface -s $dip/24 -d $dip/32 -p tcp --dport $dport -j SNAT --to-source $sip:$sport
+iptables -t nat "$kind" POSTROUTING -o $oiface -s $dip/24 -d $dip/32 -p udp --dport $dport -j SNAT --to-source $sip:$sport
 
 
-iptables -t nat -A PREROUTING -i $iiface -s $sip/32 -d $sip -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A PREROUTING -i $iiface -s $sip/32 -d $sip -p udp --dport $sport -j DNAT --to-destination $dip:$dport
-iptables -t nat -A POSTROUTING -o $oiface -s $sip/32 -d $dip/32 -p tcp --dport $dport -j SNAT --to-source $sip:$sport
-iptables -t nat -A POSTROUTING -o $oiface -s $sip/32 -d $dip/32 -p udp --dport $dport -j SNAT --to-source $sip:$sport
+iptables -t nat "$kind" PREROUTING -i $iiface -s $sip/32 -d $sip -p tcp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" PREROUTING -i $iiface -s $sip/32 -d $sip -p udp --dport $sport -j DNAT --to-destination $dip:$dport
+iptables -t nat "$kind" POSTROUTING -o $oiface -s $sip/32 -d $dip/32 -p tcp --dport $dport -j SNAT --to-source $sip:$sport
+iptables -t nat "$kind" POSTROUTING -o $oiface -s $sip/32 -d $dip/32 -p udp --dport $dport -j SNAT --to-source $sip:$sport

container-runtime/package-lock.json

@@ -17,6 +17,7 @@
         "isomorphic-fetch": "^3.0.0",
         "jsonpath": "^1.1.1",
         "lodash.merge": "^4.6.2",
+        "mime": "^4.0.7",
         "node-fetch": "^3.1.0",
         "ts-matches": "^6.3.2",
         "tslib": "^2.5.3",
@@ -37,7 +38,7 @@
     },
     "../sdk/dist": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.25",
+      "version": "0.4.0-beta.26",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",
@@ -47,7 +48,7 @@
         "deep-equality-data-structures": "^2.0.0",
         "ini": "^5.0.0",
         "isomorphic-fetch": "^3.0.0",
-        "mime-types": "^3.0.1",
+        "mime": "^4.0.7",
         "ts-matches": "^6.3.2",
         "yaml": "^2.7.1"
       },
@@ -4975,15 +4976,18 @@
       }
     },
     "node_modules/mime": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
-      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",
+      "integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa"
+      ],
       "license": "MIT",
       "bin": {
-        "mime": "cli.js"
+        "mime": "bin/cli.js"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=16"
       }
     },
     "node_modules/mime-db": {
@@ -5914,6 +5918,18 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/send/node_modules/mime": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+      "license": "MIT",
+      "bin": {
+        "mime": "cli.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/send/node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",

container-runtime/package.json

@@ -26,6 +26,7 @@
     "isomorphic-fetch": "^3.0.0",
     "jsonpath": "^1.1.1",
     "lodash.merge": "^4.6.2",
+    "mime": "^4.0.7",
     "node-fetch": "^3.1.0",
     "ts-matches": "^6.3.2",
     "tslib": "^2.5.3",

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -400,9 +400,23 @@ export class SystemForEmbassy implements System {
       this.manifest.title.toLowerCase().includes("knots")
     )
       version.flavor = "knots"
+
+    if (
+      this.manifest.id === "lnd" ||
+      this.manifest.id === "ride-the-lightning" ||
+      this.manifest.id === "datum"
+    ) {
+      version.upstream.prerelease = ["beta"]
+    } else if (
+      this.manifest.id === "lightning-terminal" ||
+      this.manifest.id === "robosats"
+    ) {
+      version.upstream.prerelease = ["alpha"]
+    }
     await effects.setDataVersion({
       version: version.toString(),
     })
+    // @FullMetal: package hacks go here
   }
   async exportNetwork(effects: Effects) {
     for (const [id, interfaceValue] of Object.entries(

core/startos/src/backup/target/mod.rs

@@ -296,7 +296,7 @@ pub async fn info(
 }
 
 lazy_static::lazy_static! {
-    static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, BackupMountGuard<TmpMountGuard>>> =
+    static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, Result<BackupMountGuard<TmpMountGuard>, TmpMountGuard>>> =
         Mutex::new(BTreeMap::new());
 }
 
@@ -305,8 +305,11 @@ lazy_static::lazy_static! {
 #[command(rename_all = "kebab-case")]
 pub struct MountParams {
     target_id: BackupTargetId,
-    server_id: String,
+    #[arg(long)]
+    server_id: Option<String>,
     password: String,
+    #[arg(long)]
+    allow_partial: bool,
 }
 
 #[instrument(skip_all)]
@@ -316,24 +319,63 @@ pub async fn mount(
         target_id,
         server_id,
         password,
+        allow_partial,
     }: MountParams,
 ) -> Result<String, Error> {
+    let server_id = if let Some(server_id) = server_id {
+        server_id
+    } else {
+        ctx.db
+            .peek()
+            .await
+            .into_public()
+            .into_server_info()
+            .into_id()
+            .de()?
+    };
+
     let mut mounts = USER_MOUNTS.lock().await;
 
-    if let Some(existing) = mounts.get(&target_id) {
-        return Ok(existing.path().display().to_string());
-    }
+    let existing = mounts.get(&target_id);
 
-    let guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(&target_id.clone().load(&ctx.db.peek().await)?, ReadWrite).await?,
-        &server_id,
-        &password,
-    )
-    .await?;
+    let base = match existing {
+        Some(Ok(a)) => return Ok(a.path().display().to_string()),
+        Some(Err(e)) => e.clone(),
+        None => {
+            TmpMountGuard::mount(&target_id.clone().load(&ctx.db.peek().await)?, ReadWrite).await?
+        }
+    };
+
+    let guard = match BackupMountGuard::mount(base.clone(), &server_id, &password).await {
+        Ok(a) => a,
+        Err(e) => {
+            if allow_partial {
+                mounts.insert(target_id, Err(base.clone()));
+                let enc_key = BackupMountGuard::<TmpMountGuard>::load_metadata(
+                    base.path(),
+                    &server_id,
+                    &password,
+                )
+                .await
+                .map(|(_, k)| k);
+                return Err(e)
+                    .with_ctx(|e| (
+                        e.kind,
+                        format!(
+                            "\nThe base filesystem did successfully mount at {:?}\nWrapped Key: {:?}",
+                            base.path(),
+                            enc_key
+                        )
+                    ));
+            } else {
+                return Err(e);
+            }
+        }
+    };
 
     let res = guard.path().display().to_string();
 
-    mounts.insert(target_id, guard);
+    mounts.insert(target_id, Ok(guard));
 
     Ok(res)
 }
@@ -350,11 +392,17 @@ pub async fn umount(_: RpcContext, UmountParams { target_id }: UmountParams) ->
     let mut mounts = USER_MOUNTS.lock().await; // TODO: move to context
     if let Some(target_id) = target_id {
         if let Some(existing) = mounts.remove(&target_id) {
-            existing.unmount().await?;
+            match existing {
+                Ok(e) => e.unmount().await?,
+                Err(e) => e.unmount().await?,
+            }
         }
     } else {
         for (_, existing) in std::mem::take(&mut *mounts) {
-            existing.unmount().await?;
+            match existing {
+                Ok(e) => e.unmount().await?,
+                Err(e) => e.unmount().await?,
+            }
         }
     }
 

core/startos/src/context/cli.rs

@@ -37,6 +37,11 @@ pub struct CliContextSeed {
 }
 impl Drop for CliContextSeed {
     fn drop(&mut self) {
+        if let Some(rt) = self.runtime.take() {
+            if let Ok(rt) = Arc::try_unwrap(rt) {
+                rt.shutdown_background();
+            }
+        }
         let tmp = format!("{}.tmp", self.cookie_path.display());
         let parent_dir = self.cookie_path.parent().unwrap_or(Path::new("/"));
         if !parent_dir.exists() {

core/startos/src/control.rs

@@ -48,7 +48,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
         .await
         .as_ref()
         .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .restart(Guid::new())
+        .restart(Guid::new(), false)
         .await?;
 
     Ok(())

core/startos/src/db/model/public.rs

@@ -236,9 +236,10 @@ impl NetworkInterfaceInfo {
     }
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS, HasModel)]
 #[ts(export)]
 #[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
 pub struct IpInfo {
     #[ts(type = "string")]
     pub name: InternedString,

core/startos/src/disk/mount/backup.rs

@@ -29,12 +29,11 @@ pub struct BackupMountGuard<G: GenericMountGuard> {
 }
 impl<G: GenericMountGuard> BackupMountGuard<G> {
     #[instrument(skip_all)]
-    pub async fn mount(
-        backup_disk_mount_guard: G,
+    pub async fn load_metadata(
+        backup_disk_path: &Path,
         server_id: &str,
         password: &str,
-    ) -> Result<Self, Error> {
-        let backup_disk_path = backup_disk_mount_guard.path();
+    ) -> Result<(StartOsRecoveryInfo, String), Error> {
         let backup_dir = backup_disk_path.join("StartOSBackups").join(server_id);
         let unencrypted_metadata_path = backup_dir.join("unencrypted-metadata.json");
         let crypt_path = backup_dir.join("crypt");
@@ -79,7 +78,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
                 &rand::random::<[u8; 32]>()[..],
             )
         };
-
         if unencrypted_metadata.password_hash.is_none() {
             unencrypted_metadata.password_hash = Some(
                 argon2::hash_encoded(
@@ -96,6 +94,20 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
                 &encrypt_slice(&enc_key, password),
             ));
         }
+        Ok((unencrypted_metadata, enc_key))
+    }
+    #[instrument(skip_all)]
+    pub async fn mount(
+        backup_disk_mount_guard: G,
+        server_id: &str,
+        password: &str,
+    ) -> Result<Self, Error> {
+        let backup_disk_path = backup_disk_mount_guard.path();
+        let (unencrypted_metadata, enc_key) =
+            Self::load_metadata(backup_disk_path, server_id, password).await?;
+        let backup_dir = backup_disk_path.join("StartOSBackups").join(server_id);
+        let unencrypted_metadata_path = backup_dir.join("unencrypted-metadata.json");
+        let crypt_path = backup_dir.join("crypt");
 
         if tokio::fs::metadata(&crypt_path).await.is_err() {
             tokio::fs::create_dir_all(&crypt_path).await.with_ctx(|_| {

core/startos/src/disk/mount/filesystem/idmapped.rs

@@ -1,5 +1,6 @@
 use std::ffi::OsStr;
 use std::fmt::Display;
+use std::os::unix::fs::MetadataExt;
 use std::path::Path;
 
 use digest::generic_array::GenericArray;
@@ -54,7 +55,30 @@ impl<Fs: FileSystem> FileSystem for IdMapped<Fs> {
         self.filesystem.source().await
     }
     async fn pre_mount(&self, mountpoint: &Path) -> Result<(), Error> {
-        self.filesystem.pre_mount(mountpoint).await
+        self.filesystem.pre_mount(mountpoint).await?;
+        let info = tokio::fs::metadata(mountpoint).await?;
+        let uid_in_range = self.from_id <= info.uid() && self.from_id + self.range > info.uid();
+        let gid_in_range = self.from_id <= info.gid() && self.from_id + self.range > info.gid();
+        if uid_in_range || gid_in_range {
+            Command::new("chown")
+                .arg(format!(
+                    "{uid}:{gid}",
+                    uid = if uid_in_range {
+                        self.to_id + info.uid() - self.from_id
+                    } else {
+                        info.uid()
+                    },
+                    gid = if gid_in_range {
+                        self.to_id + info.gid() - self.from_id
+                    } else {
+                        info.gid()
+                    },
+                ))
+                .arg(&mountpoint)
+                .invoke(crate::ErrorKind::Filesystem)
+                .await?;
+        }
+        Ok(())
     }
     async fn mount<P: AsRef<Path> + Send>(
         &self,

core/startos/src/lxc/mod.rs

@@ -179,12 +179,6 @@ impl LxcContainer {
         .await?;
         // TODO: append config
         let rootfs_dir = container_dir.join("rootfs");
-        tokio::fs::create_dir_all(&rootfs_dir).await?;
-        Command::new("chown")
-            .arg("100000:100000")
-            .arg(&rootfs_dir)
-            .invoke(ErrorKind::Filesystem)
-            .await?;
         let rootfs = OverlayGuard::mount(
             TmpMountGuard::mount(
                 &IdMapped::new(

core/startos/src/net/forward.rs

@@ -1,5 +1,5 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::net::SocketAddr;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::sync::{Arc, Weak};
 
 use futures::channel::oneshot;
@@ -52,10 +52,13 @@ struct ForwardState {
     current: BTreeMap<u16, BTreeMap<InternedString, SocketAddr>>,
 }
 impl ForwardState {
-    async fn sync(&mut self, interfaces: &BTreeMap<InternedString, bool>) -> Result<(), Error> {
+    async fn sync(
+        &mut self,
+        interfaces: &BTreeMap<InternedString, (bool, Vec<Ipv4Addr>)>,
+    ) -> Result<(), Error> {
         let private_interfaces = interfaces
             .iter()
-            .filter(|(_, public)| !*public)
+            .filter(|(_, (public, _))| !*public)
             .map(|(i, _)| i)
             .collect::<BTreeSet<_>>();
         let all_interfaces = interfaces.keys().collect::<BTreeSet<_>>();
@@ -81,26 +84,30 @@ impl ForwardState {
                     let mut to_rm = actual
                         .difference(expected)
                         .copied()
-                        .cloned()
-                        .collect::<BTreeSet<_>>();
+                        .map(|i| (i.clone(), &interfaces[i].1))
+                        .collect::<BTreeMap<_, _>>();
                     let mut to_add = expected
                         .difference(&actual)
                         .copied()
-                        .cloned()
-                        .collect::<BTreeSet<_>>();
+                        .map(|i| (i.clone(), &interfaces[i].1))
+                        .collect::<BTreeMap<_, _>>();
                     for interface in actual.intersection(expected).copied() {
                         if cur[interface] != req.target {
-                            to_rm.insert(interface.clone());
-                            to_add.insert(interface.clone());
+                            to_rm.insert(interface.clone(), &interfaces[interface].1);
+                            to_add.insert(interface.clone(), &interfaces[interface].1);
                         }
                     }
-                    for interface in to_rm {
-                        unforward(external, &*interface, cur[&interface]).await?;
+                    for (interface, ips) in to_rm {
+                        for ip in ips {
+                            unforward(&*interface, (*ip, external).into(), cur[&interface]).await?;
+                        }
                         cur.remove(&interface);
                     }
-                    for interface in to_add {
-                        forward(external, &*interface, req.target).await?;
-                        cur.insert(interface, req.target);
+                    for (interface, ips) in to_add {
+                        cur.insert(interface.clone(), req.target);
+                        for ip in ips {
+                            forward(&*interface, (*ip, external).into(), cur[&interface]).await?;
+                        }
                     }
                 }
                 (Some(req), None) => {
@@ -112,16 +119,19 @@ impl ForwardState {
                     }
                     .into_iter()
                     .copied()
-                    .cloned()
                     {
-                        forward(external, &*interface, req.target).await?;
-                        cur.insert(interface, req.target);
+                        cur.insert(interface.clone(), req.target);
+                        for ip in &interfaces[interface].1 {
+                            forward(&**interface, (*ip, external).into(), req.target).await?;
+                        }
                     }
                 }
                 (None, Some(cur)) => {
                     let to_rm = cur.keys().cloned().collect::<BTreeSet<_>>();
                     for interface in to_rm {
-                        unforward(external, &*interface, cur[&interface]).await?;
+                        for ip in &interfaces[&interface].1 {
+                            unforward(&*interface, (*ip, external).into(), cur[&interface]).await?;
+                        }
                         cur.remove(&interface);
                     }
                     self.current.remove(&external);
@@ -155,7 +165,26 @@ impl LanPortForwardController {
             let mut interfaces = ip_info.peek_and_mark_seen(|ip_info| {
                 ip_info
                     .iter()
-                    .map(|(iface, info)| (iface.clone(), info.inbound()))
+                    .map(|(iface, info)| {
+                        (
+                            iface.clone(),
+                            (
+                                info.inbound(),
+                                info.ip_info.as_ref().map_or(Vec::new(), |i| {
+                                    i.subnets
+                                        .iter()
+                                        .filter_map(|s| {
+                                            if let IpAddr::V4(ip) = s.addr() {
+                                                Some(ip)
+                                            } else {
+                                                None
+                                            }
+                                        })
+                                        .collect()
+                                }),
+                            ),
+                        )
+                    })
                     .collect()
             });
             let mut reply: Option<oneshot::Sender<Result<(), Error>>> = None;
@@ -175,7 +204,21 @@ impl LanPortForwardController {
                         interfaces = ip_info.peek(|ip_info| {
                             ip_info
                                 .iter()
-                                .map(|(iface, info)| (iface.clone(), info.inbound()))
+                                .map(|(iface, info)| (iface.clone(), (
+                                    info.inbound(),
+                                    info.ip_info.as_ref().map_or(Vec::new(), |i| {
+                                        i.subnets
+                                            .iter()
+                                            .filter_map(|s| {
+                                                if let IpAddr::V4(ip) = s.addr() {
+                                                    Some(ip)
+                                                } else {
+                                                    None
+                                                }
+                                            })
+                                            .collect()
+                                    }),
+                                )))
                                 .collect()
                         });
                     }
@@ -222,87 +265,29 @@ impl LanPortForwardController {
     }
 }
 
-// iptables -t nat -A POSTROUTING -s 10.59.0.0/24 ! -d 10.59.0.0/24 -j SNAT --to $ip
-// iptables -I INPUT -p udp --dport $port -j ACCEPT
-// iptables -I FORWARD -s 10.59.0.0/24 -j ACCEPT
-async fn forward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> {
-    for proto in ["tcp", "udp"] {
-        Command::new("iptables")
-            .arg("-I")
-            .arg("FORWARD")
-            .arg("-i")
-            .arg(interface)
-            .arg("-o")
-            .arg(START9_BRIDGE_IFACE)
-            .arg("-p")
-            .arg(proto)
-            .arg("-d")
-            .arg(target.ip().to_string())
-            .arg("--dport")
-            .arg(target.port().to_string())
-            .arg("-j")
-            .arg("ACCEPT")
-            .invoke(crate::ErrorKind::Network)
-            .await?;
-        Command::new("iptables")
-            .arg("-t")
-            .arg("nat")
-            .arg("-I")
-            .arg("PREROUTING")
-            .arg("-i")
-            .arg(interface)
-            .arg("-p")
-            .arg(proto)
-            .arg("--dport")
-            .arg(external.to_string())
-            .arg("-j")
-            .arg("DNAT")
-            .arg("--to")
-            .arg(target.to_string())
-            .invoke(crate::ErrorKind::Network)
-            .await?;
-    }
+async fn forward(interface: &str, source: SocketAddr, target: SocketAddr) -> Result<(), Error> {
+    Command::new("/usr/lib/startos/scripts/forward-port")
+        .env("iiface", interface)
+        .env("oiface", START9_BRIDGE_IFACE)
+        .env("sip", source.ip().to_string())
+        .env("dip", target.ip().to_string())
+        .env("sport", source.port().to_string())
+        .env("dport", target.port().to_string())
+        .invoke(ErrorKind::Network)
+        .await?;
     Ok(())
 }
 
-// iptables -D FORWARD -o br-start9 -p tcp -d 172.18.0.2 --dport 8333 -j ACCEPT
-// iptables -t nat -D PREROUTING -p tcp --dport 32768 -j DNAT --to 172.18.0.2:8333
-async fn unforward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> {
-    for proto in ["tcp", "udp"] {
-        Command::new("iptables")
-            .arg("-D")
-            .arg("FORWARD")
-            .arg("-i")
-            .arg(interface)
-            .arg("-o")
-            .arg(START9_BRIDGE_IFACE)
-            .arg("-p")
-            .arg(proto)
-            .arg("-d")
-            .arg(target.ip().to_string())
-            .arg("--dport")
-            .arg(target.port().to_string())
-            .arg("-j")
-            .arg("ACCEPT")
-            .invoke(crate::ErrorKind::Network)
-            .await?;
-        Command::new("iptables")
-            .arg("-t")
-            .arg("nat")
-            .arg("-D")
-            .arg("PREROUTING")
-            .arg("-i")
-            .arg(interface)
-            .arg("-p")
-            .arg(proto)
-            .arg("--dport")
-            .arg(external.to_string())
-            .arg("-j")
-            .arg("DNAT")
-            .arg("--to")
-            .arg(target.to_string())
-            .invoke(crate::ErrorKind::Network)
-            .await?;
-    }
+async fn unforward(interface: &str, source: SocketAddr, target: SocketAddr) -> Result<(), Error> {
+    Command::new("/usr/lib/startos/scripts/forward-port")
+        .env("UNDO", "1")
+        .env("iiface", interface)
+        .env("oiface", START9_BRIDGE_IFACE)
+        .env("sip", source.ip().to_string())
+        .env("dip", target.ip().to_string())
+        .env("sport", source.port().to_string())
+        .env("dport", target.port().to_string())
+        .invoke(ErrorKind::Network)
+        .await?;
     Ok(())
 }

core/startos/src/net/net_controller.rs

@@ -492,7 +492,7 @@ impl NetServiceData {
                 let mut bind_hostname_info = hostname_info.remove(internal).unwrap_or_default();
                 bind_hostname_info.push(HostnameInfo::Onion {
                     hostname: OnionHostname {
-                        value: tor_addr.to_string(),
+                        value: InternedString::from_display(tor_addr),
                         port: ports.non_ssl,
                         ssl_port: ports.ssl,
                     },

core/startos/src/net/service_interface.rs

@@ -1,6 +1,7 @@
 use std::net::{Ipv4Addr, Ipv6Addr};
 
 use imbl_value::InternedString;
+use lazy_format::lazy_format;
 use models::{HostId, ServiceInterfaceId};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
@@ -21,15 +22,29 @@ pub enum HostnameInfo {
         hostname: OnionHostname,
     },
 }
+impl HostnameInfo {
+    pub fn to_san_hostname(&self) -> InternedString {
+        match self {
+            Self::Ip { hostname, .. } => hostname.to_san_hostname(),
+            Self::Onion { hostname } => hostname.to_san_hostname(),
+        }
+    }
+}
 
 #[derive(Clone, Debug, Deserialize, Serialize, TS)]
 #[ts(export)]
 #[serde(rename_all = "camelCase")]
 pub struct OnionHostname {
-    pub value: String,
+    #[ts(type = "string")]
+    pub value: InternedString,
     pub port: Option<u16>,
     pub ssl_port: Option<u16>,
 }
+impl OnionHostname {
+    pub fn to_san_hostname(&self) -> InternedString {
+        self.value.clone()
+    }
+}
 
 #[derive(Clone, Debug, Deserialize, Serialize, TS)]
 #[ts(export)]
@@ -64,6 +79,24 @@ pub enum IpHostname {
         ssl_port: Option<u16>,
     },
 }
+impl IpHostname {
+    pub fn to_san_hostname(&self) -> InternedString {
+        match self {
+            Self::Ipv4 { value, .. } => InternedString::from_display(value),
+            Self::Ipv6 { value, .. } => InternedString::from_display(value),
+            Self::Local { value, .. } => value.clone(),
+            Self::Domain {
+                domain, subdomain, ..
+            } => {
+                if let Some(subdomain) = subdomain {
+                    InternedString::from_display(&lazy_format!("{subdomain}.{domain}"))
+                } else {
+                    domain.clone()
+                }
+            }
+        }
+    }
+}
 
 #[derive(Clone, Debug, Deserialize, Serialize, TS)]
 #[ts(export)]

core/startos/src/s9pk/v2/compat.rs

@@ -188,14 +188,22 @@ impl TryFrom<ManifestV1> for Manifest {
     type Error = Error;
     fn try_from(value: ManifestV1) -> Result<Self, Self::Error> {
         let default_url = value.upstream_repo.clone();
+        let mut version = ExtendedVersion::from(
+            exver::emver::Version::from_str(&value.version)
+                .with_kind(ErrorKind::Deserialization)?,
+        );
+        if &*value.id == "bitcoind" && value.title.to_ascii_lowercase().contains("knots") {
+            version = version.with_flavor("knots");
+        } else if &*value.id == "lnd" || &*value.id == "ride-the-lightning" || &*value.id == "datum"
+        {
+            version = version.map_upstream(|mut v| v.with_prerelease(["beta".into()]));
+        } else if &*value.id == "lightning-terminal" || &*value.id == "robosats" {
+            version = version.map_upstream(|mut v| v.with_prerelease(["alpha".into()]));
+        }
         Ok(Self {
             id: value.id,
             title: format!("{} (Legacy)", value.title).into(),
-            version: ExtendedVersion::from(
-                exver::emver::Version::from_str(&value.version)
-                    .with_kind(ErrorKind::Deserialization)?,
-            )
-            .into(),
+            version: version.into(),
             satisfies: BTreeSet::new(),
             release_notes: value.release_notes,
             can_migrate_from: VersionRange::any(),

core/startos/src/service/effects/control.rs

@@ -26,7 +26,7 @@ pub async fn restart(
     ProcedureId { procedure_id }: ProcedureId,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    context.restart(procedure_id).await?;
+    context.restart(procedure_id, false).await?;
     Ok(())
 }
 

core/startos/src/service/effects/dependency.rs

@@ -71,11 +71,6 @@ pub async fn mount(
     if is_mountpoint(&mountpoint).await? {
         unmount(&mountpoint, true).await?;
     }
-    Command::new("chown")
-        .arg("100000:100000")
-        .arg(&mountpoint)
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
     IdMapped::new(Bind::new(source).with_type(filetype), 0, 100000, 65536)
         .mount(
             mountpoint,

core/startos/src/service/effects/net/ssl.rs

@@ -1,6 +1,8 @@
 use std::collections::BTreeSet;
+use std::net::IpAddr;
 
 use imbl_value::InternedString;
+use ipnet::IpNet;
 use itertools::Itertools;
 use openssl::pkey::{PKey, Private};
 
@@ -8,6 +10,7 @@ use crate::service::effects::callbacks::CallbackHandler;
 use crate::service::effects::prelude::*;
 use crate::service::rpc::CallbackId;
 use crate::util::serde::Pem;
+use crate::HOST_IP;
 
 #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, TS, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
@@ -57,6 +60,13 @@ pub async fn get_ssl_certificate(
                         .iter()
                         .map(InternedString::from_display)
                         .chain(m.as_domains().keys()?)
+                        .chain(
+                            m.as_hostname_info()
+                                .de()?
+                                .values()
+                                .flatten()
+                                .map(|h| h.to_san_hostname()),
+                        )
                         .collect::<Vec<_>>())
                 })
                 .map(|a| a.and_then(|a| a))
@@ -70,6 +80,28 @@ pub async fn get_ssl_certificate(
                     if !packages.contains(internal) {
                         return Err(errfn(&*hostname));
                     }
+                } else if let Ok(ip) = hostname.parse::<IpAddr>() {
+                    if IpNet::new(HOST_IP.into(), 24)
+                        .with_kind(ErrorKind::ParseNetAddress)?
+                        .contains(&ip)
+                    {
+                        Ok(())
+                    } else if db
+                        .as_public()
+                        .as_server_info()
+                        .as_network()
+                        .as_network_interfaces()
+                        .as_entries()?
+                        .into_iter()
+                        .flat_map(|(_, net)| net.as_ip_info().transpose_ref())
+                        .flat_map(|net| net.as_subnets().de().log_err())
+                        .flatten()
+                        .any(|s| s.addr() == ip)
+                    {
+                        Ok(())
+                    } else {
+                        Err(errfn(&*hostname))
+                    }?;
                 } else {
                     if !allowed_hostnames.contains(hostname) {
                         return Err(errfn(&*hostname));
@@ -128,6 +160,11 @@ pub async fn get_ssl_key(
     let context = context.deref()?;
     let package_id = &context.seed.id;
     let algorithm = algorithm.unwrap_or(Algorithm::Ecdsa);
+    let container_ip = if let Some(lxc) = context.seed.persistent_container.lxc_container.get() {
+        Some(lxc.ip().await?)
+    } else {
+        None
+    };
 
     let cert = context
         .seed
@@ -135,7 +172,7 @@ pub async fn get_ssl_key(
         .db
         .mutate(|db| {
             let errfn = |h: &str| Error::new(eyre!("unknown hostname: {h}"), ErrorKind::NotFound);
-            let allowed_hostnames = db
+            let mut allowed_hostnames = db
                 .as_public()
                 .as_package_data()
                 .as_idx(package_id)
@@ -148,11 +185,19 @@ pub async fn get_ssl_key(
                         .iter()
                         .map(InternedString::from_display)
                         .chain(m.as_domains().keys()?)
+                        .chain(
+                            m.as_hostname_info()
+                                .de()?
+                                .values()
+                                .flatten()
+                                .map(|h| h.to_san_hostname()),
+                        )
                         .collect::<Vec<_>>())
                 })
                 .map(|a| a.and_then(|a| a))
                 .flatten_ok()
                 .try_collect::<_, BTreeSet<_>, _>()?;
+            allowed_hostnames.extend(container_ip.as_ref().map(InternedString::from_display));
             for hostname in &hostnames {
                 if let Some(internal) = hostname
                     .strip_suffix(".embassy")

core/startos/src/service/fake.cert.key

@@ -1,5 +0,0 @@
------BEGIN EC PRIVATE KEY-----
-MHcCAQEEINn5jiv9VFgEwdUJsDksSTAjPKwkl2DCmCmumu4D1GnNoAoGCCqGSM49
-AwEHoUQDQgAE5KuqP+Wdn8pzmNMxK2hya6mKj1H0j5b47y97tIXqf5ajTi8koRPl
-yao3YcqdtBtN37aw4rVlXVwEJIozZgyiyA==
------END EC PRIVATE KEY-----

core/startos/src/service/fake.cert.pem

@@ -1,13 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIB9DCCAZmgAwIBAgIUIWsFiA8JqIqeUo+Psn91oCQIcdwwCgYIKoZIzj0EAwIw
-TzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNPMRowGAYDVQQKDBFTdGFydDkgTGFi
-cywgSW5jLjEXMBUGA1UEAwwOZmFrZW5hbWUubG9jYWwwHhcNMjQwMjE0MTk1MTUz
-WhcNMjUwMjEzMTk1MTUzWjBPMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ08xGjAY
-BgNVBAoMEVN0YXJ0OSBMYWJzLCBJbmMuMRcwFQYDVQQDDA5mYWtlbmFtZS5sb2Nh
-bDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABOSrqj/lnZ/Kc5jTMStocmupio9R
-9I+W+O8ve7SF6n+Wo04vJKET5cmqN2HKnbQbTd+2sOK1ZV1cBCSKM2YMosijUzBR
-MB0GA1UdDgQWBBR+qd4W//H34Eg90yAPjYz3nZK79DAfBgNVHSMEGDAWgBR+qd4W
-//H34Eg90yAPjYz3nZK79DAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0kA
-MEYCIQDNSN9YWkGbntG+nC+NzEyqE9FcvYZ8TaF3sOnthqSVKwIhAM2N+WJG/p4C
-cPl4HSPPgDaOIhVZzxSje2ycb7wvFtpH
------END CERTIFICATE-----

core/startos/src/service/mod.rs

@@ -1142,23 +1142,6 @@ pub async fn cli_attach(
         None
     };
 
-    let (kill, thread_kill) = tokio::sync::oneshot::channel();
-    let (thread_send, recv) = tokio::sync::mpsc::channel(4 * CAP_1_KiB);
-    let stdin_thread: NonDetachingJoinHandle<()> = tokio::task::spawn_blocking(move || {
-        use std::io::Read;
-        let mut stdin = stdin.lock().bytes();
-
-        while thread_kill.is_empty() {
-            if let Some(b) = stdin.next() {
-                thread_send.blocking_send(b).unwrap();
-            } else {
-                break;
-            }
-        }
-    })
-    .into();
-    let mut stdin = Some(recv);
-
     let guid: Guid = from_value(
         context
             .call_remote::<RpcContext>(
@@ -1178,6 +1161,25 @@ pub async fn cli_attach(
     )?;
     let mut ws = context.ws_continuation(guid).await?;
 
+    let (kill, thread_kill) = tokio::sync::oneshot::channel();
+    let (thread_send, recv) = tokio::sync::mpsc::channel(4 * CAP_1_KiB);
+    let stdin_thread: NonDetachingJoinHandle<()> = tokio::task::spawn_blocking(move || {
+        use std::io::Read;
+        let mut stdin = stdin.lock().bytes();
+
+        while thread_kill.is_empty() {
+            if let Some(b) = stdin.next() {
+                if let Err(_) = thread_send.blocking_send(b) {
+                    break;
+                }
+            } else {
+                break;
+            }
+        }
+    })
+    .into();
+    let mut stdin = Some(recv);
+
     let mut current_in = "stdin";
     let mut current_out = "stdout".to_owned();
     ws.send(Message::Text(current_in.into()))

core/startos/src/service/persistent_container.rs

@@ -161,12 +161,6 @@ impl PersistentContainer {
                 .rootfs_dir()
                 .join("media/startos/volumes")
                 .join(volume);
-            tokio::fs::create_dir_all(&mountpoint).await?;
-            Command::new("chown")
-                .arg("100000:100000")
-                .arg(&mountpoint)
-                .invoke(crate::ErrorKind::Filesystem)
-                .await?;
             let mount = MountGuard::mount(
                 &IdMapped::new(
                     Bind::new(data_dir(DATA_DIR, &s9pk.as_manifest().id, volume)),
@@ -182,12 +176,6 @@ impl PersistentContainer {
         }
 
         let mountpoint = lxc_container.rootfs_dir().join("media/startos/assets");
-        tokio::fs::create_dir_all(&mountpoint).await?;
-        Command::new("chown")
-            .arg("100000:100000")
-            .arg(&mountpoint)
-            .invoke(crate::ErrorKind::Filesystem)
-            .await?;
         let assets = if let Some(sqfs) = s9pk
             .as_archive()
             .contents()
@@ -215,12 +203,6 @@ impl PersistentContainer {
                     .rootfs_dir()
                     .join("media/startos/assets")
                     .join(Path::new(asset).with_extension(""));
-                tokio::fs::create_dir_all(&mountpoint).await?;
-                Command::new("chown")
-                    .arg("100000:100000")
-                    .arg(&mountpoint)
-                    .invoke(crate::ErrorKind::Filesystem)
-                    .await?;
                 let Some(sqfs) = entry.as_file() else {
                     continue;
                 };
@@ -271,12 +253,6 @@ impl PersistentContainer {
                 .and_then(|e| e.as_file())
                 .or_not_found(sqfs_path.display())?;
             let mountpoint = image_path.join(image);
-            tokio::fs::create_dir_all(&mountpoint).await?;
-            Command::new("chown")
-                .arg("100000:100000")
-                .arg(&mountpoint)
-                .invoke(ErrorKind::Filesystem)
-                .await?;
             images.insert(
                 image.clone(),
                 Arc::new(

core/startos/src/service/transition/restart.rs

@@ -1,3 +1,4 @@
+use futures::future::BoxFuture;
 use futures::FutureExt;
 
 use super::TempDesiredRestore;
@@ -12,7 +13,7 @@ use crate::util::future::RemoteCancellable;
 
 pub(super) struct Restart;
 impl Handler<Restart> for ServiceActor {
-    type Response = ();
+    type Response = BoxFuture<'static, Option<()>>;
     fn conflicts_with(_: &Restart) -> ConflictBuilder<Self> {
         ConflictBuilder::everything().except::<GetActionInput>()
     }
@@ -65,14 +66,18 @@ impl Handler<Restart> for ServiceActor {
         if let Some(t) = old {
             t.abort().await;
         }
-        if transition.await.is_none() {
-            tracing::warn!("Service {} has been cancelled", &self.0.id);
-        }
+        transition.boxed()
     }
 }
 impl Service {
     #[instrument(skip_all)]
-    pub async fn restart(&self, id: Guid) -> Result<(), Error> {
-        self.actor.send(id, Restart).await
+    pub async fn restart(&self, id: Guid, wait: bool) -> Result<(), Error> {
+        let fut = self.actor.send(id, Restart).await?;
+        if wait {
+            if fut.await.is_none() {
+                tracing::warn!("Restart has been cancelled");
+            }
+        }
+        Ok(())
     }
 }

core/startos/src/util/actor/concurrent.rs

@@ -19,6 +19,7 @@ struct ConcurrentRunner<A> {
     waiting: Vec<Request<A>>,
     recv: mpsc::UnboundedReceiver<Request<A>>,
     handlers: Vec<(
+        &'static str,
         Guid,
         Arc<ConflictFn<A>>,
         oneshot::Sender<Box<dyn Any + Send>>,
@@ -47,13 +48,26 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                 if this
                     .handlers
                     .iter()
-                    .any(|(hid, f, _, _)| &id != hid && f(&*msg))
+                    .any(|(_, hid, f, _, _)| &id != hid && f(&*msg))
                 {
+                    #[cfg(feature = "unstable")]
+                    {
+                        tracing::debug!("{} must wait...", msg.type_name());
+                        tracing::debug!(
+                            "waiting on {:?}",
+                            this.handlers
+                                .iter()
+                                .filter(|h| h.2(&*msg))
+                                .map(|h| (h.0, &h.1))
+                                .collect::<Vec<_>>()
+                        );
+                    }
                     this.waiting.push((id, msg, reply));
                 } else {
                     let mut actor = this.actor.clone();
                     let queue = this.queue.clone();
                     this.handlers.push((
+                        msg.type_name(),
                         id.clone(),
                         msg.conflicts_with(),
                         reply,
@@ -69,15 +83,15 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                 .handlers
                 .iter_mut()
                 .enumerate()
-                .filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) {
+                .filter_map(|(i, (_, _, _, _, f))| match f.poll_unpin(cx) {
                     std::task::Poll::Pending => None,
                     std::task::Poll::Ready(res) => Some((i, res)),
                 })
                 .collect::<Vec<_>>();
             for (idx, res) in complete.into_iter().rev() {
                 #[allow(clippy::let_underscore_future)]
-                let (_, f, reply, _) = this.handlers.swap_remove(idx);
-                let _ = reply.send(res);
+                let (_, _, f, reply, _) = this.handlers.swap_remove(idx);
+                reply.send(res).ok();
                 // TODO: replace with Vec::extract_if once stable
                 if this.shutdown.is_some() {
                     let mut i = 0;
@@ -86,12 +100,13 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                             && !this
                                 .handlers
                                 .iter()
-                                .any(|(_, f, _, _)| f(&*this.waiting[i].1))
+                                .any(|(_, _, f, _, _)| f(&*this.waiting[i].1))
                         {
                             let (id, msg, reply) = this.waiting.remove(i);
                             let mut actor = this.actor.clone();
                             let queue = this.queue.clone();
                             this.handlers.push((
+                                msg.type_name(),
                                 id.clone(),
                                 msg.conflicts_with(),
                                 reply,
@@ -100,6 +115,18 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                             ));
                             cont = true;
                         } else {
+                            #[cfg(feature = "unstable")]
+                            {
+                                tracing::debug!("{} must wait...", this.waiting[i].1.type_name());
+                                tracing::debug!(
+                                    "waiting on {:?}",
+                                    this.handlers
+                                        .iter()
+                                        .filter(|h| h.2(&*this.waiting[i].1))
+                                        .map(|h| (h.0, &h.1))
+                                        .collect::<Vec<_>>()
+                                );
+                            }
                             i += 1;
                         }
                     }
@@ -219,3 +246,77 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use std::time::Duration;
+
+    use crate::rpc_continuations::Guid;
+    use crate::util::actor::background::BackgroundJobQueue;
+    use crate::util::actor::{Actor, ConflictBuilder, Handler};
+
+    #[derive(Clone)]
+    struct CActor;
+    impl Actor for CActor {
+        fn init(&mut self, jobs: &BackgroundJobQueue) {}
+    }
+    struct Pending;
+    impl Handler<Pending> for CActor {
+        type Response = ();
+        fn conflicts_with(_: &Pending) -> ConflictBuilder<Self> {
+            ConflictBuilder::everything().except::<NoConflicts>()
+        }
+        async fn handle(&mut self, _: Guid, _: Pending, _: &BackgroundJobQueue) -> Self::Response {
+            futures::future::pending().await
+        }
+    }
+    struct Conflicts;
+    impl Handler<Conflicts> for CActor {
+        type Response = ();
+        fn conflicts_with(_: &Conflicts) -> ConflictBuilder<Self> {
+            ConflictBuilder::everything().except::<NoConflicts>()
+        }
+        async fn handle(
+            &mut self,
+            _: Guid,
+            _: Conflicts,
+            _: &BackgroundJobQueue,
+        ) -> Self::Response {
+        }
+    }
+    struct NoConflicts;
+    impl Handler<NoConflicts> for CActor {
+        type Response = ();
+        fn conflicts_with(_: &NoConflicts) -> ConflictBuilder<Self> {
+            ConflictBuilder::nothing()
+        }
+        async fn handle(
+            &mut self,
+            _: Guid,
+            _: NoConflicts,
+            _: &BackgroundJobQueue,
+        ) -> Self::Response {
+        }
+    }
+    #[tokio::test]
+    async fn test_conflicts() {
+        let actor = super::ConcurrentActor::new(CActor);
+        let guid = Guid::new();
+        actor.queue(guid.clone(), Pending);
+        assert!(
+            tokio::time::timeout(Duration::from_secs(1), actor.send(Guid::new(), Conflicts))
+                .await
+                .is_err()
+        );
+        assert!(
+            tokio::time::timeout(Duration::from_secs(1), actor.send(Guid::new(), NoConflicts))
+                .await
+                .is_ok()
+        );
+        assert!(
+            tokio::time::timeout(Duration::from_secs(1), actor.send(guid, Conflicts))
+                .await
+                .is_ok()
+        );
+    }
+}

core/startos/src/util/actor/mod.rs

@@ -45,6 +45,9 @@ trait Message<A>: Send + Any {
         actor: &'a mut A,
         jobs: &'a BackgroundJobQueue,
     ) -> BoxFuture<'a, Box<dyn Any + Send>>;
+    fn type_name(&self) -> &'static str {
+        std::any::type_name_of_val(self)
+    }
 }
 impl<M: Send + Any, A: Actor> Message<A> for M
 where

core/startos/src/version/mod.rs

@@ -378,6 +378,7 @@ fn rollback_to_unchecked<VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>
     Ok(())
 }
 
+#[allow(unused_variables)]
 pub trait VersionT
 where
     Self: Default + Copy + Sized + RefUnwindSafe + Send + Sync + 'static,

core/startos/src/version/v0_4_0_alpha_5.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_4_0_alpha_4, VersionT};
-use crate::context::RpcContext;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {

sdk/base/lib/dependencies/dependencies.ts

@@ -81,9 +81,13 @@ export async function checkDependencies<
     ) {
       throw new Error(`Unknown HealthCheckId ${healthCheckId}`)
     }
-    const errors = Object.entries(dep.result.healthChecks)
-      .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
-      .filter(([_, res]) => res.result !== "success")
+    const errors =
+      dep.requirement.kind === "running"
+        ? dep.requirement.healthChecks
+            .map((id) => [id, dep.result.healthChecks[id] ?? null] as const)
+            .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
+            .filter(([_, res]) => res?.result !== "success")
+        : []
     return errors.length === 0
   }
   const pkgSatisfied = (packageId: DependencyId) =>
@@ -153,15 +157,20 @@ export async function checkDependencies<
     ) {
       throw new Error(`Unknown HealthCheckId ${healthCheckId}`)
     }
-    const errors = Object.entries(dep.result.healthChecks)
-      .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
-      .filter(([_, res]) => res.result !== "success")
+    const errors =
+      dep.requirement.kind === "running"
+        ? dep.requirement.healthChecks
+            .map((id) => [id, dep.result.healthChecks[id] ?? null] as const)
+            .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
+            .filter(([_, res]) => res?.result !== "success")
+        : []
     if (errors.length) {
       throw new Error(
         errors
-          .map(
-            ([_, e]) =>
-              `Health Check ${e.name} of ${dep.result.title || packageId} failed with status ${e.result}${e.message ? `: ${e.message}` : ""}`,
+          .map(([id, e]) =>
+            e
+              ? `Health Check ${e.name} of ${dep.result.title || packageId} failed with status ${e.result}${e.message ? `: ${e.message}` : ""}`
+              : `Health Check ${id} of ${dep.result.title} does not exist`,
           )
           .join("; "),
       )

sdk/base/lib/interfaces/Host.ts

@@ -134,19 +134,26 @@ export class MultiHost {
     const preferredExternalPort =
       options.preferredExternalPort ||
       knownProtocols[options.protocol].defaultPort
-    const sslProto = this.getSslProto(options, protoInfo)
-    const addSsl =
-      sslProto && "alpn" in protoInfo
+    const sslProto = this.getSslProto(options)
+    const addSsl = sslProto
+      ? {
+          // addXForwardedHeaders: null,
+          preferredExternalPort: knownProtocols[sslProto].defaultPort,
+          scheme: sslProto,
+          alpn: "alpn" in protoInfo ? protoInfo.alpn : null,
+          ...("addSsl" in options ? options.addSsl : null),
+        }
+      : options.addSsl
         ? {
             // addXForwardedHeaders: null,
-            preferredExternalPort: knownProtocols[sslProto].defaultPort,
+            preferredExternalPort: 443,
             scheme: sslProto,
-            alpn: protoInfo.alpn,
+            alpn: null,
             ...("addSsl" in options ? options.addSsl : null),
           }
         : null
 
-    const secure: Security | null = !protoInfo.secure ? null : { ssl: false }
+    const secure: Security | null = protoInfo.secure ?? null
 
     await this.options.effects.bind({
       id: this.options.id,
@@ -159,12 +166,12 @@ export class MultiHost {
     return new Origin(this, internalPort, options.protocol, sslProto)
   }
 
-  private getSslProto(
-    options: BindOptionsByKnownProtocol,
-    protoInfo: KnownProtocols[keyof KnownProtocols],
-  ) {
+  private getSslProto(options: BindOptionsByKnownProtocol) {
+    const proto = options.protocol
+    const protoInfo = knownProtocols[proto]
     if (inObject("noAddSsl", options) && options.noAddSsl) return null
     if ("withSsl" in protoInfo && protoInfo.withSsl) return protoInfo.withSsl
+    if (protoInfo.secure?.ssl) return proto
     return null
   }
 }

sdk/base/lib/util/GetSystemSmtp.ts

@@ -71,4 +71,30 @@ export class GetSystemSmtp {
         ),
       )
   }
+
+  /**
+   * Watches the system SMTP credentials. Returns when the predicate is true
+   */
+  async waitFor(pred: (value: T.SmtpValue | null) => boolean) {
+    const resolveCell = { resolve: () => {} }
+    this.effects.onLeaveContext(() => {
+      resolveCell.resolve()
+    })
+    while (this.effects.isInContext) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+        resolveCell.resolve = resolve
+      })
+      const res = await this.effects.getSystemSmtp({
+        callback: () => callback(),
+      })
+      if (pred(res)) {
+        resolveCell.resolve()
+        return res
+      }
+      await waitForNext
+    }
+    return null
+  }
 }

sdk/base/lib/util/getServiceInterface.ts

@@ -366,6 +366,36 @@ export class GetServiceInterface {
         ),
       )
   }
+
+  /**
+   * Watches the requested service interface. Returns when the predicate is true
+   */
+  async waitFor(pred: (value: ServiceInterfaceFilled | null) => boolean) {
+    const { id, packageId } = this.opts
+    const resolveCell = { resolve: () => {} }
+    this.effects.onLeaveContext(() => {
+      resolveCell.resolve()
+    })
+    while (this.effects.isInContext) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+        resolveCell.resolve = resolve
+      })
+      const res = await makeInterfaceFilled({
+        effects: this.effects,
+        id,
+        packageId,
+        callback,
+      })
+      if (pred(res)) {
+        resolveCell.resolve()
+        return res
+      }
+      await waitForNext
+    }
+    return null
+  }
 }
 export function getServiceInterface(
   effects: Effects,

sdk/base/lib/util/getServiceInterfaces.ts

@@ -130,6 +130,35 @@ export class GetServiceInterfaces {
         ),
       )
   }
+
+  /**
+   * Watches the service interfaces for the package. Returns when the predicate is true
+   */
+  async waitFor(pred: (value: ServiceInterfaceFilled[] | null) => boolean) {
+    const { packageId } = this.opts
+    const resolveCell = { resolve: () => {} }
+    this.effects.onLeaveContext(() => {
+      resolveCell.resolve()
+    })
+    while (this.effects.isInContext) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+        resolveCell.resolve = resolve
+      })
+      const res = await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+      if (pred(res)) {
+        resolveCell.resolve()
+        return res
+      }
+      await waitForNext
+    }
+    return null
+  }
 }
 export function getServiceInterfaces(
   effects: Effects,

sdk/package/lib/StartSdk.ts

@@ -249,6 +249,26 @@ export class StartSdk<Manifest extends T.SDKManifest> {
                 ),
               )
           },
+          waitFor: async (pred: (value: string | null) => boolean) => {
+            const resolveCell = { resolve: () => {} }
+            effects.onLeaveContext(() => {
+              resolveCell.resolve()
+            })
+            while (effects.isInContext) {
+              let callback: () => void = () => {}
+              const waitForNext = new Promise<void>((resolve) => {
+                callback = resolve
+                resolveCell.resolve = resolve
+              })
+              const res = await effects.getContainerIp({ ...options, callback })
+              if (pred(res)) {
+                resolveCell.resolve()
+                return res
+              }
+              await waitForNext
+            }
+            return null
+          },
         }
       },
 

sdk/package/lib/backup/Backups.ts

@@ -189,6 +189,8 @@ async function runRsync(rsyncOptions: {
 }> {
   const { srcPath, dstPath, options } = rsyncOptions
 
+  await fs.mkdir(dstPath, { recursive: true })
+
   const command = "rsync"
   const args: string[] = []
   if (options.delete) {

sdk/package/lib/util/GetSslCertificate.ts

@@ -82,4 +82,32 @@ export class GetSslCertificate {
         ),
       )
   }
+
+  /**
+   * Watches the SSL Certificate for the given hostnames if permitted. Returns when the predicate is true
+   */
+  async waitFor(pred: (value: [string, string, string] | null) => boolean) {
+    const resolveCell = { resolve: () => {} }
+    this.effects.onLeaveContext(() => {
+      resolveCell.resolve()
+    })
+    while (this.effects.isInContext) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+        resolveCell.resolve = resolve
+      })
+      const res = await this.effects.getSslCertificate({
+        hostnames: this.hostnames,
+        algorithm: this.algorithm,
+        callback: () => callback(),
+      })
+      if (pred(res)) {
+        resolveCell.resolve()
+        return res
+      }
+      await waitForNext
+    }
+    return null
+  }
 }

sdk/package/lib/util/fileHelper.ts

@@ -26,13 +26,17 @@ async function onCreated(path: string) {
   await onCreated(parent)
   const ctrl = new AbortController()
   const watch = fs.watch(parent, { persistent: false, signal: ctrl.signal })
+  if (await exists(path)) {
+    ctrl.abort()
+    return
+  }
   if (
     await fs.access(path).then(
       () => true,
       () => false,
     )
   ) {
-    ctrl.abort("finished")
+    ctrl.abort()
     return
   }
   for await (let event of watch) {
@@ -100,6 +104,10 @@ type ReadType<A> = {
     effects: T.Effects,
     callback: (value: A | null, error?: Error) => void | Promise<void>,
   ) => void
+  waitFor: (
+    effects: T.Effects,
+    pred: (value: A | null) => boolean,
+  ) => Promise<A | null>
 }
 
 /**
@@ -228,7 +236,7 @@ export class FileHelper<A> {
         const listen = Promise.resolve()
           .then(async () => {
             for await (const _ of watch) {
-              ctrl.abort("finished")
+              ctrl.abort()
               return null
             }
           })
@@ -271,6 +279,40 @@ export class FileHelper<A> {
       )
   }
 
+  private async readWaitFor<B>(
+    effects: T.Effects,
+    pred: (value: B | null, error?: Error) => boolean,
+    map: (value: A) => B,
+  ): Promise<B | null> {
+    while (effects.isInContext) {
+      if (await exists(this.path)) {
+        const ctrl = new AbortController()
+        const watch = fs.watch(this.path, {
+          persistent: false,
+          signal: ctrl.signal,
+        })
+        const newRes = await this.readOnce(map)
+        const listen = Promise.resolve()
+          .then(async () => {
+            for await (const _ of watch) {
+              ctrl.abort()
+              return null
+            }
+          })
+          .catch((e) => console.error(asError(e)))
+        if (pred(newRes)) {
+          ctrl.abort()
+          return newRes
+        }
+        await listen
+      } else {
+        if (pred(null)) return null
+        await onCreated(this.path).catch((e) => console.error(asError(e)))
+      }
+    }
+    return null
+  }
+
   read(): ReadType<A>
   read<B>(
     map: (value: A) => B,
@@ -290,6 +332,8 @@ export class FileHelper<A> {
         effects: T.Effects,
         callback: (value: A | null, error?: Error) => void | Promise<void>,
       ) => this.readOnChange(effects, callback, map, eq),
+      waitFor: (effects: T.Effects, pred: (value: A | null) => boolean) =>
+        this.readWaitFor(effects, pred, map),
     }
   }
 

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.25",
+  "version": "0.4.0-beta.26",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.25",
+      "version": "0.4.0-beta.26",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",
@@ -16,7 +16,7 @@
         "deep-equality-data-structures": "^2.0.0",
         "ini": "^5.0.0",
         "isomorphic-fetch": "^3.0.0",
-        "mime-types": "^3.0.1",
+        "mime": "^4.0.7",
         "ts-matches": "^6.3.2",
         "yaml": "^2.7.1"
       },
@@ -3865,25 +3865,19 @@
         "node": ">=8.6"
       }
     },
-    "node_modules/mime-db": {
-      "version": "1.54.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
-      "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
+    "node_modules/mime": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",
+      "integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa"
+      ],
       "license": "MIT",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz",
-      "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==",
-      "license": "MIT",
-      "dependencies": {
-        "mime-db": "^1.54.0"
+      "bin": {
+        "mime": "bin/cli.js"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">=16"
       }
     },
     "node_modules/mimic-fn": {

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.25",
+  "version": "0.4.0-beta.26",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",
@@ -32,7 +32,7 @@
   "homepage": "https://github.com/Start9Labs/start-sdk#readme",
   "dependencies": {
     "isomorphic-fetch": "^3.0.0",
-    "mime-types": "^3.0.1",
+    "mime": "^4.0.7",
     "ts-matches": "^6.3.2",
     "yaml": "^2.7.1",
     "deep-equality-data-structures": "^2.0.0",