add clearnet functionality to frontend (#2814)

* add clearnet functionality to frontend

* add pattern and add sync db on rpcs

* add domain pattern

* show acme name instead of url if known

* dont blow up if domain not present after delete

* use common name for letsencrypt

* normalize urls

* refactor start-os ui net service

* backend migration and rpcs for serverInfo.host

* fix cors

* implement clearnet for main startos ui

* ability to add and remove tor addresses, including vanity

* add guard to prevent duplicate addresses

* misc bugfixes

* better heuristics for launching UIs

* fix ipv6 mocks

* fix ipv6 display bug

* rewrite url selection for launch ui

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

core/startos/src/service/effects/net/bind.rs

@@ -1,14 +1,12 @@
 use models::{HostId, PackageId};
 
 use crate::net::host::binding::{BindId, BindOptions, NetInfo};
-use crate::net::host::HostKind;
 use crate::service::effects::prelude::*;
 
 #[derive(Debug, Clone, Serialize, Deserialize, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export)]
 pub struct BindParams {
-    kind: HostKind,
     id: HostId,
     internal_port: u16,
     #[serde(flatten)]
@@ -17,15 +15,18 @@ pub struct BindParams {
 pub async fn bind(
     context: EffectContext,
     BindParams {
-        kind,
         id,
         internal_port,
         options,
     }: BindParams,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    let mut svc = context.seed.persistent_container.net_service.lock().await;
-    svc.bind(kind, id, internal_port, options).await
+    context
+        .seed
+        .persistent_container
+        .net_service
+        .bind(id, internal_port, options)
+        .await
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
@@ -41,8 +42,12 @@ pub async fn clear_bindings(
     ClearBindingsParams { except }: ClearBindingsParams,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    let mut svc = context.seed.persistent_container.net_service.lock().await;
-    svc.clear_bindings(except.into_iter().collect()).await?;
+    context
+        .seed
+        .persistent_container
+        .net_service
+        .clear_bindings(except.into_iter().collect())
+        .await?;
     Ok(())
 }
 

core/startos/src/service/effects/net/info.rs

@@ -4,6 +4,5 @@ use crate::service::effects::prelude::*;
 
 pub async fn get_container_ip(context: EffectContext) -> Result<Ipv4Addr, Error> {
     let context = context.deref()?;
-    let net_service = context.seed.persistent_container.net_service.lock().await;
-    Ok(net_service.get_ip())
+    Ok(context.seed.persistent_container.net_service.get_ip().await)
 }

core/startos/src/service/mod.rs

@@ -605,23 +605,12 @@ impl Service {
         })
     }
 
-    pub async fn update_host(&self, host_id: HostId) -> Result<(), Error> {
-        let mut service = self.seed.persistent_container.net_service.lock().await;
-        let host = self
-            .seed
-            .ctx
-            .db
-            .peek()
+    pub async fn sync_host(&self, host_id: HostId) -> Result<(), Error> {
+        self.seed
+            .persistent_container
+            .net_service
+            .sync_host(host_id)
             .await
-            .as_public()
-            .as_package_data()
-            .as_idx(&self.seed.id)
-            .or_not_found(&self.seed.id)?
-            .as_hosts()
-            .as_idx(&host_id)
-            .or_not_found(&host_id)?
-            .de()?;
-        service.update(host_id, host).await
     }
 }
 

core/startos/src/service/persistent_container.rs

@@ -110,7 +110,7 @@ pub struct PersistentContainer {
     pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
     pub(super) subcontainers: Arc<Mutex<BTreeMap<Guid, Subcontainer>>>,
     pub(super) state: Arc<watch::Sender<ServiceState>>,
-    pub(super) net_service: Mutex<NetService>,
+    pub(super) net_service: NetService,
     destroyed: bool,
 }
 
@@ -285,7 +285,7 @@ impl PersistentContainer {
             images,
             subcontainers: Arc::new(Mutex::new(BTreeMap::new())),
             state: Arc::new(watch::channel(ServiceState::new(start)).0),
-            net_service: Mutex::new(net_service),
+            net_service,
             destroyed: false,
         })
     }

core/startos/src/service/service_actor.rs

@@ -37,38 +37,6 @@ impl Actor for ServiceActor {
                 }
             }
         });
-        let seed = self.0.clone();
-        let mut ip_info = seed.ctx.net_controller.net_iface.subscribe();
-        jobs.add_job(async move {
-            loop {
-                if let Err(e) = async {
-                    let mut service = seed.persistent_container.net_service.lock().await;
-                    let hosts = seed
-                        .ctx
-                        .db
-                        .peek()
-                        .await
-                        .as_public()
-                        .as_package_data()
-                        .as_idx(&seed.id)
-                        .or_not_found(&seed.id)?
-                        .as_hosts()
-                        .de()?;
-                    for (host_id, host) in hosts.0 {
-                        service.update(host_id, host).await?;
-                    }
-
-                    Ok::<_, Error>(())
-                }
-                .await
-                {
-                    tracing::error!("Error syncronizing net host after network change: {e}");
-                    tracing::debug!("{e:?}");
-                }
-
-                ip_info.changed().await;
-            }
-        });
     }
 }
 

core/startos/src/service/service_map.rs

@@ -4,7 +4,8 @@ use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
-use futures::{Future, FutureExt};
+use futures::stream::FuturesUnordered;
+use futures::{Future, FutureExt, StreamExt};
 use helpers::NonDetachingJoinHandle;
 use imbl::OrdMap;
 use imbl_value::InternedString;
@@ -68,8 +69,12 @@ impl ServiceMap {
         progress.start();
         let ids = ctx.db.peek().await.as_public().as_package_data().keys()?;
         progress.set_total(ids.len() as u64);
-        for id in ids {
-            if let Err(e) = self.load(ctx, &id, LoadDisposition::Retry).await {
+        let mut jobs = FuturesUnordered::new();
+        for id in &ids {
+            jobs.push(self.load(ctx, id, LoadDisposition::Retry));
+        }
+        while let Some(res) = jobs.next().await {
+            if let Err(e) = res {
                 tracing::error!("Error loading installed package as service: {e}");
                 tracing::debug!("{e:?}");
             }