rename frontend to web and update contributing guide (#2509)

* rename frontend to web and update contributing guide

* rename this time

* fix build

* restructure rust code

* update documentation

* update descriptions

* Update CONTRIBUTING.md

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

core/startos/src/net/cert-local.csr.conf.template

@@ -0,0 +1,10 @@
+[req]
+default_bits = 4096
+default_md = sha256
+distinguished_name = req_distinguished_name
+prompt = no
+
+[req_distinguished_name]
+CN = {hostname}.local
+O = Start9 Labs
+OU = StartOS

core/startos/src/net/dhcp.rs

@@ -0,0 +1,82 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::net::IpAddr;
+
+use futures::TryStreamExt;
+use rpc_toolkit::command;
+use tokio::sync::RwLock;
+
+use crate::context::RpcContext;
+use crate::db::model::IpInfo;
+use crate::net::utils::{iface_is_physical, list_interfaces};
+use crate::prelude::*;
+use crate::util::display_none;
+use crate::Error;
+
+lazy_static::lazy_static! {
+    static ref CACHED_IPS: RwLock<BTreeSet<IpAddr>> = RwLock::new(BTreeSet::new());
+}
+
+async fn _ips() -> Result<BTreeSet<IpAddr>, Error> {
+    Ok(init_ips()
+        .await?
+        .values()
+        .flat_map(|i| {
+            std::iter::empty()
+                .chain(i.ipv4.map(IpAddr::from))
+                .chain(i.ipv6.map(IpAddr::from))
+        })
+        .collect())
+}
+
+pub async fn ips() -> Result<BTreeSet<IpAddr>, Error> {
+    let ips = CACHED_IPS.read().await.clone();
+    if !ips.is_empty() {
+        return Ok(ips);
+    }
+    let ips = _ips().await?;
+    *CACHED_IPS.write().await = ips.clone();
+    Ok(ips)
+}
+
+pub async fn init_ips() -> Result<BTreeMap<String, IpInfo>, Error> {
+    let mut res = BTreeMap::new();
+    let mut ifaces = list_interfaces();
+    while let Some(iface) = ifaces.try_next().await? {
+        if iface_is_physical(&iface).await {
+            let ip_info = IpInfo::for_interface(&iface).await?;
+            res.insert(iface, ip_info);
+        }
+    }
+    Ok(res)
+}
+
+#[command(subcommands(update))]
+pub async fn dhcp() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn update(#[context] ctx: RpcContext, #[arg] interface: String) -> Result<(), Error> {
+    if iface_is_physical(&interface).await {
+        let ip_info = IpInfo::for_interface(&interface).await?;
+        ctx.db
+            .mutate(|db| {
+                db.as_server_info_mut()
+                    .as_ip_info_mut()
+                    .insert(&interface, &ip_info)
+            })
+            .await?;
+
+        let mut cached = CACHED_IPS.write().await;
+        if cached.is_empty() {
+            *cached = _ips().await?;
+        } else {
+            cached.extend(
+                std::iter::empty()
+                    .chain(ip_info.ipv4.map(IpAddr::from))
+                    .chain(ip_info.ipv6.map(IpAddr::from)),
+            );
+        }
+    }
+    Ok(())
+}

core/startos/src/net/dns.rs

@@ -0,0 +1,228 @@
+use std::borrow::Borrow;
+use std::collections::BTreeMap;
+use std::net::{Ipv4Addr, SocketAddr};
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use futures::TryFutureExt;
+use helpers::NonDetachingJoinHandle;
+use models::PackageId;
+use tokio::net::{TcpListener, UdpSocket};
+use tokio::process::Command;
+use tokio::sync::RwLock;
+use tracing::instrument;
+use trust_dns_server::authority::MessageResponseBuilder;
+use trust_dns_server::proto::op::{Header, ResponseCode};
+use trust_dns_server::proto::rr::{Name, Record, RecordType};
+use trust_dns_server::server::{Request, RequestHandler, ResponseHandler, ResponseInfo};
+use trust_dns_server::ServerFuture;
+
+use crate::util::Invoke;
+use crate::{Error, ErrorKind, ResultExt};
+
+pub struct DnsController {
+    services: Weak<RwLock<BTreeMap<Option<PackageId>, BTreeMap<Ipv4Addr, Weak<()>>>>>,
+    #[allow(dead_code)]
+    dns_server: NonDetachingJoinHandle<Result<(), Error>>,
+}
+
+struct Resolver {
+    services: Arc<RwLock<BTreeMap<Option<PackageId>, BTreeMap<Ipv4Addr, Weak<()>>>>>,
+}
+impl Resolver {
+    async fn resolve(&self, name: &Name) -> Option<Vec<Ipv4Addr>> {
+        match name.iter().next_back() {
+            Some(b"embassy") => {
+                if let Some(pkg) = name.iter().rev().skip(1).next() {
+                    if let Some(ip) = self.services.read().await.get(&Some(
+                        std::str::from_utf8(pkg)
+                            .unwrap_or_default()
+                            .parse()
+                            .unwrap_or_default(),
+                    )) {
+                        Some(
+                            ip.iter()
+                                .filter(|(_, rc)| rc.strong_count() > 0)
+                                .map(|(ip, _)| *ip)
+                                .collect(),
+                        )
+                    } else {
+                        None
+                    }
+                } else if let Some(ip) = self.services.read().await.get(&None) {
+                    Some(
+                        ip.iter()
+                            .filter(|(_, rc)| rc.strong_count() > 0)
+                            .map(|(ip, _)| *ip)
+                            .collect(),
+                    )
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl RequestHandler for Resolver {
+    async fn handle_request<R: ResponseHandler>(
+        &self,
+        request: &Request,
+        mut response_handle: R,
+    ) -> ResponseInfo {
+        let query = request.request_info().query;
+        if let Some(ip) = self.resolve(query.name().borrow()).await {
+            match query.query_type() {
+                RecordType::A => {
+                    response_handle
+                        .send_response(
+                            MessageResponseBuilder::from_message_request(&*request).build(
+                                Header::response_from_request(request.header()),
+                                &ip.into_iter()
+                                    .map(|ip| {
+                                        Record::from_rdata(
+                                            request.request_info().query.name().to_owned().into(),
+                                            0,
+                                            trust_dns_server::proto::rr::RData::A(ip.into()),
+                                        )
+                                    })
+                                    .collect::<Vec<_>>(),
+                                [],
+                                [],
+                                [],
+                            ),
+                        )
+                        .await
+                }
+                a => {
+                    if a != RecordType::AAAA {
+                        tracing::warn!(
+                            "Non A-Record requested for {}: {:?}",
+                            query.name(),
+                            query.query_type()
+                        );
+                    }
+                    let mut res = Header::response_from_request(request.header());
+                    res.set_response_code(ResponseCode::NXDomain);
+                    response_handle
+                        .send_response(
+                            MessageResponseBuilder::from_message_request(&*request).build(
+                                res.into(),
+                                [],
+                                [],
+                                [],
+                                [],
+                            ),
+                        )
+                        .await
+                }
+            }
+        } else {
+            let mut res = Header::response_from_request(request.header());
+            res.set_response_code(ResponseCode::NXDomain);
+            response_handle
+                .send_response(
+                    MessageResponseBuilder::from_message_request(&*request).build(
+                        res.into(),
+                        [],
+                        [],
+                        [],
+                        [],
+                    ),
+                )
+                .await
+        }
+        .unwrap_or_else(|e| {
+            tracing::error!("{}", e);
+            tracing::debug!("{:?}", e);
+            let mut res = Header::response_from_request(request.header());
+            res.set_response_code(ResponseCode::ServFail);
+            res.into()
+        })
+    }
+}
+
+impl DnsController {
+    #[instrument(skip_all)]
+    pub async fn init(bind: &[SocketAddr]) -> Result<Self, Error> {
+        let services = Arc::new(RwLock::new(BTreeMap::new()));
+
+        let mut server = ServerFuture::new(Resolver {
+            services: services.clone(),
+        });
+        server.register_listener(
+            TcpListener::bind(bind)
+                .await
+                .with_kind(ErrorKind::Network)?,
+            Duration::from_secs(30),
+        );
+        server.register_socket(UdpSocket::bind(bind).await.with_kind(ErrorKind::Network)?);
+
+        Command::new("resolvectl")
+            .arg("dns")
+            .arg("br-start9")
+            .arg("127.0.0.1")
+            .invoke(ErrorKind::Network)
+            .await?;
+        Command::new("resolvectl")
+            .arg("domain")
+            .arg("br-start9")
+            .arg("embassy")
+            .invoke(ErrorKind::Network)
+            .await?;
+
+        let dns_server = tokio::spawn(
+            server
+                .block_until_done()
+                .map_err(|e| Error::new(e, ErrorKind::Network)),
+        )
+        .into();
+
+        Ok(Self {
+            services: Arc::downgrade(&services),
+            dns_server,
+        })
+    }
+
+    pub async fn add(&self, pkg_id: Option<PackageId>, ip: Ipv4Addr) -> Result<Arc<()>, Error> {
+        if let Some(services) = Weak::upgrade(&self.services) {
+            let mut writable = services.write().await;
+            let mut ips = writable.remove(&pkg_id).unwrap_or_default();
+            let rc = if let Some(rc) = Weak::upgrade(&ips.remove(&ip).unwrap_or_default()) {
+                rc
+            } else {
+                Arc::new(())
+            };
+            ips.insert(ip, Arc::downgrade(&rc));
+            writable.insert(pkg_id, ips);
+            Ok(rc)
+        } else {
+            Err(Error::new(
+                eyre!("DNS Server Thread has exited"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+
+    pub async fn gc(&self, pkg_id: Option<PackageId>, ip: Ipv4Addr) -> Result<(), Error> {
+        if let Some(services) = Weak::upgrade(&self.services) {
+            let mut writable = services.write().await;
+            let mut ips = writable.remove(&pkg_id).unwrap_or_default();
+            if let Some(rc) = Weak::upgrade(&ips.remove(&ip).unwrap_or_default()) {
+                ips.insert(ip, Arc::downgrade(&rc));
+            }
+            if !ips.is_empty() {
+                writable.insert(pkg_id, ips);
+            }
+            Ok(())
+        } else {
+            Err(Error::new(
+                eyre!("DNS Server Thread has exited"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+}

core/startos/src/net/interface.rs

@@ -0,0 +1,122 @@
+use std::collections::BTreeMap;
+
+use indexmap::IndexSet;
+pub use models::InterfaceId;
+use serde::{Deserialize, Deserializer, Serialize};
+use sqlx::{Executor, Postgres};
+use tracing::instrument;
+
+use crate::db::model::{InterfaceAddressMap, InterfaceAddresses};
+use crate::net::keys::Key;
+use crate::s9pk::manifest::PackageId;
+use crate::util::serde::Port;
+use crate::{Error, ResultExt};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Interfaces(pub BTreeMap<InterfaceId, Interface>); // TODO
+impl Interfaces {
+    #[instrument(skip_all)]
+    pub fn validate(&self) -> Result<(), Error> {
+        for (_, interface) in &self.0 {
+            interface.validate().with_ctx(|_| {
+                (
+                    crate::ErrorKind::ValidateS9pk,
+                    format!("Interface {}", interface.name),
+                )
+            })?;
+        }
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn install<Ex>(
+        &self,
+        secrets: &mut Ex,
+        package_id: &PackageId,
+    ) -> Result<InterfaceAddressMap, Error>
+    where
+        for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
+    {
+        let mut interface_addresses = InterfaceAddressMap(BTreeMap::new());
+        for (id, iface) in &self.0 {
+            let mut addrs = InterfaceAddresses {
+                tor_address: None,
+                lan_address: None,
+            };
+            if iface.tor_config.is_some() || iface.lan_config.is_some() {
+                let key =
+                    Key::for_interface(secrets, Some((package_id.clone(), id.clone()))).await?;
+                if iface.tor_config.is_some() {
+                    addrs.tor_address = Some(key.tor_address().to_string());
+                }
+                if iface.lan_config.is_some() {
+                    addrs.lan_address = Some(key.local_address());
+                }
+            }
+            interface_addresses.0.insert(id.clone(), addrs);
+        }
+        Ok(interface_addresses)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Interface {
+    pub name: String,
+    pub description: String,
+    pub tor_config: Option<TorConfig>,
+    pub lan_config: Option<BTreeMap<Port, LanPortConfig>>,
+    pub ui: bool,
+    pub protocols: IndexSet<String>,
+}
+impl Interface {
+    #[instrument(skip_all)]
+    pub fn validate(&self) -> Result<(), color_eyre::eyre::Report> {
+        if self.tor_config.is_some() && !self.protocols.contains("tcp") {
+            color_eyre::eyre::bail!("must support tcp to set up a tor hidden service");
+        }
+        if self.lan_config.is_some() && !self.protocols.contains("http") {
+            color_eyre::eyre::bail!("must support http to set up a lan service");
+        }
+        if self.ui && !(self.protocols.contains("http") || self.protocols.contains("https")) {
+            color_eyre::eyre::bail!("must support http or https to serve a ui");
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct TorConfig {
+    pub port_mapping: BTreeMap<Port, Port>,
+}
+
+#[derive(Clone, Debug, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct LanPortConfig {
+    pub ssl: bool,
+    pub internal: u16,
+}
+impl<'de> Deserialize<'de> for LanPortConfig {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        #[serde(rename_all = "kebab-case")]
+        struct PermissiveLanPortConfig {
+            ssl: bool,
+            internal: Option<u16>,
+            mapping: Option<u16>,
+        }
+
+        let config = PermissiveLanPortConfig::deserialize(deserializer)?;
+        Ok(LanPortConfig {
+            ssl: config.ssl,
+            internal: config
+                .internal
+                .or(config.mapping)
+                .ok_or_else(|| serde::de::Error::missing_field("internal"))?,
+        })
+    }
+}

core/startos/src/net/keys.rs

@@ -0,0 +1,273 @@
+use color_eyre::eyre::eyre;
+use models::{Id, InterfaceId, PackageId};
+use openssl::pkey::{PKey, Private};
+use openssl::sha::Sha256;
+use openssl::x509::X509;
+use p256::elliptic_curve::pkcs8::EncodePrivateKey;
+use sqlx::PgExecutor;
+use ssh_key::private::Ed25519PrivateKey;
+use torut::onion::{OnionAddressV3, TorSecretKeyV3};
+use zeroize::Zeroize;
+
+use crate::net::ssl::CertPair;
+use crate::prelude::*;
+use crate::util::crypto::ed25519_expand_key;
+
+// TODO: delete once we may change tor addresses
+async fn compat(
+    secrets: impl PgExecutor<'_>,
+    interface: &Option<(PackageId, InterfaceId)>,
+) -> Result<Option<[u8; 64]>, Error> {
+    if let Some((package, interface)) = interface {
+        if let Some(r) = sqlx::query!(
+            "SELECT key FROM tor WHERE package = $1 AND interface = $2",
+            package,
+            interface
+        )
+        .fetch_optional(secrets)
+        .await?
+        {
+            Ok(Some(<[u8; 64]>::try_from(r.key).map_err(|e| {
+                Error::new(
+                    eyre!("expected vec of len 64, got len {}", e.len()),
+                    ErrorKind::ParseDbField,
+                )
+            })?))
+        } else {
+            Ok(None)
+        }
+    } else if let Some(key) = sqlx::query!("SELECT tor_key FROM account WHERE id = 0")
+        .fetch_one(secrets)
+        .await?
+        .tor_key
+    {
+        Ok(Some(<[u8; 64]>::try_from(key).map_err(|e| {
+            Error::new(
+                eyre!("expected vec of len 64, got len {}", e.len()),
+                ErrorKind::ParseDbField,
+            )
+        })?))
+    } else {
+        Ok(None)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Key {
+    interface: Option<(PackageId, InterfaceId)>,
+    base: [u8; 32],
+    tor_key: [u8; 64], // Does NOT necessarily match base
+}
+impl Key {
+    pub fn interface(&self) -> Option<(PackageId, InterfaceId)> {
+        self.interface.clone()
+    }
+    pub fn as_bytes(&self) -> [u8; 32] {
+        self.base
+    }
+    pub fn internal_address(&self) -> String {
+        self.interface
+            .as_ref()
+            .map(|(pkg_id, _)| format!("{}.embassy", pkg_id))
+            .unwrap_or_else(|| "embassy".to_owned())
+    }
+    pub fn tor_key(&self) -> TorSecretKeyV3 {
+        self.tor_key.into()
+    }
+    pub fn tor_address(&self) -> OnionAddressV3 {
+        self.tor_key().public().get_onion_address()
+    }
+    pub fn base_address(&self) -> String {
+        self.tor_key()
+            .public()
+            .get_onion_address()
+            .get_address_without_dot_onion()
+    }
+    pub fn local_address(&self) -> String {
+        self.base_address() + ".local"
+    }
+    pub fn openssl_key_ed25519(&self) -> PKey<Private> {
+        PKey::private_key_from_raw_bytes(&self.base, openssl::pkey::Id::ED25519).unwrap()
+    }
+    pub fn openssl_key_nistp256(&self) -> PKey<Private> {
+        let mut buf = self.base;
+        loop {
+            if let Ok(k) = p256::SecretKey::from_slice(&buf) {
+                return PKey::private_key_from_pkcs8(&*k.to_pkcs8_der().unwrap().as_bytes())
+                    .unwrap();
+            }
+            let mut sha = Sha256::new();
+            sha.update(&buf);
+            buf = sha.finish();
+        }
+    }
+    pub fn ssh_key(&self) -> Ed25519PrivateKey {
+        Ed25519PrivateKey::from_bytes(&self.base)
+    }
+    pub(crate) fn from_pair(
+        interface: Option<(PackageId, InterfaceId)>,
+        bytes: [u8; 32],
+        tor_key: [u8; 64],
+    ) -> Self {
+        Self {
+            interface,
+            tor_key,
+            base: bytes,
+        }
+    }
+    pub fn from_bytes(interface: Option<(PackageId, InterfaceId)>, bytes: [u8; 32]) -> Self {
+        Self::from_pair(interface, bytes, ed25519_expand_key(&bytes))
+    }
+    pub fn new(interface: Option<(PackageId, InterfaceId)>) -> Self {
+        Self::from_bytes(interface, rand::random())
+    }
+    pub(super) fn with_certs(self, certs: CertPair, int: X509, root: X509) -> KeyInfo {
+        KeyInfo {
+            key: self,
+            certs,
+            int,
+            root,
+        }
+    }
+    pub async fn for_package(
+        secrets: impl PgExecutor<'_>,
+        package: &PackageId,
+    ) -> Result<Vec<Self>, Error> {
+        sqlx::query!(
+            r#"
+                SELECT
+                    network_keys.package,
+                    network_keys.interface,
+                    network_keys.key,
+                    tor.key AS "tor_key?"
+                FROM
+                    network_keys
+                LEFT JOIN
+                    tor
+                ON
+                    network_keys.package = tor.package
+                AND
+                    network_keys.interface = tor.interface
+                WHERE
+                    network_keys.package = $1
+            "#,
+            package
+        )
+        .fetch_all(secrets)
+        .await?
+        .into_iter()
+        .map(|row| {
+            let interface = Some((
+                package.clone(),
+                InterfaceId::from(Id::try_from(row.interface)?),
+            ));
+            let bytes = row.key.try_into().map_err(|e: Vec<u8>| {
+                Error::new(
+                    eyre!("Invalid length for network key {} expected 32", e.len()),
+                    crate::ErrorKind::Database,
+                )
+            })?;
+            Ok(match row.tor_key {
+                Some(tor_key) => Key::from_pair(
+                    interface,
+                    bytes,
+                    tor_key.try_into().map_err(|e: Vec<u8>| {
+                        Error::new(
+                            eyre!("Invalid length for tor key {} expected 64", e.len()),
+                            crate::ErrorKind::Database,
+                        )
+                    })?,
+                ),
+                None => Key::from_bytes(interface, bytes),
+            })
+        })
+        .collect()
+    }
+    pub async fn for_interface<Ex>(
+        secrets: &mut Ex,
+        interface: Option<(PackageId, InterfaceId)>,
+    ) -> Result<Self, Error>
+    where
+        for<'a> &'a mut Ex: PgExecutor<'a>,
+    {
+        let tentative = rand::random::<[u8; 32]>();
+        let actual = if let Some((pkg, iface)) = &interface {
+            let k = tentative.as_slice();
+            let actual = sqlx::query!(
+                "INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET package = EXCLUDED.package RETURNING key",
+                pkg,
+                iface,
+                k,
+            )
+            .fetch_one(&mut *secrets)
+            .await?.key;
+            let mut bytes = tentative;
+            bytes.clone_from_slice(actual.get(0..32).ok_or_else(|| {
+                Error::new(
+                    eyre!("Invalid key size returned from DB"),
+                    crate::ErrorKind::Database,
+                )
+            })?);
+            bytes
+        } else {
+            let actual = sqlx::query!("SELECT network_key FROM account WHERE id = 0")
+                .fetch_one(&mut *secrets)
+                .await?
+                .network_key;
+            let mut bytes = tentative;
+            bytes.clone_from_slice(actual.get(0..32).ok_or_else(|| {
+                Error::new(
+                    eyre!("Invalid key size returned from DB"),
+                    crate::ErrorKind::Database,
+                )
+            })?);
+            bytes
+        };
+        let mut res = Self::from_bytes(interface, actual);
+        if let Some(tor_key) = compat(secrets, &res.interface).await? {
+            res.tor_key = tor_key;
+        }
+        Ok(res)
+    }
+}
+impl Drop for Key {
+    fn drop(&mut self) {
+        self.base.zeroize();
+        self.tor_key.zeroize();
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct KeyInfo {
+    key: Key,
+    certs: CertPair,
+    int: X509,
+    root: X509,
+}
+impl KeyInfo {
+    pub fn key(&self) -> &Key {
+        &self.key
+    }
+    pub fn certs(&self) -> &CertPair {
+        &self.certs
+    }
+    pub fn int_ca(&self) -> &X509 {
+        &self.int
+    }
+    pub fn root_ca(&self) -> &X509 {
+        &self.root
+    }
+    pub fn fullchain_ed25519(&self) -> Vec<&X509> {
+        vec![&self.certs.ed25519, &self.int, &self.root]
+    }
+    pub fn fullchain_nistp256(&self) -> Vec<&X509> {
+        vec![&self.certs.nistp256, &self.int, &self.root]
+    }
+}
+
+#[test]
+pub fn test_keygen() {
+    let key = Key::new(None);
+    key.tor_key();
+    key.openssl_key_nistp256();
+}

core/startos/src/net/mdns.rs

@@ -0,0 +1,100 @@
+use std::collections::BTreeMap;
+use std::net::Ipv4Addr;
+use std::sync::{Arc, Weak};
+
+use color_eyre::eyre::eyre;
+use tokio::process::{Child, Command};
+use tokio::sync::Mutex;
+use tracing::instrument;
+
+use crate::util::Invoke;
+use crate::{Error, ResultExt};
+
+pub async fn resolve_mdns(hostname: &str) -> Result<Ipv4Addr, Error> {
+    Ok(String::from_utf8(
+        Command::new("avahi-resolve-host-name")
+            .kill_on_drop(true)
+            .arg("-4")
+            .arg(hostname)
+            .invoke(crate::ErrorKind::Network)
+            .await?,
+    )?
+    .split_once("\t")
+    .ok_or_else(|| {
+        Error::new(
+            eyre!("Failed to resolve hostname: {}", hostname),
+            crate::ErrorKind::Network,
+        )
+    })?
+    .1
+    .trim()
+    .parse()?)
+}
+
+pub struct MdnsController(Mutex<MdnsControllerInner>);
+impl MdnsController {
+    pub async fn init() -> Result<Self, Error> {
+        Ok(MdnsController(Mutex::new(
+            MdnsControllerInner::init().await?,
+        )))
+    }
+    pub async fn add(&self, alias: String) -> Result<Arc<()>, Error> {
+        self.0.lock().await.add(alias).await
+    }
+    pub async fn gc(&self, alias: String) -> Result<(), Error> {
+        self.0.lock().await.gc(alias).await
+    }
+}
+
+pub struct MdnsControllerInner {
+    alias_cmd: Option<Child>,
+    services: BTreeMap<String, Weak<()>>,
+}
+
+impl MdnsControllerInner {
+    #[instrument(skip_all)]
+    async fn init() -> Result<Self, Error> {
+        let mut res = MdnsControllerInner {
+            alias_cmd: None,
+            services: BTreeMap::new(),
+        };
+        res.sync().await?;
+        Ok(res)
+    }
+    #[instrument(skip_all)]
+    async fn sync(&mut self) -> Result<(), Error> {
+        if let Some(mut cmd) = self.alias_cmd.take() {
+            cmd.kill().await.with_kind(crate::ErrorKind::Network)?;
+        }
+        self.alias_cmd = Some(
+            Command::new("avahi-alias")
+                .kill_on_drop(true)
+                .args(
+                    self.services
+                        .iter()
+                        .filter(|(_, rc)| rc.strong_count() > 0)
+                        .map(|(s, _)| s),
+                )
+                .spawn()?,
+        );
+        Ok(())
+    }
+    async fn add(&mut self, alias: String) -> Result<Arc<()>, Error> {
+        let rc = if let Some(rc) = Weak::upgrade(&self.services.remove(&alias).unwrap_or_default())
+        {
+            rc
+        } else {
+            Arc::new(())
+        };
+        self.services.insert(alias, Arc::downgrade(&rc));
+        self.sync().await?;
+        Ok(rc)
+    }
+    async fn gc(&mut self, alias: String) -> Result<(), Error> {
+        if let Some(rc) = Weak::upgrade(&self.services.remove(&alias).unwrap_or_default()) {
+            self.services.insert(alias, Arc::downgrade(&rc));
+        }
+        self.sync().await?;
+        Ok(())
+    }
+}

core/startos/src/net/mod.rs

@@ -0,0 +1,32 @@
+use std::sync::Arc;
+
+use futures::future::BoxFuture;
+use hyper::{Body, Error as HyperError, Request, Response};
+use rpc_toolkit::command;
+
+use crate::Error;
+
+pub mod dhcp;
+pub mod dns;
+pub mod interface;
+pub mod keys;
+pub mod mdns;
+pub mod net_controller;
+pub mod ssl;
+pub mod static_server;
+pub mod tor;
+pub mod utils;
+pub mod vhost;
+pub mod web_server;
+pub mod wifi;
+
+pub const PACKAGE_CERT_PATH: &str = "/var/lib/embassy/ssl";
+
+#[command(subcommands(tor::tor, dhcp::dhcp, ssl::ssl))]
+pub fn net() -> Result<(), Error> {
+    Ok(())
+}
+
+pub type HttpHandler = Arc<
+    dyn Fn(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, HyperError>> + Send + Sync,
+>;

core/startos/src/net/net_controller.rs

@@ -0,0 +1,369 @@
+use std::collections::BTreeMap;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::sync::{Arc, Weak};
+
+use color_eyre::eyre::eyre;
+use models::InterfaceId;
+use sqlx::PgExecutor;
+use tracing::instrument;
+
+use crate::error::ErrorCollection;
+use crate::hostname::Hostname;
+use crate::net::dns::DnsController;
+use crate::net::keys::Key;
+use crate::net::mdns::MdnsController;
+use crate::net::ssl::{export_cert, export_key, SslManager};
+use crate::net::tor::TorController;
+use crate::net::vhost::{AlpnInfo, VHostController};
+use crate::s9pk::manifest::PackageId;
+use crate::volume::cert_dir;
+use crate::{Error, HOST_IP};
+
+pub struct NetController {
+    pub(super) tor: TorController,
+    pub(super) mdns: MdnsController,
+    pub(super) vhost: VHostController,
+    pub(super) dns: DnsController,
+    pub(super) ssl: Arc<SslManager>,
+    pub(super) os_bindings: Vec<Arc<()>>,
+}
+
+impl NetController {
+    #[instrument(skip_all)]
+    pub async fn init(
+        tor_control: SocketAddr,
+        tor_socks: SocketAddr,
+        dns_bind: &[SocketAddr],
+        ssl: SslManager,
+        hostname: &Hostname,
+        os_key: &Key,
+    ) -> Result<Self, Error> {
+        let ssl = Arc::new(ssl);
+        let mut res = Self {
+            tor: TorController::new(tor_control, tor_socks),
+            mdns: MdnsController::init().await?,
+            vhost: VHostController::new(ssl.clone()),
+            dns: DnsController::init(dns_bind).await?,
+            ssl,
+            os_bindings: Vec::new(),
+        };
+        res.add_os_bindings(hostname, os_key).await?;
+        Ok(res)
+    }
+
+    async fn add_os_bindings(&mut self, hostname: &Hostname, key: &Key) -> Result<(), Error> {
+        let alpn = Err(AlpnInfo::Specified(vec!["http/1.1".into(), "h2".into()]));
+
+        // Internal DNS
+        self.vhost
+            .add(
+                key.clone(),
+                Some("embassy".into()),
+                443,
+                ([127, 0, 0, 1], 80).into(),
+                alpn.clone(),
+            )
+            .await?;
+        self.os_bindings
+            .push(self.dns.add(None, HOST_IP.into()).await?);
+
+        // LAN IP
+        self.os_bindings.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    None,
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
+                .await?,
+        );
+
+        // localhost
+        self.os_bindings.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    Some("localhost".into()),
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
+                .await?,
+        );
+        self.os_bindings.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    Some(hostname.no_dot_host_name()),
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
+                .await?,
+        );
+
+        // LAN mDNS
+        self.os_bindings.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    Some(hostname.local_domain_name()),
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
+                .await?,
+        );
+
+        // Tor (http)
+        self.os_bindings.push(
+            self.tor
+                .add(key.tor_key(), 80, ([127, 0, 0, 1], 80).into())
+                .await?,
+        );
+
+        // Tor (https)
+        self.os_bindings.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    Some(key.tor_address().to_string()),
+                    443,
+                    ([127, 0, 0, 1], 80).into(),
+                    alpn.clone(),
+                )
+                .await?,
+        );
+        self.os_bindings.push(
+            self.tor
+                .add(key.tor_key(), 443, ([127, 0, 0, 1], 443).into())
+                .await?,
+        );
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn create_service(
+        self: &Arc<Self>,
+        package: PackageId,
+        ip: Ipv4Addr,
+    ) -> Result<NetService, Error> {
+        let dns = self.dns.add(Some(package.clone()), ip).await?;
+
+        Ok(NetService {
+            shutdown: false,
+            id: package,
+            ip,
+            dns,
+            controller: Arc::downgrade(self),
+            tor: BTreeMap::new(),
+            lan: BTreeMap::new(),
+        })
+    }
+
+    async fn add_tor(
+        &self,
+        key: &Key,
+        external: u16,
+        target: SocketAddr,
+    ) -> Result<Vec<Arc<()>>, Error> {
+        let mut rcs = Vec::with_capacity(1);
+        rcs.push(self.tor.add(key.tor_key(), external, target).await?);
+        Ok(rcs)
+    }
+
+    async fn remove_tor(&self, key: &Key, external: u16, rcs: Vec<Arc<()>>) -> Result<(), Error> {
+        drop(rcs);
+        self.tor.gc(Some(key.tor_key()), Some(external)).await
+    }
+
+    async fn add_lan(
+        &self,
+        key: Key,
+        external: u16,
+        target: SocketAddr,
+        connect_ssl: Result<(), AlpnInfo>,
+    ) -> Result<Vec<Arc<()>>, Error> {
+        let mut rcs = Vec::with_capacity(2);
+        rcs.push(
+            self.vhost
+                .add(
+                    key.clone(),
+                    Some(key.local_address()),
+                    external,
+                    target.into(),
+                    connect_ssl,
+                )
+                .await?,
+        );
+        rcs.push(self.mdns.add(key.base_address()).await?);
+        Ok(rcs)
+    }
+
+    async fn remove_lan(&self, key: &Key, external: u16, rcs: Vec<Arc<()>>) -> Result<(), Error> {
+        drop(rcs);
+        self.mdns.gc(key.base_address()).await?;
+        self.vhost.gc(Some(key.local_address()), external).await
+    }
+}
+
+pub struct NetService {
+    shutdown: bool,
+    id: PackageId,
+    ip: Ipv4Addr,
+    dns: Arc<()>,
+    controller: Weak<NetController>,
+    tor: BTreeMap<(InterfaceId, u16), (Key, Vec<Arc<()>>)>,
+    lan: BTreeMap<(InterfaceId, u16), (Key, Vec<Arc<()>>)>,
+}
+impl NetService {
+    fn net_controller(&self) -> Result<Arc<NetController>, Error> {
+        Weak::upgrade(&self.controller).ok_or_else(|| {
+            Error::new(
+                eyre!("NetController is shutdown"),
+                crate::ErrorKind::Network,
+            )
+        })
+    }
+    pub async fn add_tor<Ex>(
+        &mut self,
+        secrets: &mut Ex,
+        id: InterfaceId,
+        external: u16,
+        internal: u16,
+    ) -> Result<(), Error>
+    where
+        for<'a> &'a mut Ex: PgExecutor<'a>,
+    {
+        let key = Key::for_interface(secrets, Some((self.id.clone(), id.clone()))).await?;
+        let ctrl = self.net_controller()?;
+        let tor_idx = (id, external);
+        let mut tor = self
+            .tor
+            .remove(&tor_idx)
+            .unwrap_or_else(|| (key.clone(), Vec::new()));
+        tor.1.append(
+            &mut ctrl
+                .add_tor(&key, external, SocketAddr::new(self.ip.into(), internal))
+                .await?,
+        );
+        self.tor.insert(tor_idx, tor);
+        Ok(())
+    }
+    pub async fn remove_tor(&mut self, id: InterfaceId, external: u16) -> Result<(), Error> {
+        let ctrl = self.net_controller()?;
+        if let Some((key, rcs)) = self.tor.remove(&(id, external)) {
+            ctrl.remove_tor(&key, external, rcs).await?;
+        }
+        Ok(())
+    }
+    pub async fn add_lan<Ex>(
+        &mut self,
+        secrets: &mut Ex,
+        id: InterfaceId,
+        external: u16,
+        internal: u16,
+        connect_ssl: Result<(), AlpnInfo>,
+    ) -> Result<(), Error>
+    where
+        for<'a> &'a mut Ex: PgExecutor<'a>,
+    {
+        let key = Key::for_interface(secrets, Some((self.id.clone(), id.clone()))).await?;
+        let ctrl = self.net_controller()?;
+        let lan_idx = (id, external);
+        let mut lan = self
+            .lan
+            .remove(&lan_idx)
+            .unwrap_or_else(|| (key.clone(), Vec::new()));
+        lan.1.append(
+            &mut ctrl
+                .add_lan(
+                    key,
+                    external,
+                    SocketAddr::new(self.ip.into(), internal),
+                    connect_ssl,
+                )
+                .await?,
+        );
+        self.lan.insert(lan_idx, lan);
+        Ok(())
+    }
+    pub async fn remove_lan(&mut self, id: InterfaceId, external: u16) -> Result<(), Error> {
+        let ctrl = self.net_controller()?;
+        if let Some((key, rcs)) = self.lan.remove(&(id, external)) {
+            ctrl.remove_lan(&key, external, rcs).await?;
+        }
+        Ok(())
+    }
+    pub async fn export_cert<Ex>(
+        &self,
+        secrets: &mut Ex,
+        id: &InterfaceId,
+        ip: IpAddr,
+    ) -> Result<(), Error>
+    where
+        for<'a> &'a mut Ex: PgExecutor<'a>,
+    {
+        let key = Key::for_interface(secrets, Some((self.id.clone(), id.clone()))).await?;
+        let ctrl = self.net_controller()?;
+        let cert = ctrl.ssl.with_certs(key, ip).await?;
+        let cert_dir = cert_dir(&self.id, id);
+        tokio::fs::create_dir_all(&cert_dir).await?;
+        export_key(
+            &cert.key().openssl_key_nistp256(),
+            &cert_dir.join(format!("{id}.key.pem")),
+        )
+        .await?;
+        export_cert(
+            &cert.fullchain_nistp256(),
+            &cert_dir.join(format!("{id}.cert.pem")),
+        )
+        .await?; // TODO: can upgrade to ed25519?
+        Ok(())
+    }
+    pub async fn remove_all(mut self) -> Result<(), Error> {
+        self.shutdown = true;
+        let mut errors = ErrorCollection::new();
+        if let Some(ctrl) = Weak::upgrade(&self.controller) {
+            for ((_, external), (key, rcs)) in std::mem::take(&mut self.lan) {
+                errors.handle(ctrl.remove_lan(&key, external, rcs).await);
+            }
+            for ((_, external), (key, rcs)) in std::mem::take(&mut self.tor) {
+                errors.handle(ctrl.remove_tor(&key, external, rcs).await);
+            }
+            std::mem::take(&mut self.dns);
+            errors.handle(ctrl.dns.gc(Some(self.id.clone()), self.ip).await);
+            errors.into_result()
+        } else {
+            tracing::warn!("NetService dropped after NetController is shutdown");
+            Err(Error::new(
+                eyre!("NetController is shutdown"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+}
+
+impl Drop for NetService {
+    fn drop(&mut self) {
+        if !self.shutdown {
+            tracing::debug!("Dropping NetService for {}", self.id);
+            let svc = std::mem::replace(
+                self,
+                NetService {
+                    shutdown: true,
+                    id: Default::default(),
+                    ip: Ipv4Addr::new(0, 0, 0, 0),
+                    dns: Default::default(),
+                    controller: Default::default(),
+                    tor: Default::default(),
+                    lan: Default::default(),
+                },
+            );
+            tokio::spawn(async move { svc.remove_all().await.unwrap() });
+        }
+    }
+}

core/startos/src/net/ssl.rs

@@ -0,0 +1,458 @@
+use std::cmp::Ordering;
+use std::collections::{BTreeMap, BTreeSet};
+use std::net::IpAddr;
+use std::path::Path;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use futures::FutureExt;
+use libc::time_t;
+use openssl::asn1::{Asn1Integer, Asn1Time};
+use openssl::bn::{BigNum, MsbOption};
+use openssl::ec::{EcGroup, EcKey};
+use openssl::hash::MessageDigest;
+use openssl::nid::Nid;
+use openssl::pkey::{PKey, Private};
+use openssl::x509::{X509Builder, X509Extension, X509NameBuilder, X509};
+use openssl::*;
+use rpc_toolkit::command;
+use tokio::sync::{Mutex, RwLock};
+use tracing::instrument;
+
+use crate::account::AccountInfo;
+use crate::context::RpcContext;
+use crate::hostname::Hostname;
+use crate::init::check_time_is_synchronized;
+use crate::net::dhcp::ips;
+use crate::net::keys::{Key, KeyInfo};
+use crate::{Error, ErrorKind, ResultExt, SOURCE_DATE};
+
+static CERTIFICATE_VERSION: i32 = 2; // X509 version 3 is actually encoded as '2' in the cert because fuck you.
+
+fn unix_time(time: SystemTime) -> time_t {
+    time.duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs() as time_t)
+        .or_else(|_| UNIX_EPOCH.elapsed().map(|d| -(d.as_secs() as time_t)))
+        .unwrap_or_default()
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct CertPair {
+    pub ed25519: X509,
+    pub nistp256: X509,
+}
+impl CertPair {
+    fn updated(
+        pair: Option<&Self>,
+        hostname: &Hostname,
+        signer: (&PKey<Private>, &X509),
+        applicant: &Key,
+        ip: BTreeSet<IpAddr>,
+    ) -> Result<(Self, bool), Error> {
+        let mut updated = false;
+        let mut updated_cert = |cert: Option<&X509>, osk: PKey<Private>| -> Result<X509, Error> {
+            let mut ips = BTreeSet::new();
+            if let Some(cert) = cert {
+                ips.extend(
+                    cert.subject_alt_names()
+                        .iter()
+                        .flatten()
+                        .filter_map(|a| a.ipaddress())
+                        .filter_map(|a| match a.len() {
+                            4 => Some::<IpAddr>(<[u8; 4]>::try_from(a).unwrap().into()),
+                            16 => Some::<IpAddr>(<[u8; 16]>::try_from(a).unwrap().into()),
+                            _ => None,
+                        }),
+                );
+                if cert
+                    .not_before()
+                    .compare(Asn1Time::days_from_now(0)?.as_ref())?
+                    == Ordering::Less
+                    && cert
+                        .not_after()
+                        .compare(Asn1Time::days_from_now(30)?.as_ref())?
+                        == Ordering::Greater
+                    && ips.is_superset(&ip)
+                {
+                    return Ok(cert.clone());
+                }
+            }
+            ips.extend(ip.iter().copied());
+            updated = true;
+            make_leaf_cert(signer, (&osk, &SANInfo::new(&applicant, hostname, ips)))
+        };
+        Ok((
+            Self {
+                ed25519: updated_cert(pair.map(|c| &c.ed25519), applicant.openssl_key_ed25519())?,
+                nistp256: updated_cert(
+                    pair.map(|c| &c.nistp256),
+                    applicant.openssl_key_nistp256(),
+                )?,
+            },
+            updated,
+        ))
+    }
+}
+
+pub async fn root_ca_start_time() -> Result<SystemTime, Error> {
+    Ok(if check_time_is_synchronized().await? {
+        SystemTime::now()
+    } else {
+        *SOURCE_DATE
+    })
+}
+
+#[derive(Debug)]
+pub struct SslManager {
+    hostname: Hostname,
+    root_cert: X509,
+    int_key: PKey<Private>,
+    int_cert: X509,
+    cert_cache: RwLock<BTreeMap<Key, CertPair>>,
+}
+impl SslManager {
+    pub fn new(account: &AccountInfo, start_time: SystemTime) -> Result<Self, Error> {
+        let int_key = generate_key()?;
+        let int_cert = make_int_cert(
+            (&account.root_ca_key, &account.root_ca_cert),
+            &int_key,
+            start_time,
+        )?;
+        Ok(Self {
+            hostname: account.hostname.clone(),
+            root_cert: account.root_ca_cert.clone(),
+            int_key,
+            int_cert,
+            cert_cache: RwLock::new(BTreeMap::new()),
+        })
+    }
+    pub async fn with_certs(&self, key: Key, ip: IpAddr) -> Result<KeyInfo, Error> {
+        let mut ips = ips().await?;
+        ips.insert(ip);
+        let (pair, updated) = CertPair::updated(
+            self.cert_cache.read().await.get(&key),
+            &self.hostname,
+            (&self.int_key, &self.int_cert),
+            &key,
+            ips,
+        )?;
+        if updated {
+            self.cert_cache
+                .write()
+                .await
+                .insert(key.clone(), pair.clone());
+        }
+
+        Ok(key.with_certs(pair, self.int_cert.clone(), self.root_cert.clone()))
+    }
+}
+
+const EC_CURVE_NAME: nid::Nid = nid::Nid::X9_62_PRIME256V1;
+lazy_static::lazy_static! {
+    static ref EC_GROUP: EcGroup = EcGroup::from_curve_name(EC_CURVE_NAME).unwrap();
+    static ref SSL_MUTEX: Mutex<()> = Mutex::new(()); // TODO: make thread safe
+}
+
+pub async fn export_key(key: &PKey<Private>, target: &Path) -> Result<(), Error> {
+    tokio::fs::write(target, key.private_key_to_pem_pkcs8()?)
+        .map(|res| res.with_ctx(|_| (ErrorKind::Filesystem, target.display().to_string())))
+        .await?;
+    Ok(())
+}
+pub async fn export_cert(chain: &[&X509], target: &Path) -> Result<(), Error> {
+    tokio::fs::write(
+        target,
+        chain
+            .into_iter()
+            .flat_map(|c| c.to_pem().unwrap())
+            .collect::<Vec<u8>>(),
+    )
+    .await?;
+    Ok(())
+}
+
+#[instrument(skip_all)]
+fn rand_serial() -> Result<Asn1Integer, Error> {
+    let mut bn = BigNum::new()?;
+    bn.rand(64, MsbOption::MAYBE_ZERO, false)?;
+    let asn1 = Asn1Integer::from_bn(&bn)?;
+    Ok(asn1)
+}
+#[instrument(skip_all)]
+pub fn generate_key() -> Result<PKey<Private>, Error> {
+    let new_key = EcKey::generate(EC_GROUP.as_ref())?;
+    let key = PKey::from_ec_key(new_key)?;
+    Ok(key)
+}
+
+#[instrument(skip_all)]
+pub fn make_root_cert(
+    root_key: &PKey<Private>,
+    hostname: &Hostname,
+    start_time: SystemTime,
+) -> Result<X509, Error> {
+    let mut builder = X509Builder::new()?;
+    builder.set_version(CERTIFICATE_VERSION)?;
+
+    let unix_start_time = unix_time(start_time);
+
+    let embargo = Asn1Time::from_unix(unix_start_time - 86400)?;
+    builder.set_not_before(&embargo)?;
+
+    let expiration = Asn1Time::from_unix(unix_start_time + (10 * 364 * 86400))?;
+    builder.set_not_after(&expiration)?;
+
+    builder.set_serial_number(&*rand_serial()?)?;
+
+    let mut subject_name_builder = X509NameBuilder::new()?;
+    subject_name_builder.append_entry_by_text("CN", &format!("{} Local Root CA", &*hostname.0))?;
+    subject_name_builder.append_entry_by_text("O", "Start9")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
+    let subject_name = subject_name_builder.build();
+    builder.set_subject_name(&subject_name)?;
+
+    builder.set_issuer_name(&subject_name)?;
+
+    builder.set_pubkey(&root_key)?;
+
+    // Extensions
+    let cfg = conf::Conf::new(conf::ConfMethod::default())?;
+    let ctx = builder.x509v3_context(None, Some(&cfg));
+    // subjectKeyIdentifier = hash
+    let subject_key_identifier =
+        X509Extension::new_nid(Some(&cfg), Some(&ctx), Nid::SUBJECT_KEY_IDENTIFIER, "hash")?;
+    // basicConstraints = critical, CA:true, pathlen:0
+    let basic_constraints = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::BASIC_CONSTRAINTS,
+        "critical,CA:true",
+    )?;
+    // keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+    let key_usage = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::KEY_USAGE,
+        "critical,digitalSignature,cRLSign,keyCertSign",
+    )?;
+    builder.append_extension(subject_key_identifier)?;
+    builder.append_extension(basic_constraints)?;
+    builder.append_extension(key_usage)?;
+    builder.sign(&root_key, MessageDigest::sha256())?;
+    let cert = builder.build();
+    Ok(cert)
+}
+#[instrument(skip_all)]
+pub fn make_int_cert(
+    signer: (&PKey<Private>, &X509),
+    applicant: &PKey<Private>,
+    start_time: SystemTime,
+) -> Result<X509, Error> {
+    let mut builder = X509Builder::new()?;
+    builder.set_version(CERTIFICATE_VERSION)?;
+
+    let unix_start_time = unix_time(start_time);
+
+    let embargo = Asn1Time::from_unix(unix_start_time - 86400)?;
+    builder.set_not_before(&embargo)?;
+
+    let expiration = Asn1Time::from_unix(unix_start_time + (10 * 364 * 86400))?;
+    builder.set_not_after(&expiration)?;
+
+    builder.set_serial_number(&*rand_serial()?)?;
+
+    let mut subject_name_builder = X509NameBuilder::new()?;
+    subject_name_builder.append_entry_by_text("CN", "StartOS Local Intermediate CA")?;
+    subject_name_builder.append_entry_by_text("O", "Start9")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
+    let subject_name = subject_name_builder.build();
+    builder.set_subject_name(&subject_name)?;
+
+    builder.set_issuer_name(signer.1.subject_name())?;
+
+    builder.set_pubkey(&applicant)?;
+
+    let cfg = conf::Conf::new(conf::ConfMethod::default())?;
+    let ctx = builder.x509v3_context(Some(&signer.1), Some(&cfg));
+    // subjectKeyIdentifier = hash
+    let subject_key_identifier =
+        X509Extension::new_nid(Some(&cfg), Some(&ctx), Nid::SUBJECT_KEY_IDENTIFIER, "hash")?;
+    // authorityKeyIdentifier = keyid:always,issuer
+    let authority_key_identifier = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::AUTHORITY_KEY_IDENTIFIER,
+        "keyid:always,issuer",
+    )?;
+    // basicConstraints = critical, CA:true, pathlen:0
+    let basic_constraints = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::BASIC_CONSTRAINTS,
+        "critical,CA:true,pathlen:0",
+    )?;
+    // keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+    let key_usage = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::KEY_USAGE,
+        "critical,digitalSignature,cRLSign,keyCertSign",
+    )?;
+    builder.append_extension(subject_key_identifier)?;
+    builder.append_extension(authority_key_identifier)?;
+    builder.append_extension(basic_constraints)?;
+    builder.append_extension(key_usage)?;
+    builder.sign(&signer.0, MessageDigest::sha256())?;
+    let cert = builder.build();
+    Ok(cert)
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum MaybeWildcard {
+    WithWildcard(String),
+    WithoutWildcard(String),
+}
+impl MaybeWildcard {
+    pub fn as_str(&self) -> &str {
+        match self {
+            MaybeWildcard::WithWildcard(s) => s.as_str(),
+            MaybeWildcard::WithoutWildcard(s) => s.as_str(),
+        }
+    }
+}
+impl std::fmt::Display for MaybeWildcard {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            MaybeWildcard::WithWildcard(dns) => write!(f, "DNS:{dns},DNS:*.{dns}"),
+            MaybeWildcard::WithoutWildcard(dns) => write!(f, "DNS:{dns}"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct SANInfo {
+    pub dns: BTreeSet<MaybeWildcard>,
+    pub ips: BTreeSet<IpAddr>,
+}
+impl SANInfo {
+    pub fn new(key: &Key, hostname: &Hostname, ips: BTreeSet<IpAddr>) -> Self {
+        let mut dns = BTreeSet::new();
+        if let Some((id, _)) = key.interface() {
+            dns.insert(MaybeWildcard::WithWildcard(format!("{id}.embassy")));
+            dns.insert(MaybeWildcard::WithWildcard(key.local_address().to_string()));
+        } else {
+            dns.insert(MaybeWildcard::WithoutWildcard("embassy".to_owned()));
+            dns.insert(MaybeWildcard::WithWildcard(hostname.local_domain_name()));
+            dns.insert(MaybeWildcard::WithoutWildcard(hostname.no_dot_host_name()));
+            dns.insert(MaybeWildcard::WithoutWildcard("localhost".to_owned()));
+        }
+        dns.insert(MaybeWildcard::WithWildcard(key.tor_address().to_string()));
+        Self { dns, ips }
+    }
+}
+impl std::fmt::Display for SANInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut written = false;
+        for dns in &self.dns {
+            if written {
+                write!(f, ",")?;
+            }
+            written = true;
+            write!(f, "{dns}")?;
+        }
+        for ip in &self.ips {
+            if written {
+                write!(f, ",")?;
+            }
+            written = true;
+            write!(f, "IP:{ip}")?;
+        }
+        Ok(())
+    }
+}
+
+#[instrument(skip_all)]
+pub fn make_leaf_cert(
+    signer: (&PKey<Private>, &X509),
+    applicant: (&PKey<Private>, &SANInfo),
+) -> Result<X509, Error> {
+    let mut builder = X509Builder::new()?;
+    builder.set_version(CERTIFICATE_VERSION)?;
+
+    let embargo = Asn1Time::from_unix(unix_time(SystemTime::now()) - 86400)?;
+    builder.set_not_before(&embargo)?;
+
+    // Google Apple and Mozilla reject certificate horizons longer than 398 days
+    // https://techbeacon.com/security/google-apple-mozilla-enforce-1-year-max-security-certifications
+    let expiration = Asn1Time::days_from_now(397)?;
+    builder.set_not_after(&expiration)?;
+
+    builder.set_serial_number(&*rand_serial()?)?;
+
+    let mut subject_name_builder = X509NameBuilder::new()?;
+    subject_name_builder.append_entry_by_text(
+        "CN",
+        applicant
+            .1
+            .dns
+            .first()
+            .map(MaybeWildcard::as_str)
+            .unwrap_or("localhost"),
+    )?;
+    subject_name_builder.append_entry_by_text("O", "Start9")?;
+    subject_name_builder.append_entry_by_text("OU", "StartOS")?;
+    let subject_name = subject_name_builder.build();
+    builder.set_subject_name(&subject_name)?;
+
+    builder.set_issuer_name(signer.1.subject_name())?;
+
+    builder.set_pubkey(&applicant.0)?;
+
+    // Extensions
+    let cfg = conf::Conf::new(conf::ConfMethod::default())?;
+    let ctx = builder.x509v3_context(Some(&signer.1), Some(&cfg));
+    // subjectKeyIdentifier = hash
+    let subject_key_identifier =
+        X509Extension::new_nid(Some(&cfg), Some(&ctx), Nid::SUBJECT_KEY_IDENTIFIER, "hash")?;
+    // authorityKeyIdentifier = keyid:always,issuer
+    let authority_key_identifier = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::AUTHORITY_KEY_IDENTIFIER,
+        "keyid,issuer:always",
+    )?;
+    let basic_constraints =
+        X509Extension::new_nid(Some(&cfg), Some(&ctx), Nid::BASIC_CONSTRAINTS, "CA:FALSE")?;
+    let key_usage = X509Extension::new_nid(
+        Some(&cfg),
+        Some(&ctx),
+        Nid::KEY_USAGE,
+        "critical,digitalSignature,keyEncipherment",
+    )?;
+
+    let san_string = applicant.1.to_string();
+    let subject_alt_name =
+        X509Extension::new_nid(Some(&cfg), Some(&ctx), Nid::SUBJECT_ALT_NAME, &san_string)?;
+    builder.append_extension(subject_key_identifier)?;
+    builder.append_extension(authority_key_identifier)?;
+    builder.append_extension(subject_alt_name)?;
+    builder.append_extension(basic_constraints)?;
+    builder.append_extension(key_usage)?;
+
+    builder.sign(&signer.0, MessageDigest::sha256())?;
+
+    let cert = builder.build();
+    Ok(cert)
+}
+
+#[command(subcommands(size))]
+pub async fn ssl() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command]
+pub async fn size(#[context] ctx: RpcContext) -> Result<String, Error> {
+    Ok(format!(
+        "Cert Catch size: {}",
+        ctx.net_controller.ssl.cert_cache.read().await.len()
+    ))
+}

core/startos/src/net/static_server.rs

@@ -0,0 +1,580 @@
+use std::fs::Metadata;
+use std::future::Future;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::UNIX_EPOCH;
+
+use async_compression::tokio::bufread::GzipEncoder;
+use color_eyre::eyre::eyre;
+use digest::Digest;
+use futures::FutureExt;
+use http::header::ACCEPT_ENCODING;
+use http::request::Parts as RequestParts;
+use hyper::{Body, Method, Request, Response, StatusCode};
+use include_dir::{include_dir, Dir};
+use new_mime_guess::MimeGuess;
+use openssl::hash::MessageDigest;
+use openssl::x509::X509;
+use rpc_toolkit::rpc_handler;
+use tokio::fs::File;
+use tokio::io::BufReader;
+use tokio_util::io::ReaderStream;
+
+use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext};
+use crate::core::rpc_continuations::RequestGuid;
+use crate::db::subscribe;
+use crate::install::PKG_PUBLIC_DIR;
+use crate::middleware::auth::{auth as auth_middleware, HasValidSession};
+use crate::middleware::cors::cors;
+use crate::middleware::db::db as db_middleware;
+use crate::middleware::diagnostic::diagnostic as diagnostic_middleware;
+use crate::net::HttpHandler;
+use crate::{diagnostic_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt};
+
+static NOT_FOUND: &[u8] = b"Not Found";
+static METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
+static NOT_AUTHORIZED: &[u8] = b"Not Authorized";
+
+static EMBEDDED_UIS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../../web/dist/static");
+
+const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "user-agent"];
+
+fn status_fn(_: i32) -> StatusCode {
+    StatusCode::OK
+}
+
+#[derive(Clone)]
+pub enum UiMode {
+    Setup,
+    Diag,
+    Install,
+    Main,
+}
+
+impl UiMode {
+    fn path(&self, path: &str) -> PathBuf {
+        match self {
+            Self::Setup => Path::new("setup-wizard").join(path),
+            Self::Diag => Path::new("diagnostic-ui").join(path),
+            Self::Install => Path::new("install-wizard").join(path),
+            Self::Main => Path::new("ui").join(path),
+        }
+    }
+}
+
+pub async fn setup_ui_file_router(ctx: SetupContext) -> Result<HttpHandler, Error> {
+    let handler: HttpHandler = Arc::new(move |req| {
+        let ctx = ctx.clone();
+
+        let ui_mode = UiMode::Setup;
+        async move {
+            let res = match req.uri().path() {
+                path if path.starts_with("/rpc/") => {
+                    let rpc_handler = rpc_handler!({
+                        command: setup_api,
+                        context: ctx,
+                        status: status_fn,
+                        middleware: [
+                            cors,
+                        ]
+                    });
+
+                    rpc_handler(req)
+                        .await
+                        .map_err(|err| Error::new(eyre!("{}", err), crate::ErrorKind::Network))
+                }
+                _ => alt_ui(req, ui_mode).await,
+            };
+
+            match res {
+                Ok(data) => Ok(data),
+                Err(err) => Ok(server_error(err)),
+            }
+        }
+        .boxed()
+    });
+
+    Ok(handler)
+}
+
+pub async fn diag_ui_file_router(ctx: DiagnosticContext) -> Result<HttpHandler, Error> {
+    let handler: HttpHandler = Arc::new(move |req| {
+        let ctx = ctx.clone();
+        let ui_mode = UiMode::Diag;
+        async move {
+            let res = match req.uri().path() {
+                path if path.starts_with("/rpc/") => {
+                    let rpc_handler = rpc_handler!({
+                        command: diagnostic_api,
+                        context: ctx,
+                        status: status_fn,
+                        middleware: [
+                            cors,
+                            diagnostic_middleware,
+                        ]
+                    });
+
+                    rpc_handler(req)
+                        .await
+                        .map_err(|err| Error::new(eyre!("{}", err), crate::ErrorKind::Network))
+                }
+                _ => alt_ui(req, ui_mode).await,
+            };
+
+            match res {
+                Ok(data) => Ok(data),
+                Err(err) => Ok(server_error(err)),
+            }
+        }
+        .boxed()
+    });
+
+    Ok(handler)
+}
+
+pub async fn install_ui_file_router(ctx: InstallContext) -> Result<HttpHandler, Error> {
+    let handler: HttpHandler = Arc::new(move |req| {
+        let ctx = ctx.clone();
+        let ui_mode = UiMode::Install;
+        async move {
+            let res = match req.uri().path() {
+                path if path.starts_with("/rpc/") => {
+                    let rpc_handler = rpc_handler!({
+                        command: install_api,
+                        context: ctx,
+                        status: status_fn,
+                        middleware: [
+                            cors,
+                        ]
+                    });
+
+                    rpc_handler(req)
+                        .await
+                        .map_err(|err| Error::new(eyre!("{}", err), crate::ErrorKind::Network))
+                }
+                _ => alt_ui(req, ui_mode).await,
+            };
+
+            match res {
+                Ok(data) => Ok(data),
+                Err(err) => Ok(server_error(err)),
+            }
+        }
+        .boxed()
+    });
+
+    Ok(handler)
+}
+
+pub async fn main_ui_server_router(ctx: RpcContext) -> Result<HttpHandler, Error> {
+    let handler: HttpHandler = Arc::new(move |req| {
+        let ctx = ctx.clone();
+
+        async move {
+            let res = match req.uri().path() {
+                path if path.starts_with("/rpc/") => {
+                    let auth_middleware = auth_middleware(ctx.clone());
+                    let db_middleware = db_middleware(ctx.clone());
+                    let rpc_handler = rpc_handler!({
+                        command: main_api,
+                        context: ctx,
+                        status: status_fn,
+                        middleware: [
+                            cors,
+                            auth_middleware,
+                            db_middleware,
+                        ]
+                    });
+
+                    rpc_handler(req)
+                        .await
+                        .map_err(|err| Error::new(eyre!("{}", err), crate::ErrorKind::Network))
+                }
+                "/ws/db" => subscribe(ctx, req).await,
+                path if path.starts_with("/ws/rpc/") => {
+                    match RequestGuid::from(path.strip_prefix("/ws/rpc/").unwrap()) {
+                        None => {
+                            tracing::debug!("No Guid Path");
+                            Ok::<_, Error>(bad_request())
+                        }
+                        Some(guid) => match ctx.get_ws_continuation_handler(&guid).await {
+                            Some(cont) => match cont(req).await {
+                                Ok::<_, Error>(r) => Ok::<_, Error>(r),
+                                Err(err) => Ok::<_, Error>(server_error(err)),
+                            },
+                            _ => Ok::<_, Error>(not_found()),
+                        },
+                    }
+                }
+                path if path.starts_with("/rest/rpc/") => {
+                    match RequestGuid::from(path.strip_prefix("/rest/rpc/").unwrap()) {
+                        None => {
+                            tracing::debug!("No Guid Path");
+                            Ok::<_, Error>(bad_request())
+                        }
+                        Some(guid) => match ctx.get_rest_continuation_handler(&guid).await {
+                            None => Ok::<_, Error>(not_found()),
+                            Some(cont) => match cont(req).await {
+                                Ok::<_, Error>(r) => Ok::<_, Error>(r),
+                                Err(e) => Ok::<_, Error>(server_error(e)),
+                            },
+                        },
+                    }
+                }
+                _ => main_embassy_ui(req, ctx).await,
+            };
+
+            match res {
+                Ok(data) => Ok(data),
+                Err(err) => Ok(server_error(err)),
+            }
+        }
+        .boxed()
+    });
+
+    Ok(handler)
+}
+
+async fn alt_ui(req: Request<Body>, ui_mode: UiMode) -> Result<Response<Body>, Error> {
+    let (request_parts, _body) = req.into_parts();
+    match &request_parts.method {
+        &Method::GET => {
+            let uri_path = ui_mode.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
+
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
+                    .await
+            } else {
+                Ok(not_found())
+            }
+        }
+        _ => Ok(method_not_allowed()),
+    }
+}
+
+async fn if_authorized<
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = Result<Response<Body>, Error>> + Send + Sync,
+>(
+    ctx: &RpcContext,
+    parts: &RequestParts,
+    f: F,
+) -> Result<Response<Body>, Error> {
+    if let Err(e) = HasValidSession::from_request_parts(parts, ctx).await {
+        un_authorized(e, parts.uri.path())
+    } else {
+        f().await
+    }
+}
+
+async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response<Body>, Error> {
+    let (request_parts, _body) = req.into_parts();
+    match (
+        &request_parts.method,
+        request_parts
+            .uri
+            .path()
+            .strip_prefix('/')
+            .unwrap_or(request_parts.uri.path())
+            .split_once('/'),
+    ) {
+        (&Method::GET, Some(("public", path))) => {
+            if_authorized(&ctx, &request_parts, || async {
+                let sub_path = Path::new(path);
+                if let Ok(rest) = sub_path.strip_prefix("package-data") {
+                    FileData::from_path(
+                        &request_parts,
+                        &ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
+                    )
+                    .await?
+                    .into_response(&request_parts)
+                    .await
+                } else {
+                    Ok(not_found())
+                }
+            })
+            .await
+        }
+        (&Method::GET, Some(("proxy", target))) => {
+            if_authorized(&ctx, &request_parts, || async {
+                let target = urlencoding::decode(target)?;
+                let res = ctx
+                    .client
+                    .get(target.as_ref())
+                    .headers(
+                        request_parts
+                            .headers
+                            .iter()
+                            .filter(|(h, _)| {
+                                !PROXY_STRIP_HEADERS
+                                    .iter()
+                                    .any(|bad| h.as_str().eq_ignore_ascii_case(bad))
+                            })
+                            .map(|(h, v)| (h.clone(), v.clone()))
+                            .collect(),
+                    )
+                    .send()
+                    .await
+                    .with_kind(crate::ErrorKind::Network)?;
+                let mut hres = Response::builder().status(res.status());
+                for (h, v) in res.headers().clone() {
+                    if let Some(h) = h {
+                        hres = hres.header(h, v);
+                    }
+                }
+                hres.body(Body::wrap_stream(res.bytes_stream()))
+                    .with_kind(crate::ErrorKind::Network)
+            })
+            .await
+        }
+        (&Method::GET, Some(("eos", "local.crt"))) => {
+            cert_send(&ctx.account.read().await.root_ca_cert)
+        }
+        (&Method::GET, _) => {
+            let uri_path = UiMode::Main.path(
+                request_parts
+                    .uri
+                    .path()
+                    .strip_prefix('/')
+                    .unwrap_or(request_parts.uri.path()),
+            );
+
+            let file = EMBEDDED_UIS
+                .get_file(&*uri_path)
+                .or_else(|| EMBEDDED_UIS.get_file(&*UiMode::Main.path("index.html")));
+
+            if let Some(file) = file {
+                FileData::from_embedded(&request_parts, file)
+                    .into_response(&request_parts)
+                    .await
+            } else {
+                Ok(not_found())
+            }
+        }
+        _ => Ok(method_not_allowed()),
+    }
+}
+
+fn un_authorized(err: Error, path: &str) -> Result<Response<Body>, Error> {
+    tracing::warn!("unauthorized for {} @{:?}", err, path);
+    tracing::debug!("{:?}", err);
+    Ok(Response::builder()
+        .status(StatusCode::UNAUTHORIZED)
+        .body(NOT_AUTHORIZED.into())
+        .unwrap())
+}
+
+/// HTTP status code 404
+fn not_found() -> Response<Body> {
+    Response::builder()
+        .status(StatusCode::NOT_FOUND)
+        .body(NOT_FOUND.into())
+        .unwrap()
+}
+
+/// HTTP status code 405
+fn method_not_allowed() -> Response<Body> {
+    Response::builder()
+        .status(StatusCode::METHOD_NOT_ALLOWED)
+        .body(METHOD_NOT_ALLOWED.into())
+        .unwrap()
+}
+
+fn server_error(err: Error) -> Response<Body> {
+    Response::builder()
+        .status(StatusCode::INTERNAL_SERVER_ERROR)
+        .body(err.to_string().into())
+        .unwrap()
+}
+
+fn bad_request() -> Response<Body> {
+    Response::builder()
+        .status(StatusCode::BAD_REQUEST)
+        .body(Body::empty())
+        .unwrap()
+}
+
+fn cert_send(cert: &X509) -> Result<Response<Body>, Error> {
+    let pem = cert.to_pem()?;
+    Response::builder()
+        .status(StatusCode::OK)
+        .header(
+            http::header::ETAG,
+            base32::encode(
+                base32::Alphabet::RFC4648 { padding: false },
+                &*cert.digest(MessageDigest::sha256())?,
+            )
+            .to_lowercase(),
+        )
+        .header(http::header::CONTENT_TYPE, "application/x-pem-file")
+        .header(http::header::CONTENT_LENGTH, pem.len())
+        .body(Body::from(pem))
+        .with_kind(ErrorKind::Network)
+}
+
+struct FileData {
+    data: Body,
+    len: Option<u64>,
+    encoding: Option<&'static str>,
+    e_tag: String,
+    mime: Option<String>,
+}
+impl FileData {
+    fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self {
+        let path = file.path();
+        let (encoding, data) = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .fold((None, file.contents()), |acc, e| {
+                if let Some(file) = (e == "br")
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.br", path.display())))
+                {
+                    (Some("br"), file.contents())
+                } else if let Some(file) = (e == "gzip" && acc.0 != Some("br"))
+                    .then_some(())
+                    .and_then(|_| EMBEDDED_UIS.get_file(format!("{}.gz", path.display())))
+                {
+                    (Some("gzip"), file.contents())
+                } else {
+                    acc
+                }
+            });
+
+        Self {
+            len: Some(data.len() as u64),
+            encoding,
+            data: data.into(),
+            e_tag: e_tag(path, None),
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        }
+    }
+
+    async fn from_path(req: &RequestParts, path: &Path) -> Result<Self, Error> {
+        let encoding = req
+            .headers
+            .get_all(ACCEPT_ENCODING)
+            .into_iter()
+            .filter_map(|h| h.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .filter_map(|s| s.split(";").next())
+            .map(|s| s.trim())
+            .any(|e| e == "gzip")
+            .then_some("gzip");
+
+        let file = File::open(path)
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+        let metadata = file
+            .metadata()
+            .await
+            .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
+
+        let e_tag = e_tag(path, Some(&metadata));
+
+        let (len, data) = if encoding == Some("gzip") {
+            (
+                None,
+                Body::wrap_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file)))),
+            )
+        } else {
+            (
+                Some(metadata.len()),
+                Body::wrap_stream(ReaderStream::new(file)),
+            )
+        };
+
+        Ok(Self {
+            data,
+            len,
+            encoding,
+            e_tag,
+            mime: MimeGuess::from_path(path)
+                .first()
+                .map(|m| m.essence_str().to_owned()),
+        })
+    }
+
+    async fn into_response(self, req: &RequestParts) -> Result<Response<Body>, Error> {
+        let mut builder = Response::builder();
+        if let Some(mime) = self.mime {
+            builder = builder.header(http::header::CONTENT_TYPE, &*mime);
+        }
+        builder = builder.header(http::header::ETAG, &*self.e_tag);
+        builder = builder.header(
+            http::header::CACHE_CONTROL,
+            "public, max-age=21000000, immutable",
+        );
+
+        if req
+            .headers
+            .get_all(http::header::CONNECTION)
+            .iter()
+            .flat_map(|s| s.to_str().ok())
+            .flat_map(|s| s.split(","))
+            .any(|s| s.trim() == "keep-alive")
+        {
+            builder = builder.header(http::header::CONNECTION, "keep-alive");
+        }
+
+        if req
+            .headers
+            .get("if-none-match")
+            .and_then(|h| h.to_str().ok())
+            == Some(self.e_tag.as_ref())
+        {
+            builder = builder.status(StatusCode::NOT_MODIFIED);
+            builder.body(Body::empty())
+        } else {
+            if let Some(len) = self.len {
+                builder = builder.header(http::header::CONTENT_LENGTH, len);
+            }
+            if let Some(encoding) = self.encoding {
+                builder = builder.header(http::header::CONTENT_ENCODING, encoding);
+            }
+
+            builder.body(self.data)
+        }
+        .with_kind(ErrorKind::Network)
+    }
+}
+
+fn e_tag(path: &Path, metadata: Option<&Metadata>) -> String {
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(format!("{:?}", path).as_bytes());
+    if let Some(modified) = metadata.and_then(|m| m.modified().ok()) {
+        hasher.update(
+            format!(
+                "{}",
+                modified
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap_or_default()
+                    .as_secs()
+            )
+            .as_bytes(),
+        );
+    }
+    let res = hasher.finalize();
+    format!(
+        "\"{}\"",
+        base32::encode(base32::Alphabet::RFC4648 { padding: false }, res.as_slice()).to_lowercase()
+    )
+}

core/startos/src/net/tor.rs

@@ -0,0 +1,740 @@
+use std::collections::BTreeMap;
+use std::net::SocketAddr;
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+
+use clap::ArgMatches;
+use color_eyre::eyre::eyre;
+use futures::future::BoxFuture;
+use futures::{FutureExt, TryStreamExt};
+use helpers::NonDetachingJoinHandle;
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
+use rpc_toolkit::command;
+use rpc_toolkit::yajrc::RpcError;
+use tokio::net::TcpStream;
+use tokio::process::Command;
+use tokio::sync::{mpsc, oneshot};
+use tokio::time::Instant;
+use torut::control::{AsyncEvent, AuthenticatedConn, ConnError};
+use torut::onion::{OnionAddressV3, TorSecretKeyV3};
+use tracing::instrument;
+
+use crate::context::{CliContext, RpcContext};
+use crate::logs::{
+    cli_logs_generic_follow, cli_logs_generic_nofollow, fetch_logs, follow_logs, journalctl,
+    LogFollowResponse, LogResponse, LogSource,
+};
+use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::{display_none, Invoke};
+use crate::{Error, ErrorKind, ResultExt as _};
+
+pub const SYSTEMD_UNIT: &str = "tor@default";
+const STARTING_HEALTH_TIMEOUT: u64 = 120; // 2min
+
+enum ErrorLogSeverity {
+    Fatal { wipe_state: bool },
+    Unknown { wipe_state: bool },
+}
+
+lazy_static! {
+    static ref LOG_REGEXES: Vec<(Regex, ErrorLogSeverity)> = vec![(
+        Regex::new("This could indicate a route manipulation attack, network overload, bad local network connectivity, or a bug\\.").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: true }
+    ),(
+        Regex::new("died due to an invalid selected path").unwrap(),
+        ErrorLogSeverity::Fatal { wipe_state: false }
+    ),(
+        Regex::new("Tor has not observed any network activity for the past").unwrap(),
+        ErrorLogSeverity::Unknown { wipe_state: false }
+    )];
+    static ref PROGRESS_REGEX: Regex = Regex::new("PROGRESS=([0-9]+)").unwrap();
+}
+
+#[test]
+fn random_key() {
+    println!("x'{}'", hex::encode(rand::random::<[u8; 32]>()));
+}
+
+#[command(subcommands(list_services, logs, reset))]
+pub fn tor() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(display(display_none))]
+pub async fn reset(
+    #[context] ctx: RpcContext,
+    #[arg(rename = "wipe-state", short = 'w', long = "wipe-state")] wipe_state: bool,
+    #[arg] reason: String,
+) -> Result<(), Error> {
+    ctx.net_controller
+        .tor
+        .reset(wipe_state, Error::new(eyre!("{reason}"), ErrorKind::Tor))
+        .await
+}
+
+fn display_services(services: Vec<OnionAddressV3>, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(services, matches);
+    }
+
+    let mut table = Table::new();
+    for service in services {
+        let row = row![&service.to_string()];
+        table.add_row(row);
+    }
+    table.print_tty(false).unwrap();
+}
+
+#[command(rename = "list-services", display(display_services))]
+pub async fn list_services(
+    #[context] ctx: RpcContext,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<Vec<OnionAddressV3>, Error> {
+    ctx.net_controller.tor.list_services().await
+}
+
+#[command(
+    custom_cli(cli_logs(async, context(CliContext))),
+    subcommands(self(logs_nofollow(async)), logs_follow),
+    display(display_none)
+)]
+pub async fn logs(
+    #[arg(short = 'l', long = "limit")] limit: Option<usize>,
+    #[arg(short = 'c', long = "cursor")] cursor: Option<String>,
+    #[arg(short = 'B', long = "before", default)] before: bool,
+    #[arg(short = 'f', long = "follow", default)] follow: bool,
+) -> Result<(Option<usize>, Option<String>, bool, bool), Error> {
+    Ok((limit, cursor, before, follow))
+}
+pub async fn cli_logs(
+    ctx: CliContext,
+    (limit, cursor, before, follow): (Option<usize>, Option<String>, bool, bool),
+) -> Result<(), RpcError> {
+    if follow {
+        if cursor.is_some() {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        if before {
+            return Err(RpcError::from(Error::new(
+                eyre!("The argument '--before' cannot be used with '--follow'"),
+                crate::ErrorKind::InvalidRequest,
+            )));
+        }
+        cli_logs_generic_follow(ctx, "net.tor.logs.follow", None, limit).await
+    } else {
+        cli_logs_generic_nofollow(ctx, "net.tor.logs", None, limit, cursor, before).await
+    }
+}
+pub async fn logs_nofollow(
+    _ctx: (),
+    (limit, cursor, before, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogResponse, Error> {
+    fetch_logs(LogSource::Unit(SYSTEMD_UNIT), limit, cursor, before).await
+}
+
+#[command(rpc_only, rename = "follow", display(display_none))]
+pub async fn logs_follow(
+    #[context] ctx: RpcContext,
+    #[parent_data] (limit, _, _, _): (Option<usize>, Option<String>, bool, bool),
+) -> Result<LogFollowResponse, Error> {
+    follow_logs(ctx, LogSource::Unit(SYSTEMD_UNIT), limit).await
+}
+
+fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> {
+    async move { Ok(()) }.boxed()
+}
+
+pub struct TorController(TorControl);
+impl TorController {
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        TorController(TorControl::new(tor_control, tor_socks))
+    }
+
+    pub async fn add(
+        &self,
+        key: TorSecretKeyV3,
+        external: u16,
+        target: SocketAddr,
+    ) -> Result<Arc<()>, Error> {
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::AddOnion {
+                key,
+                external,
+                target,
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
+    }
+
+    pub async fn gc(
+        &self,
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    ) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::GC { key, external })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
+    }
+
+    pub async fn reset(&self, wipe_state: bool, context: Error) -> Result<(), Error> {
+        self.0
+            .send
+            .send(TorCommand::Reset {
+                wipe_state,
+                context,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))
+    }
+
+    pub async fn list_services(&self) -> Result<Vec<OnionAddressV3>, Error> {
+        let (reply, res) = oneshot::channel();
+        self.0
+            .send
+            .send(TorCommand::GetInfo {
+                query: "onions/current".into(),
+                reply,
+            })
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))?;
+        res.await
+            .ok()
+            .ok_or_else(|| Error::new(eyre!("TorControl died"), ErrorKind::Tor))??
+            .lines()
+            .map(|l| l.trim())
+            .filter(|l| !l.is_empty())
+            .map(|l| l.parse().with_kind(ErrorKind::Tor))
+            .collect()
+    }
+}
+
+type AuthenticatedConnection = AuthenticatedConn<
+    TcpStream,
+    Box<dyn Fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> + Send + Sync>,
+>;
+
+enum TorCommand {
+    AddOnion {
+        key: TorSecretKeyV3,
+        external: u16,
+        target: SocketAddr,
+        reply: oneshot::Sender<Arc<()>>,
+    },
+    GC {
+        key: Option<TorSecretKeyV3>,
+        external: Option<u16>,
+    },
+    GetInfo {
+        query: String,
+        reply: oneshot::Sender<Result<String, Error>>,
+    },
+    Reset {
+        wipe_state: bool,
+        context: Error,
+    },
+}
+
+#[instrument(skip_all)]
+async fn torctl(
+    tor_control: SocketAddr,
+    tor_socks: SocketAddr,
+    recv: &mut mpsc::UnboundedReceiver<TorCommand>,
+    services: &mut BTreeMap<[u8; 64], BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
+    wipe_state: &AtomicBool,
+    health_timeout: &mut Duration,
+) -> Result<(), Error> {
+    let bootstrap = async {
+        if Command::new("systemctl")
+            .arg("is-active")
+            .arg("--quiet")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await
+            .is_ok()
+        {
+            Command::new("systemctl")
+                .arg("stop")
+                .arg("tor")
+                .invoke(ErrorKind::Tor)
+                .await?;
+            for _ in 0..30 {
+                if TcpStream::connect(tor_control).await.is_err() {
+                    break;
+                }
+                tokio::time::sleep(Duration::from_secs(1)).await;
+            }
+            if TcpStream::connect(tor_control).await.is_ok() {
+                return Err(Error::new(
+                    eyre!("Tor is failing to shut down"),
+                    ErrorKind::Tor,
+                ));
+            }
+        }
+        if wipe_state.load(std::sync::atomic::Ordering::SeqCst) {
+            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+            wipe_state.store(false, std::sync::atomic::Ordering::SeqCst);
+        }
+        tokio::fs::create_dir_all("/var/lib/tor").await?;
+        Command::new("chown")
+            .arg("-R")
+            .arg("debian-tor")
+            .arg("/var/lib/tor")
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        Command::new("systemctl")
+            .arg("start")
+            .arg("tor")
+            .invoke(ErrorKind::Tor)
+            .await?;
+
+        let logs = journalctl(LogSource::Unit(SYSTEMD_UNIT), 0, None, false, true).await?;
+
+        let mut tcp_stream = None;
+        for _ in 0..60 {
+            if let Ok(conn) = TcpStream::connect(tor_control).await {
+                tcp_stream = Some(conn);
+                break;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        let tcp_stream = tcp_stream.ok_or_else(|| {
+            Error::new(eyre!("Timed out waiting for tor to start"), ErrorKind::Tor)
+        })?;
+        tracing::info!("Tor is started");
+
+        let mut conn = torut::control::UnauthenticatedConn::new(tcp_stream);
+        let auth = conn
+            .load_protocol_info()
+            .await?
+            .make_auth_data()?
+            .ok_or_else(|| eyre!("Cookie Auth Not Available"))
+            .with_kind(crate::ErrorKind::Tor)?;
+        conn.authenticate(&auth).await?;
+        let mut connection: AuthenticatedConnection = conn.into_authenticated().await;
+        connection.set_async_event_handler(Some(Box::new(|event| event_handler(event))));
+
+        let mut bootstrapped = false;
+        let mut last_increment = (String::new(), Instant::now());
+        for _ in 0..300 {
+            match connection.get_info("status/bootstrap-phase").await {
+                Ok(a) => {
+                    if a.contains("TAG=done") {
+                        bootstrapped = true;
+                        break;
+                    }
+                    if let Some(p) = PROGRESS_REGEX.captures(&a) {
+                        if let Some(p) = p.get(1) {
+                            if p.as_str() != &*last_increment.0 {
+                                last_increment = (p.as_str().into(), Instant::now());
+                            }
+                        }
+                    }
+                }
+                Err(e) => {
+                    let e = Error::from(e);
+                    tracing::error!("{}", e);
+                    tracing::debug!("{:?}", e);
+                }
+            }
+            if last_increment.1.elapsed() > Duration::from_secs(30) {
+                return Err(Error::new(
+                    eyre!("Tor stuck bootstrapping at {}%", last_increment.0),
+                    ErrorKind::Tor,
+                ));
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+        if !bootstrapped {
+            return Err(Error::new(
+                eyre!("Timed out waiting for tor to bootstrap"),
+                ErrorKind::Tor,
+            ));
+        }
+        Ok((connection, logs))
+    };
+    let pre_handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                }
+                TorCommand::GetInfo { reply, .. } => {
+                    reply
+                        .send(Err(Error::new(
+                            eyre!("Tor has not finished bootstrapping..."),
+                            ErrorKind::Tor,
+                        )))
+                        .unwrap_or_default();
+                }
+                TorCommand::GC { .. } => (),
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+
+    let (mut connection, mut logs) = tokio::select! {
+        res = bootstrap => res?,
+        res = pre_handler => return res,
+    };
+
+    let hck_key = TorSecretKeyV3::generate();
+    connection
+        .add_onion_v3(
+            &hck_key,
+            false,
+            false,
+            false,
+            None,
+            &mut [(80, SocketAddr::from(([127, 0, 0, 1], 80)))].iter(),
+        )
+        .await?;
+
+    for (key, service) in std::mem::take(services) {
+        let key = TorSecretKeyV3::from(key);
+        let bindings = service
+            .iter()
+            .flat_map(|(ext, int)| {
+                int.iter()
+                    .find(|(_, rc)| rc.strong_count() > 0)
+                    .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+            })
+            .collect::<Vec<_>>();
+        if !bindings.is_empty() {
+            services.insert(key.as_bytes(), service);
+            connection
+                .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                .await?;
+        }
+    }
+
+    let handler = async {
+        while let Some(command) = recv.recv().await {
+            match command {
+                TorCommand::AddOnion {
+                    key,
+                    external,
+                    target,
+                    reply,
+                } => {
+                    let mut rm_res = Ok(());
+                    let onion_base = key
+                        .public()
+                        .get_onion_address()
+                        .get_address_without_dot_onion();
+                    let mut service = if let Some(service) = services.remove(&key.as_bytes()) {
+                        rm_res = connection.del_onion(&onion_base).await;
+                        service
+                    } else {
+                        BTreeMap::new()
+                    };
+                    let mut binding = service.remove(&external).unwrap_or_default();
+                    let rc = if let Some(rc) =
+                        Weak::upgrade(&binding.remove(&target).unwrap_or_default())
+                    {
+                        rc
+                    } else {
+                        Arc::new(())
+                    };
+                    binding.insert(target, Arc::downgrade(&rc));
+                    service.insert(external, binding);
+                    let bindings = service
+                        .iter()
+                        .flat_map(|(ext, int)| {
+                            int.iter()
+                                .find(|(_, rc)| rc.strong_count() > 0)
+                                .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                        })
+                        .collect::<Vec<_>>();
+                    services.insert(key.as_bytes(), service);
+                    reply.send(rc).unwrap_or_default();
+                    rm_res?;
+                    connection
+                        .add_onion_v3(&key, false, false, false, None, &mut bindings.iter())
+                        .await?;
+                }
+                TorCommand::GC { key, external } => {
+                    for key in if key.is_some() {
+                        itertools::Either::Left(key.into_iter().map(|k| k.as_bytes()))
+                    } else {
+                        itertools::Either::Right(services.keys().cloned().collect_vec().into_iter())
+                    } {
+                        let key = TorSecretKeyV3::from(key);
+                        let onion_base = key
+                            .public()
+                            .get_onion_address()
+                            .get_address_without_dot_onion();
+                        if let Some(mut service) = services.remove(&key.as_bytes()) {
+                            for external in if external.is_some() {
+                                itertools::Either::Left(external.into_iter())
+                            } else {
+                                itertools::Either::Right(
+                                    service.keys().copied().collect_vec().into_iter(),
+                                )
+                            } {
+                                if let Some(mut binding) = service.remove(&external) {
+                                    binding = binding
+                                        .into_iter()
+                                        .filter(|(_, rc)| rc.strong_count() > 0)
+                                        .collect();
+                                    if !binding.is_empty() {
+                                        service.insert(external, binding);
+                                    }
+                                }
+                            }
+                            let rm_res = connection.del_onion(&onion_base).await;
+                            if !service.is_empty() {
+                                let bindings = service
+                                    .iter()
+                                    .flat_map(|(ext, int)| {
+                                        int.iter()
+                                            .find(|(_, rc)| rc.strong_count() > 0)
+                                            .map(|(addr, _)| (*ext, SocketAddr::from(*addr)))
+                                    })
+                                    .collect::<Vec<_>>();
+                                if !bindings.is_empty() {
+                                    services.insert(key.as_bytes(), service);
+                                }
+                                rm_res?;
+                                if !bindings.is_empty() {
+                                    connection
+                                        .add_onion_v3(
+                                            &key,
+                                            false,
+                                            false,
+                                            false,
+                                            None,
+                                            &mut bindings.iter(),
+                                        )
+                                        .await?;
+                                }
+                            } else {
+                                rm_res?;
+                            }
+                        }
+                    }
+                }
+                TorCommand::GetInfo { query, reply } => {
+                    reply
+                        .send(connection.get_info(&query).await.with_kind(ErrorKind::Tor))
+                        .unwrap_or_default();
+                }
+                TorCommand::Reset {
+                    wipe_state: new_wipe_state,
+                    context,
+                } => {
+                    wipe_state.fetch_or(new_wipe_state, std::sync::atomic::Ordering::SeqCst);
+                    return Err(context);
+                }
+            }
+        }
+        Ok(())
+    };
+    let log_parser = async {
+        while let Some(log) = logs.try_next().await? {
+            for (regex, severity) in &*LOG_REGEXES {
+                if regex.is_match(&log.message) {
+                    let (check, wipe_state) = match severity {
+                        ErrorLogSeverity::Fatal { wipe_state } => (false, *wipe_state),
+                        ErrorLogSeverity::Unknown { wipe_state } => (true, *wipe_state),
+                    };
+                    if !check
+                        || tokio::time::timeout(
+                            Duration::from_secs(30),
+                            tokio_socks::tcp::Socks5Stream::connect(
+                                tor_socks,
+                                (hck_key.public().get_onion_address().to_string(), 80),
+                            ),
+                        )
+                        .await
+                        .map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        .and_then(|a| {
+                            a.map_err(|e| tracing::warn!("Tor is confirmed to be down: {e}"))
+                        })
+                        .is_err()
+                    {
+                        if wipe_state {
+                            Command::new("systemctl")
+                                .arg("stop")
+                                .arg("tor")
+                                .invoke(ErrorKind::Tor)
+                                .await?;
+                            tokio::fs::remove_dir_all("/var/lib/tor").await?;
+                        }
+                        return Err(Error::new(eyre!("{}", log.message), ErrorKind::Tor));
+                    }
+                }
+            }
+        }
+        Err(Error::new(eyre!("Log stream terminated"), ErrorKind::Tor))
+    };
+    let health_checker = async {
+        let mut last_success = Instant::now();
+        loop {
+            tokio::time::sleep(Duration::from_secs(30)).await;
+            if tokio::time::timeout(
+                Duration::from_secs(30),
+                tokio_socks::tcp::Socks5Stream::connect(
+                    tor_socks,
+                    (hck_key.public().get_onion_address().to_string(), 80),
+                ),
+            )
+            .await
+            .map_err(|e| e.to_string())
+            .and_then(|e| e.map_err(|e| e.to_string()))
+            .is_err()
+            {
+                if last_success.elapsed() > *health_timeout {
+                    let err = Error::new(eyre!("Tor health check failed for longer than current timeout ({health_timeout:?})"), crate::ErrorKind::Tor);
+                    *health_timeout *= 2;
+                    wipe_state.store(true, std::sync::atomic::Ordering::SeqCst);
+                    return Err(err);
+                }
+            } else {
+                last_success = Instant::now();
+            }
+        }
+    };
+
+    tokio::select! {
+        res = handler => res?,
+        res = log_parser => res?,
+        res = health_checker => res?,
+    }
+
+    Ok(())
+}
+
+struct TorControl {
+    _thread: NonDetachingJoinHandle<()>,
+    send: mpsc::UnboundedSender<TorCommand>,
+}
+impl TorControl {
+    pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
+        let (send, mut recv) = mpsc::unbounded_channel();
+        Self {
+            _thread: tokio::spawn(async move {
+                let mut services = BTreeMap::new();
+                let wipe_state = AtomicBool::new(false);
+                let mut health_timeout = Duration::from_secs(STARTING_HEALTH_TIMEOUT);
+                while let Err(e) = torctl(
+                    tor_control,
+                    tor_socks,
+                    &mut recv,
+                    &mut services,
+                    &wipe_state,
+                    &mut health_timeout,
+                )
+                .await
+                {
+                    tracing::error!("{e}: Restarting tor");
+                    tracing::debug!("{e:?}");
+                }
+                tracing::info!("TorControl is shut down.")
+            })
+            .into(),
+            send,
+        }
+    }
+}
+
+#[tokio::test]
+async fn test() {
+    let mut conn = torut::control::UnauthenticatedConn::new(
+        TcpStream::connect(SocketAddr::from(([127, 0, 0, 1], 9051)))
+            .await
+            .unwrap(), // TODO
+    );
+    let auth = conn
+        .load_protocol_info()
+        .await
+        .unwrap()
+        .make_auth_data()
+        .unwrap()
+        .ok_or_else(|| eyre!("Cookie Auth Not Available"))
+        .with_kind(crate::ErrorKind::Tor)
+        .unwrap();
+    conn.authenticate(&auth).await.unwrap();
+    let mut connection: AuthenticatedConn<
+        TcpStream,
+        fn(AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>>,
+    > = conn.into_authenticated().await;
+    let tor_key = torut::onion::TorSecretKeyV3::generate();
+    connection.get_conf("SocksPort").await.unwrap();
+    connection
+        .add_onion_v3(
+            &tor_key,
+            false,
+            false,
+            false,
+            None,
+            &mut [(443_u16, SocketAddr::from(([127, 0, 0, 1], 8443)))].iter(),
+        )
+        .await
+        .unwrap();
+    connection
+        .del_onion(
+            &tor_key
+                .public()
+                .get_onion_address()
+                .get_address_without_dot_onion(),
+        )
+        .await
+        .unwrap();
+    connection
+        .add_onion_v3(
+            &tor_key,
+            false,
+            false,
+            false,
+            None,
+            &mut [(8443_u16, SocketAddr::from(([127, 0, 0, 1], 8443)))].iter(),
+        )
+        .await
+        .unwrap();
+}

core/startos/src/net/utils.rs

@@ -0,0 +1,166 @@
+use std::convert::Infallible;
+use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
+use std::path::Path;
+
+use async_stream::try_stream;
+use color_eyre::eyre::eyre;
+use futures::stream::BoxStream;
+use futures::{StreamExt, TryStreamExt};
+use ipnet::{Ipv4Net, Ipv6Net};
+use tokio::net::{TcpListener, TcpStream};
+use tokio::process::Command;
+
+use crate::util::Invoke;
+use crate::Error;
+
+fn parse_iface_ip(output: &str) -> Result<Vec<&str>, Error> {
+    let output = output.trim();
+    if output.is_empty() {
+        return Ok(Vec::new());
+    }
+    let mut res = Vec::new();
+    for line in output.lines() {
+        if let Some(ip) = line.split_ascii_whitespace().nth(3) {
+            res.push(ip)
+        } else {
+            return Err(Error::new(
+                eyre!("malformed output from `ip`"),
+                crate::ErrorKind::Network,
+            ));
+        }
+    }
+    Ok(res)
+}
+
+pub async fn get_iface_ipv4_addr(iface: &str) -> Result<Option<(Ipv4Addr, Ipv4Net)>, Error> {
+    Ok(parse_iface_ip(&String::from_utf8(
+        Command::new("ip")
+            .arg("-4")
+            .arg("-o")
+            .arg("addr")
+            .arg("show")
+            .arg(iface)
+            .invoke(crate::ErrorKind::Network)
+            .await?,
+    )?)?
+    .into_iter()
+    .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
+    .next()
+    .transpose()?)
+}
+
+pub async fn get_iface_ipv6_addr(iface: &str) -> Result<Option<(Ipv6Addr, Ipv6Net)>, Error> {
+    Ok(parse_iface_ip(&String::from_utf8(
+        Command::new("ip")
+            .arg("-6")
+            .arg("-o")
+            .arg("addr")
+            .arg("show")
+            .arg(iface)
+            .invoke(crate::ErrorKind::Network)
+            .await?,
+    )?)?
+    .into_iter()
+    .find(|ip| !ip.starts_with("fe80::"))
+    .map(|s| Ok::<_, Error>((s.split("/").next().unwrap().parse()?, s.parse()?)))
+    .transpose()?)
+}
+
+pub async fn iface_is_physical(iface: &str) -> bool {
+    tokio::fs::metadata(Path::new("/sys/class/net").join(iface).join("device"))
+        .await
+        .is_ok()
+}
+
+pub async fn iface_is_wireless(iface: &str) -> bool {
+    tokio::fs::metadata(Path::new("/sys/class/net").join(iface).join("wireless"))
+        .await
+        .is_ok()
+}
+
+pub fn list_interfaces() -> BoxStream<'static, Result<String, Error>> {
+    try_stream! {
+        let mut ifaces = tokio::fs::read_dir("/sys/class/net").await?;
+        while let Some(iface) = ifaces.next_entry().await? {
+            if let Some(iface) = iface.file_name().into_string().ok() {
+                yield iface;
+            }
+        }
+    }
+    .boxed()
+}
+
+pub async fn find_wifi_iface() -> Result<Option<String>, Error> {
+    let mut ifaces = list_interfaces();
+    while let Some(iface) = ifaces.try_next().await? {
+        if iface_is_wireless(&iface).await {
+            return Ok(Some(iface));
+        }
+    }
+    Ok(None)
+}
+
+pub async fn find_eth_iface() -> Result<String, Error> {
+    let mut ifaces = list_interfaces();
+    while let Some(iface) = ifaces.try_next().await? {
+        if iface_is_physical(&iface).await && !iface_is_wireless(&iface).await {
+            return Ok(iface);
+        }
+    }
+    Err(Error::new(
+        eyre!("Could not detect ethernet interface"),
+        crate::ErrorKind::Network,
+    ))
+}
+
+#[pin_project::pin_project]
+pub struct SingleAccept<T>(Option<T>);
+impl<T> SingleAccept<T> {
+    pub fn new(conn: T) -> Self {
+        Self(Some(conn))
+    }
+}
+impl<T> hyper::server::accept::Accept for SingleAccept<T> {
+    type Conn = T;
+    type Error = Infallible;
+    fn poll_accept(
+        self: std::pin::Pin<&mut Self>,
+        _cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<Self::Conn, Self::Error>>> {
+        std::task::Poll::Ready(self.project().0.take().map(Ok))
+    }
+}
+
+pub struct TcpListeners {
+    listeners: Vec<TcpListener>,
+}
+impl TcpListeners {
+    pub fn new(listeners: impl IntoIterator<Item = TcpListener>) -> Self {
+        Self {
+            listeners: listeners.into_iter().collect(),
+        }
+    }
+
+    pub async fn accept(&self) -> std::io::Result<(TcpStream, SocketAddr)> {
+        futures::future::select_all(self.listeners.iter().map(|l| Box::pin(l.accept())))
+            .await
+            .0
+    }
+}
+impl hyper::server::accept::Accept for TcpListeners {
+    type Conn = TcpStream;
+    type Error = std::io::Error;
+
+    fn poll_accept(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<Self::Conn, Self::Error>>> {
+        for listener in self.listeners.iter() {
+            let poll = listener.poll_accept(cx);
+            if poll.is_ready() {
+                return poll.map(|a| a.map(|a| a.0)).map(Some);
+            }
+        }
+        std::task::Poll::Pending
+    }
+}

core/startos/src/net/vhost.rs

@@ -0,0 +1,412 @@
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::net::{IpAddr, Ipv6Addr, SocketAddr};
+use std::str::FromStr;
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use helpers::NonDetachingJoinHandle;
+use http::{Response, Uri};
+use hyper::service::{make_service_fn, service_fn};
+use hyper::Body;
+use models::ResultExt;
+use tokio::net::{TcpListener, TcpStream};
+use tokio::sync::{Mutex, RwLock};
+use tokio_rustls::rustls::server::Acceptor;
+use tokio_rustls::rustls::{RootCertStore, ServerConfig};
+use tokio_rustls::{LazyConfigAcceptor, TlsConnector};
+use tracing::instrument;
+
+use crate::net::keys::Key;
+use crate::net::ssl::SslManager;
+use crate::net::utils::SingleAccept;
+use crate::prelude::*;
+use crate::util::io::{BackTrackingReader, TimeoutStream};
+
+// not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
+
+pub struct VHostController {
+    ssl: Arc<SslManager>,
+    servers: Mutex<BTreeMap<u16, VHostServer>>,
+}
+impl VHostController {
+    pub fn new(ssl: Arc<SslManager>) -> Self {
+        Self {
+            ssl,
+            servers: Mutex::new(BTreeMap::new()),
+        }
+    }
+    #[instrument(skip_all)]
+    pub async fn add(
+        &self,
+        key: Key,
+        hostname: Option<String>,
+        external: u16,
+        target: SocketAddr,
+        connect_ssl: Result<(), AlpnInfo>,
+    ) -> Result<Arc<()>, Error> {
+        let mut writable = self.servers.lock().await;
+        let server = if let Some(server) = writable.remove(&external) {
+            server
+        } else {
+            VHostServer::new(external, self.ssl.clone()).await?
+        };
+        let rc = server
+            .add(
+                hostname,
+                TargetInfo {
+                    addr: target,
+                    connect_ssl,
+                    key,
+                },
+            )
+            .await;
+        writable.insert(external, server);
+        Ok(rc?)
+    }
+    #[instrument(skip_all)]
+    pub async fn gc(&self, hostname: Option<String>, external: u16) -> Result<(), Error> {
+        let mut writable = self.servers.lock().await;
+        if let Some(server) = writable.remove(&external) {
+            server.gc(hostname).await?;
+            if !server.is_empty().await? {
+                writable.insert(external, server);
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+struct TargetInfo {
+    addr: SocketAddr,
+    connect_ssl: Result<(), AlpnInfo>,
+    key: Key,
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum AlpnInfo {
+    Reflect,
+    Specified(Vec<Vec<u8>>),
+}
+
+struct VHostServer {
+    mapping: Weak<RwLock<BTreeMap<Option<String>, BTreeMap<TargetInfo, Weak<()>>>>>,
+    _thread: NonDetachingJoinHandle<()>,
+}
+impl VHostServer {
+    #[instrument(skip_all)]
+    async fn new(port: u16, ssl: Arc<SslManager>) -> Result<Self, Error> {
+        // check if port allowed
+        let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port))
+            .await
+            .with_kind(crate::ErrorKind::Network)?;
+        let mapping = Arc::new(RwLock::new(BTreeMap::new()));
+        Ok(Self {
+            mapping: Arc::downgrade(&mapping),
+            _thread: tokio::spawn(async move {
+                loop {
+                    match listener.accept().await {
+                        Ok((stream, _)) => {
+                            let stream =
+                                Box::pin(TimeoutStream::new(stream, Duration::from_secs(300)));
+                            let mut stream = BackTrackingReader::new(stream);
+                            stream.start_buffering();
+                            let mapping = mapping.clone();
+                            let ssl = ssl.clone();
+                            tokio::spawn(async move {
+                                if let Err(e) = async {
+                                    let mid = match LazyConfigAcceptor::new(
+                                        Acceptor::default(),
+                                        &mut stream,
+                                    )
+                                    .await
+                                    {
+                                        Ok(a) => a,
+                                        Err(_) => {
+                                            stream.rewind();
+                                            return hyper::server::Server::builder(
+                                                SingleAccept::new(stream),
+                                            )
+                                            .serve(make_service_fn(|_| async {
+                                                Ok::<_, Infallible>(service_fn(|req| async move {
+                                                    let host = req
+                                                        .headers()
+                                                        .get(http::header::HOST)
+                                                        .and_then(|host| host.to_str().ok());
+                                                    let uri = Uri::from_parts({
+                                                        let mut parts =
+                                                            req.uri().to_owned().into_parts();
+                                                        parts.authority = host
+                                                            .map(FromStr::from_str)
+                                                            .transpose()?;
+                                                        parts
+                                                    })?;
+                                                    Response::builder()
+                                                        .status(
+                                                            http::StatusCode::TEMPORARY_REDIRECT,
+                                                        )
+                                                        .header(
+                                                            http::header::LOCATION,
+                                                            uri.to_string(),
+                                                        )
+                                                        .body(Body::default())
+                                                }))
+                                            }))
+                                            .await
+                                            .with_kind(crate::ErrorKind::Network);
+                                        }
+                                    };
+                                    let target_name =
+                                        mid.client_hello().server_name().map(|s| s.to_owned());
+                                    let target = {
+                                        let mapping = mapping.read().await;
+                                        mapping
+                                            .get(&target_name)
+                                            .into_iter()
+                                            .flatten()
+                                            .find(|(_, rc)| rc.strong_count() > 0)
+                                            .or_else(|| {
+                                                if target_name
+                                                    .map(|s| s.parse::<IpAddr>().is_ok())
+                                                    .unwrap_or(true)
+                                                {
+                                                    mapping
+                                                        .get(&None)
+                                                        .into_iter()
+                                                        .flatten()
+                                                        .find(|(_, rc)| rc.strong_count() > 0)
+                                                } else {
+                                                    None
+                                                }
+                                            })
+                                            .map(|(target, _)| target.clone())
+                                    };
+                                    if let Some(target) = target {
+                                        let mut tcp_stream =
+                                            TcpStream::connect(target.addr).await?;
+                                        let key =
+                                            ssl.with_certs(target.key, target.addr.ip()).await?;
+                                        let cfg = ServerConfig::builder()
+                                            .with_safe_defaults()
+                                            .with_no_client_auth();
+                                        let mut cfg =
+                                            if mid.client_hello().signature_schemes().contains(
+                                                &tokio_rustls::rustls::SignatureScheme::ED25519,
+                                            ) {
+                                                cfg.with_single_cert(
+                                                    key.fullchain_ed25519()
+                                                        .into_iter()
+                                                        .map(|c| {
+                                                            Ok(tokio_rustls::rustls::Certificate(
+                                                                c.to_der()?,
+                                                            ))
+                                                        })
+                                                        .collect::<Result<_, Error>>()?,
+                                                    tokio_rustls::rustls::PrivateKey(
+                                                        key.key()
+                                                            .openssl_key_ed25519()
+                                                            .private_key_to_der()?,
+                                                    ),
+                                                )
+                                            } else {
+                                                cfg.with_single_cert(
+                                                    key.fullchain_nistp256()
+                                                        .into_iter()
+                                                        .map(|c| {
+                                                            Ok(tokio_rustls::rustls::Certificate(
+                                                                c.to_der()?,
+                                                            ))
+                                                        })
+                                                        .collect::<Result<_, Error>>()?,
+                                                    tokio_rustls::rustls::PrivateKey(
+                                                        key.key()
+                                                            .openssl_key_nistp256()
+                                                            .private_key_to_der()?,
+                                                    ),
+                                                )
+                                            }
+                                            .with_kind(crate::ErrorKind::OpenSsl)?;
+                                        match target.connect_ssl {
+                                            Ok(()) => {
+                                                let mut client_cfg =
+                                                    tokio_rustls::rustls::ClientConfig::builder()
+                                                        .with_safe_defaults()
+                                                        .with_root_certificates({
+                                                            let mut store = RootCertStore::empty();
+                                                            store.add(
+                                                        &tokio_rustls::rustls::Certificate(
+                                                            key.root_ca().to_der()?,
+                                                        ),
+                                                    ).with_kind(crate::ErrorKind::OpenSsl)?;
+                                                            store
+                                                        })
+                                                        .with_no_client_auth();
+                                                client_cfg.alpn_protocols = mid
+                                                    .client_hello()
+                                                    .alpn()
+                                                    .into_iter()
+                                                    .flatten()
+                                                    .map(|x| x.to_vec())
+                                                    .collect();
+                                                let mut target_stream =
+                                                    TlsConnector::from(Arc::new(client_cfg))
+                                                        .connect_with(
+                                                            key.key()
+                                                                .internal_address()
+                                                                .as_str()
+                                                                .try_into()
+                                                                .with_kind(
+                                                                    crate::ErrorKind::OpenSsl,
+                                                                )?,
+                                                            tcp_stream,
+                                                            |conn| {
+                                                                cfg.alpn_protocols.extend(
+                                                                    conn.alpn_protocol()
+                                                                        .into_iter()
+                                                                        .map(|p| p.to_vec()),
+                                                                )
+                                                            },
+                                                        )
+                                                        .await
+                                                        .with_kind(crate::ErrorKind::OpenSsl)?;
+                                                let mut tls_stream =
+                                                    match mid.into_stream(Arc::new(cfg)).await {
+                                                        Ok(a) => a,
+                                                        Err(e) => {
+                                                            tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}");
+                                                            tracing::trace!("{e:?}");
+                                                            return Ok(())
+                                                        }
+                                                    };
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut target_stream,
+                                                )
+                                                .await
+                                            }
+                                            Err(AlpnInfo::Reflect) => {
+                                                for proto in
+                                                    mid.client_hello().alpn().into_iter().flatten()
+                                                {
+                                                    cfg.alpn_protocols.push(proto.into());
+                                                }
+                                                let mut tls_stream =
+                                                    match mid.into_stream(Arc::new(cfg)).await {
+                                                        Ok(a) => a,
+                                                        Err(e) => {
+                                                            tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}");
+                                                            tracing::trace!("{e:?}");
+                                                            return Ok(())
+                                                        }
+                                                    };
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
+                                            Err(AlpnInfo::Specified(alpn)) => {
+                                                cfg.alpn_protocols = alpn;
+                                                let mut tls_stream =
+                                                    match mid.into_stream(Arc::new(cfg)).await {
+                                                        Ok(a) => a,
+                                                        Err(e) => {
+                                                            tracing::trace!( "VHostController: failed to accept TLS connection on port {port}: {e}");
+                                                            tracing::trace!("{e:?}");
+                                                            return Ok(())
+                                                        }
+                                                    };
+                                                tls_stream.get_mut().0.stop_buffering();
+                                                tokio::io::copy_bidirectional(
+                                                    &mut tls_stream,
+                                                    &mut tcp_stream,
+                                                )
+                                                .await
+                                            }
+                                        }
+                                        .map_or_else(
+                                            |e| {
+                                                use std::io::ErrorKind as E;
+                                                match e.kind() {
+                                                    E::UnexpectedEof | E::BrokenPipe | E::ConnectionAborted | E::ConnectionReset | E::ConnectionRefused | E::TimedOut | E::Interrupted | E::NotConnected => Ok(()),
+                                                _ => Err(e),
+                                            }},
+                                            |_| Ok(()),
+                                        )?;
+                                    } else {
+                                        // 503
+                                    }
+                                    Ok::<_, Error>(())
+                                }
+                                .await
+                                {
+                                    tracing::error!("Error in VHostController on port {port}: {e}");
+                                    tracing::debug!("{e:?}")
+                                }
+                            });
+                        }
+                        Err(e) => {
+                            tracing::trace!(
+                                "VHostController: failed to accept connection on port {port}: {e}"
+                            );
+                            tracing::trace!("{e:?}");
+                        }
+                    }
+                }
+            })
+            .into(),
+        })
+    }
+    async fn add(&self, hostname: Option<String>, target: TargetInfo) -> Result<Arc<()>, Error> {
+        if let Some(mapping) = Weak::upgrade(&self.mapping) {
+            let mut writable = mapping.write().await;
+            let mut targets = writable.remove(&hostname).unwrap_or_default();
+            let rc = if let Some(rc) = Weak::upgrade(&targets.remove(&target).unwrap_or_default()) {
+                rc
+            } else {
+                Arc::new(())
+            };
+            targets.insert(target, Arc::downgrade(&rc));
+            writable.insert(hostname, targets);
+            Ok(rc)
+        } else {
+            Err(Error::new(
+                eyre!("VHost Service Thread has exited"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+    async fn gc(&self, hostname: Option<String>) -> Result<(), Error> {
+        if let Some(mapping) = Weak::upgrade(&self.mapping) {
+            let mut writable = mapping.write().await;
+            let mut targets = writable.remove(&hostname).unwrap_or_default();
+            targets = targets
+                .into_iter()
+                .filter(|(_, rc)| rc.strong_count() > 0)
+                .collect();
+            if !targets.is_empty() {
+                writable.insert(hostname, targets);
+            }
+            Ok(())
+        } else {
+            Err(Error::new(
+                eyre!("VHost Service Thread has exited"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+    async fn is_empty(&self) -> Result<bool, Error> {
+        if let Some(mapping) = Weak::upgrade(&self.mapping) {
+            Ok(mapping.read().await.is_empty())
+        } else {
+            Err(Error::new(
+                eyre!("VHost Service Thread has exited"),
+                crate::ErrorKind::Network,
+            ))
+        }
+    }
+}

core/startos/src/net/web_server.rs

@@ -0,0 +1,61 @@
+use std::convert::Infallible;
+use std::net::SocketAddr;
+
+use futures::future::ready;
+use futures::FutureExt;
+use helpers::NonDetachingJoinHandle;
+use hyper::service::{make_service_fn, service_fn};
+use hyper::Server;
+use tokio::sync::oneshot;
+
+use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext};
+use crate::net::static_server::{
+    diag_ui_file_router, install_ui_file_router, main_ui_server_router, setup_ui_file_router,
+};
+use crate::net::HttpHandler;
+use crate::Error;
+
+pub struct WebServer {
+    shutdown: oneshot::Sender<()>,
+    thread: NonDetachingJoinHandle<()>,
+}
+impl WebServer {
+    pub fn new(bind: SocketAddr, router: HttpHandler) -> Self {
+        let (shutdown, shutdown_recv) = oneshot::channel();
+        let thread = NonDetachingJoinHandle::from(tokio::spawn(async move {
+            let server = Server::bind(&bind)
+                .http1_preserve_header_case(true)
+                .http1_title_case_headers(true)
+                .serve(make_service_fn(move |_| {
+                    let router = router.clone();
+                    ready(Ok::<_, Infallible>(service_fn(move |req| router(req))))
+                }))
+                .with_graceful_shutdown(shutdown_recv.map(|_| ()));
+            if let Err(e) = server.await {
+                tracing::error!("Spawning hyper server error: {}", e);
+            }
+        }));
+        Self { shutdown, thread }
+    }
+
+    pub async fn shutdown(self) {
+        self.shutdown.send(()).unwrap_or_default();
+        self.thread.await.unwrap()
+    }
+
+    pub async fn main(bind: SocketAddr, ctx: RpcContext) -> Result<Self, Error> {
+        Ok(Self::new(bind, main_ui_server_router(ctx).await?))
+    }
+
+    pub async fn setup(bind: SocketAddr, ctx: SetupContext) -> Result<Self, Error> {
+        Ok(Self::new(bind, setup_ui_file_router(ctx).await?))
+    }
+
+    pub async fn diagnostic(bind: SocketAddr, ctx: DiagnosticContext) -> Result<Self, Error> {
+        Ok(Self::new(bind, diag_ui_file_router(ctx).await?))
+    }
+
+    pub async fn install(bind: SocketAddr, ctx: InstallContext) -> Result<Self, Error> {
+        Ok(Self::new(bind, install_ui_file_router(ctx).await?))
+    }
+}

core/startos/src/net/wifi.rs

@@ -0,0 +1,828 @@
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+
+use clap::ArgMatches;
+use isocountry::CountryCode;
+use lazy_static::lazy_static;
+use regex::Regex;
+use rpc_toolkit::command;
+use tokio::process::Command;
+use tokio::sync::RwLock;
+use tracing::instrument;
+
+use crate::context::RpcContext;
+use crate::prelude::*;
+use crate::util::serde::{display_serializable, IoFormat};
+use crate::util::{display_none, Invoke};
+use crate::{Error, ErrorKind};
+
+type WifiManager = Arc<RwLock<WpaCli>>;
+
+pub fn wifi_manager(ctx: &RpcContext) -> Result<&WifiManager, Error> {
+    if let Some(wifi_manager) = ctx.wifi_manager.as_ref() {
+        Ok(wifi_manager)
+    } else {
+        Err(Error::new(
+            color_eyre::eyre::eyre!("No WiFi interface available"),
+            ErrorKind::Wifi,
+        ))
+    }
+}
+
+#[command(subcommands(add, connect, delete, get, country, available))]
+pub async fn wifi() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(subcommands(get_available))]
+pub async fn available() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(subcommands(set_country))]
+pub async fn country() -> Result<(), Error> {
+    Ok(())
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn add(
+    #[context] ctx: RpcContext,
+    #[arg] ssid: String,
+    #[arg] password: String,
+) -> Result<(), Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    if !ssid.is_ascii() {
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("SSID may not have special characters"),
+            ErrorKind::Wifi,
+        ));
+    }
+    if !password.is_ascii() {
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("WiFi Password may not have special characters"),
+            ErrorKind::Wifi,
+        ));
+    }
+    async fn add_procedure(
+        db: PatchDb,
+        wifi_manager: WifiManager,
+        ssid: &Ssid,
+        password: &Psk,
+    ) -> Result<(), Error> {
+        tracing::info!("Adding new WiFi network: '{}'", ssid.0);
+        let mut wpa_supplicant = wifi_manager.write().await;
+        wpa_supplicant.add_network(db, ssid, password).await?;
+        drop(wpa_supplicant);
+        Ok(())
+    }
+    if let Err(err) = add_procedure(
+        ctx.db.clone(),
+        wifi_manager.clone(),
+        &Ssid(ssid.clone()),
+        &Psk(password.clone()),
+    )
+    .await
+    {
+        tracing::error!("Failed to add new WiFi network '{}': {}", ssid, err);
+        tracing::debug!("{:?}", err);
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("Failed adding {}", ssid),
+            ErrorKind::Wifi,
+        ));
+    }
+    Ok(())
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(), Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    if !ssid.is_ascii() {
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("SSID may not have special characters"),
+            ErrorKind::Wifi,
+        ));
+    }
+    async fn connect_procedure(
+        db: PatchDb,
+        wifi_manager: WifiManager,
+        ssid: &Ssid,
+    ) -> Result<(), Error> {
+        let wpa_supplicant = wifi_manager.read().await;
+        let current = wpa_supplicant.get_current_network().await?;
+        drop(wpa_supplicant);
+        let mut wpa_supplicant = wifi_manager.write().await;
+        let connected = wpa_supplicant.select_network(db.clone(), ssid).await?;
+        if connected {
+            tracing::info!("Successfully connected to WiFi: '{}'", ssid.0);
+        } else {
+            tracing::info!("Failed to connect to WiFi: '{}'", ssid.0);
+            match current {
+                None => {
+                    tracing::info!("No WiFi to revert to!");
+                }
+                Some(current) => {
+                    wpa_supplicant.select_network(db, &current).await?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    if let Err(err) =
+        connect_procedure(ctx.db.clone(), wifi_manager.clone(), &Ssid(ssid.clone())).await
+    {
+        tracing::error!("Failed to connect to WiFi network '{}': {}", &ssid, err);
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("Can't connect to {}", ssid),
+            ErrorKind::Wifi,
+        ));
+    }
+    Ok(())
+}
+
+#[command(display(display_none))]
+#[instrument(skip_all)]
+pub async fn delete(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(), Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    if !ssid.is_ascii() {
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("SSID may not have special characters"),
+            ErrorKind::Wifi,
+        ));
+    }
+    let wpa_supplicant = wifi_manager.read().await;
+    let current = wpa_supplicant.get_current_network().await?;
+    drop(wpa_supplicant);
+    let mut wpa_supplicant = wifi_manager.write().await;
+    let ssid = Ssid(ssid);
+    let is_current_being_removed = matches!(current, Some(current) if current == ssid);
+    let is_current_removed_and_no_hardwire =
+        is_current_being_removed && !interface_connected(&ctx.ethernet_interface).await?;
+    if is_current_removed_and_no_hardwire {
+        return Err(Error::new(color_eyre::eyre::eyre!("Forbidden: Deleting this network would make your server unreachable. Either connect to ethernet or connect to a different WiFi network to remedy this."), ErrorKind::Wifi));
+    }
+
+    wpa_supplicant.remove_network(ctx.db.clone(), &ssid).await?;
+    Ok(())
+}
+#[derive(serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct WiFiInfo {
+    ssids: HashMap<Ssid, SignalStrength>,
+    connected: Option<Ssid>,
+    country: Option<CountryCode>,
+    ethernet: bool,
+    available_wifi: Vec<WifiListOut>,
+}
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub struct WifiListInfo {
+    strength: SignalStrength,
+    security: Vec<String>,
+}
+#[derive(serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct WifiListOut {
+    ssid: Ssid,
+    strength: SignalStrength,
+    security: Vec<String>,
+}
+pub type WifiList = HashMap<Ssid, WifiListInfo>;
+fn display_wifi_info(info: WiFiInfo, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(info, matches);
+    }
+
+    let mut table_global = Table::new();
+    table_global.add_row(row![bc =>
+        "CONNECTED",
+        "SIGNAL_STRENGTH",
+        "COUNTRY",
+        "ETHERNET",
+    ]);
+    table_global.add_row(row![
+        &info
+            .connected
+            .as_ref()
+            .map_or("[N/A]".to_owned(), |c| c.0.clone()),
+        &info
+            .connected
+            .as_ref()
+            .and_then(|x| info.ssids.get(x))
+            .map_or("[N/A]".to_owned(), |ss| format!("{}", ss.0)),
+        info.country.as_ref().map(|c| c.alpha2()).unwrap_or("00"),
+        &format!("{}", info.ethernet)
+    ]);
+    table_global.print_tty(false).unwrap();
+
+    let mut table_ssids = Table::new();
+    table_ssids.add_row(row![bc => "SSID", "STRENGTH"]);
+    for (ssid, signal_strength) in &info.ssids {
+        let mut row = row![&ssid.0, format!("{}", signal_strength.0)];
+        row.iter_mut()
+            .map(|c| {
+                c.style(Attr::ForegroundColor(match &signal_strength.0 {
+                    x if x >= &90 => color::GREEN,
+                    x if x == &50 => color::MAGENTA,
+                    x if x == &0 => color::RED,
+                    _ => color::YELLOW,
+                }))
+            })
+            .for_each(drop);
+        table_ssids.add_row(row);
+    }
+    table_ssids.print_tty(false).unwrap();
+
+    let mut table_global = Table::new();
+    table_global.add_row(row![bc =>
+        "SSID",
+        "STRENGTH",
+        "SECURITY",
+    ]);
+    for table_info in info.available_wifi {
+        table_global.add_row(row![
+            &table_info.ssid.0,
+            &format!("{}", table_info.strength.0),
+            &table_info.security.join(" ")
+        ]);
+    }
+
+    table_global.print_tty(false).unwrap();
+}
+
+fn display_wifi_list(info: Vec<WifiListOut>, matches: &ArgMatches) {
+    use prettytable::*;
+
+    if matches.is_present("format") {
+        return display_serializable(info, matches);
+    }
+
+    let mut table_global = Table::new();
+    table_global.add_row(row![bc =>
+        "SSID",
+        "STRENGTH",
+        "SECURITY",
+    ]);
+    for table_info in info {
+        table_global.add_row(row![
+            &table_info.ssid.0,
+            &format!("{}", table_info.strength.0),
+            &table_info.security.join(" ")
+        ]);
+    }
+
+    table_global.print_tty(false).unwrap();
+}
+
+#[command(display(display_wifi_info))]
+#[instrument(skip_all)]
+pub async fn get(
+    #[context] ctx: RpcContext,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<WiFiInfo, Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    let wpa_supplicant = wifi_manager.read().await;
+    let (list_networks, current_res, country_res, ethernet_res, signal_strengths) = tokio::join!(
+        wpa_supplicant.list_networks_low(),
+        wpa_supplicant.get_current_network(),
+        wpa_supplicant.get_country_low(),
+        interface_connected(&ctx.ethernet_interface),
+        wpa_supplicant.list_wifi_low()
+    );
+    let signal_strengths = signal_strengths?;
+    let list_networks: BTreeSet<_> = list_networks?.into_iter().map(|(_, x)| x.ssid).collect();
+    let available_wifi = {
+        let mut wifi_list: Vec<WifiListOut> = signal_strengths
+            .clone()
+            .into_iter()
+            .filter(|(ssid, _)| !list_networks.contains(ssid))
+            .map(|(ssid, info)| WifiListOut {
+                ssid,
+                strength: info.strength,
+                security: info.security,
+            })
+            .collect();
+        wifi_list.sort_by_key(|x| x.strength);
+        wifi_list.reverse();
+        wifi_list
+    };
+    let ssids: HashMap<Ssid, SignalStrength> = list_networks
+        .into_iter()
+        .map(|ssid| {
+            let signal_strength = signal_strengths
+                .get(&ssid)
+                .map(|x| x.strength)
+                .unwrap_or_default();
+            (ssid, signal_strength)
+        })
+        .collect();
+    let current = current_res?;
+    Ok(WiFiInfo {
+        ssids,
+        connected: current,
+        country: country_res?,
+        ethernet: ethernet_res?,
+        available_wifi,
+    })
+}
+
+#[command(rename = "get", display(display_wifi_list))]
+#[instrument(skip_all)]
+pub async fn get_available(
+    #[context] ctx: RpcContext,
+    #[allow(unused_variables)]
+    #[arg(long = "format")]
+    format: Option<IoFormat>,
+) -> Result<Vec<WifiListOut>, Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    let wpa_supplicant = wifi_manager.read().await;
+    let (wifi_list, network_list) = tokio::join!(
+        wpa_supplicant.list_wifi_low(),
+        wpa_supplicant.list_networks_low()
+    );
+    let network_list = network_list?
+        .into_iter()
+        .map(|(_, info)| info.ssid)
+        .collect::<BTreeSet<_>>();
+    let mut wifi_list: Vec<WifiListOut> = wifi_list?
+        .into_iter()
+        .filter(|(ssid, _)| !network_list.contains(ssid))
+        .map(|(ssid, info)| WifiListOut {
+            ssid,
+            strength: info.strength,
+            security: info.security,
+        })
+        .collect();
+    wifi_list.sort_by_key(|x| x.strength);
+    wifi_list.reverse();
+    Ok(wifi_list)
+}
+
+#[command(rename = "set", display(display_none))]
+pub async fn set_country(
+    #[context] ctx: RpcContext,
+    #[arg(parse(country_code_parse))] country: CountryCode,
+) -> Result<(), Error> {
+    let wifi_manager = wifi_manager(&ctx)?;
+    if !interface_connected(&ctx.ethernet_interface).await? {
+        return Err(Error::new(
+            color_eyre::eyre::eyre!("Won't change country without hardwire connection"),
+            crate::ErrorKind::Wifi,
+        ));
+    }
+    let mut wpa_supplicant = wifi_manager.write().await;
+    wpa_supplicant.set_country_low(country.alpha2()).await?;
+    for (network_id, _wifi_info) in wpa_supplicant.list_networks_low().await? {
+        wpa_supplicant.remove_network_low(network_id).await?;
+    }
+    wpa_supplicant.remove_all_connections().await?;
+
+    wpa_supplicant.save_config(ctx.db.clone()).await?;
+
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct WpaCli {
+    interface: String,
+}
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct NetworkId(String);
+
+/// Ssid are the names of the wifis, usually human readable.
+#[derive(
+    Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize,
+)]
+pub struct Ssid(String);
+
+/// So a signal strength is a number between 0-100, I want the null option to be 0 since there is no signal
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    serde::Serialize,
+    serde::Deserialize,
+)]
+pub struct SignalStrength(u8);
+
+impl SignalStrength {
+    fn new(size: Option<u8>) -> Self {
+        let size = match size {
+            None => return Self(0),
+            Some(x) => x,
+        };
+        if size >= 100 {
+            return Self(100);
+        }
+        Self(size)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct WifiInfo {
+    ssid: Ssid,
+    device: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Psk(String);
+impl WpaCli {
+    pub fn init(interface: String) -> Self {
+        WpaCli { interface }
+    }
+
+    #[instrument(skip_all)]
+    pub async fn set_add_network_low(&mut self, ssid: &Ssid, psk: &Psk) -> Result<(), Error> {
+        let _ = Command::new("nmcli")
+            .arg("-a")
+            .arg("-w")
+            .arg("30")
+            .arg("d")
+            .arg("wifi")
+            .arg("con")
+            .arg(&ssid.0)
+            .arg("password")
+            .arg(&psk.0)
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn add_network_low(&mut self, ssid: &Ssid, psk: &Psk) -> Result<(), Error> {
+        if self.find_networks(ssid).await?.is_empty() {
+            Command::new("nmcli")
+                .arg("con")
+                .arg("add")
+                .arg("con-name")
+                .arg(&ssid.0)
+                .arg("type")
+                .arg("wifi")
+                .arg("ssid")
+                .arg(&ssid.0)
+                .invoke(ErrorKind::Wifi)
+                .await?;
+        }
+        Command::new("nmcli")
+            .arg("con")
+            .arg("modify")
+            .arg(&ssid.0)
+            .arg("wifi-sec.key-mgmt")
+            .arg("wpa-psk")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Command::new("nmcli")
+            .arg("con")
+            .arg("modify")
+            .arg(&ssid.0)
+            .arg("ifname")
+            .arg(&self.interface)
+            .invoke(ErrorKind::Wifi)
+            .await
+            .map(|_| ())
+            .unwrap_or_else(|e| {
+                tracing::warn!("Failed to set interface {} for {}", self.interface, ssid.0);
+                tracing::debug!("{:?}", e);
+            });
+        Command::new("nmcli")
+            .arg("con")
+            .arg("modify")
+            .arg(&ssid.0)
+            .arg("wifi-sec.psk")
+            .arg(&psk.0)
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Ok(())
+    }
+    pub async fn set_country_low(&mut self, country_code: &str) -> Result<(), Error> {
+        let _ = Command::new("iw")
+            .arg("reg")
+            .arg("set")
+            .arg(country_code)
+            .invoke(ErrorKind::Wifi)
+            .await?;
+
+        Ok(())
+    }
+    pub async fn get_country_low(&self) -> Result<Option<CountryCode>, Error> {
+        let r = Command::new("iw")
+            .arg("reg")
+            .arg("get")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        let r = String::from_utf8(r)?;
+        lazy_static! {
+            static ref RE: Regex = Regex::new("country (\\w+):").unwrap();
+        }
+        let first_country = r.lines().find(|s| s.contains("country")).ok_or_else(|| {
+            Error::new(
+                color_eyre::eyre::eyre!("Could not find a country config lines"),
+                ErrorKind::Wifi,
+            )
+        })?;
+        let country = &RE.captures(first_country).ok_or_else(|| {
+            Error::new(
+                color_eyre::eyre::eyre!("Could not find a country config with regex"),
+                ErrorKind::Wifi,
+            )
+        })?[1];
+        if country == "00" {
+            Ok(None)
+        } else {
+            Ok(Some(CountryCode::for_alpha2(country).map_err(|_| {
+                Error::new(
+                    color_eyre::eyre::eyre!("Invalid Country Code: {}", country),
+                    ErrorKind::Wifi,
+                )
+            })?))
+        }
+    }
+    pub async fn remove_network_low(&mut self, id: NetworkId) -> Result<(), Error> {
+        let _ = Command::new("nmcli")
+            .arg("c")
+            .arg("del")
+            .arg(&id.0)
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn list_networks_low(&self) -> Result<BTreeMap<NetworkId, WifiInfo>, Error> {
+        let r = Command::new("nmcli")
+            .arg("-t")
+            .arg("c")
+            .arg("show")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        let r = String::from_utf8(r)?;
+        tracing::info!("JCWM: all the networks: {:?}", r);
+        Ok(r.lines()
+            .filter_map(|l| {
+                let mut cs = l.split(':');
+                let name = Ssid(cs.next()?.to_owned());
+                let uuid = NetworkId(cs.next()?.to_owned());
+                let connection_type = cs.next()?;
+                let device = cs.next();
+                if !connection_type.contains("wireless") {
+                    return None;
+                }
+                let info = WifiInfo {
+                    ssid: name,
+                    device: device.map(|x| x.to_owned()),
+                };
+                Some((uuid, info))
+            })
+            .collect::<BTreeMap<NetworkId, WifiInfo>>())
+    }
+
+    #[instrument(skip_all)]
+    pub async fn list_wifi_low(&self) -> Result<WifiList, Error> {
+        let r = Command::new("nmcli")
+            .arg("-g")
+            .arg("SSID,SIGNAL,security")
+            .arg("d")
+            .arg("wifi")
+            .arg("list")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Ok(String::from_utf8(r)?
+            .lines()
+            .filter_map(|l| {
+                let mut values = l.split(':');
+                let ssid = Ssid(values.next()?.to_owned());
+                let signal = SignalStrength::new(std::str::FromStr::from_str(values.next()?).ok());
+                let security: Vec<String> =
+                    values.next()?.split(' ').map(|x| x.to_owned()).collect();
+                Some((
+                    ssid,
+                    WifiListInfo {
+                        strength: signal,
+                        security,
+                    },
+                ))
+            })
+            .collect::<WifiList>())
+    }
+    pub async fn select_network_low(&mut self, id: &NetworkId) -> Result<(), Error> {
+        let _ = Command::new("nmcli")
+            .arg("c")
+            .arg("up")
+            .arg(&id.0)
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        Ok(())
+    }
+    pub async fn remove_all_connections(&mut self) -> Result<(), Error> {
+        let location_connections = Path::new("/etc/NetworkManager/system-connections");
+        let mut connections = tokio::fs::read_dir(&location_connections).await?;
+        while let Some(connection) = connections.next_entry().await? {
+            let path = connection.path();
+            if path.is_file() {
+                let _ = tokio::fs::remove_file(&path).await?;
+            }
+        }
+
+        Ok(())
+    }
+    pub async fn save_config(&mut self, db: PatchDb) -> Result<(), Error> {
+        let new_country = self.get_country_low().await?;
+        db.mutate(|d| {
+            d.as_server_info_mut()
+                .as_last_wifi_region_mut()
+                .ser(&new_country)
+        })
+        .await
+    }
+    async fn check_active_network(&self, ssid: &Ssid) -> Result<Option<NetworkId>, Error> {
+        Ok(self
+            .list_networks_low()
+            .await?
+            .iter()
+            .find_map(|(network_id, wifi_info)| {
+                wifi_info.device.as_ref()?;
+                if wifi_info.ssid == *ssid {
+                    Some(network_id.clone())
+                } else {
+                    None
+                }
+            }))
+    }
+    pub async fn find_networks(&self, ssid: &Ssid) -> Result<Vec<NetworkId>, Error> {
+        Ok(self
+            .list_networks_low()
+            .await?
+            .iter()
+            .filter_map(|(network_id, wifi_info)| {
+                if wifi_info.ssid == *ssid {
+                    Some(network_id.clone())
+                } else {
+                    None
+                }
+            })
+            .collect())
+    }
+    #[instrument(skip_all)]
+    pub async fn select_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
+        let m_id = self.check_active_network(ssid).await?;
+        match m_id {
+            None => Err(Error::new(
+                color_eyre::eyre::eyre!("SSID Not Found"),
+                ErrorKind::Wifi,
+            )),
+            Some(x) => {
+                self.select_network_low(&x).await?;
+                self.save_config(db).await?;
+                let connect = async {
+                    let mut current;
+                    loop {
+                        current = self.get_current_network().await;
+                        if let Ok(Some(ssid)) = &current {
+                            tracing::debug!("Connected to: {}", ssid.0);
+                            break;
+                        }
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                        tracing::debug!("Retrying...");
+                    }
+                    current
+                };
+                let res = match tokio::time::timeout(Duration::from_secs(20), connect).await {
+                    Err(_) => None,
+                    Ok(net) => net?,
+                };
+                tracing::debug!("{:?}", res);
+                Ok(match res {
+                    None => false,
+                    Some(net) => &net == ssid,
+                })
+            }
+        }
+    }
+    #[instrument(skip_all)]
+    pub async fn get_current_network(&self) -> Result<Option<Ssid>, Error> {
+        let r = Command::new("iwgetid")
+            .arg(&self.interface)
+            .arg("--raw")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+        let output = String::from_utf8(r)?;
+        let network = output.trim();
+        tracing::debug!("Current Network: \"{}\"", network);
+        if network.is_empty() {
+            Ok(None)
+        } else {
+            Ok(Some(Ssid(network.to_owned())))
+        }
+    }
+    #[instrument(skip_all)]
+    pub async fn remove_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
+        let found_networks = self.find_networks(ssid).await?;
+        if found_networks.is_empty() {
+            return Ok(true);
+        }
+        for network_id in found_networks {
+            self.remove_network_low(network_id).await?;
+        }
+        self.save_config(db).await?;
+        Ok(true)
+    }
+    #[instrument(skip_all)]
+    pub async fn set_add_network(
+        &mut self,
+        db: PatchDb,
+        ssid: &Ssid,
+        psk: &Psk,
+    ) -> Result<(), Error> {
+        self.set_add_network_low(ssid, psk).await?;
+        self.save_config(db).await?;
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn add_network(&mut self, db: PatchDb, ssid: &Ssid, psk: &Psk) -> Result<(), Error> {
+        self.add_network_low(ssid, psk).await?;
+        self.save_config(db).await?;
+        Ok(())
+    }
+}
+
+#[instrument(skip_all)]
+pub async fn interface_connected(interface: &str) -> Result<bool, Error> {
+    let out = Command::new("ifconfig")
+        .arg(interface)
+        .invoke(ErrorKind::Wifi)
+        .await?;
+    let v = std::str::from_utf8(&out)?
+        .lines()
+        .find(|s| s.contains("inet"));
+    Ok(v.is_some())
+}
+
+pub fn country_code_parse(code: &str, _matches: &ArgMatches) -> Result<CountryCode, Error> {
+    CountryCode::for_alpha2(code).map_err(|_| {
+        Error::new(
+            color_eyre::eyre::eyre!("Invalid Country Code: {}", code),
+            ErrorKind::Wifi,
+        )
+    })
+}
+
+#[instrument(skip_all)]
+pub async fn synchronize_wpa_supplicant_conf<P: AsRef<Path>>(
+    main_datadir: P,
+    wifi_iface: &str,
+    last_country_code: &Option<CountryCode>,
+) -> Result<(), Error> {
+    let persistent = main_datadir.as_ref().join("system-connections");
+    tracing::debug!("persistent: {:?}", persistent);
+    // let supplicant = Path::new("/etc/wpa_supplicant.conf");
+
+    if tokio::fs::metadata(&persistent).await.is_err() {
+        tokio::fs::create_dir_all(&persistent).await?;
+    }
+    crate::disk::mount::util::bind(&persistent, "/etc/NetworkManager/system-connections", false)
+        .await?;
+    // if tokio::fs::metadata(&supplicant).await.is_err() {
+    //     tokio::fs::write(&supplicant, include_str!("wpa_supplicant.conf.base")).await?;
+    // }
+
+    Command::new("systemctl")
+        .arg("restart")
+        .arg("NetworkManager")
+        .invoke(ErrorKind::Wifi)
+        .await?;
+    Command::new("ifconfig")
+        .arg(wifi_iface)
+        .arg("up")
+        .invoke(ErrorKind::Wifi)
+        .await?;
+    if let Some(last_country_code) = last_country_code {
+        tracing::info!("Setting the region");
+        let _ = Command::new("iw")
+            .arg("reg")
+            .arg("set")
+            .arg(last_country_code.alpha2())
+            .invoke(ErrorKind::Wifi)
+            .await?;
+    } else {
+        tracing::info!("Setting the region fallback");
+        let _ = Command::new("iw")
+            .arg("reg")
+            .arg("set")
+            .arg("US")
+            .invoke(ErrorKind::Wifi)
+            .await?;
+    }
+    Ok(())
+}

core/startos/src/net/wpa_supplicant.conf.base

@@ -0,0 +1,3 @@
+ctrl_interface=DIR=/run/wpa_supplicant GROUP=netdev
+update_config=1
+country=US

core/startos/src/net/ws_server.rs

@@ -0,0 +1,94 @@
+use crate::context::RpcContext;
+
+pub async fn ws_server_handle(rpc_ctx: RpcContext) {
+
+    let ws_ctx = rpc_ctx.clone();
+    let ws_server_handle = {
+        let builder = Server::bind(&ws_ctx.bind_ws);
+
+        let make_svc = ::rpc_toolkit::hyper::service::make_service_fn(move |_| {
+            let ctx = ws_ctx.clone();
+            async move {
+                Ok::<_, ::rpc_toolkit::hyper::Error>(::rpc_toolkit::hyper::service::service_fn(
+                    move |req| {
+                        let ctx = ctx.clone();
+                        async move {
+                            tracing::debug!("Request to {}", req.uri().path());
+                            match req.uri().path() {
+                                "/ws/db" => {
+                                    Ok(subscribe(ctx, req).await.unwrap_or_else(err_to_500))
+                                }
+                                path if path.starts_with("/ws/rpc/") => {
+                                     match RequestGuid::from(
+                                        path.strip_prefix("/ws/rpc/").unwrap(),
+                                    ) {
+                                        None => {
+                                            tracing::debug!("No Guid Path");
+                                            Response::builder()
+                                                .status(StatusCode::BAD_REQUEST)
+                                                .body(Body::empty())
+                                        }
+                                        Some(guid) => {
+                                            match ctx.get_ws_continuation_handler(&guid).await {
+                                                Some(cont) => match cont(req).await {
+                                                    Ok(r) => Ok(r),
+                                                    Err(e) => Response::builder()
+                                                        .status(
+                                                            StatusCode::INTERNAL_SERVER_ERROR,
+                                                        )
+                                                        .body(Body::from(format!("{}", e))),
+                                                },
+                                                _ => Response::builder()
+                                                    .status(StatusCode::NOT_FOUND)
+                                                    .body(Body::empty()),
+                                            }
+                                        }
+                                    }
+                                }
+                                path if path.starts_with("/rest/rpc/") => {
+                                    match RequestGuid::from(
+                                        path.strip_prefix("/rest/rpc/").unwrap(),
+                                    ) {
+                                        None => {
+                                            tracing::debug!("No Guid Path");
+                                            Response::builder()
+                                                .status(StatusCode::BAD_REQUEST)
+                                                .body(Body::empty())
+                                        }
+                                        Some(guid) => {
+                                            match ctx.get_rest_continuation_handler(&guid).await
+                                            {
+                                                None => Response::builder()
+                                                    .status(StatusCode::NOT_FOUND)
+                                                    .body(Body::empty()),
+                                                Some(cont) => match cont(req).await {
+                                                    Ok(r) => Ok(r),
+                                                    Err(e) => Response::builder()
+                                                        .status(
+                                                            StatusCode::INTERNAL_SERVER_ERROR,
+                                                        )
+                                                        .body(Body::from(format!("{}", e))),
+                                                },
+                                            }
+                                        }
+                                    }
+                                }
+                                _ => Response::builder()
+                                    .status(StatusCode::NOT_FOUND)
+                                    .body(Body::empty()),
+                            }
+                        }
+                    },
+                ))
+            }
+        });
+        builder.serve(make_svc)
+    }
+    .with_graceful_shutdown({
+        let mut shutdown = rpc_ctx.shutdown.subscribe();
+        async move {
+            shutdown.recv().await.expect("context dropped");
+        }
+    });
+
+}