Feature/start tunnel (#3037)

* fix live-build resolv.conf

* improved debuggability

* wip: start-tunnel

* fixes for trixie and tor

* non-free-firmware on trixie

* wip

* web server WIP

* wip: tls refactor

* FE patchdb, mocks, and most endpoints

* fix editing records and patch mocks

* refactor complete

* finish api

* build and formatter update

* minor change toi viewing addresses and fix build

* fixes

* more providers

* endpoint for getting config

* fix tests

* api fixes

* wip: separate port forward controller into parts

* simplify iptables rules

* bump sdk

* misc fixes

* predict next subnet and ip, use wan ips, and form validation

* refactor: break big components apart and address todos (#3043)

* refactor: break big components apart and address todos

* starttunnel readme, fix pf mocks, fix adding tor domain in startos

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* better tui

* tui tweaks

* fix: address comments

* better regex for subnet

* fixes

* better validation

* handle rpc errors

* build fixes

* fix: address comments (#3044)

* fix: address comments

* fix unread notification mocks

* fix row click for notification

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix raspi build

* fix build

* fix build

* fix build

* fix build

* try to fix build

* fix tests

* fix tests

* fix rsync tests

* delete useless effectful test

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>

core/startos/src/net/acme.rs

@@ -1,9 +1,12 @@
 use std::collections::{BTreeMap, BTreeSet};
+use std::net::IpAddr;
 use std::str::FromStr;
+use std::sync::Arc;
 
-use async_acme::acme::Identifier;
+use async_acme::acme::{ACME_TLS_ALPN_NAME, Identifier};
 use clap::Parser;
 use clap::builder::ValueParserFactory;
+use futures::StreamExt;
 use imbl_value::InternedString;
 use itertools::Itertools;
 use models::{ErrorData, FromStrParser};
@@ -11,14 +14,209 @@ use openssl::pkey::{PKey, Private};
 use openssl::x509::X509;
 use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
 use serde::{Deserialize, Serialize};
+use tokio_rustls::rustls::ServerConfig;
+use tokio_rustls::rustls::crypto::CryptoProvider;
+use tokio_rustls::rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
+use tokio_rustls::rustls::server::ClientHello;
+use tokio_rustls::rustls::sign::CertifiedKey;
 use ts_rs::TS;
 use url::Url;
 
 use crate::context::{CliContext, RpcContext};
 use crate::db::model::Database;
 use crate::db::model::public::AcmeSettings;
+use crate::db::{DbAccess, DbAccessByKey, DbAccessMut};
+use crate::net::tls::{SingleCertResolver, TlsHandler};
+use crate::net::web_server::Accept;
 use crate::prelude::*;
 use crate::util::serde::{Pem, Pkcs8Doc};
+use crate::util::sync::{SyncMutex, Watch};
+
+pub type AcmeTlsAlpnCache =
+    Arc<SyncMutex<BTreeMap<InternedString, Watch<Option<Arc<CertifiedKey>>>>>>;
+
+pub struct AcmeTlsHandler<M: HasModel, S> {
+    pub db: TypedPatchDb<M>,
+    pub acme_cache: AcmeTlsAlpnCache,
+    pub crypto_provider: Arc<CryptoProvider>,
+    pub get_provider: S,
+    pub in_progress: Watch<BTreeSet<BTreeSet<InternedString>>>,
+}
+impl<M, S> AcmeTlsHandler<M, S>
+where
+    for<'a> M: DbAccessByKey<AcmeSettings, Key<'a> = &'a AcmeProvider>
+        + DbAccessMut<AcmeCertStore>
+        + HasModel<Model = Model<M>>
+        + Send
+        + Sync,
+    S: GetAcmeProvider + Clone,
+{
+    pub async fn get_cert(&self, san_info: &BTreeSet<InternedString>) -> Option<CertifiedKey> {
+        let provider = self.get_provider.get_provider(san_info).await?;
+        let provider = provider.as_ref();
+        loop {
+            let peek = self.db.peek().await;
+            let store = <M as DbAccess<AcmeCertStore>>::access(&peek);
+            if let Some(cert) = store
+                .as_certs()
+                .as_idx(&provider.0)
+                .and_then(|p| p.as_idx(JsonKey::new_ref(san_info)))
+            {
+                let cert = cert.de().log_err()?;
+                return Some(
+                    CertifiedKey::from_der(
+                        cert.fullchain
+                            .into_iter()
+                            .map(|c| Ok(CertificateDer::from(c.to_der()?)))
+                            .collect::<Result<_, Error>>()
+                            .log_err()?,
+                        PrivateKeyDer::from(PrivatePkcs8KeyDer::from(
+                            cert.key.0.private_key_to_pkcs8().log_err()?,
+                        )),
+                        &*self.crypto_provider,
+                    )
+                    .log_err()?,
+                );
+            }
+
+            if !self.in_progress.send_if_modified(|x| {
+                if !x.contains(san_info) {
+                    x.insert(san_info.clone());
+                    true
+                } else {
+                    false
+                }
+            }) {
+                self.in_progress
+                    .clone()
+                    .wait_for(|x| !x.contains(san_info))
+                    .await;
+                continue;
+            }
+
+            let contact = <M as DbAccessByKey<AcmeSettings>>::access_by_key(&peek, &provider)?
+                .as_contact()
+                .de()
+                .log_err()?;
+
+            let identifiers: Vec<_> = san_info
+                .iter()
+                .map(|d| match d.parse::<IpAddr>() {
+                    Ok(a) => Identifier::Ip(a),
+                    _ => Identifier::Dns((&**d).into()),
+                })
+                .collect::<Vec<_>>();
+
+            let cache_entries = san_info
+                .iter()
+                .cloned()
+                .map(|d| (d, Watch::new(None)))
+                .collect::<BTreeMap<_, _>>();
+            self.acme_cache.mutate(|c| {
+                c.extend(cache_entries.iter().map(|(k, v)| (k.clone(), v.clone())));
+            });
+
+            let cert = async_acme::rustls_helper::order(
+                |identifier, cert| {
+                    let domain = InternedString::from_display(&identifier);
+                    if let Some(entry) = cache_entries.get(&domain) {
+                        entry.send(Some(Arc::new(cert)));
+                    }
+                    Ok(())
+                },
+                provider.0.as_str(),
+                &identifiers,
+                Some(&AcmeCertCache(&self.db)),
+                &contact,
+            )
+            .await
+            .log_err()?;
+
+            self.acme_cache
+                .mutate(|c| c.retain(|c, _| !cache_entries.contains_key(c)));
+
+            self.in_progress.send_modify(|i| i.remove(san_info));
+
+            return Some(cert);
+        }
+    }
+}
+
+pub trait GetAcmeProvider {
+    fn get_provider<'a, 'b: 'a>(
+        &'b self,
+        san_info: &'a BTreeSet<InternedString>,
+    ) -> impl Future<Output = Option<impl AsRef<AcmeProvider> + Send + 'b>> + Send + 'a;
+}
+
+impl<'a, A, M, S> TlsHandler<'a, A> for Arc<AcmeTlsHandler<M, S>>
+where
+    A: Accept + 'a,
+    <A as Accept>::Metadata: Send + Sync,
+    for<'m> M: DbAccessByKey<AcmeSettings, Key<'m> = &'m AcmeProvider>
+        + DbAccessMut<AcmeCertStore>
+        + HasModel<Model = Model<M>>
+        + Send
+        + Sync,
+    S: GetAcmeProvider + Clone + Send + Sync,
+{
+    async fn get_config(
+        &'a mut self,
+        hello: &'a ClientHello<'a>,
+        _: &'a <A as Accept>::Metadata,
+    ) -> Option<ServerConfig> {
+        let domain = hello.server_name()?;
+        if hello
+            .alpn()
+            .into_iter()
+            .flatten()
+            .any(|a| a == ACME_TLS_ALPN_NAME)
+        {
+            let cert = self
+                .acme_cache
+                .peek(|c| c.get(domain).cloned())
+                .ok_or_else(|| {
+                    Error::new(
+                        eyre!("No challenge recv available for {domain}"),
+                        ErrorKind::OpenSsl,
+                    )
+                })
+                .log_err()?;
+            tracing::info!("Waiting for verification cert for {domain}");
+            let cert = cert
+                .filter(|c| futures::future::ready(c.is_some()))
+                .next()
+                .await
+                .flatten()?;
+            tracing::info!("Verification cert received for {domain}");
+            let mut cfg = ServerConfig::builder_with_provider(self.crypto_provider.clone())
+                .with_safe_default_protocol_versions()
+                .log_err()?
+                .with_no_client_auth()
+                .with_cert_resolver(Arc::new(SingleCertResolver(cert)));
+
+            cfg.alpn_protocols = vec![ACME_TLS_ALPN_NAME.to_vec()];
+            tracing::info!("performing ACME auth challenge");
+
+            return Some(cfg);
+        }
+
+        let domains: BTreeSet<InternedString> = [domain.into()].into_iter().collect();
+
+        let crypto_provider = self.crypto_provider.clone();
+        if let Some(cert) = self.get_cert(&domains).await {
+            return Some(
+                ServerConfig::builder_with_provider(crypto_provider)
+                    .with_safe_default_protocol_versions()
+                    .log_err()?
+                    .with_no_client_auth()
+                    .with_cert_resolver(Arc::new(SingleCertResolver(Arc::new(cert)))),
+            );
+        }
+
+        None
+    }
+}
 
 #[derive(Debug, Default, Deserialize, Serialize, HasModel)]
 #[model = "Model<Self>"]
@@ -32,29 +230,35 @@ impl AcmeCertStore {
     }
 }
 
+impl DbAccess<AcmeCertStore> for Database {
+    fn access<'a>(db: &'a Model<Self>) -> &'a Model<AcmeCertStore> {
+        db.as_private().as_key_store().as_acme()
+    }
+}
+impl DbAccessMut<AcmeCertStore> for Database {
+    fn access_mut<'a>(db: &'a mut Model<Self>) -> &'a mut Model<AcmeCertStore> {
+        db.as_private_mut().as_key_store_mut().as_acme_mut()
+    }
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct AcmeCert {
     pub key: Pem<PKey<Private>>,
     pub fullchain: Vec<Pem<X509>>,
 }
 
-pub struct AcmeCertCache<'a>(pub &'a TypedPatchDb<Database>);
+pub struct AcmeCertCache<'a, M: HasModel>(pub &'a TypedPatchDb<M>);
 #[async_trait::async_trait]
-impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
+impl<'a, M> async_acme::cache::AcmeCache for AcmeCertCache<'a, M>
+where
+    M: HasModel<Model = Model<M>> + DbAccessMut<AcmeCertStore> + Send + Sync,
+{
     type Error = ErrorData;
 
     async fn read_account(&self, contacts: &[&str]) -> Result<Option<Vec<u8>>, Self::Error> {
         let contacts = JsonKey::new(contacts.into_iter().map(|s| (*s).to_owned()).collect_vec());
-        let Some(account) = self
-            .0
-            .peek()
-            .await
-            .into_private()
-            .into_key_store()
-            .into_acme()
-            .into_accounts()
-            .into_idx(&contacts)
-        else {
+        let peek = self.0.peek().await;
+        let Some(account) = M::access(&peek).as_accounts().as_idx(&contacts) else {
             return Ok(None);
         };
         Ok(Some(account.de()?.0.document.into_vec()))
@@ -68,9 +272,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
         };
         self.0
             .mutate(|db| {
-                db.as_private_mut()
-                    .as_key_store_mut()
-                    .as_acme_mut()
+                M::access_mut(db)
                     .as_accounts_mut()
                     .insert(&contacts, &Pem::new(key))
             })
@@ -96,16 +298,11 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
         let directory_url = directory_url
             .parse::<Url>()
             .with_kind(ErrorKind::ParseUrl)?;
-        let Some(cert) = self
-            .0
-            .peek()
-            .await
-            .into_private()
-            .into_key_store()
-            .into_acme()
-            .into_certs()
-            .into_idx(&directory_url)
-            .and_then(|a| a.into_idx(&identifiers))
+        let peek = self.0.peek().await;
+        let Some(cert) = M::access(&peek)
+            .as_certs()
+            .as_idx(&directory_url)
+            .and_then(|a| a.as_idx(&identifiers))
         else {
             return Ok(None);
         };
@@ -160,9 +357,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
         };
         self.0
             .mutate(|db| {
-                db.as_private_mut()
-                    .as_key_store_mut()
-                    .as_acme_mut()
+                M::access_mut(db)
                     .as_certs_mut()
                     .upsert(&directory_url, || Ok(BTreeMap::new()))?
                     .insert(&identifiers, &cert)
@@ -235,6 +430,11 @@ impl AsRef<str> for AcmeProvider {
         self.0.as_str()
     }
 }
+impl AsRef<AcmeProvider> for AcmeProvider {
+    fn as_ref(&self) -> &AcmeProvider {
+        self
+    }
+}
 impl ValueParserFactory for AcmeProvider {
     type Parser = FromStrParser<Self>;
     fn value_parser() -> Self::Parser {