Feature/start tunnel (#3037)

* fix live-build resolv.conf

* improved debuggability

* wip: start-tunnel

* fixes for trixie and tor

* non-free-firmware on trixie

* wip

* web server WIP

* wip: tls refactor

* FE patchdb, mocks, and most endpoints

* fix editing records and patch mocks

* refactor complete

* finish api

* build and formatter update

* minor change toi viewing addresses and fix build

* fixes

* more providers

* endpoint for getting config

* fix tests

* api fixes

* wip: separate port forward controller into parts

* simplify iptables rules

* bump sdk

* misc fixes

* predict next subnet and ip, use wan ips, and form validation

* refactor: break big components apart and address todos (#3043)

* refactor: break big components apart and address todos

* starttunnel readme, fix pf mocks, fix adding tor domain in startos

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* better tui

* tui tweaks

* fix: address comments

* better regex for subnet

* fixes

* better validation

* handle rpc errors

* build fixes

* fix: address comments (#3044)

* fix: address comments

* fix unread notification mocks

* fix row click for notification

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix raspi build

* fix build

* fix build

* fix build

* fix build

* try to fix build

* fix tests

* fix tests

* fix rsync tests

* delete useless effectful test

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>

core/startos/src/net/web_server.rs

@@ -1,70 +1,200 @@
+use std::any::Any;
+use std::collections::BTreeMap;
 use std::future::Future;
 use std::net::SocketAddr;
 use std::ops::Deref;
+use std::pin::Pin;
 use std::sync::Arc;
-use std::task::Poll;
+use std::task::{Poll, ready};
 use std::time::Duration;
 
 use axum::Router;
 use futures::future::Either;
-use futures::FutureExt;
+use futures::{FutureExt, TryFutureExt};
 use helpers::NonDetachingJoinHandle;
+use http::Extensions;
 use hyper_util::rt::{TokioIo, TokioTimer};
-use tokio::net::{TcpListener, TcpStream};
+use tokio::net::TcpListener;
 use tokio::sync::oneshot;
+use visit_rs::{Visit, VisitFields, Visitor};
 
-use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext};
-use crate::net::gateway::{
-    lookup_info_by_addr, NetworkInterfaceListener, SelfContainedNetworkInterfaceListener,
-};
-use crate::net::static_server::{
-    diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, redirecter, refresher,
-    setup_ui_router,
-};
+use crate::net::static_server::{UiContext, ui_router};
 use crate::prelude::*;
 use crate::util::actor::background::BackgroundJobQueue;
+use crate::util::io::ReadWriter;
 use crate::util::sync::{SyncRwLock, Watch};
 
-pub struct Accepted {
-    pub https_redirect: bool,
-    pub stream: TcpStream,
+pub type AcceptStream = Pin<Box<dyn ReadWriter + Send + 'static>>;
+
+pub trait MetadataVisitor: Visitor<Result = ()> {
+    fn visit<M: Clone + Send + Sync + 'static>(&mut self, metadata: &M) -> Self::Result;
+}
+
+pub struct ExtensionVisitor<'a>(&'a mut Extensions);
+impl<'a> Visitor for ExtensionVisitor<'a> {
+    type Result = ();
+}
+impl<'a> MetadataVisitor for ExtensionVisitor<'a> {
+    fn visit<M: Clone + Send + Sync + 'static>(&mut self, metadata: &M) -> Self::Result {
+        self.0.insert(metadata.clone());
+    }
+}
+impl<'a> Visit<ExtensionVisitor<'a>>
+    for Box<dyn for<'x> Visit<ExtensionVisitor<'x>> + Send + Sync + 'static>
+{
+    fn visit(
+        &self,
+        visitor: &mut ExtensionVisitor<'a>,
+    ) -> <ExtensionVisitor<'a> as Visitor>::Result {
+        (&**self).visit(visitor)
+    }
+}
+
+pub struct ExtractVisitor<T>(Option<T>);
+impl<T> Visitor for ExtractVisitor<T> {
+    type Result = ();
+}
+impl<T: Clone + Send + Sync + 'static> MetadataVisitor for ExtractVisitor<T> {
+    fn visit<M: Clone + Send + Sync + 'static>(&mut self, metadata: &M) -> Self::Result {
+        if let Some(matching) = (metadata as &dyn Any).downcast_ref::<T>() {
+            self.0 = Some(matching.clone());
+        }
+    }
+}
+pub fn extract<
+    T: Clone + Send + Sync + 'static,
+    M: Visit<ExtractVisitor<T>> + Clone + Send + Sync + 'static,
+>(
+    metadata: &M,
+) -> Option<T> {
+    let mut visitor = ExtractVisitor(None);
+    visitor.visit(metadata);
+    visitor.0
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct TcpMetadata {
+    pub peer_addr: SocketAddr,
+    pub local_addr: SocketAddr,
+}
+impl<V: MetadataVisitor> Visit<V> for TcpMetadata {
+    fn visit(&self, visitor: &mut V) -> <V as visit_rs::Visitor>::Result {
+        visitor.visit(self)
+    }
 }
 
 pub trait Accept {
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>>;
+    type Metadata;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>>;
+    fn into_dyn(self) -> DynAccept
+    where
+        Self: Sized + Send + Sync + 'static,
+        for<'a> Self::Metadata: Visit<ExtensionVisitor<'a>> + Send + Sync + 'static,
+    {
+        DynAccept::new(self)
+    }
 }
 
-impl Accept for Vec<TcpListener> {
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
-        for listener in &*self {
-            if let Poll::Ready((stream, _)) = listener.poll_accept(cx)? {
-                return Poll::Ready(Ok(Accepted {
-                    https_redirect: false,
-                    stream,
-                }));
+impl Accept for TcpListener {
+    type Metadata = TcpMetadata;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
+        if let Poll::Ready((stream, peer_addr)) = TcpListener::poll_accept(self, cx)? {
+            if let Err(e) = socket2::SockRef::from(&stream).set_tcp_keepalive(
+                &socket2::TcpKeepalive::new()
+                    .with_time(Duration::from_secs(900))
+                    .with_interval(Duration::from_secs(60))
+                    .with_retries(5),
+            ) {
+                tracing::error!("Failed to set tcp keepalive: {e}");
+                tracing::debug!("{e:?}");
+            }
+            return Poll::Ready(Ok((
+                TcpMetadata {
+                    local_addr: self.local_addr()?,
+                    peer_addr,
+                },
+                Box::pin(stream),
+            )));
+        }
+        Poll::Pending
+    }
+}
+
+impl<A> Accept for Vec<A>
+where
+    A: Accept,
+{
+    type Metadata = A::Metadata;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
+        for listener in self {
+            if let Poll::Ready(accepted) = listener.poll_accept(cx)? {
+                return Poll::Ready(Ok(accepted));
             }
         }
         Poll::Pending
     }
 }
-impl Accept for NetworkInterfaceListener {
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
-        NetworkInterfaceListener::poll_accept(self, cx, &true).map(|res| {
-            res.map(|a| {
-                let public = self
-                    .ip_info
-                    .peek(|i| lookup_info_by_addr(i, a.bind).map_or(true, |(_, i)| i.public()));
-                Accepted {
-                    https_redirect: public,
-                    stream: a.stream,
-                }
-            })
-        })
+
+#[derive(Clone, VisitFields)]
+pub struct MapListenerMetadata<K, M> {
+    pub inner: M,
+    pub key: K,
+}
+impl<K, M, V> Visit<V> for MapListenerMetadata<K, M>
+where
+    V: MetadataVisitor,
+    K: Visit<V>,
+    M: Visit<V>,
+{
+    fn visit(&self, visitor: &mut V) -> <V as Visitor>::Result {
+        self.visit_fields(visitor).collect()
     }
 }
 
-impl<A: Accept, B: Accept> Accept for Either<A, B> {
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
+impl<K, A> Accept for BTreeMap<K, A>
+where
+    K: Clone,
+    A: Accept,
+{
+    type Metadata = MapListenerMetadata<K, A::Metadata>;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
+        for (key, listener) in self {
+            if let Poll::Ready((metadata, stream)) = listener.poll_accept(cx)? {
+                return Poll::Ready(Ok((
+                    MapListenerMetadata {
+                        inner: metadata,
+                        key: key.clone(),
+                    },
+                    stream,
+                )));
+            }
+        }
+        Poll::Pending
+    }
+}
+
+impl<A, B> Accept for Either<A, B>
+where
+    A: Accept,
+    B: Accept<Metadata = A::Metadata>,
+{
+    type Metadata = A::Metadata;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
         match self {
             Either::Left(a) => a.poll_accept(cx),
             Either::Right(b) => b.poll_accept(cx),
@@ -72,7 +202,11 @@ impl<A: Accept, B: Accept> Accept for Either<A, B> {
     }
 }
 impl<A: Accept> Accept for Option<A> {
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
+    type Metadata = A::Metadata;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
         match self {
             None => Poll::Pending,
             Some(a) => a.poll_accept(cx),
@@ -80,6 +214,68 @@ impl<A: Accept> Accept for Option<A> {
     }
 }
 
+trait DynAcceptT: Send + Sync {
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<
+        Result<
+            (
+                Box<dyn for<'a> Visit<ExtensionVisitor<'a>> + Send + Sync>,
+                AcceptStream,
+            ),
+            Error,
+        >,
+    >;
+}
+impl<A> DynAcceptT for A
+where
+    A: Accept + Send + Sync,
+    for<'a> <A as Accept>::Metadata: Visit<ExtensionVisitor<'a>> + Send + Sync + 'static,
+{
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<
+        Result<
+            (
+                Box<dyn for<'a> Visit<ExtensionVisitor<'a>> + Send + Sync>,
+                AcceptStream,
+            ),
+            Error,
+        >,
+    > {
+        let (metadata, stream) = ready!(Accept::poll_accept(self, cx)?);
+        Poll::Ready(Ok((Box::new(metadata), stream)))
+    }
+}
+pub struct DynAccept(Box<dyn DynAcceptT>);
+impl Accept for DynAccept {
+    type Metadata = Box<dyn for<'a> Visit<ExtensionVisitor<'a>> + Send + Sync>;
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(Self::Metadata, AcceptStream), Error>> {
+        DynAcceptT::poll_accept(&mut *self.0, cx)
+    }
+    fn into_dyn(self) -> DynAccept
+    where
+        Self: Sized,
+        for<'a> Self::Metadata: Visit<ExtensionVisitor<'a>> + Send + Sync + 'static,
+    {
+        self
+    }
+}
+impl DynAccept {
+    pub fn new<A>(accept: A) -> Self
+    where
+        A: Accept + Send + Sync + 'static,
+        for<'a> <A as Accept>::Metadata: Visit<ExtensionVisitor<'a>> + Send + Sync + 'static,
+    {
+        Self(Box::new(accept))
+    }
+}
+
 #[pin_project::pin_project]
 pub struct Acceptor<A: Accept> {
     acceptor: Watch<A>,
@@ -95,12 +291,15 @@ impl<A: Accept + Send + Sync + 'static> Acceptor<A> {
         self.acceptor.poll_changed(cx)
     }
 
-    fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
-        let _ = self.poll_changed(cx);
+    fn poll_accept(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(A::Metadata, AcceptStream), Error>> {
+        while self.poll_changed(cx).is_ready() {}
         self.acceptor.peek_mut(|a| a.poll_accept(cx))
     }
 
-    async fn accept(&mut self) -> Result<Accepted, Error> {
+    async fn accept(&mut self) -> Result<(A::Metadata, AcceptStream), Error> {
         std::future::poll_fn(|cx| self.poll_accept(cx)).await
     }
 }
@@ -111,20 +310,73 @@ impl Acceptor<Vec<TcpListener>> {
         ))
     }
 }
-
-pub type UpgradableListener =
-    Option<Either<SelfContainedNetworkInterfaceListener, NetworkInterfaceListener>>;
-
-impl Acceptor<UpgradableListener> {
-    pub fn bind_upgradable(listener: SelfContainedNetworkInterfaceListener) -> Self {
-        Self::new(Some(Either::Left(listener)))
+impl Acceptor<Vec<DynAccept>> {
+    pub async fn bind_dyn(listen: impl IntoIterator<Item = SocketAddr>) -> Result<Self, Error> {
+        Ok(Self::new(
+            futures::future::try_join_all(
+                listen
+                    .into_iter()
+                    .map(TcpListener::bind)
+                    .map(|f| f.map_ok(DynAccept::new)),
+            )
+            .await?,
+        ))
+    }
+}
+impl<K> Acceptor<BTreeMap<K, TcpListener>>
+where
+    K: Ord + Clone + Send + Sync + 'static,
+{
+    pub async fn bind_map(
+        listen: impl IntoIterator<Item = (K, SocketAddr)>,
+    ) -> Result<Self, Error> {
+        Ok(Self::new(
+            futures::future::try_join_all(listen.into_iter().map(|(key, addr)| async move {
+                Ok::<_, Error>((
+                    key,
+                    TcpListener::bind(addr)
+                        .await
+                        .with_kind(ErrorKind::Network)?,
+                ))
+            }))
+            .await?
+            .into_iter()
+            .collect(),
+        ))
+    }
+}
+impl<K> Acceptor<BTreeMap<K, DynAccept>>
+where
+    K: Ord + Clone + Send + Sync + 'static,
+{
+    pub async fn bind_map_dyn(
+        listen: impl IntoIterator<Item = (K, SocketAddr)>,
+    ) -> Result<Self, Error> {
+        Ok(Self::new(
+            futures::future::try_join_all(listen.into_iter().map(|(key, addr)| async move {
+                Ok::<_, Error>((
+                    key,
+                    TcpListener::bind(addr)
+                        .await
+                        .with_kind(ErrorKind::Network)?,
+                ))
+            }))
+            .await?
+            .into_iter()
+            .map(|(key, listener)| (key, listener.into_dyn()))
+            .collect(),
+        ))
     }
 }
 
 pub struct WebServerAcceptorSetter<A: Accept> {
     acceptor: Watch<A>,
 }
-impl<A: Accept, B: Accept> WebServerAcceptorSetter<Option<Either<A, B>>> {
+impl<A, B> WebServerAcceptorSetter<Option<Either<A, B>>>
+where
+    A: Accept,
+    B: Accept<Metadata = A::Metadata>,
+{
     pub fn try_upgrade<F: FnOnce(A) -> Result<B, Error>>(&self, f: F) -> Result<(), Error> {
         let mut res = Ok(());
         self.acceptor.send_modify(|a| {
@@ -151,20 +403,24 @@ impl<A: Accept> Deref for WebServerAcceptorSetter<A> {
 
 pub struct WebServer<A: Accept> {
     shutdown: oneshot::Sender<()>,
-    router: Watch<Option<Router>>,
+    router: Watch<Router>,
     acceptor: Watch<A>,
     thread: NonDetachingJoinHandle<()>,
 }
-impl<A: Accept + Send + Sync + 'static> WebServer<A> {
+impl<A> WebServer<A>
+where
+    A: Accept + Send + Sync + 'static,
+    for<'a> A::Metadata: Visit<ExtensionVisitor<'a>> + Send + Sync + 'static,
+{
     pub fn acceptor_setter(&self) -> WebServerAcceptorSetter<A> {
         WebServerAcceptorSetter {
             acceptor: self.acceptor.clone(),
         }
     }
 
-    pub fn new(mut acceptor: Acceptor<A>) -> Self {
+    pub fn new(mut acceptor: Acceptor<A>, router: Router) -> Self {
         let acceptor_send = acceptor.acceptor.clone();
-        let router = Watch::<Option<Router>>::new(None);
+        let router = Watch::new(router);
         let service = router.clone_unseen();
         let (shutdown, shutdown_recv) = oneshot::channel();
         let thread = NonDetachingJoinHandle::from(tokio::spawn(async move {
@@ -187,8 +443,14 @@ impl<A: Accept + Send + Sync + 'static> WebServer<A> {
                 }
             }
 
-            struct SwappableRouter(Watch<Option<Router>>, bool);
-            impl hyper::service::Service<hyper::Request<hyper::body::Incoming>> for SwappableRouter {
+            struct SwappableRouter<M> {
+                router: Watch<Router>,
+                metadata: M,
+            }
+            impl<M: for<'a> Visit<ExtensionVisitor<'a>> + Send + Sync + 'static>
+                hyper::service::Service<hyper::Request<hyper::body::Incoming>>
+                for SwappableRouter<M>
+            {
                 type Response = <Router as tower_service::Service<
                     hyper::Request<hyper::body::Incoming>,
                 >>::Response;
@@ -199,19 +461,13 @@ impl<A: Accept + Send + Sync + 'static> WebServer<A> {
                     hyper::Request<hyper::body::Incoming>,
                 >>::Future;
 
-                fn call(&self, req: hyper::Request<hyper::body::Incoming>) -> Self::Future {
+                fn call(&self, mut req: hyper::Request<hyper::body::Incoming>) -> Self::Future {
                     use tower_service::Service;
 
-                    if self.1 {
-                        redirecter().call(req)
-                    } else {
-                        let router = self.0.read();
-                        if let Some(mut router) = router {
-                            router.call(req)
-                        } else {
-                            refresher().call(req)
-                        }
-                    }
+                    self.metadata
+                        .visit(&mut ExtensionVisitor(req.extensions_mut()));
+
+                    self.router.read().call(req)
                 }
             }
 
@@ -238,16 +494,16 @@ impl<A: Accept + Send + Sync + 'static> WebServer<A> {
                     let mut err = None;
                     for _ in 0..5 {
                         if let Err(e) = async {
-                            let accepted = acceptor.accept().await?;
+                            let (metadata, stream) = acceptor.accept().await?;
                             queue.add_job(
                                 graceful.watch(
                                     server
                                         .serve_connection_with_upgrades(
-                                            TokioIo::new(accepted.stream),
-                                            SwappableRouter(
-                                                service.clone(),
-                                                accepted.https_redirect,
-                                            ),
+                                            TokioIo::new(stream),
+                                            SwappableRouter {
+                                                router: service.clone(),
+                                                metadata,
+                                            },
                                         )
                                         .into_owned(),
                                 ),
@@ -300,26 +556,10 @@ impl<A: Accept + Send + Sync + 'static> WebServer<A> {
     }
 
     pub fn serve_router(&mut self, router: Router) {
-        self.router.send(Some(router))
+        self.router.send(router)
     }
 
-    pub fn serve_main(&mut self, ctx: RpcContext) {
-        self.serve_router(main_ui_router(ctx))
-    }
-
-    pub fn serve_setup(&mut self, ctx: SetupContext) {
-        self.serve_router(setup_ui_router(ctx))
-    }
-
-    pub fn serve_diagnostic(&mut self, ctx: DiagnosticContext) {
-        self.serve_router(diagnostic_ui_router(ctx))
-    }
-
-    pub fn serve_install(&mut self, ctx: InstallContext) {
-        self.serve_router(install_ui_router(ctx))
-    }
-
-    pub fn serve_init(&mut self, ctx: InitContext) {
-        self.serve_router(init_ui_router(ctx))
+    pub fn serve_ui_for<C: UiContext>(&mut self, ctx: C) {
+        self.serve_router(ui_router(ctx))
     }
 }