Gateways, domains, and new service interface (#3001)

* add support for inbound proxies * backend changes * fix file type * proxy -> tunnel, implement backend apis * wip start-tunneld * add domains and gateways, remove routers, fix docs links * dont show hidden actions * show and test dns * edit instead of chnage acme and change gateway * refactor: domains page * refactor: gateways page * domains and acme refactor * certificate authorities * refactor public/private gateways * fix fe types * domains mostly finished * refactor: add file control to form service * add ip util to sdk * domains api + migration * start service interface page, WIP * different options for clearnet domains * refactor: styles for interfaces page * minor * better placeholder for no addresses * start sorting addresses * best address logic * comments * fix unnecessary export * MVP of service interface page * domains preferred * fix: address comments * only translations left * wip: start-tunnel & fix build * forms for adding domain, rework things based on new ideas * fix: dns testing * public domain, max width, descriptions for dns * nix StartOS domains, implement public and private domains at interface scope * restart tor instead of reset * better icon for restart tor * dns * fix sort functions for public and private domains * with todos * update types * clean up tech debt, bump dependencies * revert to ts-rs v9 * fix all types * fix dns form * add missing translations * it builds * fix: comments (#3009) * fix: comments * undo default --------- Co-authored-by: Matt Hill <mattnine@protonmail.com> * fix: refactor legacy components (#3010) * fix: comments * fix: refactor legacy components * remove default again --------- Co-authored-by: Matt Hill <mattnine@protonmail.com> * more translations * wip * fix deadlock * coukd work * simple renaming * placeholder for empty service interfaces table * honor hidden form values * remove logs * reason instead of description * fix dns * misc fixes * implement toggling gateways for service interface * fix showing dns records * move status column in service list * remove unnecessary truthy check * refactor: refactor forms components and remove legacy Taiga UI package (#3012) * handle wh file uploads * wip: debugging tor * socks5 proxy working * refactor: fix multiple comments (#3013) * refactor: fix multiple comments * styling changes, add documentation to sidebar * translations for dns page * refactor: subtle colors * rearrange service page --------- Co-authored-by: Matt Hill <mattnine@protonmail.com> * fix file_stream and remove non-terminating test * clean up logs * support for sccache * fix gha sccache * more marketplace translations * install wizard clarity * stub hostnameInfo in migration * fix address info after setup, fix styling on SI page, new 040 release notes * remove tor logs from os * misc fixes * reset tor still not functioning... * update ts * minor styling and wording * chore: some fixes (#3015) * fix gateway renames * different handling for public domains * styling fixes * whole navbar should not be clickable on service show page * timeout getState request * remove links from changelog * misc fixes from pairing * use custom name for gateway in more places * fix dns parsing * closes #3003 * closes #2999 * chore: some fixes (#3017) * small copy change * revert hardcoded error for testing * dont require port forward if gateway is public * use old wan ip when not available * fix .const hanging on undefined * fix test * fix doc test * fix renames * update deps * allow specifying dependency metadata directly * temporarily make dependencies not cliackable in marketplace listings * fix socks bind * fix test --------- Co-authored-by: Aiden McClelland <me@drbonez.dev> Co-authored-by: waterplea <alexander@inkin.ru>
2026-03-30 04:01:58 +00:00 · 2025-09-09 21:43:51 -06:00
parent 1cc9a1a30b
commit add01ebc68
537 changed files with 19940 additions and 20551 deletions
--- a/core/startos/src/registry/context.rs
+++ b/core/startos/src/registry/context.rs
@@ -3,26 +3,33 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;

+use chrono::Utc;
 use clap::Parser;
 use imbl_value::InternedString;
 use patch_db::PatchDb;
 use reqwest::{Client, Proxy};
 use rpc_toolkit::yajrc::RpcError;
-use rpc_toolkit::{CallRemote, Context, Empty};
+use rpc_toolkit::{CallRemote, Context, Empty, RpcRequest};
 use serde::{Deserialize, Serialize};
 use sqlx::PgPool;
 use tokio::sync::broadcast::Sender;
 use tracing::instrument;
+use ts_rs::TS;
 use url::Url;

-use crate::context::config::{ContextConfig, CONFIG_PATH};
+use crate::context::config::{CONFIG_PATH, ContextConfig};
 use crate::context::{CliContext, RpcContext};
+use crate::middleware::signature::SignatureAuthContext;
 use crate::prelude::*;
-use crate::registry::auth::{SignatureHeader, AUTH_SIG_HEADER};
-use crate::registry::device_info::{DeviceInfo, DEVICE_INFO_HEADER};
-use crate::registry::signer::sign::AnySigningKey;
 use crate::registry::RegistryDatabase;
+use crate::registry::device_info::{DEVICE_INFO_HEADER, DeviceInfo};
+use crate::registry::signer::SignerInfo;
 use crate::rpc_continuations::RpcContinuations;
+use crate::sign::AnyVerifyingKey;
+use crate::util::io::append_file;
+
+const DEFAULT_REGISTRY_LISTEN: SocketAddr =
+    SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 5959);

 #[derive(Debug, Clone, Default, Deserialize, Serialize, Parser)]
 #[serde(rename_all = "kebab-case")]
@@ -31,9 +38,9 @@ pub struct RegistryConfig {
    #[arg(short = 'c', long = "config")]
    pub config: Option<PathBuf>,
    #[arg(short = 'l', long = "listen")]
-    pub listen: Option<SocketAddr>,
-    #[arg(short = 'h', long = "hostname")]
-    pub hostname: Option<InternedString>,
+    pub registry_listen: Option<SocketAddr>,
+    #[arg(short = 'H', long = "hostname")]
+    pub registry_hostname: Vec<InternedString>,
    #[arg(short = 'p', long = "tor-proxy")]
    pub tor_proxy: Option<Url>,
    #[arg(short = 'd', long = "datadir")]
@@ -45,9 +52,9 @@ impl ContextConfig for RegistryConfig {
    fn next(&mut self) -> Option<PathBuf> {
        self.config.take()
    }
-    fn merge_with(&mut self, other: Self) {
-        self.listen = self.listen.take().or(other.listen);
-        self.hostname = self.hostname.take().or(other.hostname);
+    fn merge_with(&mut self, mut other: Self) {
+        self.registry_listen = self.registry_listen.take().or(other.registry_listen);
+        self.registry_hostname.append(&mut other.registry_hostname);
        self.tor_proxy = self.tor_proxy.take().or(other.tor_proxy);
        self.datadir = self.datadir.take().or(other.datadir);
    }
@@ -63,7 +70,7 @@ impl RegistryConfig {
 }

 pub struct RegistryContextSeed {
-    pub hostname: InternedString,
+    pub hostnames: Vec<InternedString>,
    pub listen: SocketAddr,
    pub db: TypedPatchDb<RegistryDatabase>,
    pub datadir: PathBuf,
@@ -105,20 +112,15 @@ impl RegistryContext {
            },
            None => None,
        };
+        if config.registry_hostname.is_empty() {
+            return Err(Error::new(
+                eyre!("missing required configuration: registry-hostname"),
+                ErrorKind::NotFound,
+            ));
+        }
        Ok(Self(Arc::new(RegistryContextSeed {
-            hostname: config
-                .hostname
-                .as_ref()
-                .ok_or_else(|| {
-                    Error::new(
-                        eyre!("missing required configuration: hostname"),
-                        ErrorKind::NotFound,
-                    )
-                })?
-                .clone(),
-            listen: config
-                .listen
-                .unwrap_or(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 5959)),
+            hostnames: config.registry_hostname.clone(),
+            listen: config.registry_listen.unwrap_or(DEFAULT_REGISTRY_LISTEN),
            db,
            datadir,
            rpc_continuations: RpcContinuations::new(),
@@ -163,64 +165,28 @@ impl CallRemote<RegistryContext> for CliContext {
        params: Value,
        _: Empty,
    ) -> Result<Value, RpcError> {
-        use reqwest::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
-        use reqwest::Method;
-        use rpc_toolkit::yajrc::{GenericRpcMethod, Id, RpcRequest};
-        use rpc_toolkit::RpcResponse;
-
-        let url = self
-            .registry_url
-            .clone()
-            .ok_or_else(|| Error::new(eyre!("`--registry` required"), ErrorKind::InvalidRequest))?;
-        method = method.strip_prefix("registry.").unwrap_or(method);
-
-        let rpc_req = RpcRequest {
-            id: Some(Id::Number(0.into())),
-            method: GenericRpcMethod::<_, _, Value>::new(method),
-            params,
-        };
-        let body = serde_json::to_vec(&rpc_req)?;
-        let host = url.host().or_not_found("registry hostname")?.to_string();
-        let mut req = self
-            .client
-            .request(Method::POST, url)
-            .header(CONTENT_TYPE, "application/json")
-            .header(ACCEPT, "application/json")
-            .header(CONTENT_LENGTH, body.len());
-        if let Ok(key) = self.developer_key() {
-            req = req.header(
-                AUTH_SIG_HEADER,
-                SignatureHeader::sign(&AnySigningKey::Ed25519(key.clone()), &body, &host)?
-                    .to_header(),
+        let url = if let Some(url) = self.registry_url.clone() {
+            url
+        } else if self.registry_hostname.is_some() {
+            format!(
+                "http://{}",
+                self.registry_listen.unwrap_or(DEFAULT_REGISTRY_LISTEN)
+            )
+            .parse()
+            .map_err(Error::from)?
+        } else {
+            return Err(
+                Error::new(eyre!("`--registry` required"), ErrorKind::InvalidRequest).into(),
            );
-        }
-        let res = req.body(body).send().await?;
+        };
+        method = method.strip_prefix("registry.").unwrap_or(method);
+        let sig_context = self
+            .registry_hostname
+            .clone()
+            .or(url.host().as_ref().map(InternedString::from_display))
+            .or_not_found("registry hostname")?;

-        if !res.status().is_success() {
-            let status = res.status();
-            let txt = res.text().await?;
-            let mut res = Err(Error::new(
-                eyre!("{}", status.canonical_reason().unwrap_or(status.as_str())),
-                ErrorKind::Network,
-            ));
-            if !txt.is_empty() {
-                res = res.with_ctx(|_| (ErrorKind::Network, txt));
-            }
-            return res.map_err(From::from);
-        }
-
-        match res
-            .headers()
-            .get(CONTENT_TYPE)
-            .and_then(|v| v.to_str().ok())
-        {
-            Some("application/json") => {
-                serde_json::from_slice::<RpcResponse>(&*res.bytes().await?)
-                    .with_kind(ErrorKind::Deserialization)?
-                    .result
-            }
-            _ => Err(Error::new(eyre!("unknown content type"), ErrorKind::Network).into()),
-        }
+        crate::middleware::signature::call_remote(self, url, &sig_context, method, params).await
    }
 }

@@ -231,10 +197,10 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
        params: Value,
        RegistryUrlParams { registry }: RegistryUrlParams,
    ) -> Result<Value, RpcError> {
-        use reqwest::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
        use reqwest::Method;
-        use rpc_toolkit::yajrc::{GenericRpcMethod, Id, RpcRequest};
+        use reqwest::header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE};
        use rpc_toolkit::RpcResponse;
+        use rpc_toolkit::yajrc::{GenericRpcMethod, Id, RpcRequest};

        let url = registry.join("rpc/v0")?;
        method = method.strip_prefix("registry.").unwrap_or(method);
@@ -286,3 +252,72 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
        }
    }
 }
+
+#[derive(Deserialize)]
+pub struct RegistryAuthMetadata {
+    #[serde(default)]
+    admin: bool,
+}
+
+#[derive(Serialize, Deserialize, TS)]
+pub struct AdminLogRecord {
+    pub timestamp: String,
+    pub name: String,
+    #[ts(type = "{ id: string | number | null; method: string; params: any }")]
+    pub request: RpcRequest,
+    pub key: AnyVerifyingKey,
+}
+
+impl SignatureAuthContext for RegistryContext {
+    type Database = RegistryDatabase;
+    type AdditionalMetadata = RegistryAuthMetadata;
+    type CheckPubkeyRes = Option<(AnyVerifyingKey, SignerInfo)>;
+    fn db(&self) -> &TypedPatchDb<Self::Database> {
+        &self.db
+    }
+    async fn sig_context(
+        &self,
+    ) -> impl IntoIterator<Item = Result<impl AsRef<str> + Send, Error>> + Send {
+        self.hostnames.iter().map(Ok)
+    }
+    fn check_pubkey(
+        db: &Model<Self::Database>,
+        pubkey: Option<&AnyVerifyingKey>,
+        metadata: Self::AdditionalMetadata,
+    ) -> Result<Self::CheckPubkeyRes, Error> {
+        if metadata.admin {
+            if let Some(pubkey) = pubkey {
+                let (guid, admin) = db.as_index().as_signers().get_signer_info(pubkey)?;
+                if db.as_admins().de()?.contains(&guid) {
+                    return Ok(Some((pubkey.clone(), admin)));
+                }
+            }
+            Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
+        } else {
+            Ok(None)
+        }
+    }
+    async fn post_auth_hook(
+        &self,
+        check_pubkey_res: Self::CheckPubkeyRes,
+        request: &RpcRequest,
+    ) -> Result<(), Error> {
+        use tokio::io::AsyncWriteExt;
+        if let Some((pubkey, admin)) = check_pubkey_res {
+            let mut log = append_file(self.datadir.join("admin.log")).await?;
+            log.write_all(
+                (serde_json::to_string(&AdminLogRecord {
+                    timestamp: Utc::now().to_rfc3339(),
+                    name: admin.name,
+                    request: request.clone(),
+                    key: pubkey,
+                })
+                .with_kind(ErrorKind::Serialization)?
+                    + "\n")
+                    .as_bytes(),
+            )
+            .await?;
+        }
+        Ok(())
+    }
+}