From 022f7134be3d75d7a03dfeb1fd8e148206120a57 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Sat, 9 Aug 2025 21:57:32 -0600 Subject: [PATCH] wip: start-tunnel & fix build --- .../Systems/SystemForEmbassy/index.ts | 12 +- core/Cargo.lock | 11 - core/startos/Cargo.toml | 5 +- core/startos/src/bins/mod.rs | 4 +- core/startos/src/bins/tunnel.rs | 2 +- core/startos/src/db/model/public.rs | 2 +- core/startos/src/tunnel/api.rs | 130 +++++++++++ core/startos/src/tunnel/client.conf.template | 10 + core/startos/src/tunnel/db.rs | 18 +- core/startos/src/tunnel/forward.rs | 1 + core/startos/src/tunnel/init.rs | 0 core/startos/src/tunnel/mod.rs | 22 +- .../src/tunnel/server-peer.conf.template | 4 + core/startos/src/tunnel/server.conf.template | 5 + core/startos/src/tunnel/wg.rs | 220 ++++++++++++++++++ core/startos/src/util/serde.rs | 5 + sdk/package/lib/backup/Backups.ts | 2 +- 17 files changed, 401 insertions(+), 52 deletions(-) create mode 100644 core/startos/src/tunnel/api.rs create mode 100644 core/startos/src/tunnel/client.conf.template create mode 100644 core/startos/src/tunnel/forward.rs delete mode 100644 core/startos/src/tunnel/init.rs create mode 100644 core/startos/src/tunnel/server-peer.conf.template create mode 100644 core/startos/src/tunnel/server.conf.template create mode 100644 core/startos/src/tunnel/wg.rs diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index a33a03058..7a2de376b 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -1238,14 +1238,14 @@ async function updateConfig( const url: string = filled === null || filled.addressInfo === null ? "" - : catchFn(() => - utils.hostnameInfoToAddress( - specValue.target === "lan-address" + : catchFn( + () => + (specValue.target === "lan-address" ? filled.addressInfo!.localHostnames[0] || - filled.addressInfo!.onionHostnames[0] + filled.addressInfo!.onionHostnames[0] : filled.addressInfo!.onionHostnames[0] || - filled.addressInfo!.localHostnames[0], - ), + filled.addressInfo!.localHostnames[0] + ).hostname.value, ) || "" mutConfigValue[key] = url } diff --git a/core/Cargo.lock b/core/Cargo.lock index 6d4d6e719..71063b3da 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -3136,16 +3136,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iprange" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37209be0ad225457e63814401415e748e2453a5297f9b637338f5fb8afa4ec00" -dependencies = [ - "ipnet", - "serde", -] - [[package]] name = "iri-string" version = "0.7.8" @@ -6136,7 +6126,6 @@ dependencies = [ "indicatif", "integer-encoding", "ipnet", - "iprange", "isocountry", "itertools 0.14.0", "jaq-core", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 752fc69ae..7cf4c55f7 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -90,7 +90,7 @@ der = { version = "0.7.9", features = ["derive", "pem"] } digest = "0.10.7" divrem = "1.0.0" ed25519 = { version = "2.2.3", features = ["pkcs8", "pem", "alloc"] } -ed25519-dalek = { version = "2.1.1", features = [ +ed25519-dalek = { version = "2.2.0", features = [ "serde", "zeroize", "rand_core", @@ -134,7 +134,6 @@ indexmap = { version = "2.0.2", features = ["serde"] } indicatif = { version = "0.17.7", features = ["tokio"] } integer-encoding = { version = "4.0.0", features = ["tokio_async"] } ipnet = { version = "2.8.0", features = ["serde"] } -iprange = { version = "0.6.7", features = ["serde"] } isocountry = "0.3.2" itertools = "0.14.0" jaq-core = "0.10.1" @@ -179,7 +178,7 @@ proptest = "1.3.1" proptest-derive = "0.5.0" pty-process = { version = "0.5.1", optional = true } qrcode = "0.14.1" -rand = "0.9.0" +rand = "0.9.2" regex = "1.10.2" reqwest = { version = "0.12.4", features = ["stream", "json", "socks"] } reqwest_cookie_store = "0.8.0" diff --git a/core/startos/src/bins/mod.rs b/core/startos/src/bins/mod.rs index 7230d04d5..2308e06d3 100644 --- a/core/startos/src/bins/mod.rs +++ b/core/startos/src/bins/mod.rs @@ -5,7 +5,7 @@ use std::path::Path; #[cfg(feature = "cli-container")] pub mod container_cli; pub mod deprecated; -#[cfg(feature = "registry")] +#[cfg(any(feature = "registry", feature = "cli-registry"))] pub mod registry; #[cfg(feature = "cli")] pub mod start_cli; @@ -13,7 +13,7 @@ pub mod start_cli; pub mod start_init; #[cfg(feature = "startd")] pub mod startd; -#[cfg(feature = "tunnel")] +#[cfg(any(feature = "tunnel", feature = "cli-tunnel"))] pub mod tunnel; fn select_executable(name: &str) -> Option)> { diff --git a/core/startos/src/bins/tunnel.rs b/core/startos/src/bins/tunnel.rs index bf80848a5..6aa0003af 100644 --- a/core/startos/src/bins/tunnel.rs +++ b/core/startos/src/bins/tunnel.rs @@ -94,7 +94,7 @@ pub fn cli(args: impl IntoIterator) { if let Err(e) = CliApp::new( |cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?), - crate::tunnel::tunnel_api(), + crate::tunnel::api::tunnel_api(), ) .run(args) { diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index 7ff339c0a..3e1f9c2cf 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -250,7 +250,7 @@ impl NetworkInterfaceInfo { if !ip4s.is_empty() { return ip4s.iter().all(|ip4| { ip4.is_loopback() - || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations + // || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations || ip4.is_link_local() }); } diff --git a/core/startos/src/tunnel/api.rs b/core/startos/src/tunnel/api.rs new file mode 100644 index 000000000..7ce208806 --- /dev/null +++ b/core/startos/src/tunnel/api.rs @@ -0,0 +1,130 @@ +use std::net::Ipv4Addr; + +use clap::Parser; +use ipnet::Ipv4Net; +use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler}; +use serde::{Deserialize, Serialize}; + +use crate::context::CliContext; +use crate::prelude::*; +use crate::tunnel::context::TunnelContext; +use crate::tunnel::wg::WgSubnetConfig; + +pub fn tunnel_api() -> ParentHandler { + ParentHandler::new() + .subcommand( + "db", + super::db::db_api::() + .with_about("Commands to interact with the db i.e. dump and apply"), + ) + .subcommand( + "subnet", + subnet_api::().with_about("Add, remove, or modify subnets"), + ) + // .subcommand( + // "forward", + // ParentHandler::::new() + // .subcommand( + // "add", + // from_fn_async(add_forward) + // .with_metadata("sync_db", Value::Bool(true)) + // .no_display() + // .with_about("Add a new port forward") + // .with_call_remote::(), + // ) + // .subcommand( + // "remove", + // from_fn_async(remove_forward) + // .with_metadata("sync_db", Value::Bool(true)) + // .no_display() + // .with_about("Remove a port forward") + // .with_call_remote::(), + // ), + // ) +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct SubnetParams { + subnet: Ipv4Net, +} + +pub fn subnet_api() -> ParentHandler { + ParentHandler::new() + .subcommand( + "add", + from_fn_async(add_subnet) + .with_metadata("sync_db", Value::Bool(true)) + .with_inherited(|a, _| a) + .no_display() + .with_about("Add a new subnet") + .with_call_remote::(), + ) + .subcommand( + "remove", + from_fn_async(remove_subnet) + .with_metadata("sync_db", Value::Bool(true)) + .with_inherited(|a, _| a) + .no_display() + .with_about("Remove a subnet") + .with_call_remote::(), + ) + // .subcommand( + // "set-default-forward-target", + // from_fn_async(set_default_forward_target) + // .with_metadata("sync_db", Value::Bool(true)) + // .no_display() + // .with_about("Set the default target for port forwarding") + // .with_call_remote::(), + // ) + // .subcommand( + // "add-client", + // from_fn_async(add_client) + // .with_metadata("sync_db", Value::Bool(true)) + // .no_display() + // .with_about("Add a client to a subnet") + // .with_call_remote::(), + // ) + // .subcommand( + // "remove-client", + // from_fn_async(remove_client) + // .with_metadata("sync_db", Value::Bool(true)) + // .no_display() + // .with_about("Remove a client from a subnet") + // .with_call_remote::(), + // ) +} + +pub async fn add_subnet( + ctx: TunnelContext, + _: Empty, + SubnetParams { subnet }: SubnetParams, +) -> Result<(), Error> { + let server = ctx + .db + .mutate(|db| { + let map = db.as_wg_mut().as_subnets_mut(); + if !map.contains_key(&subnet)? { + map.insert(&subnet, &WgSubnetConfig::new())?; + } + db.as_wg().de() + }) + .await + .result?; + server.sync().await +} + +pub async fn remove_subnet( + ctx: TunnelContext, + _: Empty, + SubnetParams { subnet }: SubnetParams, +) -> Result<(), Error> { + let server = ctx + .db + .mutate(|db| { + db.as_wg_mut().as_subnets_mut().remove(&subnet)?; + db.as_wg().de() + }) + .await + .result?; + server.sync().await +} diff --git a/core/startos/src/tunnel/client.conf.template b/core/startos/src/tunnel/client.conf.template new file mode 100644 index 000000000..2c7735eb9 --- /dev/null +++ b/core/startos/src/tunnel/client.conf.template @@ -0,0 +1,10 @@ +[Interface] +Address = {addr}/24 +PrivateKey = {privkey} + +[Peer] +PublicKey = {server_pubkey} +PresharedKey = {psk} +AllowedIPs = 0.0.0.0/0, ::/0 +Endpoint = {server_addr} +PersistentKeepalive = 25 \ No newline at end of file diff --git a/core/startos/src/tunnel/db.rs b/core/startos/src/tunnel/db.rs index 33e01d54b..a7f77af57 100644 --- a/core/startos/src/tunnel/db.rs +++ b/core/startos/src/tunnel/db.rs @@ -1,8 +1,10 @@ use std::collections::{BTreeMap, HashSet}; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::{Ipv4Addr, SocketAddrV4}; use std::path::PathBuf; use clap::Parser; +use imbl_value::InternedString; +use ipnet::Ipv4Net; use itertools::Itertools; use patch_db::json_ptr::{JsonPointer, ROOT}; use patch_db::Dump; @@ -17,24 +19,18 @@ use crate::context::CliContext; use crate::prelude::*; use crate::sign::AnyVerifyingKey; use crate::tunnel::context::TunnelContext; +use crate::tunnel::wg::WgServer; use crate::util::serde::{apply_expr, HandlerExtSerde}; -#[derive(Debug, Default, Deserialize, Serialize, HasModel)] +#[derive(Default, Deserialize, Serialize, HasModel)] #[serde(rename_all = "camelCase")] #[model = "Model"] pub struct TunnelDatabase { pub sessions: Sessions, pub password: String, pub auth_pubkeys: HashSet, - pub clients: BTreeMap, - pub port_forwards: BTreeMap, -} - -#[derive(Debug, Default, Deserialize, Serialize, HasModel)] -#[serde(rename_all = "camelCase")] -#[model = "Model"] -pub struct ClientInfo { - pub server: bool, + pub wg: WgServer, + pub port_forwards: BTreeMap, } pub fn db_api() -> ParentHandler { diff --git a/core/startos/src/tunnel/forward.rs b/core/startos/src/tunnel/forward.rs new file mode 100644 index 000000000..3616db216 --- /dev/null +++ b/core/startos/src/tunnel/forward.rs @@ -0,0 +1 @@ +use crate::prelude::*; diff --git a/core/startos/src/tunnel/init.rs b/core/startos/src/tunnel/init.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/startos/src/tunnel/mod.rs b/core/startos/src/tunnel/mod.rs index 2a24a56ee..acd4836ed 100644 --- a/core/startos/src/tunnel/mod.rs +++ b/core/startos/src/tunnel/mod.rs @@ -1,34 +1,24 @@ -use std::collections::{BTreeMap, HashSet}; -use std::net::{Ipv4Addr, SocketAddr}; - use axum::Router; use futures::future::ready; -use rpc_toolkit::{Context, HandlerExt, ParentHandler, Server}; -use serde::{Deserialize, Serialize}; +use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server}; -use crate::auth::Sessions; +use crate::context::CliContext; use crate::middleware::auth::Auth; use crate::middleware::cors::Cors; use crate::net::static_server::{bad_request, not_found, server_error}; use crate::net::web_server::{Accept, WebServer}; use crate::prelude::*; use crate::rpc_continuations::Guid; -use crate::sign::AnyVerifyingKey; use crate::tunnel::context::TunnelContext; +pub mod api; pub mod context; pub mod db; -pub mod init; +pub mod forward; +pub mod wg; pub const TUNNEL_DEFAULT_PORT: u16 = 5960; -pub fn tunnel_api() -> ParentHandler { - ParentHandler::new().subcommand( - "db", - db::db_api::().with_about("Commands to interact with the db i.e. dump and apply"), - ) -} - pub fn tunnel_router(ctx: TunnelContext) -> Router { use axum::extract as x; use axum::routing::{any, get}; @@ -36,7 +26,7 @@ pub fn tunnel_router(ctx: TunnelContext) -> Router { .route("/rpc/{*path}", { let ctx = ctx.clone(); any( - Server::new(move || ready(Ok(ctx.clone())), tunnel_api()) + Server::new(move || ready(Ok(ctx.clone())), api::tunnel_api()) .middleware(Cors::new()) .middleware(Auth::new()) ) diff --git a/core/startos/src/tunnel/server-peer.conf.template b/core/startos/src/tunnel/server-peer.conf.template new file mode 100644 index 000000000..387b5171f --- /dev/null +++ b/core/startos/src/tunnel/server-peer.conf.template @@ -0,0 +1,4 @@ +[Peer] +PublicKey = {pubkey} +PresharedKey = {psk} +AllowedIPs = {addr}/32 diff --git a/core/startos/src/tunnel/server.conf.template b/core/startos/src/tunnel/server.conf.template new file mode 100644 index 000000000..52b56139c --- /dev/null +++ b/core/startos/src/tunnel/server.conf.template @@ -0,0 +1,5 @@ +[Interface] +Address = {subnets} +PrivateKey = {server_privkey} +ListenPort = {server_port} + diff --git a/core/startos/src/tunnel/wg.rs b/core/startos/src/tunnel/wg.rs new file mode 100644 index 000000000..29fd427ad --- /dev/null +++ b/core/startos/src/tunnel/wg.rs @@ -0,0 +1,220 @@ +use std::collections::BTreeMap; +use std::net::{Ipv4Addr, SocketAddrV4}; + +use ed25519_dalek::{SigningKey, VerifyingKey}; +use imbl_value::InternedString; +use ipnet::Ipv4Net; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; + +use crate::prelude::*; +use crate::util::io::write_file_atomic; +use crate::util::serde::Base64; +use crate::util::Invoke; + +#[derive(Deserialize, Serialize, HasModel)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +pub struct WgServer { + pub port: u16, + pub key: Base64, + pub subnets: WgSubnetMap, +} +impl Default for WgServer { + fn default() -> Self { + Self { + port: 51820, + key: Base64(WgKey::generate()), + subnets: WgSubnetMap::default(), + } + } +} +impl WgServer { + pub fn server_config<'a>(&'a self) -> ServerConfig<'a> { + ServerConfig(self) + } + pub async fn sync(&self) -> Result<(), Error> { + Command::new("wg-quick") + .arg("down") + .arg("wg0") + .invoke(ErrorKind::Network) + .await + .or_else(|e| { + let msg = e.source.to_string(); + if msg.contains("does not exist") || msg.contains("is not a WireGuard interface") { + Ok(Vec::new()) + } else { + Err(e) + } + })?; + write_file_atomic( + "/etc/wireguard/wg0.conf", + self.server_config().to_string().as_bytes(), + ) + .await?; + Command::new("wg-quick") + .arg("up") + .arg("wg0") + .invoke(ErrorKind::Network) + .await?; + Ok(()) + } +} + +#[derive(Default, Deserialize, Serialize)] +pub struct WgSubnetMap(pub BTreeMap); +impl Map for WgSubnetMap { + type Key = Ipv4Net; + type Value = WgSubnetConfig; + fn key_str(key: &Self::Key) -> Result, Error> { + Self::key_string(key) + } + fn key_string(key: &Self::Key) -> Result { + Ok(InternedString::from_display(key)) + } +} + +#[derive(Default, Deserialize, Serialize, HasModel)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +pub struct WgSubnetConfig { + pub default_forward_target: Option, + pub clients: BTreeMap, +} +impl WgSubnetConfig { + pub fn new() -> Self { + Self::default() + } + pub fn add_client<'a>( + &'a mut self, + subnet: Ipv4Net, + ) -> Result<(Ipv4Addr, &'a WgConfig), Error> { + let addr = subnet + .hosts() + .find(|a| !self.clients.contains_key(a)) + .ok_or_else(|| Error::new(eyre!("subnet exhausted"), ErrorKind::Network))?; + let config = self.clients.entry(addr).or_insert(WgConfig::generate()); + Ok((addr, config)) + } +} + +pub struct WgKey(SigningKey); +impl WgKey { + pub fn generate() -> Self { + Self(SigningKey::generate( + &mut ssh_key::rand_core::OsRng::default(), + )) + } +} +impl AsRef<[u8]> for WgKey { + fn as_ref(&self) -> &[u8] { + self.0.as_bytes() + } +} +impl TryFrom> for WgKey { + type Error = ed25519_dalek::SignatureError; + fn try_from(value: Vec) -> Result { + Ok(Self(value.as_slice().try_into()?)) + } +} +impl std::ops::Deref for WgKey { + type Target = SigningKey; + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl Base64 { + pub fn verifying_key(&self) -> Base64 { + Base64(self.0.verifying_key()) + } +} + +#[derive(Deserialize, Serialize, HasModel)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +pub struct WgConfig { + pub key: Base64, + pub psk: Base64<[u8; 32]>, +} +impl WgConfig { + pub fn generate() -> Self { + Self { + key: Base64(WgKey::generate()), + psk: Base64(rand::random()), + } + } + pub fn server_peer_config<'a>(&'a self, addr: Ipv4Addr) -> ServerPeerConfig<'a> { + ServerPeerConfig { + client_config: self, + client_addr: addr, + } + } + pub fn client_config<'a>( + &'a self, + addr: Ipv4Addr, + server_pubkey: Base64, + server_addr: SocketAddrV4, + ) -> ClientConfig<'a> { + ClientConfig { + client_config: self, + client_addr: addr, + server_pubkey, + server_addr, + } + } +} + +pub struct ServerPeerConfig<'a> { + client_config: &'a WgConfig, + client_addr: Ipv4Addr, +} +impl<'a> std::fmt::Display for ServerPeerConfig<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + include_str!("./server-peer.conf.template"), + pubkey = self.client_config.key.verifying_key().to_padded_string(), + psk = self.client_config.psk.to_padded_string(), + addr = self.client_addr, + ) + } +} + +pub struct ClientConfig<'a> { + client_config: &'a WgConfig, + client_addr: Ipv4Addr, + server_pubkey: Base64, + server_addr: SocketAddrV4, +} +impl<'a> std::fmt::Display for ClientConfig<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + include_str!("./client.conf.template"), + privkey = self.client_config.key.to_padded_string(), + psk = self.client_config.psk, + addr = self.client_addr, + server_pubkey = self.server_pubkey.to_padded_string(), + server_addr = self.server_addr, + ) + } +} + +pub struct ServerConfig<'a>(&'a WgServer); +impl<'a> std::fmt::Display for ServerConfig<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self(server) = *self; + write!( + f, + include_str!("./server.conf.template"), + subnets = server.subnets.0.keys().join(", "), + server_port = server.port, + server_privkey = server.key.to_padded_string(), + )?; + for (addr, peer) in server.subnets.0.values().flat_map(|s| &s.clients) { + write!(f, "{}", peer.server_peer_config(*addr))?; + } + Ok(()) + } +} diff --git a/core/startos/src/util/serde.rs b/core/startos/src/util/serde.rs index ec0763bdb..2115bf153 100644 --- a/core/startos/src/util/serde.rs +++ b/core/startos/src/util/serde.rs @@ -1023,6 +1023,11 @@ pub const BASE64: base64::engine::GeneralPurpose = #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, TS)] #[ts(type = "string", concrete(T = Vec))] pub struct Base64(pub T); +impl> Base64 { + pub fn to_padded_string(&self) -> String { + base64::engine::general_purpose::STANDARD.encode(self.0.as_ref()) + } +} impl> std::fmt::Display for Base64 { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&BASE64.encode(self.0.as_ref())) diff --git a/sdk/package/lib/backup/Backups.ts b/sdk/package/lib/backup/Backups.ts index 6784ad51c..b7348e01d 100644 --- a/sdk/package/lib/backup/Backups.ts +++ b/sdk/package/lib/backup/Backups.ts @@ -197,7 +197,7 @@ async function runRsync(rsyncOptions: { for (const exclude of options.exclude) { args.push(`--exclude=${exclude}`) } - args.push("-actAXH") + args.push("-rlptgocAXH") args.push("--info=progress2") args.push("--no-inc-recursive") args.push(srcPath)