wip: start-tunnel & fix build

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -1238,14 +1238,14 @@ async function updateConfig(
         const url: string =
           filled === null || filled.addressInfo === null
             ? ""
-            : catchFn(() =>
-                utils.hostnameInfoToAddress(
-                  specValue.target === "lan-address"
+            : catchFn(
+                () =>
+                  (specValue.target === "lan-address"
                     ? filled.addressInfo!.localHostnames[0] ||
-                        filled.addressInfo!.onionHostnames[0]
+                      filled.addressInfo!.onionHostnames[0]
                     : filled.addressInfo!.onionHostnames[0] ||
-                        filled.addressInfo!.localHostnames[0],
-                ),
+                      filled.addressInfo!.localHostnames[0]
+                  ).hostname.value,
               ) || ""
         mutConfigValue[key] = url
       }

core/Cargo.lock

@@ -3136,16 +3136,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "iprange"
-version = "0.6.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37209be0ad225457e63814401415e748e2453a5297f9b637338f5fb8afa4ec00"
-dependencies = [
- "ipnet",
- "serde",
-]
-
 [[package]]
 name = "iri-string"
 version = "0.7.8"
@@ -6136,7 +6126,6 @@ dependencies = [
  "indicatif",
  "integer-encoding",
  "ipnet",
- "iprange",
  "isocountry",
  "itertools 0.14.0",
  "jaq-core",

core/startos/Cargo.toml

@@ -90,7 +90,7 @@ der = { version = "0.7.9", features = ["derive", "pem"] }
 digest = "0.10.7"
 divrem = "1.0.0"
 ed25519 = { version = "2.2.3", features = ["pkcs8", "pem", "alloc"] }
-ed25519-dalek = { version = "2.1.1", features = [
+ed25519-dalek = { version = "2.2.0", features = [
   "serde",
   "zeroize",
   "rand_core",
@@ -134,7 +134,6 @@ indexmap = { version = "2.0.2", features = ["serde"] }
 indicatif = { version = "0.17.7", features = ["tokio"] }
 integer-encoding = { version = "4.0.0", features = ["tokio_async"] }
 ipnet = { version = "2.8.0", features = ["serde"] }
-iprange = { version = "0.6.7", features = ["serde"] }
 isocountry = "0.3.2"
 itertools = "0.14.0"
 jaq-core = "0.10.1"
@@ -179,7 +178,7 @@ proptest = "1.3.1"
 proptest-derive = "0.5.0"
 pty-process = { version = "0.5.1", optional = true }
 qrcode = "0.14.1"
-rand = "0.9.0"
+rand = "0.9.2"
 regex = "1.10.2"
 reqwest = { version = "0.12.4", features = ["stream", "json", "socks"] }
 reqwest_cookie_store = "0.8.0"

core/startos/src/bins/mod.rs

@@ -5,7 +5,7 @@ use std::path::Path;
 #[cfg(feature = "cli-container")]
 pub mod container_cli;
 pub mod deprecated;
-#[cfg(feature = "registry")]
+#[cfg(any(feature = "registry", feature = "cli-registry"))]
 pub mod registry;
 #[cfg(feature = "cli")]
 pub mod start_cli;
@@ -13,7 +13,7 @@ pub mod start_cli;
 pub mod start_init;
 #[cfg(feature = "startd")]
 pub mod startd;
-#[cfg(feature = "tunnel")]
+#[cfg(any(feature = "tunnel", feature = "cli-tunnel"))]
 pub mod tunnel;
 
 fn select_executable(name: &str) -> Option<fn(VecDeque<OsString>)> {

core/startos/src/bins/tunnel.rs

@@ -94,7 +94,7 @@ pub fn cli(args: impl IntoIterator<Item = OsString>) {
 
     if let Err(e) = CliApp::new(
         |cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?),
-        crate::tunnel::tunnel_api(),
+        crate::tunnel::api::tunnel_api(),
     )
     .run(args)
     {

core/startos/src/db/model/public.rs

@@ -250,7 +250,7 @@ impl NetworkInterfaceInfo {
                 if !ip4s.is_empty() {
                     return ip4s.iter().all(|ip4| {
                         ip4.is_loopback()
-                    || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations
+                    // || (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations
                     || ip4.is_link_local()
                     });
                 }

core/startos/src/tunnel/api.rs

@@ -0,0 +1,130 @@
+use std::net::Ipv4Addr;
+
+use clap::Parser;
+use ipnet::Ipv4Net;
+use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
+use serde::{Deserialize, Serialize};
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::tunnel::context::TunnelContext;
+use crate::tunnel::wg::WgSubnetConfig;
+
+pub fn tunnel_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "db",
+            super::db::db_api::<C>()
+                .with_about("Commands to interact with the db i.e. dump and apply"),
+        )
+        .subcommand(
+            "subnet",
+            subnet_api::<C>().with_about("Add, remove, or modify subnets"),
+        )
+    // .subcommand(
+    //     "forward",
+    //     ParentHandler::<C>::new()
+    //         .subcommand(
+    //             "add",
+    //             from_fn_async(add_forward)
+    //                 .with_metadata("sync_db", Value::Bool(true))
+    //                 .no_display()
+    //                 .with_about("Add a new port forward")
+    //                 .with_call_remote::<CliContext>(),
+    //         )
+    //         .subcommand(
+    //             "remove",
+    //             from_fn_async(remove_forward)
+    //                 .with_metadata("sync_db", Value::Bool(true))
+    //                 .no_display()
+    //                 .with_about("Remove a port forward")
+    //                 .with_call_remote::<CliContext>(),
+    //         ),
+    // )
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct SubnetParams {
+    subnet: Ipv4Net,
+}
+
+pub fn subnet_api<C: Context>() -> ParentHandler<C, SubnetParams> {
+    ParentHandler::new()
+        .subcommand(
+            "add",
+            from_fn_async(add_subnet)
+                .with_metadata("sync_db", Value::Bool(true))
+                .with_inherited(|a, _| a)
+                .no_display()
+                .with_about("Add a new subnet")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_subnet)
+                .with_metadata("sync_db", Value::Bool(true))
+                .with_inherited(|a, _| a)
+                .no_display()
+                .with_about("Remove a subnet")
+                .with_call_remote::<CliContext>(),
+        )
+    // .subcommand(
+    //     "set-default-forward-target",
+    //     from_fn_async(set_default_forward_target)
+    //         .with_metadata("sync_db", Value::Bool(true))
+    //         .no_display()
+    //         .with_about("Set the default target for port forwarding")
+    //         .with_call_remote::<CliContext>(),
+    // )
+    // .subcommand(
+    //     "add-client",
+    //     from_fn_async(add_client)
+    //         .with_metadata("sync_db", Value::Bool(true))
+    //         .no_display()
+    //         .with_about("Add a client to a subnet")
+    //         .with_call_remote::<CliContext>(),
+    // )
+    // .subcommand(
+    //     "remove-client",
+    //     from_fn_async(remove_client)
+    //         .with_metadata("sync_db", Value::Bool(true))
+    //         .no_display()
+    //         .with_about("Remove a client from a subnet")
+    //         .with_call_remote::<CliContext>(),
+    // )
+}
+
+pub async fn add_subnet(
+    ctx: TunnelContext,
+    _: Empty,
+    SubnetParams { subnet }: SubnetParams,
+) -> Result<(), Error> {
+    let server = ctx
+        .db
+        .mutate(|db| {
+            let map = db.as_wg_mut().as_subnets_mut();
+            if !map.contains_key(&subnet)? {
+                map.insert(&subnet, &WgSubnetConfig::new())?;
+            }
+            db.as_wg().de()
+        })
+        .await
+        .result?;
+    server.sync().await
+}
+
+pub async fn remove_subnet(
+    ctx: TunnelContext,
+    _: Empty,
+    SubnetParams { subnet }: SubnetParams,
+) -> Result<(), Error> {
+    let server = ctx
+        .db
+        .mutate(|db| {
+            db.as_wg_mut().as_subnets_mut().remove(&subnet)?;
+            db.as_wg().de()
+        })
+        .await
+        .result?;
+    server.sync().await
+}

core/startos/src/tunnel/client.conf.template

@@ -0,0 +1,10 @@
+[Interface]
+Address = {addr}/24
+PrivateKey = {privkey}
+
+[Peer]
+PublicKey = {server_pubkey}
+PresharedKey = {psk}
+AllowedIPs = 0.0.0.0/0, ::/0
+Endpoint = {server_addr}
+PersistentKeepalive = 25

core/startos/src/tunnel/db.rs

@@ -1,8 +1,10 @@
 use std::collections::{BTreeMap, HashSet};
-use std::net::{Ipv4Addr, SocketAddr};
+use std::net::{Ipv4Addr, SocketAddrV4};
 use std::path::PathBuf;
 
 use clap::Parser;
+use imbl_value::InternedString;
+use ipnet::Ipv4Net;
 use itertools::Itertools;
 use patch_db::json_ptr::{JsonPointer, ROOT};
 use patch_db::Dump;
@@ -17,24 +19,18 @@ use crate::context::CliContext;
 use crate::prelude::*;
 use crate::sign::AnyVerifyingKey;
 use crate::tunnel::context::TunnelContext;
+use crate::tunnel::wg::WgServer;
 use crate::util::serde::{apply_expr, HandlerExtSerde};
 
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
+#[derive(Default, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "camelCase")]
 #[model = "Model<Self>"]
 pub struct TunnelDatabase {
     pub sessions: Sessions,
     pub password: String,
     pub auth_pubkeys: HashSet<AnyVerifyingKey>,
-    pub clients: BTreeMap<Ipv4Addr, ClientInfo>,
-    pub port_forwards: BTreeMap<SocketAddr, SocketAddr>,
-}
-
-#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
-#[serde(rename_all = "camelCase")]
-#[model = "Model<Self>"]
-pub struct ClientInfo {
-    pub server: bool,
+    pub wg: WgServer,
+    pub port_forwards: BTreeMap<SocketAddrV4, SocketAddrV4>,
 }
 
 pub fn db_api<C: Context>() -> ParentHandler<C> {

core/startos/src/tunnel/forward.rs

@@ -0,0 +1 @@
+use crate::prelude::*;

core/startos/src/tunnel/mod.rs

@@ -1,34 +1,24 @@
-use std::collections::{BTreeMap, HashSet};
-use std::net::{Ipv4Addr, SocketAddr};
-
 use axum::Router;
 use futures::future::ready;
-use rpc_toolkit::{Context, HandlerExt, ParentHandler, Server};
-use serde::{Deserialize, Serialize};
+use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server};
 
-use crate::auth::Sessions;
+use crate::context::CliContext;
 use crate::middleware::auth::Auth;
 use crate::middleware::cors::Cors;
 use crate::net::static_server::{bad_request, not_found, server_error};
 use crate::net::web_server::{Accept, WebServer};
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
-use crate::sign::AnyVerifyingKey;
 use crate::tunnel::context::TunnelContext;
 
+pub mod api;
 pub mod context;
 pub mod db;
-pub mod init;
+pub mod forward;
+pub mod wg;
 
 pub const TUNNEL_DEFAULT_PORT: u16 = 5960;
 
-pub fn tunnel_api<C: Context>() -> ParentHandler<C> {
-    ParentHandler::new().subcommand(
-        "db",
-        db::db_api::<C>().with_about("Commands to interact with the db i.e. dump and apply"),
-    )
-}
-
 pub fn tunnel_router(ctx: TunnelContext) -> Router {
     use axum::extract as x;
     use axum::routing::{any, get};
@@ -36,7 +26,7 @@ pub fn tunnel_router(ctx: TunnelContext) -> Router {
         .route("/rpc/{*path}", {
             let ctx = ctx.clone();
             any(
-                Server::new(move || ready(Ok(ctx.clone())), tunnel_api())
+                Server::new(move || ready(Ok(ctx.clone())), api::tunnel_api())
                     .middleware(Cors::new())
                     .middleware(Auth::new())
             )

core/startos/src/tunnel/server-peer.conf.template

@@ -0,0 +1,4 @@
+[Peer]
+PublicKey = {pubkey}
+PresharedKey = {psk}
+AllowedIPs = {addr}/32

core/startos/src/tunnel/server.conf.template

@@ -0,0 +1,5 @@
+[Interface]
+Address = {subnets}
+PrivateKey = {server_privkey}
+ListenPort = {server_port}
+

core/startos/src/tunnel/wg.rs

@@ -0,0 +1,220 @@
+use std::collections::BTreeMap;
+use std::net::{Ipv4Addr, SocketAddrV4};
+
+use ed25519_dalek::{SigningKey, VerifyingKey};
+use imbl_value::InternedString;
+use ipnet::Ipv4Net;
+use itertools::Itertools;
+use serde::{Deserialize, Serialize};
+use tokio::process::Command;
+
+use crate::prelude::*;
+use crate::util::io::write_file_atomic;
+use crate::util::serde::Base64;
+use crate::util::Invoke;
+
+#[derive(Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+pub struct WgServer {
+    pub port: u16,
+    pub key: Base64<WgKey>,
+    pub subnets: WgSubnetMap,
+}
+impl Default for WgServer {
+    fn default() -> Self {
+        Self {
+            port: 51820,
+            key: Base64(WgKey::generate()),
+            subnets: WgSubnetMap::default(),
+        }
+    }
+}
+impl WgServer {
+    pub fn server_config<'a>(&'a self) -> ServerConfig<'a> {
+        ServerConfig(self)
+    }
+    pub async fn sync(&self) -> Result<(), Error> {
+        Command::new("wg-quick")
+            .arg("down")
+            .arg("wg0")
+            .invoke(ErrorKind::Network)
+            .await
+            .or_else(|e| {
+                let msg = e.source.to_string();
+                if msg.contains("does not exist") || msg.contains("is not a WireGuard interface") {
+                    Ok(Vec::new())
+                } else {
+                    Err(e)
+                }
+            })?;
+        write_file_atomic(
+            "/etc/wireguard/wg0.conf",
+            self.server_config().to_string().as_bytes(),
+        )
+        .await?;
+        Command::new("wg-quick")
+            .arg("up")
+            .arg("wg0")
+            .invoke(ErrorKind::Network)
+            .await?;
+        Ok(())
+    }
+}
+
+#[derive(Default, Deserialize, Serialize)]
+pub struct WgSubnetMap(pub BTreeMap<Ipv4Net, WgSubnetConfig>);
+impl Map for WgSubnetMap {
+    type Key = Ipv4Net;
+    type Value = WgSubnetConfig;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Self::key_string(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(InternedString::from_display(key))
+    }
+}
+
+#[derive(Default, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+pub struct WgSubnetConfig {
+    pub default_forward_target: Option<Ipv4Addr>,
+    pub clients: BTreeMap<Ipv4Addr, WgConfig>,
+}
+impl WgSubnetConfig {
+    pub fn new() -> Self {
+        Self::default()
+    }
+    pub fn add_client<'a>(
+        &'a mut self,
+        subnet: Ipv4Net,
+    ) -> Result<(Ipv4Addr, &'a WgConfig), Error> {
+        let addr = subnet
+            .hosts()
+            .find(|a| !self.clients.contains_key(a))
+            .ok_or_else(|| Error::new(eyre!("subnet exhausted"), ErrorKind::Network))?;
+        let config = self.clients.entry(addr).or_insert(WgConfig::generate());
+        Ok((addr, config))
+    }
+}
+
+pub struct WgKey(SigningKey);
+impl WgKey {
+    pub fn generate() -> Self {
+        Self(SigningKey::generate(
+            &mut ssh_key::rand_core::OsRng::default(),
+        ))
+    }
+}
+impl AsRef<[u8]> for WgKey {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_bytes()
+    }
+}
+impl TryFrom<Vec<u8>> for WgKey {
+    type Error = ed25519_dalek::SignatureError;
+    fn try_from(value: Vec<u8>) -> Result<Self, Self::Error> {
+        Ok(Self(value.as_slice().try_into()?))
+    }
+}
+impl std::ops::Deref for WgKey {
+    type Target = SigningKey;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl Base64<WgKey> {
+    pub fn verifying_key(&self) -> Base64<VerifyingKey> {
+        Base64(self.0.verifying_key())
+    }
+}
+
+#[derive(Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+pub struct WgConfig {
+    pub key: Base64<WgKey>,
+    pub psk: Base64<[u8; 32]>,
+}
+impl WgConfig {
+    pub fn generate() -> Self {
+        Self {
+            key: Base64(WgKey::generate()),
+            psk: Base64(rand::random()),
+        }
+    }
+    pub fn server_peer_config<'a>(&'a self, addr: Ipv4Addr) -> ServerPeerConfig<'a> {
+        ServerPeerConfig {
+            client_config: self,
+            client_addr: addr,
+        }
+    }
+    pub fn client_config<'a>(
+        &'a self,
+        addr: Ipv4Addr,
+        server_pubkey: Base64<VerifyingKey>,
+        server_addr: SocketAddrV4,
+    ) -> ClientConfig<'a> {
+        ClientConfig {
+            client_config: self,
+            client_addr: addr,
+            server_pubkey,
+            server_addr,
+        }
+    }
+}
+
+pub struct ServerPeerConfig<'a> {
+    client_config: &'a WgConfig,
+    client_addr: Ipv4Addr,
+}
+impl<'a> std::fmt::Display for ServerPeerConfig<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            include_str!("./server-peer.conf.template"),
+            pubkey = self.client_config.key.verifying_key().to_padded_string(),
+            psk = self.client_config.psk.to_padded_string(),
+            addr = self.client_addr,
+        )
+    }
+}
+
+pub struct ClientConfig<'a> {
+    client_config: &'a WgConfig,
+    client_addr: Ipv4Addr,
+    server_pubkey: Base64<VerifyingKey>,
+    server_addr: SocketAddrV4,
+}
+impl<'a> std::fmt::Display for ClientConfig<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            include_str!("./client.conf.template"),
+            privkey = self.client_config.key.to_padded_string(),
+            psk = self.client_config.psk,
+            addr = self.client_addr,
+            server_pubkey = self.server_pubkey.to_padded_string(),
+            server_addr = self.server_addr,
+        )
+    }
+}
+
+pub struct ServerConfig<'a>(&'a WgServer);
+impl<'a> std::fmt::Display for ServerConfig<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self(server) = *self;
+        write!(
+            f,
+            include_str!("./server.conf.template"),
+            subnets = server.subnets.0.keys().join(", "),
+            server_port = server.port,
+            server_privkey = server.key.to_padded_string(),
+        )?;
+        for (addr, peer) in server.subnets.0.values().flat_map(|s| &s.clients) {
+            write!(f, "{}", peer.server_peer_config(*addr))?;
+        }
+        Ok(())
+    }
+}

core/startos/src/util/serde.rs

@@ -1023,6 +1023,11 @@ pub const BASE64: base64::engine::GeneralPurpose =
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, TS)]
 #[ts(type = "string", concrete(T = Vec<u8>))]
 pub struct Base64<T>(pub T);
+impl<T: AsRef<[u8]>> Base64<T> {
+    pub fn to_padded_string(&self) -> String {
+        base64::engine::general_purpose::STANDARD.encode(self.0.as_ref())
+    }
+}
 impl<T: AsRef<[u8]>> std::fmt::Display for Base64<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str(&BASE64.encode(self.0.as_ref()))

sdk/package/lib/backup/Backups.ts

@@ -197,7 +197,7 @@ async function runRsync(rsyncOptions: {
   for (const exclude of options.exclude) {
     args.push(`--exclude=${exclude}`)
   }
-  args.push("-actAXH")
+  args.push("-rlptgocAXH")
   args.push("--info=progress2")
   args.push("--no-inc-recursive")
   args.push(srcPath)