Gateways, domains, and new service interface (#3001)

* add support for inbound proxies

* backend changes

* fix file type

* proxy -> tunnel, implement backend apis

* wip start-tunneld

* add domains and gateways, remove routers, fix docs links

* dont show hidden actions

* show and test dns

* edit instead of chnage acme and change gateway

* refactor: domains page

* refactor: gateways page

* domains and acme refactor

* certificate authorities

* refactor public/private gateways

* fix fe types

* domains mostly finished

* refactor: add file control to form service

* add ip util to sdk

* domains api + migration

* start service interface page, WIP

* different options for clearnet domains

* refactor: styles for interfaces page

* minor

* better placeholder for no addresses

* start sorting addresses

* best address logic

* comments

* fix unnecessary export

* MVP of service interface page

* domains preferred

* fix: address comments

* only translations left

* wip: start-tunnel & fix build

* forms for adding domain, rework things based on new ideas

* fix: dns testing

* public domain, max width, descriptions for dns

* nix StartOS domains, implement public and private domains at interface scope

* restart tor instead of reset

* better icon for restart tor

* dns

* fix sort functions for public and private domains

* with todos

* update types

* clean up tech debt, bump dependencies

* revert to ts-rs v9

* fix all types

* fix dns form

* add missing translations

* it builds

* fix: comments (#3009)

* fix: comments

* undo default

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix: refactor legacy components (#3010)

* fix: comments

* fix: refactor legacy components

* remove default again

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* more translations

* wip

* fix deadlock

* coukd work

* simple renaming

* placeholder for empty service interfaces table

* honor hidden form values

* remove logs

* reason instead of description

* fix dns

* misc fixes

* implement toggling gateways for service interface

* fix showing dns records

* move status column in service list

* remove unnecessary truthy check

* refactor: refactor forms components and remove legacy Taiga UI package (#3012)

* handle wh file uploads

* wip: debugging tor

* socks5 proxy working

* refactor: fix multiple comments (#3013)

* refactor: fix multiple comments

* styling changes, add documentation to sidebar

* translations for dns page

* refactor: subtle colors

* rearrange service page

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

* fix file_stream and remove non-terminating test

* clean  up logs

* support for sccache

* fix gha sccache

* more marketplace translations

* install wizard clarity

* stub hostnameInfo in migration

* fix address info after setup, fix styling on SI page, new 040 release notes

* remove tor logs from os

* misc fixes

* reset tor still not functioning...

* update ts

* minor styling and wording

* chore: some fixes (#3015)

* fix gateway renames

* different handling for public domains

* styling fixes

* whole navbar should not be clickable on service show page

* timeout getState request

* remove links from changelog

* misc fixes from pairing

* use custom name for gateway in more places

* fix dns parsing

* closes #3003

* closes #2999

* chore: some fixes (#3017)

* small copy change

* revert hardcoded error for testing

* dont require port forward if gateway is public

* use old wan ip when not available

* fix .const hanging on undefined

* fix test

* fix doc test

* fix renames

* update deps

* allow specifying dependency metadata directly

* temporarily make dependencies not cliackable in marketplace listings

* fix socks bind

* fix test

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: waterplea <alexander@inkin.ru>
This commit is contained in:
Matt Hill
2025-09-09 21:43:51 -06:00
committed by GitHub
parent 1cc9a1a30b
commit add01ebc68
537 changed files with 19940 additions and 20551 deletions

View File

@@ -9,20 +9,24 @@ use ipnet::IpNet;
use models::{HostId, OptionExt, PackageId};
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use torut::onion::{OnionAddressV3, TorSecretKeyV3};
use tracing::instrument;
use crate::db::model::public::{NetworkInterfaceInfo, NetworkInterfaceType};
use crate::db::model::Database;
use crate::error::ErrorCollection;
use crate::hostname::Hostname;
use crate::net::dns::DnsController;
use crate::net::forward::LanPortForwardController;
use crate::net::forward::PortForwardController;
use crate::net::gateway::{
AndFilter, DynInterfaceFilter, IdFilter, InterfaceFilter, NetworkInterfaceController, OrFilter,
PublicFilter, SecureFilter,
};
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{AddSslOptions, BindId, BindOptions};
use crate::net::host::{host_for, Host, Hosts};
use crate::net::network_interface::NetworkInterfaceController;
use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname};
use crate::net::tor::TorController;
use crate::net::socks::SocksController;
use crate::net::tor::{OnionAddress, TorController, TorSecretKey};
use crate::net::utils::ipv6_is_local;
use crate::net::vhost::{AlpnInfo, TargetInfo, VHostController};
use crate::prelude::*;
@@ -36,7 +40,8 @@ pub struct NetController {
pub(super) vhost: VHostController,
pub(crate) net_iface: Arc<NetworkInterfaceController>,
pub(super) dns: DnsController,
pub(super) forward: LanPortForwardController,
pub(super) forward: PortForwardController,
pub(super) socks: SocksController,
pub(super) server_hostnames: Vec<Option<InternedString>>,
pub(crate) callbacks: Arc<ServiceCallbacks>,
}
@@ -44,18 +49,20 @@ pub struct NetController {
impl NetController {
pub async fn init(
db: TypedPatchDb<Database>,
tor_control: SocketAddr,
tor_socks: SocketAddr,
hostname: &Hostname,
socks_listen: SocketAddr,
) -> Result<Self, Error> {
let net_iface = Arc::new(NetworkInterfaceController::new(db.clone()));
let tor = TorController::new()?;
let socks = SocksController::new(socks_listen, tor.clone())?;
Ok(Self {
db: db.clone(),
tor: TorController::new(tor_control, tor_socks),
vhost: VHostController::new(db, net_iface.clone()),
dns: DnsController::init(net_iface.lxcbr_status()).await?,
forward: LanPortForwardController::new(net_iface.subscribe()),
tor,
vhost: VHostController::new(db.clone(), net_iface.clone()),
dns: DnsController::init(db, &net_iface.watcher).await?,
forward: PortForwardController::new(net_iface.watcher.subscribe()),
net_iface,
socks,
server_hostnames: vec![
// LAN IP
None,
@@ -78,7 +85,7 @@ impl NetController {
package: PackageId,
ip: Ipv4Addr,
) -> Result<NetService, Error> {
let dns = self.dns.add(Some(package.clone()), ip).await?;
let dns = self.dns.add_service(Some(package.clone()), ip)?;
let res = NetService::new(NetServiceData {
id: Some(package),
@@ -92,7 +99,7 @@ impl NetController {
}
pub async fn os_bindings(self: &Arc<Self>) -> Result<NetService, Error> {
let dns = self.dns.add(None, HOST_IP.into()).await?;
let dns = self.dns.add_service(None, HOST_IP.into())?;
let service = NetService::new(NetServiceData {
id: None,
@@ -126,9 +133,10 @@ impl NetController {
#[derive(Default, Debug)]
struct HostBinds {
forwards: BTreeMap<u16, (SocketAddr, bool, Arc<()>)>,
forwards: BTreeMap<u16, (SocketAddr, DynInterfaceFilter, Arc<()>)>,
vhosts: BTreeMap<(Option<InternedString>, u16), (TargetInfo, Arc<()>)>,
tor: BTreeMap<OnionAddressV3, (OrdMap<u16, SocketAddr>, Vec<Arc<()>>)>,
private_dns: BTreeMap<InternedString, Arc<()>>,
tor: BTreeMap<OnionAddress, (OrdMap<u16, SocketAddr>, Vec<Arc<()>>)>,
}
pub struct NetServiceData {
@@ -217,9 +225,10 @@ impl NetServiceData {
}
async fn update(&mut self, ctrl: &NetController, id: HostId, host: Host) -> Result<(), Error> {
let mut forwards: BTreeMap<u16, (SocketAddr, bool)> = BTreeMap::new();
let mut forwards: BTreeMap<u16, (SocketAddr, DynInterfaceFilter)> = BTreeMap::new();
let mut vhosts: BTreeMap<(Option<InternedString>, u16), TargetInfo> = BTreeMap::new();
let mut tor: BTreeMap<OnionAddressV3, (TorSecretKeyV3, OrdMap<u16, SocketAddr>)> =
let mut private_dns: BTreeSet<InternedString> = BTreeSet::new();
let mut tor: BTreeMap<OnionAddress, (TorSecretKey, OrdMap<u16, SocketAddr>)> =
BTreeMap::new();
let mut hostname_info: BTreeMap<u16, Vec<HostnameInfo>> = BTreeMap::new();
let binds = self.binds.entry(id.clone()).or_default();
@@ -228,7 +237,7 @@ impl NetServiceData {
// LAN
let server_info = peek.as_public().as_server_info();
let net_ifaces = ctrl.net_iface.ip_info();
let net_ifaces = ctrl.net_iface.watcher.ip_info();
let hostname = server_info.as_hostname().de()?;
for (port, bind) in &host.bindings {
if !bind.enabled {
@@ -255,7 +264,7 @@ impl NetServiceData {
vhosts.insert(
(hostname, external),
TargetInfo {
public: bind.net.public,
filter: bind.net.clone().into_dyn(),
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
@@ -270,87 +279,177 @@ impl NetServiceData {
vhosts.insert(
(Some(hostname), external),
TargetInfo {
public: false,
filter: OrFilter(
IdFilter(
NetworkInterfaceInfo::loopback().0.clone(),
),
IdFilter(
NetworkInterfaceInfo::lxc_bridge().0.clone(),
),
)
.into_dyn(),
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
},
);
); // TODO: wrap onion ssl stream directly in tor ctrl
}
}
HostAddress::Domain {
address,
public,
acme,
private,
} => {
if hostnames.insert(address.clone()) {
let address = Some(address.clone());
if ssl.preferred_external_port == 443 {
if public && bind.net.public {
if let Some(public) = &public {
vhosts.insert(
(address.clone(), 5443),
TargetInfo {
public: false,
acme: acme.clone(),
filter: AndFilter(
bind.net.clone(),
AndFilter(
IdFilter(public.gateway.clone()),
PublicFilter { public: false },
),
)
.into_dyn(),
acme: public.acme.clone(),
addr,
connect_ssl: connect_ssl.clone(),
},
);
vhosts.insert(
(address.clone(), 443),
TargetInfo {
filter: AndFilter(
bind.net.clone(),
if private {
OrFilter(
IdFilter(public.gateway.clone()),
PublicFilter { public: false },
)
.into_dyn()
} else {
AndFilter(
IdFilter(public.gateway.clone()),
PublicFilter { public: true },
)
.into_dyn()
},
)
.into_dyn(),
acme: public.acme.clone(),
addr,
connect_ssl: connect_ssl.clone(),
},
);
} else {
vhosts.insert(
(address.clone(), 443),
TargetInfo {
filter: AndFilter(
bind.net.clone(),
PublicFilter { public: false },
)
.into_dyn(),
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
vhosts.insert(
(address.clone(), 443),
TargetInfo {
public: public && bind.net.public,
acme,
addr,
connect_ssl: connect_ssl.clone(),
},
);
} else {
vhosts.insert(
(address.clone(), external),
TargetInfo {
public: public && bind.net.public,
acme,
addr,
connect_ssl: connect_ssl.clone(),
},
);
if let Some(public) = public {
vhosts.insert(
(address.clone(), external),
TargetInfo {
filter: AndFilter(
bind.net.clone(),
if private {
OrFilter(
IdFilter(public.gateway.clone()),
PublicFilter { public: false },
)
.into_dyn()
} else {
IdFilter(public.gateway.clone())
.into_dyn()
},
)
.into_dyn(),
acme: public.acme.clone(),
addr,
connect_ssl: connect_ssl.clone(),
},
);
} else {
vhosts.insert(
(address.clone(), external),
TargetInfo {
filter: AndFilter(
bind.net.clone(),
PublicFilter { public: false },
)
.into_dyn(),
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
}
}
}
}
}
}
if let Some(security) = bind.options.secure {
if bind.options.add_ssl.is_some() && security.ssl {
// doesn't make sense to have 2 listening ports, both with ssl
} else {
let external = bind.net.assigned_port.or_not_found("assigned lan port")?;
forwards.insert(external, ((self.ip, *port).into(), bind.net.public));
}
if bind
.options
.secure
.map_or(true, |s| !(s.ssl && bind.options.add_ssl.is_some()))
{
let external = bind.net.assigned_port.or_not_found("assigned lan port")?;
forwards.insert(
external,
(
(self.ip, *port).into(),
AndFilter(
SecureFilter {
secure: bind.options.secure.is_some(),
},
bind.net.clone(),
)
.into_dyn(),
),
);
}
let mut bind_hostname_info: Vec<HostnameInfo> =
hostname_info.remove(port).unwrap_or_default();
for (interface, public, ip_info) in
net_ifaces.iter().filter_map(|(interface, info)| {
if let Some(ip_info) = &info.ip_info {
Some((interface, info.inbound(), ip_info))
} else {
None
}
})
for (interface, info) in net_ifaces
.iter()
.filter(|(id, info)| bind.net.filter(id, info))
{
if !public {
let port = bind.net.assigned_port.filter(|_| {
bind.options.secure.map_or(false, |s| {
!(s.ssl && bind.options.add_ssl.is_some()) || info.secure()
})
});
if !info.public()
&& info.ip_info.as_ref().map_or(false, |i| {
i.device_type != Some(NetworkInterfaceType::Wireguard)
})
{
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
gateway_id: interface.clone(),
public: false,
hostname: IpHostname::Local {
value: InternedString::from_display(&{
let hostname = &hostname;
lazy_format!("{hostname}.local")
}),
port: bind.net.assigned_port,
port,
ssl_port: bind.net.assigned_ssl_port,
},
});
@@ -358,11 +457,13 @@ impl NetServiceData {
for address in host.addresses() {
if let HostAddress::Domain {
address,
public: domain_public,
..
public,
private,
} = address
{
if !public || (domain_public && bind.net.public) {
let private = private && !info.public();
let public = public.as_ref().map_or(false, |p| &p.gateway == interface);
if public || private {
if bind
.options
.add_ssl
@@ -370,23 +471,21 @@ impl NetServiceData {
.map_or(false, |ssl| ssl.preferred_external_port == 443)
{
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: public && domain_public && bind.net.public, // TODO: check if port forward is active
gateway_id: interface.clone(),
public,
hostname: IpHostname::Domain {
domain: address.clone(),
subdomain: None,
value: address.clone(),
port: None,
ssl_port: Some(443),
},
});
} else {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
gateway_id: interface.clone(),
public,
hostname: IpHostname::Domain {
domain: address.clone(),
subdomain: None,
port: bind.net.assigned_port,
value: address.clone(),
port,
ssl_port: bind.net.assigned_ssl_port,
},
});
@@ -394,14 +493,15 @@ impl NetServiceData {
}
}
}
if !public || bind.net.public {
if let Some(wan_ip) = ip_info.wan_ip.filter(|_| public) {
if let Some(ip_info) = &info.ip_info {
let public = info.public();
if let Some(wan_ip) = ip_info.wan_ip {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public,
gateway_id: interface.clone(),
public: true,
hostname: IpHostname::Ipv4 {
value: wan_ip,
port: bind.net.assigned_port,
port,
ssl_port: bind.net.assigned_ssl_port,
},
});
@@ -411,11 +511,11 @@ impl NetServiceData {
IpNet::V4(net) => {
if !public {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
gateway_id: interface.clone(),
public,
hostname: IpHostname::Ipv4 {
value: net.addr(),
port: bind.net.assigned_port,
port,
ssl_port: bind.net.assigned_ssl_port,
},
});
@@ -423,12 +523,12 @@ impl NetServiceData {
}
IpNet::V6(net) => {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
gateway_id: interface.clone(),
public: public && !ipv6_is_local(net.addr()),
hostname: IpHostname::Ipv6 {
value: net.addr(),
scope_id: ip_info.scope_id,
port: bind.net.assigned_port,
port,
ssl_port: bind.net.assigned_ssl_port,
},
});
@@ -438,6 +538,7 @@ impl NetServiceData {
}
}
hostname_info.insert(*port, bind_hostname_info);
private_dns.append(&mut hostnames);
}
}
@@ -487,7 +588,7 @@ impl NetServiceData {
.as_key_store()
.as_onion()
.get_key(tor_addr)?;
tor.insert(key.public().get_onion_address(), (key, tor_binds.clone()));
tor.insert(key.onion_address(), (key, tor_binds.clone()));
for (internal, ports) in &tor_hostname_ports {
let mut bind_hostname_info = hostname_info.remove(internal).unwrap_or_default();
bind_hostname_info.push(HostnameInfo::Onion {
@@ -509,8 +610,8 @@ impl NetServiceData {
.collect::<BTreeSet<_>>();
for external in all {
let mut prev = binds.forwards.remove(&external);
if let Some((internal, public)) = forwards.remove(&external) {
prev = prev.filter(|(i, p, _)| i == &internal && *p == public);
if let Some((internal, filter)) = forwards.remove(&external) {
prev = prev.filter(|(i, f, _)| i == &internal && *f == filter);
binds.forwards.insert(
external,
if let Some(prev) = prev {
@@ -518,8 +619,8 @@ impl NetServiceData {
} else {
(
internal,
public,
ctrl.forward.add(external, public, internal).await?,
filter.clone(),
ctrl.forward.add(external, filter, internal).await?,
)
},
);
@@ -553,6 +654,22 @@ impl NetServiceData {
}
}
let mut rm = BTreeSet::new();
binds.private_dns.retain(|fqdn, _| {
if private_dns.remove(fqdn) {
true
} else {
rm.insert(fqdn.clone());
false
}
});
for fqdn in private_dns {
binds
.private_dns
.insert(fqdn.clone(), ctrl.dns.add_private_domain(fqdn)?);
}
ctrl.dns.gc_private_domains(&rm)?;
let all = binds
.tor
.keys()
@@ -568,17 +685,15 @@ impl NetServiceData {
if let Some(prev) = prev {
prev
} else {
let rcs = ctrl
.tor
.add(key, tor_binds.iter().map(|(k, v)| (*k, *v)).collect())
.await?;
let service = ctrl.tor.service(key)?;
let rcs = service.proxy_all(tor_binds.iter().map(|(k, v)| (*k, *v)));
(tor_binds, rcs)
},
);
} else {
if let Some((_, rc)) = prev {
drop(rc);
ctrl.tor.gc(Some(onion), None).await?;
ctrl.tor.gc(Some(onion)).await?;
}
}
}
@@ -662,7 +777,7 @@ impl NetService {
}
fn new(data: NetServiceData) -> Result<Self, Error> {
let mut ip_info = data.net_controller()?.net_iface.subscribe();
let mut ip_info = data.net_controller()?.net_iface.watcher.subscribe();
let data = Arc::new(Mutex::new(data));
let thread_data = data.clone();
let sync_task = tokio::spawn(async move {