Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

add clearnet functionality to frontend (#2814)

Browse Source 
* add clearnet functionality to frontend

* add pattern and add sync db on rpcs

* add domain pattern

* show acme name instead of url if known

* dont blow up if domain not present after delete

* use common name for letsencrypt

* normalize urls

* refactor start-os ui net service

* backend migration and rpcs for serverInfo.host

* fix cors

* implement clearnet for main startos ui

* ability to add and remove tor addresses, including vanity

* add guard to prevent duplicate addresses

* misc bugfixes

* better heuristics for launching UIs

* fix ipv6 mocks

* fix ipv6 display bug

* rewrite url selection for launch ui

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
This commit is contained in:
Matt Hill
2025-01-21 20:46:36 -07:00
committed by GitHub
parent 0a9f1d2a27
commit 479797361e
90 changed files with 2838 additions and 1203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/startos/Cargo.toml
View File

@@ -14,7 +14,7 @@ keywords = [
name = "start-os"
readme = "README.md"
repository = "https://github.com/Start9Labs/start-os"
version = "0.3.6-alpha.11"
version = "0.3.6-alpha.12"
license = "MIT"
[lib]
@@ -117,7 +117,7 @@ id-pool = { version = "0.2.2", default-features = false, features = [
"u16",
] }
imbl = "2.0.3"
imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" }
imbl-value = "0.1.2"
include_dir = { version = "0.7.3", features = ["metadata"] }
indexmap = { version = "2.0.2", features = ["serde"] }
indicatif = { version = "0.17.7", features = ["tokio"] }
@@ -172,7 +172,7 @@ regex = "1.10.2"
reqwest = { version = "0.12.4", features = ["stream", "json", "socks"] }
reqwest_cookie_store = "0.8.0"
rpassword = "7.2.0"
rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "refactor/no-dyn-ctx" }
rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "master" }
rust-argon2 = "2.0.0"
rustyline-async = "0.4.1"
semver = { version = "1.0.20", features = ["serde"] }

34
core/startos/src/account.rs
View File

@@ -24,7 +24,7 @@ pub struct AccountInfo {
pub server_id: String,
pub hostname: Hostname,
pub password: String,
pub tor_key: TorSecretKeyV3,
pub tor_keys: Vec<TorSecretKeyV3>,
pub root_ca_key: PKey<Private>,
pub root_ca_cert: X509,
pub ssh_key: ssh_key::PrivateKey,
@@ -34,7 +34,7 @@ impl AccountInfo {
pub fn new(password: &str, start_time: SystemTime) -> Result<Self, Error> {
let server_id = generate_id();
let hostname = generate_hostname();
let tor_key = TorSecretKeyV3::generate();
let tor_key = vec![TorSecretKeyV3::generate()];
let root_ca_key = generate_key()?;
let root_ca_cert = make_root_cert(&root_ca_key, &hostname, start_time)?;
let ssh_key = ssh_key::PrivateKey::from(ssh_key::private::Ed25519Keypair::random(
@@ -45,7 +45,7 @@ impl AccountInfo {
server_id,
hostname,
password: hash_password(password)?,
tor_key,
tor_keys: tor_key,
root_ca_key,
root_ca_cert,
ssh_key,
@@ -58,8 +58,11 @@ impl AccountInfo {
let hostname = Hostname(db.as_public().as_server_info().as_hostname().de()?);
let password = db.as_private().as_password().de()?;
let key_store = db.as_private().as_key_store();
let tor_addr = db.as_public().as_server_info().as_onion_address().de()?;
let tor_key = key_store.as_onion().get_key(&tor_addr)?;
let tor_addrs = db.as_public().as_server_info().as_host().as_onions().de()?;
let tor_keys = tor_addrs
.into_iter()
.map(|tor_addr| key_store.as_onion().get_key(&tor_addr))
.collect::<Result<_, _>>()?;
let cert_store = key_store.as_local_certs();
let root_ca_key = cert_store.as_root_key().de()?.0;
let root_ca_cert = cert_store.as_root_cert().de()?.0;
@@ -70,7 +73,7 @@ impl AccountInfo {
server_id,
hostname,
password,
tor_key,
tor_keys,
root_ca_key,
root_ca_cert,
ssh_key,
@@ -82,17 +85,16 @@ impl AccountInfo {
let server_info = db.as_public_mut().as_server_info_mut();
server_info.as_id_mut().ser(&self.server_id)?;
server_info.as_hostname_mut().ser(&self.hostname.0)?;
server_info
.as_lan_address_mut()
.ser(&self.hostname.lan_address().parse()?)?;
server_info
.as_pubkey_mut()
.ser(&self.ssh_key.public_key().to_openssh()?)?;
let onion_address = self.tor_key.public().get_onion_address();
server_info.as_onion_address_mut().ser(&onion_address)?;
server_info
.as_tor_address_mut()
.ser(&format!("https://{onion_address}").parse()?)?;
server_info.as_host_mut().as_onions_mut().ser(
&self
.tor_keys
.iter()
.map(|tor_key| tor_key.public().get_onion_address())
.collect(),
)?;
db.as_private_mut().as_password_mut().ser(&self.password)?;
db.as_private_mut()
.as_ssh_privkey_mut()
@@ -101,7 +103,9 @@ impl AccountInfo {
.as_compat_s9pk_key_mut()
.ser(Pem::new_ref(&self.compat_s9pk_key))?;
let key_store = db.as_private_mut().as_key_store_mut();
key_store.as_onion_mut().insert_key(&self.tor_key)?;
for tor_key in &self.tor_keys {
key_store.as_onion_mut().insert_key(tor_key)?;
}
let cert_store = key_store.as_local_certs_mut();
cert_store
.as_root_key_mut()

10
core/startos/src/backup/os.rs
View File

@@ -85,7 +85,7 @@ impl OsBackupV0 {
&mut rand::thread_rng(),
ssh_key::Algorithm::Ed25519,
)?,
tor_key: TorSecretKeyV3::from(self.tor_key.0),
tor_keys: vec![TorSecretKeyV3::from(self.tor_key.0)],
compat_s9pk_key: ed25519_dalek::SigningKey::generate(&mut rand::thread_rng()),
},
ui: self.ui,
@@ -114,7 +114,7 @@ impl OsBackupV1 {
root_ca_key: self.root_ca_key.0,
root_ca_cert: self.root_ca_cert.0,
ssh_key: ssh_key::PrivateKey::from(Ed25519Keypair::from_seed(&self.net_key.0)),
tor_key: TorSecretKeyV3::from(ed25519_expand_key(&self.net_key.0)),
tor_keys: vec![TorSecretKeyV3::from(ed25519_expand_key(&self.net_key.0))],
compat_s9pk_key: ed25519_dalek::SigningKey::from_bytes(&self.net_key),
},
ui: self.ui,
@@ -132,7 +132,7 @@ struct OsBackupV2 {
root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
root_ca_cert: Pem<X509>, // PEM Encoded OpenSSL X509 Certificate
ssh_key: Pem<ssh_key::PrivateKey>, // PEM Encoded OpenSSH Key
tor_key: TorSecretKeyV3, // Base64 Encoded Ed25519 Expanded Secret Key
tor_keys: Vec<TorSecretKeyV3>, // Base64 Encoded Ed25519 Expanded Secret Key
compat_s9pk_key: Pem<ed25519_dalek::SigningKey>, // PEM Encoded ED25519 Key
ui: Value, // JSON Value
}
@@ -146,7 +146,7 @@ impl OsBackupV2 {
root_ca_key: self.root_ca_key.0,
root_ca_cert: self.root_ca_cert.0,
ssh_key: self.ssh_key.0,
tor_key: self.tor_key,
tor_keys: self.tor_keys,
compat_s9pk_key: self.compat_s9pk_key.0,
},
ui: self.ui,
@@ -159,7 +159,7 @@ impl OsBackupV2 {
root_ca_key: Pem(backup.account.root_ca_key.clone()),
root_ca_cert: Pem(backup.account.root_ca_cert.clone()),
ssh_key: Pem(backup.account.ssh_key.clone()),
tor_key: backup.account.tor_key.clone(),
tor_keys: backup.account.tor_keys.clone(),
compat_s9pk_key: Pem(backup.account.compat_s9pk_key.clone()),
ui: backup.ui.clone(),
}

6
core/startos/src/backup/restore.rs
View File

@@ -18,7 +18,7 @@ use crate::db::model::Database;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::init::{init, InitResult};
use crate::init::init;
use crate::prelude::*;
use crate::s9pk::S9pk;
use crate::service::service_map::DownloadInstallFuture;
@@ -109,13 +109,13 @@ pub async fn recover_full_embassy(
db.put(&ROOT, &Database::init(&os_backup.account)?).await?;
drop(db);
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
let init_result = init(&ctx.webserver, &ctx.config, init_phases).await?;
let rpc_ctx = RpcContext::init(
&ctx.webserver,
&ctx.config,
disk_guid.clone(),
Some(net_ctrl),
Some(init_result),
rpc_ctx_phases,
)
.await?;

6
core/startos/src/bins/start_init.rs
View File

@@ -11,7 +11,7 @@ use crate::disk::fsck::RepairStrategy;
use crate::disk::main::DEFAULT_PASSWORD;
use crate::disk::REPAIR_DISK_PATH;
use crate::firmware::{check_for_firmware_update, update_firmware};
use crate::init::{InitPhases, InitResult, STANDBY_MODE_PATH};
use crate::init::{InitPhases, STANDBY_MODE_PATH};
use crate::net::web_server::{UpgradableListener, WebServer};
use crate::prelude::*;
use crate::progress::FullProgressTracker;
@@ -188,14 +188,14 @@ async fn setup_or_init(
}));
}
let InitResult { net_ctrl } =
let init_result =
crate::init::init(&server.acceptor_setter(), config, init_phases).await?;
let rpc_ctx = RpcContext::init(
&server.acceptor_setter(),
config,
disk_guid,
Some(net_ctrl),
Some(init_result),
rpc_ctx_phases,
)
.await?;

3
core/startos/src/context/config.rs
View File

@@ -108,8 +108,6 @@ pub struct ServerConfig {
#[arg(long)]
pub tor_socks: Option<SocketAddr>,
#[arg(long)]
pub dns_bind: Option<Vec<SocketAddr>>,
#[arg(long)]
pub revision_cache_size: Option<usize>,
#[arg(long)]
pub disable_encryption: Option<bool>,
@@ -125,7 +123,6 @@ impl ContextConfig for ServerConfig {
self.os_partitions = self.os_partitions.take().or(other.os_partitions);
self.tor_control = self.tor_control.take().or(other.tor_control);
self.tor_socks = self.tor_socks.take().or(other.tor_socks);
self.dns_bind = self.dns_bind.take().or(other.dns_bind);
self.revision_cache_size = self
.revision_cache_size
.take()

57
core/startos/src/context/rpc.rs
View File

@@ -26,9 +26,9 @@ use crate::auth::Sessions;
use crate::context::config::ServerConfig;
use crate::db::model::Database;
use crate::disk::OsPartitionInfo;
use crate::init::check_time_is_synchronized;
use crate::init::{check_time_is_synchronized, InitResult};
use crate::lxc::{ContainerId, LxcContainer, LxcManager};
use crate::net::net_controller::{NetController, PreInitNetController};
use crate::net::net_controller::{NetController, NetService};
use crate::net::utils::{find_eth_iface, find_wifi_iface};
use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter};
use crate::net::wifi::WpaCli;
@@ -53,6 +53,7 @@ pub struct RpcContextSeed {
pub sync_db: watch::Sender<u64>,
pub account: RwLock<AccountInfo>,
pub net_controller: Arc<NetController>,
pub os_net_service: NetService,
pub s9pk_arch: Option<&'static str>,
pub services: ServiceMap,
pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
@@ -119,7 +120,7 @@ impl RpcContext {
webserver: &WebServerAcceptorSetter<UpgradableListener>,
config: &ServerConfig,
disk_guid: Arc<String>,
net_ctrl: Option<PreInitNetController>,
init_result: Option<InitResult>,
InitRpcContextPhases {
mut load_db,
mut init_net_ctrl,
@@ -133,7 +134,7 @@ impl RpcContext {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
load_db.start();
let db = if let Some(net_ctrl) = &net_ctrl {
let db = if let Some(InitResult { net_ctrl, .. }) = &init_result {
net_ctrl.db.clone()
} else {
TypedPatchDb::<Database>::load(config.db().await?).await?
@@ -144,31 +145,28 @@ impl RpcContext {
tracing::info!("Opened PatchDB");
init_net_ctrl.start();
let net_controller = Arc::new(
NetController::init(
if let Some(net_ctrl) = net_ctrl {
net_ctrl
} else {
let net_ctrl = PreInitNetController::init(
db.clone(),
config
.tor_control
.unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
tor_proxy,
&account.hostname,
account.tor_key.clone(),
)
.await?;
webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
net_ctrl
},
config
.dns_bind
.as_deref()
.unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
)
.await?,
);
let (net_controller, os_net_service) = if let Some(InitResult {
net_ctrl,
os_net_service,
}) = init_result
{
(net_ctrl, os_net_service)
} else {
let net_ctrl = Arc::new(
NetController::init(
db.clone(),
config
.tor_control
.unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
tor_proxy,
&account.hostname,
)
.await?,
);
webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
let os_net_service = net_ctrl.os_bindings().await?;
(net_ctrl, os_net_service)
};
init_net_ctrl.complete();
tracing::info!("Initialized Net Controller");
@@ -230,6 +228,7 @@ impl RpcContext {
db,
account: RwLock::new(account),
net_controller,
os_net_service,
s9pk_arch: if config.multi_arch_s9pks.unwrap_or(false) {
None
} else {

8
core/startos/src/context/setup.rs
View File

@@ -40,7 +40,7 @@ lazy_static::lazy_static! {
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct SetupResult {
pub tor_address: String,
pub tor_addresses: Vec<String>,
#[ts(type = "string")]
pub hostname: Hostname,
#[ts(type = "string")]
@@ -51,7 +51,11 @@ impl TryFrom<&AccountInfo> for SetupResult {
type Error = Error;
fn try_from(value: &AccountInfo) -> Result<Self, Self::Error> {
Ok(Self {
tor_address: format!("https://{}", value.tor_key.public().get_onion_address()),
tor_addresses: value
.tor_keys
.iter()
.map(|tor_key| format!("https://{}", tor_key.public().get_onion_address()))
.collect(),
hostname: value.hostname.clone(),
lan_address: value.hostname.lan_address(),
root_ca: String::from_utf8(value.root_ca_cert.to_pem()?)?,

54
core/startos/src/db/model/public.rs
View File

@@ -10,19 +10,22 @@ use itertools::Itertools;
use models::PackageId;
use openssl::hash::MessageDigest;
use patch_db::{HasModel, Value};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use ts_rs::TS;
use crate::account::AccountInfo;
use crate::db::model::package::AllPackageData;
use crate::net::acme::AcmeProvider;
use crate::net::host::address::DomainConfig;
use crate::net::host::binding::{AddSslOptions, BindInfo, BindOptions, NetInfo};
use crate::net::host::Host;
use crate::net::vhost::AlpnInfo;
use crate::prelude::*;
use crate::progress::FullProgress;
use crate::system::SmtpValue;
use crate::util::cpupower::Governor;
use crate::util::lshw::LshwDevice;
use crate::util::serde::MaybeUtf8String;
use crate::version::{Current, VersionT};
use crate::{ARCH, PLATFORM};
@@ -38,7 +41,6 @@ pub struct Public {
}
impl Public {
pub fn init(account: &AccountInfo) -> Result<Self, Error> {
let lan_address = account.hostname.lan_address().parse().unwrap();
Ok(Self {
server_info: ServerInfo {
arch: get_arch(),
@@ -46,14 +48,42 @@ impl Public {
id: account.server_id.clone(),
version: Current::default().semver(),
hostname: account.hostname.no_dot_host_name(),
host: Host {
bindings: [(
80,
BindInfo {
enabled: false,
options: BindOptions {
preferred_external_port: 80,
add_ssl: Some(AddSslOptions {
preferred_external_port: 443,
alpn: Some(AlpnInfo::Specified(vec![
MaybeUtf8String("http/1.1".into()),
MaybeUtf8String("h2".into()),
])),
}),
secure: None,
},
net: NetInfo {
assigned_port: None,
assigned_ssl_port: Some(443),
public: false,
},
},
)]
.into_iter()
.collect(),
onions: account
.tor_keys
.iter()
.map(|k| k.public().get_onion_address())
.collect(),
domains: BTreeMap::new(),
hostname_info: BTreeMap::new(),
},
last_backup: None,
package_version_compat: Current::default().compat().clone(),
post_init_migration_todos: BTreeSet::new(),
lan_address,
onion_address: account.tor_key.public().get_onion_address(),
tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
.parse()
.unwrap(),
network_interfaces: BTreeMap::new(),
acme: BTreeMap::new(),
status_info: ServerStatus {
@@ -115,6 +145,7 @@ pub struct ServerInfo {
pub id: String,
#[ts(type = "string")]
pub hostname: InternedString,
pub host: Host,
#[ts(type = "string")]
pub version: Version,
#[ts(type = "string")]
@@ -123,13 +154,6 @@ pub struct ServerInfo {
pub post_init_migration_todos: BTreeSet<Version>,
#[ts(type = "string | null")]
pub last_backup: Option<DateTime<Utc>>,
#[ts(type = "string")]
pub lan_address: Url,
#[ts(type = "string")]
pub onion_address: OnionAddressV3,
/// for backwards compatibility
#[ts(type = "string")]
pub tor_address: Url,
#[ts(as = "BTreeMap::<String, NetworkInterfaceInfo>")]
#[serde(default)]
pub network_interfaces: BTreeMap<InternedString, NetworkInterfaceInfo>,

80
core/startos/src/init.rs
View File

@@ -3,6 +3,7 @@ use std::io::Cursor;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use axum::extract::ws::{self};
@@ -25,7 +26,8 @@ use crate::db::model::public::ServerStatus;
use crate::db::model::Database;
use crate::disk::mount::util::unmount;
use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
use crate::net::net_controller::PreInitNetController;
use crate::net::net_controller::{NetController, NetService};
use crate::net::utils::find_wifi_iface;
use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter};
use crate::prelude::*;
use crate::progress::{
@@ -197,7 +199,8 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
}
pub struct InitResult {
pub net_ctrl: PreInitNetController,
pub net_ctrl: Arc<NetController>,
pub os_net_service: NetService,
}
pub struct InitPhases {
@@ -347,19 +350,21 @@ pub async fn init(
let account = AccountInfo::load(&peek)?;
start_net.start();
let net_ctrl = PreInitNetController::init(
db.clone(),
cfg.tor_control
.unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
cfg.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
Ipv4Addr::new(127, 0, 0, 1),
9050,
))),
&account.hostname,
account.tor_key,
)
.await?;
let net_ctrl = Arc::new(
NetController::init(
db.clone(),
cfg.tor_control
.unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
cfg.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
Ipv4Addr::new(127, 0, 0, 1),
9050,
))),
&account.hostname,
)
.await?,
);
webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
let os_net_service = net_ctrl.os_bindings().await?;
start_net.complete();
mount_logs.start();
@@ -394,8 +399,6 @@ pub async fn init(
mount_logs.complete();
tracing::info!("Mounted Logs");
let mut server_info = peek.as_public().as_server_info().de()?;
load_ca_cert.start();
// write to ca cert store
tokio::fs::write(
@@ -423,7 +426,15 @@ pub async fn init(
load_ca_cert.complete();
load_wifi.start();
crate::net::wifi::synchronize_network_manager(MAIN_DATA, &mut server_info.wifi).await?;
let wifi_interface = find_wifi_iface().await?;
let wifi = db
.mutate(|db| {
let wifi = db.as_public_mut().as_server_info_mut().as_wifi_mut();
wifi.as_interface_mut().ser(&wifi_interface)?;
wifi.de()
})
.await?;
crate::net::wifi::synchronize_network_manager(MAIN_DATA, &wifi).await?;
load_wifi.complete();
tracing::info!("Synchronized WiFi");
@@ -448,8 +459,10 @@ pub async fn init(
crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?;
init_tmp.complete();
let server_info = db.peek().await.into_public().into_server_info();
set_governor.start();
let governor = if let Some(governor) = &server_info.governor {
let selected_governor = server_info.as_governor().de()?;
let governor = if let Some(governor) = &selected_governor {
if cpupower::get_available_governors()
.await?
.contains(governor)
@@ -470,11 +483,11 @@ pub async fn init(
set_governor.complete();
sync_clock.start();
server_info.ntp_synced = false;
let mut ntp_synced = false;
let mut not_made_progress = 0u32;
for _ in 0..1800 {
if check_time_is_synchronized().await? {
server_info.ntp_synced = true;
ntp_synced = true;
break;
}
let t = SystemTime::now();
@@ -491,7 +504,7 @@ pub async fn init(
break;
}
}
if !server_info.ntp_synced {
if !ntp_synced {
tracing::warn!("Timed out waiting for system time to synchronize");
} else {
tracing::info!("Syncronized system clock");
@@ -499,15 +512,16 @@ pub async fn init(
sync_clock.complete();
enable_zram.start();
if server_info.zram {
crate::system::enable_zram().await?
if server_info.as_zram().de()? {
crate::system::enable_zram().await?;
tracing::info!("Enabled ZRAM");
}
enable_zram.complete();
update_server_info.start();
server_info.ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
server_info.devices = lshw().await?;
server_info.status_info = ServerStatus {
let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
let devices = lshw().await?;
let status_info = ServerStatus {
updated: false,
update_progress: None,
backup_progress: None,
@@ -515,10 +529,15 @@ pub async fn init(
restarting: false,
};
db.mutate(|v| {
v.as_public_mut().as_server_info_mut().ser(&server_info)?;
let server_info = v.as_public_mut().as_server_info_mut();
server_info.as_ntp_synced_mut().ser(&ntp_synced)?;
server_info.as_ram_mut().ser(&ram)?;
server_info.as_devices_mut().ser(&devices)?;
server_info.as_status_info_mut().ser(&status_info)?;
Ok(())
})
.await?;
tracing::info!("Updated server info");
update_server_info.complete();
launch_service_network.start();
@@ -527,6 +546,7 @@ pub async fn init(
.arg("lxc-net.service")
.invoke(ErrorKind::Lxc)
.await?;
tracing::info!("Launched service intranet");
launch_service_network.complete();
validate_db.start();
@@ -535,6 +555,7 @@ pub async fn init(
d.ser(&model)
})
.await?;
tracing::info!("Validated database");
validate_db.complete();
if let Some(progress) = postinit {
@@ -543,7 +564,10 @@ pub async fn init(
tracing::info!("System initialized.");
Ok(InitResult { net_ctrl })
Ok(InitResult {
net_ctrl,
os_net_service,
})
}
pub fn init_api<C: Context>() -> ParentHandler<C> {

5
core/startos/src/lib.rs
View File

@@ -87,6 +87,7 @@ use crate::context::{
CliContext, DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext,
};
use crate::disk::fsck::RequiresReboot;
use crate::net::net;
use crate::registry::context::{RegistryContext, RegistryUrlParams};
use crate::util::serde::HandlerExtSerde;
@@ -313,7 +314,7 @@ pub fn server<C: Context>() -> ParentHandler<C> {
.no_display()
.with_about("Remove system smtp server and credentials")
.with_call_remote::<CliContext>()
)
).subcommand("host", net::host::server_host_api::<C>().with_about("Commands for modifying the host for the system ui"))
}
pub fn package<C: Context>() -> ParentHandler<C> {
@@ -427,7 +428,7 @@ pub fn package<C: Context>() -> ParentHandler<C> {
.subcommand("attach", from_fn_async(service::cli_attach).no_display())
.subcommand(
"host",
net::host::host::<C>().with_about("Manage network hosts for a package"),
net::host::host_api::<C>().with_about("Manage network hosts for a package"),
)
}

10
core/startos/src/middleware/cors.rs
View File

@@ -1,6 +1,7 @@
use axum::body::Body;
use axum::extract::Request;
use axum::response::Response;
use http::{HeaderMap, HeaderValue};
use http::{HeaderMap, HeaderValue, Method};
use rpc_toolkit::{Empty, Middleware};
#[derive(Clone)]
@@ -52,6 +53,13 @@ impl<Context: Send + Sync + 'static> Middleware<Context> for Cors {
request: &mut Request,
) -> Result<(), Response> {
self.get_cors_headers(request);
if request.method() == Method::OPTIONS {
let mut response = Response::new(Body::empty());
response
.headers_mut()
.extend(std::mem::take(&mut self.headers));
return Err(response);
}
Ok(())
}
async fn process_http_response(&mut self, _: &Context, response: &mut Response) {

69
core/startos/src/net/acme.rs
View File

@@ -173,16 +173,26 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
}
pub fn acme<C: Context>() -> ParentHandler<C> {
ParentHandler::new().subcommand(
"init",
from_fn_async(init)
.no_display()
.with_about("Setup ACME certificate acquisition")
.with_call_remote::<CliContext>(),
)
ParentHandler::new()
.subcommand(
"init",
from_fn_async(init)
.with_metadata("sync_db", Value::Bool(true))
.no_display()
.with_about("Setup ACME certificate acquisition")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove)
.with_metadata("sync_db", Value::Bool(true))
.no_display()
.with_about("Setup ACME certificate acquisition")
.with_call_remote::<CliContext>(),
)
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, TS)]
#[ts(type = "string")]
pub struct AcmeProvider(pub Url);
impl FromStr for AcmeProvider {
@@ -193,9 +203,31 @@ impl FromStr for AcmeProvider {
"letsencrypt-staging" => async_acme::acme::LETS_ENCRYPT_STAGING_DIRECTORY.parse(),
s => s.parse(),
}
.map(|mut u: Url| {
let path = u
.path_segments()
.into_iter()
.flatten()
.filter(|p| !p.is_empty())
.map(|p| p.to_owned())
.collect::<Vec<_>>();
if let Ok(mut path_mut) = u.path_segments_mut() {
path_mut.clear();
path_mut.extend(path);
}
u
})
.map(Self)
}
}
impl<'de> Deserialize<'de> for AcmeProvider {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
crate::util::serde::deserialize_from_str(deserializer)
}
}
impl AsRef<str> for AcmeProvider {
fn as_ref(&self) -> &str {
self.0.as_str()
@@ -230,3 +262,24 @@ pub async fn init(
.await?;
Ok(())
}
#[derive(Deserialize, Serialize, Parser)]
pub struct RemoveAcmeParams {
#[arg(long)]
pub provider: AcmeProvider,
}
pub async fn remove(
ctx: RpcContext,
RemoveAcmeParams { provider }: RemoveAcmeParams,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_server_info_mut()
.as_acme_mut()
.remove(&provider)
})
.await?;
Ok(())
}

57
core/startos/src/net/dns.rs
View File

@@ -1,6 +1,6 @@
use std::borrow::Borrow;
use std::collections::BTreeMap;
use std::net::{Ipv4Addr, SocketAddr};
use std::net::Ipv4Addr;
use std::sync::{Arc, Weak};
use std::time::Duration;
@@ -19,6 +19,7 @@ use trust_dns_server::server::{Request, RequestHandler, ResponseHandler, Respons
use trust_dns_server::ServerFuture;
use crate::net::forward::START9_BRIDGE_IFACE;
use crate::util::sync::Watch;
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt};
@@ -140,38 +141,46 @@ impl RequestHandler for Resolver {
impl DnsController {
#[instrument(skip_all)]
pub async fn init(bind: &[SocketAddr]) -> Result<Self, Error> {
pub async fn init(mut lxcbr_status: Watch<bool>) -> Result<Self, Error> {
let services = Arc::new(RwLock::new(BTreeMap::new()));
let mut server = ServerFuture::new(Resolver {
services: services.clone(),
});
server.register_listener(
TcpListener::bind(bind)
.await
.with_kind(ErrorKind::Network)?,
Duration::from_secs(30),
);
server.register_socket(UdpSocket::bind(bind).await.with_kind(ErrorKind::Network)?);
Command::new("resolvectl")
.arg("dns")
.arg(START9_BRIDGE_IFACE)
.arg("127.0.0.1")
.invoke(ErrorKind::Network)
.await?;
Command::new("resolvectl")
.arg("domain")
.arg(START9_BRIDGE_IFACE)
.arg("embassy")
.invoke(ErrorKind::Network)
.await?;
let dns_server = tokio::spawn(async move {
server.register_listener(
TcpListener::bind((Ipv4Addr::LOCALHOST, 53))
.await
.with_kind(ErrorKind::Network)?,
Duration::from_secs(30),
);
server.register_socket(
UdpSocket::bind((Ipv4Addr::LOCALHOST, 53))
.await
.with_kind(ErrorKind::Network)?,
);
lxcbr_status.wait_for(|a| *a).await;
Command::new("resolvectl")
.arg("dns")
.arg(START9_BRIDGE_IFACE)
.arg("127.0.0.1")
.invoke(ErrorKind::Network)
.await?;
Command::new("resolvectl")
.arg("domain")
.arg(START9_BRIDGE_IFACE)
.arg("embassy")
.invoke(ErrorKind::Network)
.await?;
let dns_server = tokio::spawn(
server
.block_until_done()
.map_err(|e| Error::new(e, ErrorKind::Network)),
)
.await
.map_err(|e| Error::new(e, ErrorKind::Network))
})
.into();
Ok(Self {

158
core/startos/src/net/host/address.rs
View File

@@ -1,13 +1,16 @@
use std::collections::BTreeSet;
use clap::Parser;
use imbl_value::InternedString;
use models::{HostId, PackageId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::db::model::DatabaseModel;
use crate::net::acme::AcmeProvider;
use crate::net::host::{all_hosts, HostApiKind};
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde};
@@ -35,19 +38,51 @@ pub struct DomainConfig {
pub acme: Option<AcmeProvider>,
}
#[derive(Deserialize, Serialize, Parser)]
pub struct AddressApiParams {
host: HostId,
fn check_duplicates(db: &DatabaseModel) -> Result<(), Error> {
let mut onions = BTreeSet::<OnionAddressV3>::new();
let mut domains = BTreeSet::<InternedString>::new();
let mut check_onion = |onion: OnionAddressV3| {
if onions.contains(&onion) {
return Err(Error::new(
eyre!("onion address {onion} is already in use"),
ErrorKind::InvalidRequest,
));
}
onions.insert(onion);
Ok(())
};
let mut check_domain = |domain: InternedString| {
if domains.contains(&domain) {
return Err(Error::new(
eyre!("domain {domain} is already in use"),
ErrorKind::InvalidRequest,
));
}
domains.insert(domain);
Ok(())
};
for host in all_hosts(db) {
let host = host?;
for onion in host.as_onions().de()? {
check_onion(onion)?;
}
for domain in host.as_domains().keys()? {
check_domain(domain)?;
}
}
Ok(())
}
pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
ParentHandler::<C, AddressApiParams, PackageId>::new()
pub fn address_api<C: Context, Kind: HostApiKind>(
) -> ParentHandler<C, Kind::Params, Kind::InheritedParams> {
ParentHandler::<C, Kind::Params, Kind::InheritedParams>::new()
.subcommand(
"domain",
ParentHandler::<C, Empty, (PackageId, HostId)>::new()
ParentHandler::<C, Empty, Kind::Inheritance>::new()
.subcommand(
"add",
from_fn_async(add_domain)
from_fn_async(add_domain::<Kind>)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(|_, a| a)
.no_display()
.with_about("Add an address to this host")
@@ -55,20 +90,22 @@ pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
)
.subcommand(
"remove",
from_fn_async(remove_domain)
from_fn_async(remove_domain::<Kind>)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(|_, a| a)
.no_display()
.with_about("Remove an address from this host")
.with_call_remote::<CliContext>(),
)
.with_inherited(|AddressApiParams { host }, package| (package, host)),
.with_inherited(Kind::inheritance),
)
.subcommand(
"onion",
ParentHandler::<C, Empty, (PackageId, HostId)>::new()
ParentHandler::<C, Empty, Kind::Inheritance>::new()
.subcommand(
"add",
from_fn_async(add_onion)
from_fn_async(add_onion::<Kind>)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(|_, a| a)
.no_display()
.with_about("Add an address to this host")
@@ -76,18 +113,19 @@ pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
)
.subcommand(
"remove",
from_fn_async(remove_onion)
from_fn_async(remove_onion::<Kind>)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(|_, a| a)
.no_display()
.with_about("Remove an address from this host")
.with_call_remote::<CliContext>(),
)
.with_inherited(|AddressApiParams { host }, package| (package, host)),
.with_inherited(Kind::inheritance),
)
.subcommand(
"list",
from_fn_async(list_addresses)
.with_inherited(|AddressApiParams { host }, package| (package, host))
from_fn_async(list_addresses::<Kind>)
.with_inherited(Kind::inheritance)
.with_display_serializable()
.with_custom_display_fn(|HandlerArgs { params, .. }, res| {
use prettytable::*;
@@ -136,14 +174,14 @@ pub struct AddDomainParams {
pub acme: Option<AcmeProvider>,
}
pub async fn add_domain(
pub async fn add_domain<Kind: HostApiKind>(
ctx: RpcContext,
AddDomainParams {
domain,
private,
acme,
}: AddDomainParams,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
@@ -153,13 +191,7 @@ pub async fn add_domain(
}
}
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
Kind::host_for(&inheritance, db)?
.as_domains_mut()
.insert(
&domain,
@@ -167,12 +199,11 @@ pub async fn add_domain(
public: !private,
acme,
},
)
)?;
check_duplicates(db)
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Kind::sync_host(&ctx, inheritance).await?;
Ok(())
}
@@ -182,27 +213,19 @@ pub struct RemoveDomainParams {
pub domain: InternedString,
}
pub async fn remove_domain(
pub async fn remove_domain<Kind: HostApiKind>(
ctx: RpcContext,
RemoveDomainParams { domain }: RemoveDomainParams,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
Kind::host_for(&inheritance, db)?
.as_domains_mut()
.remove(&domain)
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Kind::sync_host(&ctx, inheritance).await?;
Ok(())
}
@@ -212,10 +235,10 @@ pub struct OnionParams {
pub onion: String,
}
pub async fn add_onion(
pub async fn add_onion<Kind: HostApiKind>(
ctx: RpcContext,
OnionParams { onion }: OnionParams,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
@@ -230,28 +253,22 @@ pub async fn add_onion(
.mutate(|db| {
db.as_private().as_key_store().as_onion().get_key(&onion)?;
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
Kind::host_for(&inheritance, db)?
.as_onions_mut()
.mutate(|a| Ok(a.insert(onion)))
.mutate(|a| Ok(a.insert(onion)))?;
check_duplicates(db)
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Kind::sync_host(&ctx, inheritance).await?;
Ok(())
}
pub async fn remove_onion(
pub async fn remove_onion<Kind: HostApiKind>(
ctx: RpcContext,
OnionParams { onion }: OnionParams,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
@@ -264,40 +281,23 @@ pub async fn remove_onion(
.parse::<OnionAddressV3>()?;
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
Kind::host_for(&inheritance, db)?
.as_onions_mut()
.mutate(|a| Ok(a.remove(&onion)))
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Kind::sync_host(&ctx, inheritance).await?;
Ok(())
}
pub async fn list_addresses(
pub async fn list_addresses<Kind: HostApiKind>(
ctx: RpcContext,
_: Empty,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<Vec<HostAddress>, Error> {
Ok(ctx
.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(&package)
.or_not_found(&package)?
.into_hosts()
.into_idx(&host)
.or_not_found(&host)?
Ok(Kind::host_for(&inheritance, &mut ctx.db.peek().await)?
.de()?
.addresses()
.collect())

59
core/startos/src/net/host/binding.rs
View File

@@ -3,13 +3,14 @@ use std::str::FromStr;
use clap::builder::ValueParserFactory;
use clap::Parser;
use models::{FromStrParser, HostId, PackageId};
use models::{FromStrParser, HostId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::net::forward::AvailablePorts;
use crate::net::host::HostApiKind;
use crate::net::vhost::AlpnInfo;
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde};
@@ -146,17 +147,13 @@ pub struct AddSslOptions {
pub alpn: Option<AlpnInfo>,
}
#[derive(Deserialize, Serialize, Parser)]
pub struct BindingApiParams {
host: HostId,
}
pub fn binding<C: Context>() -> ParentHandler<C, BindingApiParams, PackageId> {
ParentHandler::<C, BindingApiParams, PackageId>::new()
pub fn binding<C: Context, Kind: HostApiKind>(
) -> ParentHandler<C, Kind::Params, Kind::InheritedParams> {
ParentHandler::<C, Kind::Params, Kind::InheritedParams>::new()
.subcommand(
"list",
from_fn_async(list_bindings)
.with_inherited(|BindingApiParams { host }, package| (package, host))
from_fn_async(list_bindings::<Kind>)
.with_inherited(Kind::inheritance)
.with_display_serializable()
.with_custom_display_fn(|HandlerArgs { params, .. }, res| {
use prettytable::*;
@@ -194,30 +191,22 @@ pub fn binding<C: Context>() -> ParentHandler<C, BindingApiParams, PackageId> {
)
.subcommand(
"set-public",
from_fn_async(set_public)
.with_inherited(|BindingApiParams { host }, package| (package, host))
from_fn_async(set_public::<Kind>)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(Kind::inheritance)
.no_display()
.with_about("Add an binding to this host")
.with_call_remote::<CliContext>(),
)
}
pub async fn list_bindings(
pub async fn list_bindings<Kind: HostApiKind>(
ctx: RpcContext,
_: Empty,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<BTreeMap<u16, BindInfo>, Error> {
ctx.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(&package)
.or_not_found(&package)?
.into_hosts()
.into_idx(&host)
.or_not_found(&host)?
.into_bindings()
Kind::host_for(&inheritance, &mut ctx.db.peek().await)?
.as_bindings()
.de()
}
@@ -230,23 +219,17 @@ pub struct BindingSetPublicParams {
public: Option<bool>,
}
pub async fn set_public(
pub async fn set_public<Kind: HostApiKind>(
ctx: RpcContext,
BindingSetPublicParams {
internal_port,
public,
}: BindingSetPublicParams,
(package, host): (PackageId, HostId),
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
Kind::host_for(&inheritance, db)?
.as_bindings_mut()
.mutate(|b| {
b.get_mut(&internal_port)
@@ -257,11 +240,5 @@ pub async fn set_public(
})
})
.await?;
ctx.services
.get(&package)
.await
.as_ref()
.or_not_found(&package)?
.update_host(host)
.await
Kind::sync_host(&ctx, inheritance).await
}

136
core/startos/src/net/host/mod.rs
View File

@@ -1,9 +1,12 @@
use std::collections::{BTreeMap, BTreeSet};
use std::future::Future;
use std::panic::RefUnwindSafe;
use clap::Parser;
use imbl_value::InternedString;
use itertools::Itertools;
use models::{HostId, PackageId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, OrEmpty, ParentHandler};
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use ts_rs::TS;
@@ -11,7 +14,7 @@ use ts_rs::TS;
use crate::context::RpcContext;
use crate::db::model::DatabaseModel;
use crate::net::forward::AvailablePorts;
use crate::net::host::address::{address, DomainConfig, HostAddress};
use crate::net::host::address::{address_api, DomainConfig, HostAddress};
use crate::net::host::binding::{binding, BindInfo, BindOptions};
use crate::net::service_interface::HostnameInfo;
use crate::prelude::*;
@@ -19,12 +22,11 @@ use crate::prelude::*;
pub mod address;
pub mod binding;
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct Host {
pub kind: HostKind,
pub bindings: BTreeMap<u16, BindInfo>,
#[ts(type = "string[]")]
pub onions: BTreeSet<OnionAddressV3>,
@@ -39,14 +41,8 @@ impl AsRef<Host> for Host {
}
}
impl Host {
pub fn new(kind: HostKind) -> Self {
Self {
kind,
bindings: BTreeMap::new(),
onions: BTreeSet::new(),
domains: BTreeMap::new(),
hostname_info: BTreeMap::new(),
}
pub fn new() -> Self {
Self::default()
}
pub fn addresses<'a>(&'a self) -> impl Iterator<Item = HostAddress> + 'a {
self.onions
@@ -67,15 +63,6 @@ impl Host {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub enum HostKind {
Multi,
// Single,
// Static,
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
#[model = "Model<Self>"]
#[ts(export)]
@@ -94,10 +81,12 @@ impl Map for Hosts {
pub fn host_for<'a>(
db: &'a mut DatabaseModel,
package_id: &PackageId,
package_id: Option<&PackageId>,
host_id: &HostId,
host_kind: HostKind,
) -> Result<&'a mut Model<Host>, Error> {
let Some(package_id) = package_id else {
return Ok(db.as_public_mut().as_server_info_mut().as_host_mut());
};
fn host_info<'a>(
db: &'a mut DatabaseModel,
package_id: &PackageId,
@@ -121,7 +110,7 @@ pub fn host_for<'a>(
None
};
host_info(db, package_id)?.upsert(host_id, || {
let mut h = Host::new(host_kind);
let mut h = Host::new();
h.onions.insert(
tor_key
.or_not_found("generated tor key")?
@@ -132,12 +121,20 @@ pub fn host_for<'a>(
})
}
pub fn all_hosts(db: &DatabaseModel) -> impl Iterator<Item = Result<&Model<Host>, Error>> {
[Ok(db.as_public().as_server_info().as_host())]
.into_iter()
.chain(
[db.as_public().as_package_data().as_entries()]
.into_iter()
.flatten_ok()
.map(|entry| entry.and_then(|(_, v)| v.as_hosts().as_entries()))
.flatten_ok()
.map_ok(|(_, v)| v),
)
}
impl Model<Host> {
pub fn set_kind(&mut self, kind: HostKind) -> Result<(), Error> {
match (self.as_kind().de()?, kind) {
(HostKind::Multi, HostKind::Multi) => Ok(()),
}
}
pub fn add_binding(
&mut self,
available_ports: &mut AvailablePorts,
@@ -157,16 +154,78 @@ impl Model<Host> {
}
#[derive(Deserialize, Serialize, Parser)]
pub struct HostParams {
pub struct RequiresPackageId {
package: PackageId,
}
pub fn host<C: Context>() -> ParentHandler<C, HostParams> {
ParentHandler::<C, HostParams>::new()
#[derive(Deserialize, Serialize, Parser)]
pub struct RequiresHostId {
host: HostId,
}
pub trait HostApiKind: 'static {
type Params: Send + Sync + 'static;
type InheritedParams: Send + Sync + 'static;
type Inheritance: RefUnwindSafe + OrEmpty<Self::Inheritance> + Send + Sync + 'static;
fn inheritance(params: Self::Params, inherited: Self::InheritedParams) -> Self::Inheritance;
fn host_for<'a>(
inheritance: &Self::Inheritance,
db: &'a mut DatabaseModel,
) -> Result<&'a mut Model<Host>, Error>;
fn sync_host(
ctx: &RpcContext,
inheritance: Self::Inheritance,
) -> impl Future<Output = Result<(), Error>> + Send;
}
pub struct ForPackage;
impl HostApiKind for ForPackage {
type Params = RequiresHostId;
type InheritedParams = PackageId;
type Inheritance = (PackageId, HostId);
fn inheritance(
RequiresHostId { host }: Self::Params,
package: Self::InheritedParams,
) -> Self::Inheritance {
(package, host)
}
fn host_for<'a>(
(package, host): &Self::Inheritance,
db: &'a mut DatabaseModel,
) -> Result<&'a mut Model<Host>, Error> {
host_for(db, Some(package), host)
}
async fn sync_host(ctx: &RpcContext, (package, host): Self::Inheritance) -> Result<(), Error> {
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.sync_host(host).await?;
Ok(())
}
}
pub struct ForServer;
impl HostApiKind for ForServer {
type Params = Empty;
type InheritedParams = Empty;
type Inheritance = Empty;
fn inheritance(_: Self::Params, _: Self::InheritedParams) -> Self::Inheritance {
Empty {}
}
fn host_for<'a>(
_: &Self::Inheritance,
db: &'a mut DatabaseModel,
) -> Result<&'a mut Model<Host>, Error> {
host_for(db, None, &HostId::default())
}
async fn sync_host(ctx: &RpcContext, _: Self::Inheritance) -> Result<(), Error> {
ctx.os_net_service.sync_host(HostId::default()).await
}
}
pub fn host_api<C: Context>() -> ParentHandler<C, RequiresPackageId> {
ParentHandler::<C, RequiresPackageId>::new()
.subcommand(
"list",
from_fn_async(list_hosts)
.with_inherited(|HostParams { package }, _| package)
.with_inherited(|RequiresPackageId { package }, _| package)
.with_custom_display_fn(|_, ids| {
for id in ids {
println!("{id}")
@@ -177,14 +236,21 @@ pub fn host<C: Context>() -> ParentHandler<C, HostParams> {
)
.subcommand(
"address",
address::<C>().with_inherited(|HostParams { package }, _| package),
address_api::<C, ForPackage>()
.with_inherited(|RequiresPackageId { package }, _| package),
)
.subcommand(
"binding",
binding::<C>().with_inherited(|HostParams { package }, _| package),
binding::<C, ForPackage>().with_inherited(|RequiresPackageId { package }, _| package),
)
}
pub fn server_host_api<C: Context>() -> ParentHandler<C> {
ParentHandler::<C>::new()
.subcommand("address", address_api::<C, ForServer>())
.subcommand("binding", binding::<C, ForServer>())
}
pub async fn list_hosts(
ctx: RpcContext,
_: Empty,

4
core/startos/src/net/keys.rs
View File

@@ -21,7 +21,9 @@ impl KeyStore {
local_certs: CertStore::new(account)?,
acme: AcmeCertStore::new(),
};
res.onion.insert(account.tor_key.clone());
for tor_key in account.tor_keys.iter().cloned() {
res.onion.insert(tor_key);
}
Ok(res)
}
}

428
core/startos/src/net/net_controller.rs
View File

@@ -7,6 +7,8 @@ use imbl::OrdMap;
use imbl_value::InternedString;
use ipnet::IpNet;
use models::{HostId, OptionExt, PackageId};
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use torut::onion::{OnionAddressV3, TorSecretKeyV3};
use tracing::instrument;
@@ -16,8 +18,8 @@ use crate::hostname::Hostname;
use crate::net::dns::DnsController;
use crate::net::forward::LanPortForwardController;
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{BindId, BindOptions};
use crate::net::host::{host_for, Host, HostKind, Hosts};
use crate::net::host::binding::{AddSslOptions, BindId, BindOptions};
use crate::net::host::{host_for, Host, Hosts};
use crate::net::network_interface::NetworkInterfaceController;
use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname};
use crate::net::tor::TorController;
@@ -27,129 +29,44 @@ use crate::prelude::*;
use crate::util::serde::MaybeUtf8String;
use crate::HOST_IP;
pub struct PreInitNetController {
pub db: TypedPatchDb<Database>,
tor: TorController,
vhost: VHostController,
pub net_iface: Arc<NetworkInterfaceController>,
os_bindings: Vec<Arc<()>>,
server_hostnames: Vec<Option<InternedString>>,
}
impl PreInitNetController {
#[instrument(skip_all)]
pub async fn init(
db: TypedPatchDb<Database>,
tor_control: SocketAddr,
tor_socks: SocketAddr,
hostname: &Hostname,
os_tor_key: TorSecretKeyV3,
) -> Result<Self, Error> {
let net_iface = Arc::new(NetworkInterfaceController::new(db.clone()));
let mut res = Self {
db: db.clone(),
tor: TorController::new(tor_control, tor_socks),
vhost: VHostController::new(db, net_iface.clone()),
net_iface,
os_bindings: Vec::new(),
server_hostnames: Vec::new(),
};
res.add_os_bindings(hostname, os_tor_key).await?;
Ok(res)
}
async fn add_os_bindings(
&mut self,
hostname: &Hostname,
tor_key: TorSecretKeyV3,
) -> Result<(), Error> {
self.server_hostnames = vec![
// LAN IP
None,
// Internal DNS
Some("embassy".into()),
Some("startos".into()),
// localhost
Some("localhost".into()),
Some(hostname.no_dot_host_name()),
// LAN mDNS
Some(hostname.local_domain_name()),
];
let vhost_target = TargetInfo {
public: false,
acme: None,
addr: ([127, 0, 0, 1], 80).into(),
connect_ssl: Err(AlpnInfo::Specified(vec![
MaybeUtf8String("http/1.1".into()),
MaybeUtf8String("h2".into()),
])),
};
for hostname in self.server_hostnames.iter().cloned() {
self.os_bindings
.push(self.vhost.add(hostname, 443, vhost_target.clone())?);
}
// Tor
self.os_bindings.push(self.vhost.add(
Some(InternedString::from_display(
&tor_key.public().get_onion_address(),
)),
443,
vhost_target,
)?);
self.os_bindings.extend(
self.tor
.add(
tor_key,
vec![
(80, ([127, 0, 0, 1], 80).into()), // http
(443, ([127, 0, 0, 1], 443).into()), // https
],
)
.await?,
);
Ok(())
}
}
pub struct NetController {
db: TypedPatchDb<Database>,
pub(crate) db: TypedPatchDb<Database>,
pub(super) tor: TorController,
pub(super) vhost: VHostController,
pub net_iface: Arc<NetworkInterfaceController>,
pub(crate) net_iface: Arc<NetworkInterfaceController>,
pub(super) dns: DnsController,
pub(super) forward: LanPortForwardController,
pub(super) os_bindings: Vec<Arc<()>>,
pub(super) server_hostnames: Vec<Option<InternedString>>,
}
impl NetController {
pub async fn init(
PreInitNetController {
db,
tor,
vhost,
net_iface,
os_bindings,
server_hostnames,
}: PreInitNetController,
dns_bind: &[SocketAddr],
db: TypedPatchDb<Database>,
tor_control: SocketAddr,
tor_socks: SocketAddr,
hostname: &Hostname,
) -> Result<Self, Error> {
let mut res = Self {
db,
tor,
vhost,
dns: DnsController::init(dns_bind).await?,
let net_iface = Arc::new(NetworkInterfaceController::new(db.clone()));
Ok(Self {
db: db.clone(),
tor: TorController::new(tor_control, tor_socks),
vhost: VHostController::new(db, net_iface.clone()),
dns: DnsController::init(net_iface.lxcbr_status()).await?,
forward: LanPortForwardController::new(net_iface.subscribe()),
net_iface,
os_bindings,
server_hostnames,
};
res.os_bindings
.push(res.dns.add(None, HOST_IP.into()).await?);
Ok(res)
server_hostnames: vec![
// LAN IP
None,
// Internal DNS
Some("embassy".into()),
Some("startos".into()),
// localhost
Some("localhost".into()),
Some(hostname.no_dot_host_name()),
// LAN mDNS
Some(hostname.local_domain_name()),
],
})
}
#[instrument(skip_all)]
@@ -160,17 +77,48 @@ impl NetController {
) -> Result<NetService, Error> {
let dns = self.dns.add(Some(package.clone()), ip).await?;
let mut res = NetService {
shutdown: false,
id: package,
let res = NetService::new(NetServiceData {
id: Some(package),
ip,
dns,
controller: Arc::downgrade(self),
binds: BTreeMap::new(),
};
})?;
res.clear_bindings(Default::default()).await?;
Ok(res)
}
pub async fn os_bindings(self: &Arc<Self>) -> Result<NetService, Error> {
let dns = self.dns.add(None, HOST_IP.into()).await?;
let service = NetService::new(NetServiceData {
id: None,
ip: [127, 0, 0, 1].into(),
dns,
controller: Arc::downgrade(self),
binds: BTreeMap::new(),
})?;
service.clear_bindings(Default::default()).await?;
service
.bind(
HostId::default(),
80,
BindOptions {
preferred_external_port: 80,
add_ssl: Some(AddSslOptions {
preferred_external_port: 443,
alpn: Some(AlpnInfo::Specified(vec![
MaybeUtf8String("http/1.1".into()),
MaybeUtf8String("h2".into()),
])),
}),
secure: None,
},
)
.await?;
Ok(service)
}
}
#[derive(Default, Debug)]
@@ -180,15 +128,14 @@ struct HostBinds {
tor: BTreeMap<OnionAddressV3, (OrdMap<u16, SocketAddr>, Vec<Arc<()>>)>,
}
pub struct NetService {
shutdown: bool,
id: PackageId,
pub struct NetServiceData {
id: Option<PackageId>,
ip: Ipv4Addr,
dns: Arc<()>,
controller: Weak<NetController>,
binds: BTreeMap<HostId, HostBinds>,
}
impl NetService {
impl NetServiceData {
fn net_controller(&self) -> Result<Arc<NetController>, Error> {
Weak::upgrade(&self.controller).ok_or_else(|| {
Error::new(
@@ -198,49 +145,54 @@ impl NetService {
})
}
pub async fn bind(
async fn clear_bindings(
&mut self,
kind: HostKind,
id: HostId,
internal_port: u16,
options: BindOptions,
ctrl: &NetController,
except: BTreeSet<BindId>,
) -> Result<(), Error> {
crate::dbg!("bind", &kind, &id, internal_port, &options);
let pkg_id = &self.id;
let host = self
.net_controller()?
.db
.mutate(|db| {
let mut ports = db.as_private().as_available_ports().de()?;
let host = host_for(db, pkg_id, &id, kind)?;
host.add_binding(&mut ports, internal_port, options)?;
let host = host.de()?;
db.as_private_mut().as_available_ports_mut().ser(&ports)?;
Ok(host)
})
.await?;
self.update(id, host).await
}
pub async fn clear_bindings(&mut self, except: BTreeSet<BindId>) -> Result<(), Error> {
let pkg_id = &self.id;
let hosts = self
.net_controller()?
.db
.mutate(|db| {
let mut res = Hosts::default();
for (host_id, host) in db
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(pkg_id)
.or_not_found(pkg_id)?
.as_hosts_mut()
.as_entries_mut()?
{
if let Some(pkg_id) = &self.id {
let hosts = ctrl
.db
.mutate(|db| {
let mut res = Hosts::default();
for (host_id, host) in db
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(pkg_id)
.or_not_found(pkg_id)?
.as_hosts_mut()
.as_entries_mut()?
{
host.as_bindings_mut().mutate(|b| {
for (internal_port, info) in b {
if !except.contains(&BindId {
id: host_id.clone(),
internal_port: *internal_port,
}) {
info.disable();
}
}
Ok(())
})?;
res.0.insert(host_id, host.de()?);
}
Ok(res)
})
.await?;
let mut errors = ErrorCollection::new();
for (id, host) in hosts.0 {
errors.handle(self.update(ctrl, id, host).await);
}
errors.into_result()
} else {
let host = ctrl
.db
.mutate(|db| {
let host = db.as_public_mut().as_server_info_mut().as_host_mut();
host.as_bindings_mut().mutate(|b| {
for (internal_port, info) in b {
if !except.contains(&BindId {
id: host_id.clone(),
id: HostId::default(),
internal_port: *internal_port,
}) {
info.disable();
@@ -248,20 +200,14 @@ impl NetService {
}
Ok(())
})?;
res.0.insert(host_id, host.de()?);
}
Ok(res)
})
.await?;
let mut errors = ErrorCollection::new();
for (id, host) in hosts.0 {
errors.handle(self.update(id, host).await);
host.de()
})
.await?;
self.update(ctrl, HostId::default(), host).await
}
errors.into_result()
}
pub async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> {
let ctrl = self.net_controller()?;
async fn update(&mut self, ctrl: &NetController, id: HostId, host: Host) -> Result<(), Error> {
let mut forwards: BTreeMap<u16, (SocketAddr, bool)> = BTreeMap::new();
let mut vhosts: BTreeMap<(Option<InternedString>, u16), TargetInfo> = BTreeMap::new();
let mut tor: BTreeMap<OnionAddressV3, (TorSecretKeyV3, OrdMap<u16, SocketAddr>)> =
@@ -630,7 +576,7 @@ impl NetService {
ctrl.db
.mutate(|db| {
host_for(db, &self.id, &id, host.kind)?
host_for(db, self.id.as_ref(), &id)?
.as_hostname_info_mut()
.ser(&hostname_info)
})
@@ -638,10 +584,129 @@ impl NetService {
Ok(())
}
async fn update_all(&mut self) -> Result<(), Error> {
let ctrl = self.net_controller()?;
if let Some(id) = &self.id {
for (host_id, host) in ctrl
.db
.peek()
.await
.as_public()
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_hosts()
.as_entries()?
{
self.update(&*ctrl, host_id, host.de()?).await?;
}
} else {
self.update(
&*ctrl,
HostId::default(),
ctrl.db
.peek()
.await
.as_public()
.as_server_info()
.as_host()
.de()?,
)
.await?;
}
Ok(())
}
}
pub struct NetService {
shutdown: bool,
data: Arc<Mutex<NetServiceData>>,
sync_task: JoinHandle<()>,
}
impl NetService {
fn dummy() -> Self {
Self {
shutdown: true,
data: Arc::new(Mutex::new(NetServiceData {
id: None,
ip: Ipv4Addr::new(0, 0, 0, 0),
dns: Default::default(),
controller: Default::default(),
binds: BTreeMap::new(),
})),
sync_task: tokio::spawn(futures::future::ready(())),
}
}
fn new(data: NetServiceData) -> Result<Self, Error> {
let mut ip_info = data.net_controller()?.net_iface.subscribe();
let data = Arc::new(Mutex::new(data));
let thread_data = data.clone();
let sync_task = tokio::spawn(async move {
loop {
if let Err(e) = thread_data.lock().await.update_all().await {
tracing::error!("Failed to update network info: {e}");
tracing::debug!("{e:?}");
}
ip_info.changed().await;
}
});
Ok(Self {
shutdown: false,
data,
sync_task,
})
}
pub async fn bind(
&self,
id: HostId,
internal_port: u16,
options: BindOptions,
) -> Result<(), Error> {
let mut data = self.data.lock().await;
let pkg_id = &data.id;
let ctrl = data.net_controller()?;
let host = ctrl
.db
.mutate(|db| {
let mut ports = db.as_private().as_available_ports().de()?;
let host = host_for(db, pkg_id.as_ref(), &id)?;
host.add_binding(&mut ports, internal_port, options)?;
let host = host.de()?;
db.as_private_mut().as_available_ports_mut().ser(&ports)?;
Ok(host)
})
.await?;
data.update(&*ctrl, id, host).await
}
pub async fn clear_bindings(&self, except: BTreeSet<BindId>) -> Result<(), Error> {
let mut data = self.data.lock().await;
let ctrl = data.net_controller()?;
data.clear_bindings(&*ctrl, except).await
}
pub async fn update(&self, id: HostId, host: Host) -> Result<(), Error> {
let mut data = self.data.lock().await;
let ctrl = data.net_controller()?;
data.update(&*ctrl, id, host).await
}
pub async fn sync_host(&self, id: HostId) -> Result<(), Error> {
let mut data = self.data.lock().await;
let ctrl = data.net_controller()?;
let host = host_for(&mut ctrl.db.peek().await, data.id.as_ref(), &id)?.de()?;
data.update(&*ctrl, id, host).await
}
pub async fn remove_all(mut self) -> Result<(), Error> {
self.shutdown = true;
if let Some(ctrl) = Weak::upgrade(&self.controller) {
self.clear_bindings(Default::default()).await?;
self.sync_task.abort();
let mut data = self.data.lock().await;
if let Some(ctrl) = Weak::upgrade(&data.controller) {
self.shutdown = true;
data.clear_bindings(&*ctrl, Default::default()).await?;
drop(ctrl);
Ok(())
} else {
@@ -653,26 +718,15 @@ impl NetService {
}
}
pub fn get_ip(&self) -> Ipv4Addr {
self.ip
pub async fn get_ip(&self) -> Ipv4Addr {
self.data.lock().await.ip
}
}
impl Drop for NetService {
fn drop(&mut self) {
if !self.shutdown {
tracing::debug!("Dropping NetService for {}", self.id);
let svc = std::mem::replace(
self,
NetService {
shutdown: true,
id: Default::default(),
ip: Ipv4Addr::new(0, 0, 0, 0),
dns: Default::default(),
controller: Default::default(),
binds: BTreeMap::new(),
},
);
let svc = std::mem::replace(self, Self::dummy());
tokio::spawn(async move { svc.remove_all().await.log_err() });
}
}

58
core/startos/src/net/network_interface.rs
View File

@@ -27,6 +27,7 @@ use zbus::{proxy, Connection};
use crate::context::{CliContext, RpcContext};
use crate::db::model::public::{IpInfo, NetworkInterfaceInfo, NetworkInterfaceType};
use crate::db::model::Database;
use crate::net::forward::START9_BRIDGE_IFACE;
use crate::net::utils::{ipv6_is_link_local, ipv6_is_local};
use crate::prelude::*;
use crate::util::future::Until;
@@ -319,7 +320,10 @@ impl<'a> StubStream<'a> for SignalStream<'a> {
}
#[instrument(skip_all)]
async fn watcher(write_to: Watch<BTreeMap<InternedString, NetworkInterfaceInfo>>) {
async fn watcher(
write_to: Watch<BTreeMap<InternedString, NetworkInterfaceInfo>>,
lxcbr_status: Watch<bool>,
) {
loop {
let res: Result<(), Error> = async {
let connection = Connection::system().await?;
@@ -357,19 +361,27 @@ async fn watcher(write_to: Watch<BTreeMap<InternedString, NetworkInterfaceInfo>>
let mut ifaces = BTreeSet::new();
let mut jobs = Vec::new();
for device in devices {
use futures::future::Either;
let device_proxy =
device::DeviceProxy::new(&connection, device.clone()).await?;
let iface = InternedString::intern(device_proxy.ip_interface().await?);
if iface.is_empty() {
continue;
} else if &*iface == START9_BRIDGE_IFACE {
jobs.push(Either::Left(watch_activated(
&connection,
device_proxy.clone(),
&lxcbr_status,
)));
}
jobs.push(watch_ip(
jobs.push(Either::Right(watch_ip(
&connection,
device_proxy.clone(),
iface.clone(),
&write_to,
));
)));
ifaces.insert(iface);
}
@@ -588,13 +600,49 @@ async fn watch_ip(
}
}
#[instrument(skip(_connection, device_proxy, write_to))]
async fn watch_activated(
_connection: &Connection,
device_proxy: device::DeviceProxy<'_>,
write_to: &Watch<bool>,
) -> Result<(), Error> {
let mut until = Until::new()
.with_stream(
device_proxy
.receive_active_connection_changed()
.await
.stub(),
)
.with_stream(
device_proxy
.receive_state_changed()
.await?
.into_inner()
.stub(),
);
loop {
until
.run(async {
write_to.send(device_proxy._state().await? == 100);
Ok(())
})
.await?;
}
}
pub struct NetworkInterfaceController {
db: TypedPatchDb<Database>,
lxcbr_status: Watch<bool>,
ip_info: Watch<BTreeMap<InternedString, NetworkInterfaceInfo>>,
_watcher: NonDetachingJoinHandle<()>,
listeners: SyncMutex<BTreeMap<u16, Weak<()>>>,
}
impl NetworkInterfaceController {
pub fn lxcbr_status(&self) -> Watch<bool> {
self.lxcbr_status.clone_unseen()
}
pub fn subscribe(&self) -> Watch<BTreeMap<InternedString, NetworkInterfaceInfo>> {
self.ip_info.clone_unseen()
}
@@ -665,8 +713,10 @@ impl NetworkInterfaceController {
}
pub fn new(db: TypedPatchDb<Database>) -> Self {
let mut ip_info = Watch::new(BTreeMap::new());
let lxcbr_status = Watch::new(false);
Self {
db: db.clone(),
lxcbr_status: lxcbr_status.clone(),
ip_info: ip_info.clone(),
_watcher: tokio::spawn(async move {
match db
@@ -688,7 +738,7 @@ impl NetworkInterfaceController {
tracing::debug!("{e:?}");
}
};
tokio::join!(watcher(ip_info.clone()), async {
tokio::join!(watcher(ip_info.clone(), lxcbr_status), async {
let res: Result<(), Error> = async {
loop {
if let Err(e) = async {

29
core/startos/src/net/vhost.rs
View File

@@ -1,6 +1,5 @@
use std::collections::{BTreeMap, BTreeSet};
use std::net::{IpAddr, SocketAddr};
use std::str::FromStr;
use std::sync::{Arc, Weak};
use std::time::Duration;
@@ -328,17 +327,23 @@ impl VHostServer {
.headers()
.get(http::header::HOST)
.and_then(|host| host.to_str().ok());
let uri = Uri::from_parts({
let mut parts = req.uri().to_owned().into_parts();
parts.scheme = Some("https".parse()?);
parts.authority =
host.map(FromStr::from_str).transpose()?;
parts
})?;
Response::builder()
.status(http::StatusCode::TEMPORARY_REDIRECT)
.header(http::header::LOCATION, uri.to_string())
.body(Body::default())
if let Some(host) = host {
let uri = Uri::from_parts({
let mut parts =
req.uri().to_owned().into_parts();
parts.scheme = Some("https".parse()?);
parts.authority = Some(host.parse()?);
parts
})?;
Response::builder()
.status(http::StatusCode::TEMPORARY_REDIRECT)
.header(http::header::LOCATION, uri.to_string())
.body(Body::default())
} else {
Response::builder()
.status(http::StatusCode::BAD_REQUEST)
.body(Body::from("Host header required"))
}
}
.await
{

3
core/startos/src/net/wifi.rs
View File

@@ -899,9 +899,8 @@ impl TypedValueParser for CountryCodeParser {
#[instrument(skip_all)]
pub async fn synchronize_network_manager<P: AsRef<Path>>(
main_datadir: P,
wifi: &mut WifiInfo,
wifi: &WifiInfo,
) -> Result<(), Error> {
wifi.interface = find_wifi_iface().await?;
let persistent = main_datadir.as_ref().join("system-connections");
if tokio::fs::metadata(&persistent).await.is_err() {

19
core/startos/src/service/effects/net/bind.rs
View File

@@ -1,14 +1,12 @@
use models::{HostId, PackageId};
use crate::net::host::binding::{BindId, BindOptions, NetInfo};
use crate::net::host::HostKind;
use crate::service::effects::prelude::*;
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct BindParams {
kind: HostKind,
id: HostId,
internal_port: u16,
#[serde(flatten)]
@@ -17,15 +15,18 @@ pub struct BindParams {
pub async fn bind(
context: EffectContext,
BindParams {
kind,
id,
internal_port,
options,
}: BindParams,
) -> Result<(), Error> {
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.bind(kind, id, internal_port, options).await
context
.seed
.persistent_container
.net_service
.bind(id, internal_port, options)
.await
}
#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
@@ -41,8 +42,12 @@ pub async fn clear_bindings(
ClearBindingsParams { except }: ClearBindingsParams,
) -> Result<(), Error> {
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.clear_bindings(except.into_iter().collect()).await?;
context
.seed
.persistent_container
.net_service
.clear_bindings(except.into_iter().collect())
.await?;
Ok(())
}

3
core/startos/src/service/effects/net/info.rs
View File

@@ -4,6 +4,5 @@ use crate::service::effects::prelude::*;
pub async fn get_container_ip(context: EffectContext) -> Result<Ipv4Addr, Error> {
let context = context.deref()?;
let net_service = context.seed.persistent_container.net_service.lock().await;
Ok(net_service.get_ip())
Ok(context.seed.persistent_container.net_service.get_ip().await)
}

21
core/startos/src/service/mod.rs
View File

@@ -605,23 +605,12 @@ impl Service {
})
}
pub async fn update_host(&self, host_id: HostId) -> Result<(), Error> {
let mut service = self.seed.persistent_container.net_service.lock().await;
let host = self
.seed
.ctx
.db
.peek()
pub async fn sync_host(&self, host_id: HostId) -> Result<(), Error> {
self.seed
.persistent_container
.net_service
.sync_host(host_id)
.await
.as_public()
.as_package_data()
.as_idx(&self.seed.id)
.or_not_found(&self.seed.id)?
.as_hosts()
.as_idx(&host_id)
.or_not_found(&host_id)?
.de()?;
service.update(host_id, host).await
}
}

4
core/startos/src/service/persistent_container.rs
View File

@@ -110,7 +110,7 @@ pub struct PersistentContainer {
pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
pub(super) subcontainers: Arc<Mutex<BTreeMap<Guid, Subcontainer>>>,
pub(super) state: Arc<watch::Sender<ServiceState>>,
pub(super) net_service: Mutex<NetService>,
pub(super) net_service: NetService,
destroyed: bool,
}
@@ -285,7 +285,7 @@ impl PersistentContainer {
images,
subcontainers: Arc::new(Mutex::new(BTreeMap::new())),
state: Arc::new(watch::channel(ServiceState::new(start)).0),
net_service: Mutex::new(net_service),
net_service,
destroyed: false,
})
}

32
core/startos/src/service/service_actor.rs
View File

@@ -37,38 +37,6 @@ impl Actor for ServiceActor {
}
}
});
let seed = self.0.clone();
let mut ip_info = seed.ctx.net_controller.net_iface.subscribe();
jobs.add_job(async move {
loop {
if let Err(e) = async {
let mut service = seed.persistent_container.net_service.lock().await;
let hosts = seed
.ctx
.db
.peek()
.await
.as_public()
.as_package_data()
.as_idx(&seed.id)
.or_not_found(&seed.id)?
.as_hosts()
.de()?;
for (host_id, host) in hosts.0 {
service.update(host_id, host).await?;
}
Ok::<_, Error>(())
}
.await
{
tracing::error!("Error syncronizing net host after network change: {e}");
tracing::debug!("{e:?}");
}
ip_info.changed().await;
}
});
}
}

11
core/startos/src/service/service_map.rs
View File

@@ -4,7 +4,8 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use futures::future::BoxFuture;
use futures::{Future, FutureExt};
use futures::stream::FuturesUnordered;
use futures::{Future, FutureExt, StreamExt};
use helpers::NonDetachingJoinHandle;
use imbl::OrdMap;
use imbl_value::InternedString;
@@ -68,8 +69,12 @@ impl ServiceMap {
progress.start();
let ids = ctx.db.peek().await.as_public().as_package_data().keys()?;
progress.set_total(ids.len() as u64);
for id in ids {
if let Err(e) = self.load(ctx, &id, LoadDisposition::Retry).await {
let mut jobs = FuturesUnordered::new();
for id in &ids {
jobs.push(self.load(ctx, id, LoadDisposition::Retry));
}
while let Some(res) = jobs.next().await {
if let Err(e) = res {
tracing::error!("Error loading installed package as service: {e}");
tracing::debug!("{e:?}");
}

15
core/startos/src/setup.rs
View File

@@ -31,7 +31,7 @@ use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::disk::util::{pvscan, recovery_info, DiskInfo, StartOsRecoveryInfo};
use crate::disk::REPAIR_DISK_PATH;
use crate::init::{init, InitPhases, InitResult};
use crate::net::net_controller::PreInitNetController;
use crate::net::net_controller::NetController;
use crate::net::ssl::root_ca_start_time;
use crate::prelude::*;
use crate::progress::{FullProgress, PhaseProgressTrackerHandle};
@@ -80,10 +80,11 @@ async fn setup_init(
ctx: &SetupContext,
password: Option<String>,
init_phases: InitPhases,
) -> Result<(AccountInfo, PreInitNetController), Error> {
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
) -> Result<(AccountInfo, InitResult), Error> {
let init_result = init(&ctx.webserver, &ctx.config, init_phases).await?;
let account = net_ctrl
let account = init_result
.net_ctrl
.db
.mutate(|m| {
let mut account = AccountInfo::load(m)?;
@@ -99,7 +100,7 @@ async fn setup_init(
})
.await?;
Ok((account, net_ctrl))
Ok((account, init_result))
}
#[derive(Deserialize, Serialize, TS)]
@@ -452,13 +453,13 @@ async fn fresh_setup(
db.put(&ROOT, &Database::init(&account)?).await?;
drop(db);
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
let init_result = init(&ctx.webserver, &ctx.config, init_phases).await?;
let rpc_ctx = RpcContext::init(
&ctx.webserver,
&ctx.config,
guid,
Some(net_ctrl),
Some(init_result),
rpc_ctx_phases,
)
.await?;

8
core/startos/src/util/sync.rs
View File

@@ -77,6 +77,14 @@ impl<T> Watch<T> {
pub async fn changed(&mut self) {
futures::future::poll_fn(|cx| self.poll_changed(cx)).await
}
pub async fn wait_for<F: FnMut(&T) -> bool>(&mut self, mut f: F) {
loop {
if self.peek(&mut f) {
break;
}
self.changed().await;
}
}
pub fn send_if_modified<F: FnOnce(&mut T) -> bool>(&self, modify: F) -> bool {
self.shared.mutate(|shared| {
let changed = modify(&mut shared.data);

6
core/startos/src/version/mod.rs
View File

@@ -30,8 +30,9 @@ mod v0_3_6_alpha_9;
mod v0_3_6_alpha_10;
mod v0_3_6_alpha_11;
mod v0_3_6_alpha_12;
pub type Current = v0_3_6_alpha_11::Version; // VERSION_BUMP
pub type Current = v0_3_6_alpha_12::Version; // VERSION_BUMP
impl Current {
#[instrument(skip(self, db))]
@@ -113,6 +114,7 @@ enum Version {
V0_3_6_alpha_9(Wrapper<v0_3_6_alpha_9::Version>),
V0_3_6_alpha_10(Wrapper<v0_3_6_alpha_10::Version>),
V0_3_6_alpha_11(Wrapper<v0_3_6_alpha_11::Version>),
V0_3_6_alpha_12(Wrapper<v0_3_6_alpha_12::Version>),
Other(exver::Version),
}
@@ -148,6 +150,7 @@ impl Version {
Self::V0_3_6_alpha_9(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_10(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_11(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_12(v) => DynVersion(Box::new(v.0)),
Self::Other(v) => {
return Err(Error::new(
eyre!("unknown version {v}"),
@@ -175,6 +178,7 @@ impl Version {
Version::V0_3_6_alpha_9(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_10(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_11(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_12(Wrapper(x)) => x.semver(),
Version::Other(x) => x.clone(),
}
}

4
core/startos/src/version/v0_3_6_alpha_0.rs
View File

@@ -478,7 +478,7 @@ async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<Accoun
password: account_query
.try_get("password")
.with_ctx(|_| (ErrorKind::Database, "password"))?,
tor_key: TorSecretKeyV3::try_from(
tor_keys: vec![TorSecretKeyV3::try_from(
if let Some(bytes) = account_query
.try_get::<Option<Vec<u8>>, _>("tor_key")
.with_ctx(|_| (ErrorKind::Database, "tor_key"))?
@@ -503,7 +503,7 @@ async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<Accoun
.with_ctx(|_| (ErrorKind::Database, "password.u8 32"))?,
)
},
)?,
)?],
server_id: account_query
.try_get("server_id")
.with_ctx(|_| (ErrorKind::Database, "server_id"))?,

68
core/startos/src/version/v0_3_6_alpha_12.rs Normal file
View File

@@ -0,0 +1,68 @@
use std::collections::BTreeMap;
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::json;
use super::v0_3_5::V0_3_0_COMPAT;
use super::{v0_3_6_alpha_11, VersionT};
use crate::prelude::*;
lazy_static::lazy_static! {
static ref V0_3_6_alpha_12: exver::Version = exver::Version::new(
[0, 3, 6],
[PreReleaseSegment::String("alpha".into()), 12.into()]
);
}
#[derive(Clone, Copy, Debug, Default)]
pub struct Version;
impl VersionT for Version {
type Previous = v0_3_6_alpha_11::Version;
type PreUpRes = ();
async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
Ok(())
}
fn semver(self) -> exver::Version {
V0_3_6_alpha_12.clone()
}
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
let bindings: BTreeMap<u16, Value> = [(
80,
json!({
"enabled": false,
"options": {
"preferredExternalPort": 80,
"addSsl": {
"preferredExternalPort": 443,
"alpn": { "specified": [ "http/1.1", "h2" ] },
},
"secure": null,
},
"net": {
"assignedPort": null,
"assignedSslPort": 443,
"public": false,
}
}),
)]
.into_iter()
.collect();
let onion = db["public"]["serverInfo"]["onionAddress"].clone();
db["public"]["serverInfo"]["host"] = json!({
"bindings": bindings,
"onions": [onion],
"domains": {},
"hostnameInfo": {},
});
Ok(())
}
fn down(self, _db: &mut Value) -> Result<(), Error> {
Ok(())
}
}