Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 04:01:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

enabling support for wireguard and firewall (#2713)

Browse Source 
* wip: enabling support for wireguard and firewall

* wip

* wip

* wip

* wip

* wip

* implement some things

* fix warning

* wip

* alpha.23

* misc fixes

* remove ufw since no longer required

* remove debug info

* add cli bindings

* debugging

* fixes

* individualized acme and privacy settings for domains and bindings

* sdk version bump

* migration

* misc fixes

* refactor Host::update

* debug info

* refactor webserver

* misc fixes

* misc fixes

* refactor port forwarding

* recheck interfaces every 5 min if no dbus event

* misc fixes and cleanup

* misc fixes
This commit is contained in:
Aiden McClelland
2025-01-09 16:34:34 -07:00
committed by GitHub
parent 45ca9405d3
commit 29e8210782
144 changed files with 4878 additions and 2398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1149
core/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

1
core/models/Cargo.toml
View File

@@ -40,3 +40,4 @@ tokio = { version = "1", features = ["full"] }
torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies" }
tracing = "0.1.39"
yasi = "0.1.5"
zbus = "5"

7
core/models/src/errors.rs
View File

@@ -90,6 +90,7 @@ pub enum ErrorKind {
Lxc = 72,
Cancelled = 73,
Git = 74,
DBus = 75,
}
impl ErrorKind {
pub fn as_str(&self) -> &'static str {
@@ -169,6 +170,7 @@ impl ErrorKind {
Lxc => "LXC Error",
Cancelled => "Cancelled",
Git => "Git Error",
DBus => "DBus Error",
}
}
}
@@ -327,6 +329,11 @@ impl From<torut::onion::OnionAddressParseError> for Error {
Error::new(e, ErrorKind::Tor)
}
}
impl From<zbus::Error> for Error {
fn from(e: zbus::Error) -> Self {
Error::new(e, ErrorKind::DBus)
}
}
impl From<rustls::Error> for Error {
fn from(e: rustls::Error) -> Self {
Error::new(e, ErrorKind::OpenSsl)

17
core/startos/Cargo.toml
View File

@@ -14,7 +14,7 @@ keywords = [
name = "start-os"
readme = "README.md"
repository = "https://github.com/Start9Labs/start-os"
version = "0.3.6-alpha.9"
version = "0.3.6-alpha.10"
license = "MIT"
[lib]
@@ -50,7 +50,7 @@ test = []
[dependencies]
aes = { version = "0.7.5", features = ["ctr"] }
async-acme = { version = "0.5.0", git = "https://github.com/dr-bonez/async-acme.git", features = [
async-acme = { version = "0.6.0", git = "https://github.com/dr-bonez/async-acme.git", features = [
"use_rustls",
"use_tokio",
] }
@@ -62,7 +62,6 @@ async-compression = { version = "0.4.4", features = [
async-stream = "0.3.5"
async-trait = "0.1.74"
axum = { version = "0.7.3", features = ["ws"] }
axum-server = "0.6.0"
barrage = "0.2.3"
backhand = "0.18.0"
base32 = "0.5.0"
@@ -76,6 +75,7 @@ clap = "4.4.12"
color-eyre = "0.6.2"
console = "0.15.7"
console-subscriber = { version = "0.3.0", optional = true }
const_format = "0.2.34"
cookie = "0.18.0"
cookie_store = "0.21.0"
der = { version = "0.7.9", features = ["derive", "pem"] }
@@ -102,11 +102,15 @@ hex = "0.4.3"
hmac = "0.12.1"
http = "1.0.0"
http-body-util = "0.1"
hyper-util = { version = "0.1.5", features = [
"tokio",
hyper = { version = "1.5", features = ["server", "http1", "http2"] }
hyper-util = { version = "0.1.10", features = [
"server",
"server-auto",
"server-graceful",
"service",
"http1",
"http2",
"tokio",
] }
id-pool = { version = "0.2.2", default-features = false, features = [
"serde",
@@ -131,12 +135,14 @@ lazy_format = "2.0"
lazy_static = "1.4.0"
libc = "0.2.149"
log = "0.4.20"
mio = "1"
mbrman = "0.5.2"
models = { version = "*", path = "../models" }
new_mime_guess = "4"
nix = { version = "0.29.0", features = [
"fs",
"mount",
"net",
"process",
"sched",
"signal",
@@ -216,6 +222,7 @@ unix-named-pipe = "0.2.0"
url = { version = "2.4.1", features = ["serde"] }
urlencoding = "2.1.3"
uuid = { version = "1.4.1", features = ["v4"] }
zbus = "5.1.1"
zeroize = "1.6.0"
mail-send = { git = "https://github.com/dr-bonez/mail-send.git", branch = "main" }
rustls = "0.23.20"

1
core/startos/src/action.rs
View File

@@ -1,4 +1,3 @@
use std::collections::BTreeMap;
use std::fmt;
use clap::{CommandFactory, FromArgMatches, Parser};

3
core/startos/src/auth.rs
View File

@@ -187,9 +187,8 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(
Ok(())
}
#[derive(Deserialize, Serialize, Parser, TS)]
#[derive(Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
#[ts(export)]
pub struct LoginParams {
password: Option<PasswordType>,

3
core/startos/src/backup/restore.rs
View File

@@ -109,9 +109,10 @@ pub async fn recover_full_embassy(
db.put(&ROOT, &Database::init(&os_backup.account)?).await?;
drop(db);
let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?;
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
let rpc_ctx = RpcContext::init(
&ctx.webserver,
&ctx.config,
disk_guid.clone(),
Some(net_ctrl),

4
core/startos/src/bins/container_cli.rs
View File

@@ -4,7 +4,7 @@ use rpc_toolkit::CliApp;
use serde_json::Value;
use crate::service::cli::{ContainerCliContext, ContainerClientConfig};
use crate::util::logger::EmbassyLogger;
use crate::util::logger::LOGGER;
use crate::version::{Current, VersionT};
lazy_static::lazy_static! {
@@ -12,7 +12,7 @@ lazy_static::lazy_static! {
}
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
LOGGER.enable();
if let Err(e) = CliApp::new(
|cfg: ContainerClientConfig| Ok(ContainerCliContext::init(cfg)),
crate::service::effects::handler(),

10
core/startos/src/bins/registry.rs
View File

@@ -1,20 +1,20 @@
use std::ffi::OsString;
use clap::Parser;
use futures::FutureExt;
use futures::{FutureExt};
use tokio::signal::unix::signal;
use tracing::instrument;
use crate::net::web_server::WebServer;
use crate::net::web_server::{Acceptor, WebServer};
use crate::prelude::*;
use crate::registry::context::{RegistryConfig, RegistryContext};
use crate::util::logger::EmbassyLogger;
use crate::util::logger::LOGGER;
#[instrument(skip_all)]
async fn inner_main(config: &RegistryConfig) -> Result<(), Error> {
let server = async {
let ctx = RegistryContext::init(config).await?;
let mut server = WebServer::new(ctx.listen);
let mut server = WebServer::new(Acceptor::bind([ctx.listen]).await?);
server.serve_registry(ctx.clone());
let mut shutdown_recv = ctx.shutdown.subscribe();
@@ -63,7 +63,7 @@ async fn inner_main(config: &RegistryConfig) -> Result<(), Error> {
}
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
LOGGER.enable();
let config = RegistryConfig::parse_from(args).load().unwrap();

5
core/startos/src/bins/start_cli.rs
View File

@@ -5,7 +5,7 @@ use serde_json::Value;
use crate::context::config::ClientConfig;
use crate::context::CliContext;
use crate::util::logger::EmbassyLogger;
use crate::util::logger::LOGGER;
use crate::version::{Current, VersionT};
lazy_static::lazy_static! {
@@ -13,7 +13,8 @@ lazy_static::lazy_static! {
}
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
LOGGER.enable();
if let Err(e) = CliApp::new(
|cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?),
crate::expanded_api(),

29
core/startos/src/bins/start_init.rs
View File

@@ -1,3 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use tokio::process::Command;
@@ -11,16 +12,16 @@ use crate::disk::main::DEFAULT_PASSWORD;
use crate::disk::REPAIR_DISK_PATH;
use crate::firmware::{check_for_firmware_update, update_firmware};
use crate::init::{InitPhases, InitResult, STANDBY_MODE_PATH};
use crate::net::web_server::WebServer;
use crate::net::web_server::{UpgradableListener, WebServer};
use crate::prelude::*;
use crate::progress::FullProgressTracker;
use crate::shutdown::Shutdown;
use crate::util::Invoke;
use crate::PLATFORM;
use crate::{DATA_DIR, PLATFORM};
#[instrument(skip_all)]
async fn setup_or_init(
server: &mut WebServer,
server: &mut WebServer<UpgradableListener>,
config: &ServerConfig,
) -> Result<Result<(RpcContext, FullProgressTracker), Shutdown>, Error> {
if let Some(firmware) = check_for_firmware_update()
@@ -111,7 +112,7 @@ async fn setup_or_init(
.await
.is_err()
{
let ctx = SetupContext::init(config)?;
let ctx = SetupContext::init(server, config)?;
server.serve_setup(ctx.clone());
@@ -156,7 +157,7 @@ async fn setup_or_init(
let disk_guid = Arc::new(String::from(guid_string.trim()));
let requires_reboot = crate::disk::main::import(
&**disk_guid,
config.datadir(),
DATA_DIR,
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
RepairStrategy::Aggressive
} else {
@@ -178,18 +179,26 @@ async fn setup_or_init(
tracing::info!("Loaded Disk");
if requires_reboot.0 {
tracing::info!("Rebooting...");
let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1));
reboot_phase.start();
return Ok(Err(Shutdown {
export_args: Some((disk_guid, config.datadir().to_owned())),
export_args: Some((disk_guid, Path::new(DATA_DIR).to_owned())),
restart: true,
}));
}
let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?;
let InitResult { net_ctrl } =
crate::init::init(&server.acceptor_setter(), config, init_phases).await?;
let rpc_ctx =
RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?;
let rpc_ctx = RpcContext::init(
&server.acceptor_setter(),
config,
disk_guid,
Some(net_ctrl),
rpc_ctx_phases,
)
.await?;
Ok::<_, Error>(Ok((rpc_ctx, handle)))
}
@@ -203,7 +212,7 @@ async fn setup_or_init(
#[instrument(skip_all)]
pub async fn main(
server: &mut WebServer,
server: &mut WebServer<UpgradableListener>,
config: &ServerConfig,
) -> Result<Result<(RpcContext, FullProgressTracker), Shutdown>, Error> {
if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {

30
core/startos/src/bins/startd.rs
View File

@@ -1,6 +1,6 @@
use std::cmp::max;
use std::ffi::OsString;
use std::net::{Ipv6Addr, SocketAddr};
use std::net::IpAddr;
use std::sync::Arc;
use clap::Parser;
@@ -12,21 +12,26 @@ use tracing::instrument;
use crate::context::config::ServerConfig;
use crate::context::rpc::InitRpcContextPhases;
use crate::context::{DiagnosticContext, InitContext, RpcContext};
use crate::net::web_server::WebServer;
use crate::net::utils::ipv6_is_local;
use crate::net::web_server::{Acceptor, UpgradableListener, WebServer};
use crate::shutdown::Shutdown;
use crate::system::launch_metrics_task;
use crate::util::logger::EmbassyLogger;
use crate::util::io::append_file;
use crate::util::logger::LOGGER;
use crate::{Error, ErrorKind, ResultExt};
#[instrument(skip_all)]
async fn inner_main(
server: &mut WebServer,
server: &mut WebServer<UpgradableListener>,
config: &ServerConfig,
) -> Result<Option<Shutdown>, Error> {
let rpc_ctx = if !tokio::fs::metadata("/run/startos/initialized")
.await
.is_ok()
{
LOGGER.set_logfile(Some(
append_file("/run/startos/init.log").await?.into_std().await,
));
let (ctx, handle) = match super::start_init::main(server, &config).await? {
Err(s) => return Ok(Some(s)),
Ok(ctx) => ctx,
@@ -34,6 +39,7 @@ async fn inner_main(
tokio::fs::write("/run/startos/initialized", "").await?;
server.serve_main(ctx.clone());
LOGGER.set_logfile(None);
handle.complete();
ctx
@@ -44,6 +50,7 @@ async fn inner_main(
server.serve_init(init_ctx);
let ctx = RpcContext::init(
&server.acceptor_setter(),
config,
Arc::new(
tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
@@ -131,7 +138,7 @@ async fn inner_main(
}
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
LOGGER.enable();
let config = ServerConfig::parse_from(args).load().unwrap();
@@ -142,7 +149,18 @@ pub fn main(args: impl IntoIterator<Item = OsString>) {
.build()
.expect("failed to initialize runtime");
rt.block_on(async {
let mut server = WebServer::new(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80));
let addrs = crate::net::utils::all_socket_addrs_for(80).await?;
let mut server = WebServer::new(
Acceptor::bind_upgradable(addrs.into_iter().filter(|addr| match addr.ip() {
IpAddr::V4(ip4) => {
ip4.is_loopback()
|| (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations
|| ip4.is_link_local()
}
IpAddr::V6(ip6) => ipv6_is_local(ip6),
}))
.await?,
);
match inner_main(&mut server, &config).await {
Ok(a) => {
server.shutdown().await;

16
core/startos/src/context/config.rs
View File

@@ -13,6 +13,7 @@ use crate::disk::OsPartitionInfo;
use crate::init::init_postgres;
use crate::prelude::*;
use crate::util::serde::IoFormat;
use crate::MAIN_DATA;
pub const DEVICE_CONFIG_PATH: &str = "/media/startos/config/config.yaml"; // "/media/startos/config/config.yaml";
pub const CONFIG_PATH: &str = "/etc/startos/config.yaml";
@@ -103,8 +104,6 @@ pub struct ServerConfig {
#[arg(skip)]
pub os_partitions: Option<OsPartitionInfo>,
#[arg(long)]
pub bind_rpc: Option<SocketAddr>,
#[arg(long)]
pub tor_control: Option<SocketAddr>,
#[arg(long)]
pub tor_socks: Option<SocketAddr>,
@@ -112,8 +111,6 @@ pub struct ServerConfig {
pub dns_bind: Option<Vec<SocketAddr>>,
#[arg(long)]
pub revision_cache_size: Option<usize>,
#[arg(short, long)]
pub datadir: Option<PathBuf>,
#[arg(long)]
pub disable_encryption: Option<bool>,
#[arg(long)]
@@ -126,7 +123,6 @@ impl ContextConfig for ServerConfig {
fn merge_with(&mut self, other: Self) {
self.ethernet_interface = self.ethernet_interface.take().or(other.ethernet_interface);
self.os_partitions = self.os_partitions.take().or(other.os_partitions);
self.bind_rpc = self.bind_rpc.take().or(other.bind_rpc);
self.tor_control = self.tor_control.take().or(other.tor_control);
self.tor_socks = self.tor_socks.take().or(other.tor_socks);
self.dns_bind = self.dns_bind.take().or(other.dns_bind);
@@ -134,7 +130,6 @@ impl ContextConfig for ServerConfig {
.revision_cache_size
.take()
.or(other.revision_cache_size);
self.datadir = self.datadir.take().or(other.datadir);
self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks);
}
@@ -148,13 +143,8 @@ impl ServerConfig {
self.load_path_rec(Some(CONFIG_PATH))?;
Ok(self)
}
pub fn datadir(&self) -> &Path {
self.datadir
.as_deref()
.unwrap_or_else(|| Path::new("/embassy-data"))
}
pub async fn db(&self) -> Result<PatchDb, Error> {
let db_path = self.datadir().join("main").join("embassy.db");
let db_path = Path::new(MAIN_DATA).join("embassy.db");
let db = PatchDb::open(&db_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
@@ -163,7 +153,7 @@ impl ServerConfig {
}
#[instrument(skip_all)]
pub async fn secret_store(&self) -> Result<PgPool, Error> {
init_postgres(self.datadir()).await?;
init_postgres("/media/startos/data").await?;
let secret_store =
PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
.await?;

5
core/startos/src/context/diagnostic.rs
View File

@@ -1,5 +1,4 @@
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use rpc_toolkit::yajrc::RpcError;
@@ -13,7 +12,6 @@ use crate::shutdown::Shutdown;
use crate::Error;
pub struct DiagnosticContextSeed {
pub datadir: PathBuf,
pub shutdown: Sender<Shutdown>,
pub error: Arc<RpcError>,
pub disk_guid: Option<Arc<String>>,
@@ -25,7 +23,7 @@ pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
impl DiagnosticContext {
#[instrument(skip_all)]
pub fn init(
config: &ServerConfig,
_config: &ServerConfig,
disk_guid: Option<Arc<String>>,
error: Error,
) -> Result<Self, Error> {
@@ -35,7 +33,6 @@ impl DiagnosticContext {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
Ok(Self(Arc::new(DiagnosticContextSeed {
datadir: config.datadir().to_owned(),
shutdown,
disk_guid,
error: Arc::new(error.into()),

11
core/startos/src/context/rpc.rs
View File

@@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet};
use std::future::Future;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
@@ -31,6 +30,7 @@ use crate::init::check_time_is_synchronized;
use crate::lxc::{ContainerId, LxcContainer, LxcManager};
use crate::net::net_controller::{NetController, PreInitNetController};
use crate::net::utils::{find_eth_iface, find_wifi_iface};
use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter};
use crate::net::wifi::WpaCli;
use crate::prelude::*;
use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle};
@@ -47,7 +47,6 @@ pub struct RpcContextSeed {
pub os_partitions: OsPartitionInfo,
pub wifi_interface: Option<String>,
pub ethernet_interface: String,
pub datadir: PathBuf,
pub disk_guid: Arc<String>,
pub ephemeral_sessions: SyncMutex<Sessions>,
pub db: TypedPatchDb<Database>,
@@ -117,6 +116,7 @@ pub struct RpcContext(Arc<RpcContextSeed>);
impl RpcContext {
#[instrument(skip_all)]
pub async fn init(
webserver: &WebServerAcceptorSetter<UpgradableListener>,
config: &ServerConfig,
disk_guid: Arc<String>,
net_ctrl: Option<PreInitNetController>,
@@ -149,7 +149,7 @@ impl RpcContext {
if let Some(net_ctrl) = net_ctrl {
net_ctrl
} else {
PreInitNetController::init(
let net_ctrl = PreInitNetController::init(
db.clone(),
config
.tor_control
@@ -158,7 +158,9 @@ impl RpcContext {
&account.hostname,
account.tor_key.clone(),
)
.await?
.await?;
webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
net_ctrl
},
config
.dns_bind
@@ -210,7 +212,6 @@ impl RpcContext {
let seed = Arc::new(RpcContextSeed {
is_closed: AtomicBool::new(false),
datadir: config.datadir().to_path_buf(),
os_partitions: config.os_partitions.clone().ok_or_else(|| {
Error::new(
eyre!("OS Partition Information Missing"),

19
core/startos/src/context/setup.rs
View File

@@ -1,5 +1,5 @@
use std::ops::Deref;
use std::path::PathBuf;
use std::path::{Path};
use std::sync::Arc;
use std::time::Duration;
@@ -10,8 +10,6 @@ use josekit::jwk::Jwk;
use patch_db::PatchDb;
use rpc_toolkit::Context;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgConnectOptions;
use sqlx::PgPool;
use tokio::sync::broadcast::Sender;
use tokio::sync::OnceCell;
use tracing::instrument;
@@ -22,12 +20,13 @@ use crate::context::config::ServerConfig;
use crate::context::RpcContext;
use crate::disk::OsPartitionInfo;
use crate::hostname::Hostname;
use crate::init::init_postgres;
use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter};
use crate::prelude::*;
use crate::progress::FullProgressTracker;
use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations};
use crate::setup::SetupProgress;
use crate::util::net::WebSocketExt;
use crate::MAIN_DATA;
lazy_static::lazy_static! {
pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
@@ -61,6 +60,7 @@ impl TryFrom<&AccountInfo> for SetupResult {
}
pub struct SetupContextSeed {
pub webserver: WebServerAcceptorSetter<UpgradableListener>,
pub config: ServerConfig,
pub os_partitions: OsPartitionInfo,
pub disable_encryption: bool,
@@ -68,7 +68,6 @@ pub struct SetupContextSeed {
pub task: OnceCell<NonDetachingJoinHandle<()>>,
pub result: OnceCell<Result<(SetupResult, RpcContext), Error>>,
pub shutdown: Sender<()>,
pub datadir: PathBuf,
pub rpc_continuations: RpcContinuations,
}
@@ -76,10 +75,13 @@ pub struct SetupContextSeed {
pub struct SetupContext(Arc<SetupContextSeed>);
impl SetupContext {
#[instrument(skip_all)]
pub fn init(config: &ServerConfig) -> Result<Self, Error> {
pub fn init(
webserver: &WebServer<UpgradableListener>,
config: &ServerConfig,
) -> Result<Self, Error> {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
let datadir = config.datadir().to_owned();
Ok(Self(Arc::new(SetupContextSeed {
webserver: webserver.acceptor_setter(),
config: config.clone(),
os_partitions: config.os_partitions.clone().ok_or_else(|| {
Error::new(
@@ -92,13 +94,12 @@ impl SetupContext {
task: OnceCell::new(),
result: OnceCell::new(),
shutdown,
datadir,
rpc_continuations: RpcContinuations::new(),
})))
}
#[instrument(skip_all)]
pub async fn db(&self) -> Result<PatchDb, Error> {
let db_path = self.datadir.join("main").join("embassy.db");
let db_path = Path::new(MAIN_DATA).join("embassy.db");
let db = PatchDb::open(&db_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;

81
core/startos/src/db/model/public.rs
View File

@@ -1,10 +1,10 @@
use std::collections::{BTreeMap, BTreeSet};
use std::net::{Ipv4Addr, Ipv6Addr};
use std::net::{IpAddr, Ipv4Addr};
use chrono::{DateTime, Utc};
use exver::{Version, VersionRange};
use imbl_value::InternedString;
use ipnet::{Ipv4Net, Ipv6Net};
use ipnet::IpNet;
use isocountry::CountryCode;
use itertools::Itertools;
use models::PackageId;
@@ -17,7 +17,7 @@ use ts_rs::TS;
use crate::account::AccountInfo;
use crate::db::model::package::AllPackageData;
use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
use crate::net::acme::AcmeProvider;
use crate::prelude::*;
use crate::progress::FullProgress;
use crate::system::SmtpValue;
@@ -54,8 +54,8 @@ impl Public {
tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
.parse()
.unwrap(),
ip_info: BTreeMap::new(),
acme: None,
network_interfaces: BTreeMap::new(),
acme: BTreeMap::new(),
status_info: ServerStatus {
backup_progress: None,
updated: false,
@@ -130,8 +130,11 @@ pub struct ServerInfo {
/// for backwards compatibility
#[ts(type = "string")]
pub tor_address: Url,
pub ip_info: BTreeMap<String, IpInfo>,
pub acme: Option<AcmeSettings>,
#[ts(as = "BTreeMap::<String, NetworkInterfaceInfo>")]
#[serde(default)]
pub network_interfaces: BTreeMap<InternedString, NetworkInterfaceInfo>,
#[serde(default)]
pub acme: BTreeMap<AcmeProvider, AcmeSettings>,
#[serde(default)]
pub status_info: ServerStatus,
pub wifi: WifiInfo,
@@ -151,43 +154,61 @@ pub struct ServerInfo {
pub devices: Vec<LshwDevice>,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct IpInfo {
#[ts(type = "string | null")]
pub ipv4_range: Option<Ipv4Net>,
pub ipv4: Option<Ipv4Addr>,
#[ts(type = "string | null")]
pub ipv6_range: Option<Ipv6Net>,
pub ipv6: Option<Ipv6Addr>,
pub struct NetworkInterfaceInfo {
pub public: Option<bool>,
pub ip_info: Option<IpInfo>,
}
impl IpInfo {
pub async fn for_interface(iface: &str) -> Result<Self, Error> {
let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
Ok(Self {
ipv4_range,
ipv4,
ipv6_range,
ipv6,
impl NetworkInterfaceInfo {
pub fn public(&self) -> bool {
self.public.unwrap_or_else(|| {
!self.ip_info.as_ref().map_or(true, |ip_info| {
ip_info.subnets.iter().all(|ipnet| {
match ipnet.addr() {
IpAddr::V4(ip4) => {
ip4.is_loopback()
|| (ip4.is_private() && !ip4.octets().starts_with(&[10, 59])) // reserving 10.59 for public wireguard configurations
|| ip4.is_link_local()
}
IpAddr::V6(_) => true,
}
})
})
})
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize, TS)]
#[ts(export)]
#[serde(rename_all = "camelCase")]
pub struct IpInfo {
pub scope_id: u32,
pub device_type: Option<NetworkInterfaceType>,
#[ts(type = "string[]")]
pub subnets: BTreeSet<IpNet>,
pub wan_ip: Option<Ipv4Addr>,
#[ts(type = "string[]")]
pub ntp_servers: BTreeSet<InternedString>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, TS)]
#[ts(export)]
#[serde(rename_all = "kebab-case")]
pub enum NetworkInterfaceType {
Ethernet,
Wireless,
Wireguard,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct AcmeSettings {
#[ts(type = "string")]
pub provider: Url,
/// email addresses for letsencrypt
pub contact: Vec<String>,
#[ts(type = "string[]")]
/// domains to get letsencrypt certs for
pub domains: BTreeSet<InternedString>,
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]

4
core/startos/src/diagnostic.rs
View File

@@ -10,7 +10,7 @@ use crate::context::{CliContext, DiagnosticContext, RpcContext};
use crate::init::SYSTEM_REBUILD_PATH;
use crate::shutdown::Shutdown;
use crate::util::io::delete_file;
use crate::Error;
use crate::{Error, DATA_DIR};
pub fn diagnostic<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
@@ -71,7 +71,7 @@ pub fn restart(ctx: DiagnosticContext) -> Result<(), Error> {
export_args: ctx
.disk_guid
.clone()
.map(|guid| (guid, ctx.datadir.clone())),
.map(|guid| (guid, Path::new(DATA_DIR).to_owned())),
restart: true,
})
.expect("receiver dropped");

1
core/startos/src/disk/mount/backup.rs
View File

@@ -7,7 +7,6 @@ use models::PackageId;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
use super::filesystem::ecryptfs::EcryptFS;
use super::guard::{GenericMountGuard, TmpMountGuard};
use crate::auth::check_password;
use crate::backup::target::BackupInfo;

1
core/startos/src/disk/mount/filesystem/mod.rs
View File

@@ -1,7 +1,6 @@
use std::ffi::OsStr;
use std::fmt::{Display, Write};
use std::path::Path;
use std::time::Duration;
use digest::generic_array::GenericArray;
use digest::OutputSizeUser;

29
core/startos/src/init.rs
View File

@@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime};
use axum::extract::ws::{self};
use color_eyre::eyre::eyre;
use const_format::formatcp;
use futures::{StreamExt, TryStreamExt};
use itertools::Itertools;
use models::ResultExt;
@@ -25,6 +26,7 @@ use crate::db::model::Database;
use crate::disk::mount::util::unmount;
use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
use crate::net::net_controller::PreInitNetController;
use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter};
use crate::prelude::*;
use crate::progress::{
FullProgress, FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar,
@@ -37,7 +39,7 @@ use crate::util::io::{create_file, IOHook};
use crate::util::lshw::lshw;
use crate::util::net::WebSocketExt;
use crate::util::{cpupower, Invoke};
use crate::Error;
use crate::{Error, MAIN_DATA, PACKAGE_DATA};
pub const SYSTEM_REBUILD_PATH: &str = "/media/startos/config/system-rebuild";
pub const STANDBY_MODE_PATH: &str = "/media/startos/config/standby";
@@ -274,6 +276,7 @@ pub async fn run_script<P: AsRef<Path>>(path: P, mut progress: PhaseProgressTrac
#[instrument(skip_all)]
pub async fn init(
webserver: &WebServerAcceptorSetter<UpgradableListener>,
cfg: &ServerConfig,
InitPhases {
preinit,
@@ -317,7 +320,7 @@ pub async fn init(
})?;
tokio::fs::set_permissions(LOCAL_AUTH_COOKIE_PATH, Permissions::from_mode(0o046)).await?;
Command::new("chown")
.arg("root:embassy")
.arg("root:startos")
.arg(LOCAL_AUTH_COOKIE_PATH)
.invoke(crate::ErrorKind::Filesystem)
.await?;
@@ -356,10 +359,11 @@ pub async fn init(
account.tor_key,
)
.await?;
webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
start_net.complete();
mount_logs.start();
let log_dir = cfg.datadir().join("main/logs");
let log_dir = Path::new(MAIN_DATA).join("logs");
if tokio::fs::metadata(&log_dir).await.is_err() {
tokio::fs::create_dir_all(&log_dir).await?;
}
@@ -419,36 +423,28 @@ pub async fn init(
load_ca_cert.complete();
load_wifi.start();
crate::net::wifi::synchronize_wpa_supplicant_conf(
&cfg.datadir().join("main"),
&mut server_info.wifi,
)
.await?;
crate::net::wifi::synchronize_network_manager(MAIN_DATA, &mut server_info.wifi).await?;
load_wifi.complete();
tracing::info!("Synchronized WiFi");
init_tmp.start();
let tmp_dir = cfg.datadir().join("package-data/tmp");
let tmp_dir = Path::new(PACKAGE_DATA).join("tmp");
if tokio::fs::metadata(&tmp_dir).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_dir).await?;
}
if tokio::fs::metadata(&tmp_dir).await.is_err() {
tokio::fs::create_dir_all(&tmp_dir).await?;
}
let tmp_var = cfg.datadir().join(format!("package-data/tmp/var"));
let tmp_var = Path::new(PACKAGE_DATA).join("tmp/var");
if tokio::fs::metadata(&tmp_var).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_var).await?;
}
crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?;
let downloading = cfg
.datadir()
.join(format!("package-data/archive/downloading"));
let downloading = Path::new(PACKAGE_DATA).join("archive/downloading");
if tokio::fs::metadata(&downloading).await.is_ok() {
tokio::fs::remove_dir_all(&downloading).await?;
}
let tmp_docker = cfg
.datadir()
.join(format!("package-data/tmp/{CONTAINER_TOOL}"));
let tmp_docker = Path::new(PACKAGE_DATA).join(formatcp!("tmp/{CONTAINER_TOOL}"));
crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?;
init_tmp.complete();
@@ -509,7 +505,6 @@ pub async fn init(
enable_zram.complete();
update_server_info.start();
server_info.ip_info = crate::net::dhcp::init_ips().await?;
server_info.ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
server_info.devices = lshw().await?;
server_info.status_info = ServerStatus {

3
core/startos/src/install/mod.rs
View File

@@ -202,9 +202,6 @@ pub async fn sideload(
use axum::extract::ws::Message;
async move {
if let Err(e) = async {
type RpcResponse = rpc_toolkit::yajrc::RpcResponse<
GenericRpcMethod<&'static str, (), FullProgress>,
>;
tokio::select! {
res = async {
while let Some(progress) = progress_listener.next().await {

7
core/startos/src/lib.rs
View File

@@ -1,6 +1,11 @@
use const_format::formatcp;
pub const DATA_DIR: &str = "/media/startos/data";
pub const MAIN_DATA: &str = formatcp!("{DATA_DIR}/main");
pub const PACKAGE_DATA: &str = formatcp!("{DATA_DIR}/package-data");
pub const DEFAULT_REGISTRY: &str = "https://registry.start9.com";
// pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
pub const HOST_IP: [u8; 4] = [10, 0, 3, 1];
pub use std::env::consts::ARCH;
lazy_static::lazy_static! {
pub static ref PLATFORM: String = {

1
core/startos/src/lxc/mod.rs
View File

@@ -1,5 +1,4 @@
use std::collections::BTreeSet;
use std::ffi::OsString;
use std::net::Ipv4Addr;
use std::path::Path;
use std::sync::{Arc, Weak};

174
core/startos/src/net/acme.rs
View File

@@ -1,6 +1,7 @@
use std::collections::{BTreeMap, BTreeSet};
use std::str::FromStr;
use async_acme::acme::Identifier;
use clap::builder::ValueParserFactory;
use clap::Parser;
use imbl_value::InternedString;
@@ -10,6 +11,7 @@ use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use url::Url;
use crate::context::{CliContext, RpcContext};
@@ -78,10 +80,18 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
async fn read_certificate(
&self,
domains: &[String],
identifiers: &[Identifier],
directory_url: &str,
) -> Result<Option<(String, String)>, Self::Error> {
let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect());
let identifiers = JsonKey::new(
identifiers
.into_iter()
.map(|d| match d {
Identifier::Dns(d) => d.into(),
Identifier::Ip(ip) => InternedString::from_display(ip),
})
.collect(),
);
let directory_url = directory_url
.parse::<Url>()
.with_kind(ErrorKind::ParseUrl)?;
@@ -94,7 +104,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
.into_acme()
.into_certs()
.into_idx(&directory_url)
.and_then(|a| a.into_idx(&domains))
.and_then(|a| a.into_idx(&identifiers))
else {
return Ok(None);
};
@@ -120,13 +130,21 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
async fn write_certificate(
&self,
domains: &[String],
identifiers: &[Identifier],
directory_url: &str,
key_pem: &str,
certificate_pem: &str,
) -> Result<(), Self::Error> {
tracing::info!("Saving new certificate for {domains:?}");
let domains = JsonKey::new(domains.into_iter().map(InternedString::intern).collect());
tracing::info!("Saving new certificate for {identifiers:?}");
let identifiers = JsonKey::new(
identifiers
.into_iter()
.map(|d| match d {
Identifier::Dns(d) => d.into(),
Identifier::Ip(ip) => InternedString::from_display(ip),
})
.collect(),
);
let directory_url = directory_url
.parse::<Url>()
.with_kind(ErrorKind::ParseUrl)?;
@@ -146,7 +164,7 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
.as_acme_mut()
.as_certs_mut()
.upsert(&directory_url, || Ok(BTreeMap::new()))?
.insert(&domains, &cert)
.insert(&identifiers, &cert)
})
.await?;
@@ -155,22 +173,17 @@ impl<'a> async_acme::cache::AcmeCache for AcmeCertCache<'a> {
}
pub fn acme<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
"init",
from_fn_async(init)
.no_display()
.with_about("Setup ACME certificate acquisition")
.with_call_remote::<CliContext>(),
)
.subcommand(
"domain",
domain::<C>()
.with_about("Add, remove, or view domains for which to acquire ACME certificates"),
)
ParentHandler::new().subcommand(
"init",
from_fn_async(init)
.no_display()
.with_about("Setup ACME certificate acquisition")
.with_call_remote::<CliContext>(),
)
}
#[derive(Clone, Deserialize, Serialize)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS)]
#[ts(type = "string")]
pub struct AcmeProvider(pub Url);
impl FromStr for AcmeProvider {
type Err = <Url as FromStr>::Err;
@@ -183,6 +196,11 @@ impl FromStr for AcmeProvider {
.map(Self)
}
}
impl AsRef<str> for AcmeProvider {
fn as_ref(&self) -> &str {
self.0.as_str()
}
}
impl ValueParserFactory for AcmeProvider {
type Parser = FromStrParser<Self>;
fn value_parser() -> Self::Parser {
@@ -200,125 +218,15 @@ pub struct InitAcmeParams {
pub async fn init(
ctx: RpcContext,
InitAcmeParams {
provider: AcmeProvider(provider),
contact,
}: InitAcmeParams,
InitAcmeParams { provider, contact }: InitAcmeParams,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_server_info_mut()
.as_acme_mut()
.map_mutate(|acme| {
Ok(Some(AcmeSettings {
provider,
contact,
domains: acme.map(|acme| acme.domains).unwrap_or_default(),
}))
})
.insert(&provider, &AcmeSettings { contact })
})
.await?;
Ok(())
}
pub fn domain<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
"add",
from_fn_async(add_domain)
.no_display()
.with_about("Add a domain for which to acquire ACME certificates")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove_domain)
.no_display()
.with_about("Remove a domain for which to acquire ACME certificates")
.with_call_remote::<CliContext>(),
)
.subcommand(
"list",
from_fn_async(list_domains)
.with_custom_display_fn(|_, res| {
for domain in res {
println!("{domain}")
}
Ok(())
})
.with_about("List domains for which to acquire ACME certificates")
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser)]
pub struct DomainParams {
pub domain: InternedString,
}
pub async fn add_domain(
ctx: RpcContext,
DomainParams { domain }: DomainParams,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_server_info_mut()
.as_acme_mut()
.transpose_mut()
.ok_or_else(|| {
Error::new(
eyre!("Please call `start-cli net acme init` before adding a domain"),
ErrorKind::InvalidRequest,
)
})?
.as_domains_mut()
.mutate(|domains| {
domains.insert(domain);
Ok(())
})
})
.await?;
Ok(())
}
pub async fn remove_domain(
ctx: RpcContext,
DomainParams { domain }: DomainParams,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
if let Some(acme) = db
.as_public_mut()
.as_server_info_mut()
.as_acme_mut()
.transpose_mut()
{
acme.as_domains_mut().mutate(|domains| {
domains.remove(&domain);
Ok(())
})
} else {
Ok(())
}
})
.await?;
Ok(())
}
pub async fn list_domains(ctx: RpcContext) -> Result<BTreeSet<InternedString>, Error> {
if let Some(acme) = ctx
.db
.peek()
.await
.into_public()
.into_server_info()
.into_acme()
.transpose()
{
acme.into_domains().de()
} else {
Ok(BTreeSet::new())
}
}

99
core/startos/src/net/dhcp.rs
View File

@@ -1,99 +0,0 @@
use std::collections::{BTreeMap, BTreeSet};
use std::net::IpAddr;
use clap::Parser;
use futures::TryStreamExt;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::db::model::public::IpInfo;
use crate::net::utils::{iface_is_physical, list_interfaces};
use crate::prelude::*;
use crate::Error;
lazy_static::lazy_static! {
static ref CACHED_IPS: RwLock<BTreeSet<IpAddr>> = RwLock::new(BTreeSet::new());
}
async fn _ips() -> Result<BTreeSet<IpAddr>, Error> {
Ok(init_ips()
.await?
.values()
.flat_map(|i| {
std::iter::empty()
.chain(i.ipv4.map(IpAddr::from))
.chain(i.ipv6.map(IpAddr::from))
})
.collect())
}
pub async fn ips() -> Result<BTreeSet<IpAddr>, Error> {
let ips = CACHED_IPS.read().await.clone();
if !ips.is_empty() {
return Ok(ips);
}
let ips = _ips().await?;
*CACHED_IPS.write().await = ips.clone();
Ok(ips)
}
pub async fn init_ips() -> Result<BTreeMap<String, IpInfo>, Error> {
let mut res = BTreeMap::new();
let mut ifaces = list_interfaces();
while let Some(iface) = ifaces.try_next().await? {
if iface_is_physical(&iface).await {
let ip_info = IpInfo::for_interface(&iface).await?;
res.insert(iface, ip_info);
}
}
Ok(res)
}
// #[command(subcommands(update))]
pub fn dhcp<C: Context>() -> ParentHandler<C> {
ParentHandler::new().subcommand(
"update",
from_fn_async::<_, _, (), Error, (RpcContext, UpdateParams)>(update)
.no_display()
.with_about("Update IP assigned by dhcp")
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
pub struct UpdateParams {
interface: String,
}
pub async fn update(
ctx: RpcContext,
UpdateParams { interface }: UpdateParams,
) -> Result<(), Error> {
if iface_is_physical(&interface).await {
let ip_info = IpInfo::for_interface(&interface).await?;
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_server_info_mut()
.as_ip_info_mut()
.insert(&interface, &ip_info)
})
.await?;
let mut cached = CACHED_IPS.write().await;
if cached.is_empty() {
*cached = _ips().await?;
} else {
cached.extend(
std::iter::empty()
.chain(ip_info.ipv4.map(IpAddr::from))
.chain(ip_info.ipv6.map(IpAddr::from)),
);
}
}
Ok(())
}

371
core/startos/src/net/forward.rs
View File

@@ -1,12 +1,16 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::net::SocketAddr;
use std::sync::{Arc, Weak};
use futures::channel::oneshot;
use helpers::NonDetachingJoinHandle;
use id_pool::IdPool;
use imbl_value::InternedString;
use serde::{Deserialize, Serialize};
use tokio::process::Command;
use tokio::sync::Mutex;
use tokio::sync::{mpsc, watch};
use crate::db::model::public::NetworkInterfaceInfo;
use crate::prelude::*;
use crate::util::Invoke;
@@ -34,144 +38,269 @@ impl AvailablePorts {
}
}
#[derive(Debug)]
struct ForwardRequest {
public: bool,
target: SocketAddr,
rc: Weak<()>,
}
#[derive(Debug, Default)]
struct ForwardState {
requested: BTreeMap<u16, ForwardRequest>,
current: BTreeMap<u16, BTreeMap<InternedString, SocketAddr>>,
}
impl ForwardState {
async fn sync(&mut self, interfaces: &BTreeMap<InternedString, bool>) -> Result<(), Error> {
let private_interfaces = interfaces
.iter()
.filter(|(_, public)| !*public)
.map(|(i, _)| i)
.collect::<BTreeSet<_>>();
let all_interfaces = interfaces.keys().collect::<BTreeSet<_>>();
self.requested.retain(|_, req| req.rc.strong_count() > 0);
for external in self
.requested
.keys()
.chain(self.current.keys())
.copied()
.collect::<BTreeSet<_>>()
{
match (
self.requested.get(&external),
self.current.get_mut(&external),
) {
(Some(req), Some(cur)) => {
let expected = if req.public {
&all_interfaces
} else {
&private_interfaces
};
let actual = cur.keys().collect::<BTreeSet<_>>();
let mut to_rm = actual
.difference(expected)
.copied()
.cloned()
.collect::<BTreeSet<_>>();
let mut to_add = expected
.difference(&actual)
.copied()
.cloned()
.collect::<BTreeSet<_>>();
for interface in actual.intersection(expected).copied() {
if cur[interface] != req.target {
to_rm.insert(interface.clone());
to_add.insert(interface.clone());
}
}
for interface in to_rm {
unforward(external, &*interface, cur[&interface]).await?;
cur.remove(&interface);
}
for interface in to_add {
forward(external, &*interface, req.target).await?;
cur.insert(interface, req.target);
}
}
(Some(req), None) => {
let cur = self.current.entry(external).or_default();
for interface in if req.public {
&all_interfaces
} else {
&private_interfaces
}
.into_iter()
.copied()
.cloned()
{
forward(external, &*interface, req.target).await?;
cur.insert(interface, req.target);
}
}
(None, Some(cur)) => {
let to_rm = cur.keys().cloned().collect::<BTreeSet<_>>();
for interface in to_rm {
unforward(external, &*interface, cur[&interface]).await?;
cur.remove(&interface);
}
self.current.remove(&external);
}
_ => (),
}
}
Ok(())
}
}
fn err_has_exited<T>(_: T) -> Error {
Error::new(
eyre!("PortForwardController thread has exited"),
ErrorKind::Unknown,
)
}
pub struct LanPortForwardController {
forwards: Mutex<BTreeMap<u16, BTreeMap<SocketAddr, Weak<()>>>>,
req: mpsc::UnboundedSender<(
Option<(u16, ForwardRequest)>,
oneshot::Sender<Result<(), Error>>,
)>,
_thread: NonDetachingJoinHandle<()>,
}
impl LanPortForwardController {
pub fn new() -> Self {
pub fn new(
mut net_iface: watch::Receiver<BTreeMap<InternedString, NetworkInterfaceInfo>>,
) -> Self {
let (req_send, mut req_recv) = mpsc::unbounded_channel();
let thread = NonDetachingJoinHandle::from(tokio::spawn(async move {
let mut state = ForwardState::default();
let mut interfaces = net_iface
.borrow_and_update()
.iter()
.map(|(iface, info)| (iface.clone(), info.public()))
.collect();
let mut reply: Option<oneshot::Sender<Result<(), Error>>> = None;
loop {
tokio::select! {
msg = req_recv.recv() => {
if let Some((msg, re)) = msg {
if let Some((external, req)) = msg {
state.requested.insert(external, req);
}
reply = Some(re);
} else {
break;
}
}
_ = net_iface.changed() => {
interfaces = net_iface
.borrow()
.iter()
.map(|(iface, info)| (iface.clone(), info.public()))
.collect();
}
}
let res = state.sync(&interfaces).await;
if let Err(e) = &res {
tracing::error!("Error in PortForwardController: {e}");
tracing::debug!("{e:?}");
}
if let Some(re) = reply.take() {
let _ = re.send(res);
}
}
}));
Self {
forwards: Mutex::new(BTreeMap::new()),
req: req_send,
_thread: thread,
}
}
pub async fn add(&self, port: u16, addr: SocketAddr) -> Result<Arc<()>, Error> {
let mut writable = self.forwards.lock().await;
let (prev, mut forward) = if let Some(forward) = writable.remove(&port) {
(
forward.keys().next().cloned(),
forward
.into_iter()
.filter(|(_, rc)| rc.strong_count() > 0)
.collect(),
)
} else {
(None, BTreeMap::new())
};
pub async fn add(&self, port: u16, public: bool, target: SocketAddr) -> Result<Arc<()>, Error> {
let rc = Arc::new(());
forward.insert(addr, Arc::downgrade(&rc));
let next = forward.keys().next().cloned();
if !forward.is_empty() {
writable.insert(port, forward);
}
let (send, recv) = oneshot::channel();
self.req
.send((
Some((
port,
ForwardRequest {
public,
target,
rc: Arc::downgrade(&rc),
},
)),
send,
))
.map_err(err_has_exited)?;
update_forward(port, prev, next).await?;
Ok(rc)
recv.await.map_err(err_has_exited)?.map(|_| rc)
}
pub async fn gc(&self, external: u16) -> Result<(), Error> {
let mut writable = self.forwards.lock().await;
let (prev, forward) = if let Some(forward) = writable.remove(&external) {
(
forward.keys().next().cloned(),
forward
.into_iter()
.filter(|(_, rc)| rc.strong_count() > 0)
.collect(),
)
} else {
(None, BTreeMap::new())
};
let next = forward.keys().next().cloned();
if !forward.is_empty() {
writable.insert(external, forward);
}
pub async fn gc(&self) -> Result<(), Error> {
let (send, recv) = oneshot::channel();
self.req.send((None, send)).map_err(err_has_exited)?;
update_forward(external, prev, next).await
recv.await.map_err(err_has_exited)?
}
}
async fn update_forward(
external: u16,
prev: Option<SocketAddr>,
next: Option<SocketAddr>,
) -> Result<(), Error> {
if prev != next {
if let Some(prev) = prev {
unforward(START9_BRIDGE_IFACE, external, prev).await?;
}
if let Some(next) = next {
forward(START9_BRIDGE_IFACE, external, next).await?;
}
}
Ok(())
}
// iptables -I FORWARD -o br-start9 -p tcp -d 172.18.0.2 --dport 8333 -j ACCEPT
// iptables -t nat -I PREROUTING -p tcp --dport 32768 -j DNAT --to 172.18.0.2:8333
async fn forward(iface: &str, external: u16, addr: SocketAddr) -> Result<(), Error> {
Command::new("iptables")
.arg("-I")
.arg("FORWARD")
.arg("-o")
.arg(iface)
.arg("-p")
.arg("tcp")
.arg("-d")
.arg(addr.ip().to_string())
.arg("--dport")
.arg(addr.port().to_string())
.arg("-j")
.arg("ACCEPT")
.invoke(crate::ErrorKind::Network)
.await?;
Command::new("iptables")
.arg("-t")
.arg("nat")
.arg("-I")
.arg("PREROUTING")
.arg("-p")
.arg("tcp")
.arg("--dport")
.arg(external.to_string())
.arg("-j")
.arg("DNAT")
.arg("--to")
.arg(addr.to_string())
.invoke(crate::ErrorKind::Network)
.await?;
async fn forward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> {
for proto in ["tcp", "udp"] {
Command::new("iptables")
.arg("-I")
.arg("FORWARD")
.arg("-i")
.arg(interface)
.arg("-o")
.arg(START9_BRIDGE_IFACE)
.arg("-p")
.arg(proto)
.arg("-d")
.arg(target.ip().to_string())
.arg("--dport")
.arg(target.port().to_string())
.arg("-j")
.arg("ACCEPT")
.invoke(crate::ErrorKind::Network)
.await?;
Command::new("iptables")
.arg("-t")
.arg("nat")
.arg("-I")
.arg("PREROUTING")
.arg("-i")
.arg(interface)
.arg("-p")
.arg(proto)
.arg("--dport")
.arg(external.to_string())
.arg("-j")
.arg("DNAT")
.arg("--to")
.arg(target.to_string())
.invoke(crate::ErrorKind::Network)
.await?;
}
Ok(())
}
// iptables -D FORWARD -o br-start9 -p tcp -d 172.18.0.2 --dport 8333 -j ACCEPT
// iptables -t nat -D PREROUTING -p tcp --dport 32768 -j DNAT --to 172.18.0.2:8333
async fn unforward(iface: &str, external: u16, addr: SocketAddr) -> Result<(), Error> {
Command::new("iptables")
.arg("-D")
.arg("FORWARD")
.arg("-o")
.arg(iface)
.arg("-p")
.arg("tcp")
.arg("-d")
.arg(addr.ip().to_string())
.arg("--dport")
.arg(addr.port().to_string())
.arg("-j")
.arg("ACCEPT")
.invoke(crate::ErrorKind::Network)
.await?;
Command::new("iptables")
.arg("-t")
.arg("nat")
.arg("-D")
.arg("PREROUTING")
.arg("-p")
.arg("tcp")
.arg("--dport")
.arg(external.to_string())
.arg("-j")
.arg("DNAT")
.arg("--to")
.arg(addr.to_string())
.invoke(crate::ErrorKind::Network)
.await?;
async fn unforward(external: u16, interface: &str, target: SocketAddr) -> Result<(), Error> {
for proto in ["tcp", "udp"] {
Command::new("iptables")
.arg("-D")
.arg("FORWARD")
.arg("-i")
.arg(interface)
.arg("-o")
.arg(START9_BRIDGE_IFACE)
.arg("-p")
.arg(proto)
.arg("-d")
.arg(target.ip().to_string())
.arg("--dport")
.arg(target.port().to_string())
.arg("-j")
.arg("ACCEPT")
.invoke(crate::ErrorKind::Network)
.await?;
Command::new("iptables")
.arg("-t")
.arg("nat")
.arg("-D")
.arg("PREROUTING")
.arg("-i")
.arg(interface)
.arg("-p")
.arg(proto)
.arg("--dport")
.arg(external.to_string())
.arg("-j")
.arg("DNAT")
.arg("--to")
.arg(target.to_string())
.invoke(crate::ErrorKind::Network)
.await?;
}
Ok(())
}

313
core/startos/src/net/host/address.rs
View File

@@ -1,57 +1,298 @@
use std::fmt;
use std::str::FromStr;
use clap::builder::ValueParserFactory;
use clap::Parser;
use imbl_value::InternedString;
use models::FromStrParser;
use models::{HostId, PackageId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::net::acme::AcmeProvider;
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde};
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)]
#[serde(rename_all = "camelCase")]
#[serde(tag = "kind")]
#[ts(export)]
#[derive(Clone, Debug, Deserialize, Serialize)]
pub enum HostAddress {
Onion {
#[ts(type = "string")]
address: OnionAddressV3,
},
Domain {
#[ts(type = "string")]
address: InternedString,
public: bool,
acme: Option<AcmeProvider>,
},
}
impl FromStr for HostAddress {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
if let Some(addr) = s.strip_suffix(".onion") {
Ok(HostAddress::Onion {
address: addr
.parse::<OnionAddressV3>()
.with_kind(ErrorKind::ParseUrl)?,
})
} else {
Ok(HostAddress::Domain { address: s.into() })
}
}
#[derive(Debug, Deserialize, Serialize, TS)]
pub struct DomainConfig {
pub public: bool,
pub acme: Option<AcmeProvider>,
}
impl fmt::Display for HostAddress {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Onion { address } => write!(f, "{address}"),
Self::Domain { address } => write!(f, "{address}"),
}
}
#[derive(Deserialize, Serialize, Parser)]
pub struct AddressApiParams {
host: HostId,
}
impl ValueParserFactory for HostAddress {
type Parser = FromStrParser<Self>;
fn value_parser() -> Self::Parser {
Self::Parser::new()
}
pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
ParentHandler::<C, AddressApiParams, PackageId>::new()
.subcommand(
"domain",
ParentHandler::<C, Empty, (PackageId, HostId)>::new()
.subcommand(
"add",
from_fn_async(add_domain)
.with_inherited(|_, a| a)
.no_display()
.with_about("Add an address to this host")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove_domain)
.with_inherited(|_, a| a)
.no_display()
.with_about("Remove an address from this host")
.with_call_remote::<CliContext>(),
)
.with_inherited(|AddressApiParams { host }, package| (package, host)),
)
.subcommand(
"onion",
ParentHandler::<C, Empty, (PackageId, HostId)>::new()
.subcommand(
"add",
from_fn_async(add_onion)
.with_inherited(|_, a| a)
.no_display()
.with_about("Add an address to this host")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove_onion)
.with_inherited(|_, a| a)
.no_display()
.with_about("Remove an address from this host")
.with_call_remote::<CliContext>(),
)
.with_inherited(|AddressApiParams { host }, package| (package, host)),
)
.subcommand(
"list",
from_fn_async(list_addresses)
.with_inherited(|AddressApiParams { host }, package| (package, host))
.with_display_serializable()
.with_custom_display_fn(|HandlerArgs { params, .. }, res| {
use prettytable::*;
if let Some(format) = params.format {
display_serializable(format, res);
return Ok(());
}
let mut table = Table::new();
table.add_row(row![bc => "ADDRESS", "PUBLIC", "ACME PROVIDER"]);
for address in &res {
match address {
HostAddress::Onion { address } => {
table.add_row(row![address, true, "N/A"]);
}
HostAddress::Domain {
address,
public,
acme,
} => {
table.add_row(row![
address,
*public,
acme.as_ref().map(|a| a.0.as_str()).unwrap_or("NONE")
]);
}
}
}
table.print_tty(false)?;
Ok(())
})
.with_about("List addresses for this host")
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser)]
pub struct AddDomainParams {
pub domain: InternedString,
#[arg(long)]
pub private: bool,
#[arg(long)]
pub acme: Option<AcmeProvider>,
}
pub async fn add_domain(
ctx: RpcContext,
AddDomainParams {
domain,
private,
acme,
}: AddDomainParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
if let Some(acme) = &acme {
if !db.as_public().as_server_info().as_acme().contains_key(&acme)? {
return Err(Error::new(eyre!("unknown acme provider {}, please run acme.init for this provider first", acme.0), ErrorKind::InvalidRequest));
}
}
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_domains_mut()
.insert(
&domain,
&DomainConfig {
public: !private,
acme,
},
)
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
#[derive(Deserialize, Serialize, Parser)]
pub struct RemoveDomainParams {
pub domain: InternedString,
}
pub async fn remove_domain(
ctx: RpcContext,
RemoveDomainParams { domain }: RemoveDomainParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_domains_mut()
.remove(&domain)
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
#[derive(Deserialize, Serialize, Parser)]
pub struct OnionParams {
pub onion: String,
}
pub async fn add_onion(
ctx: RpcContext,
OnionParams { onion }: OnionParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
.ok_or_else(|| {
Error::new(
eyre!("onion hostname must end in .onion"),
ErrorKind::InvalidOnionAddress,
)
})?
.parse::<OnionAddressV3>()?;
ctx.db
.mutate(|db| {
db.as_private().as_key_store().as_onion().get_key(&onion)?;
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_onions_mut()
.mutate(|a| Ok(a.insert(onion)))
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
pub async fn remove_onion(
ctx: RpcContext,
OnionParams { onion }: OnionParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
.ok_or_else(|| {
Error::new(
eyre!("onion hostname must end in .onion"),
ErrorKind::InvalidOnionAddress,
)
})?
.parse::<OnionAddressV3>()?;
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_onions_mut()
.mutate(|a| Ok(a.remove(&onion)))
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
pub async fn list_addresses(
ctx: RpcContext,
_: Empty,
(package, host): (PackageId, HostId),
) -> Result<Vec<HostAddress>, Error> {
Ok(ctx
.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(&package)
.or_not_found(&package)?
.into_hosts()
.into_idx(&host)
.or_not_found(&host)?
.de()?
.addresses()
.collect())
}

139
core/startos/src/net/host/binding.rs
View File

@@ -1,13 +1,18 @@
use std::collections::BTreeMap;
use std::str::FromStr;
use clap::builder::ValueParserFactory;
use models::{FromStrParser, HostId};
use clap::Parser;
use models::{FromStrParser, HostId, PackageId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::net::forward::AvailablePorts;
use crate::net::vhost::AlpnInfo;
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, TS)]
#[ts(export)]
@@ -41,12 +46,14 @@ impl FromStr for BindId {
pub struct BindInfo {
pub enabled: bool,
pub options: BindOptions,
pub lan: LanInfo,
pub net: NetInfo,
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize, TS, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct LanInfo {
pub struct NetInfo {
pub public: bool,
pub assigned_port: Option<u16>,
pub assigned_ssl_port: Option<u16>,
}
@@ -63,7 +70,8 @@ impl BindInfo {
Ok(Self {
enabled: true,
options,
lan: LanInfo {
net: NetInfo {
public: false,
assigned_port,
assigned_ssl_port,
},
@@ -74,7 +82,7 @@ impl BindInfo {
available_ports: &mut AvailablePorts,
options: BindOptions,
) -> Result<Self, Error> {
let Self { mut lan, .. } = self;
let Self { net: mut lan, .. } = self;
if options
.secure
.map_or(false, |s| !(s.ssl && options.add_ssl.is_some()))
@@ -104,7 +112,7 @@ impl BindInfo {
Ok(Self {
enabled: true,
options,
lan,
net: lan,
})
}
pub fn disable(&mut self) {
@@ -137,3 +145,122 @@ pub struct AddSslOptions {
// pub add_x_forwarded_headers: bool, // TODO
pub alpn: Option<AlpnInfo>,
}
#[derive(Deserialize, Serialize, Parser)]
pub struct BindingApiParams {
host: HostId,
}
pub fn binding<C: Context>() -> ParentHandler<C, BindingApiParams, PackageId> {
ParentHandler::<C, BindingApiParams, PackageId>::new()
.subcommand(
"list",
from_fn_async(list_bindings)
.with_inherited(|BindingApiParams { host }, package| (package, host))
.with_display_serializable()
.with_custom_display_fn(|HandlerArgs { params, .. }, res| {
use prettytable::*;
if let Some(format) = params.format {
return Ok(display_serializable(format, res));
}
let mut table = Table::new();
table.add_row(row![bc => "INTERNAL PORT", "ENABLED", "PUBLIC", "EXTERNAL PORT", "EXTERNAL SSL PORT"]);
for (internal, info) in res {
table.add_row(row![
internal,
info.enabled,
info.net.public,
if let Some(port) = info.net.assigned_port {
port.to_string()
} else {
"N/A".to_owned()
},
if let Some(port) = info.net.assigned_ssl_port {
port.to_string()
} else {
"N/A".to_owned()
},
]);
}
table.print_tty(false).unwrap();
Ok(())
})
.with_about("List bindinges for this host")
.with_call_remote::<CliContext>(),
)
.subcommand(
"set-public",
from_fn_async(set_public)
.with_inherited(|BindingApiParams { host }, package| (package, host))
.no_display()
.with_about("Add an binding to this host")
.with_call_remote::<CliContext>(),
)
}
pub async fn list_bindings(
ctx: RpcContext,
_: Empty,
(package, host): (PackageId, HostId),
) -> Result<BTreeMap<u16, BindInfo>, Error> {
ctx.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(&package)
.or_not_found(&package)?
.into_hosts()
.into_idx(&host)
.or_not_found(&host)?
.into_bindings()
.de()
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "camelCase")]
pub struct SetPublicParams {
internal_port: u16,
#[arg(long)]
public: Option<bool>,
}
pub async fn set_public(
ctx: RpcContext,
SetPublicParams {
internal_port,
public,
}: SetPublicParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_bindings_mut()
.mutate(|b| {
b.get_mut(&internal_port)
.or_not_found(internal_port)?
.net
.public = public.unwrap_or(true);
Ok(())
})
})
.await?;
ctx.services
.get(&package)
.await
.as_ref()
.or_not_found(&package)?
.update_host(host)
.await
}

163
core/startos/src/net/host/mod.rs
View File

@@ -5,13 +5,14 @@ use imbl_value::InternedString;
use models::{HostId, PackageId};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::context::RpcContext;
use crate::db::model::DatabaseModel;
use crate::net::forward::AvailablePorts;
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{BindInfo, BindOptions};
use crate::net::host::address::{address, DomainConfig, HostAddress};
use crate::net::host::binding::{binding, BindInfo, BindOptions};
use crate::net::service_interface::HostnameInfo;
use crate::prelude::*;
@@ -25,7 +26,10 @@ pub mod binding;
pub struct Host {
pub kind: HostKind,
pub bindings: BTreeMap<u16, BindInfo>,
pub addresses: BTreeSet<HostAddress>,
#[ts(type = "string[]")]
pub onions: BTreeSet<OnionAddressV3>,
#[ts(as = "BTreeMap::<String, DomainConfig>")]
pub domains: BTreeMap<InternedString, DomainConfig>,
/// COMPUTED: NetService::update
pub hostname_info: BTreeMap<u16, Vec<HostnameInfo>>, // internal port -> Hostnames
}
@@ -39,13 +43,27 @@ impl Host {
Self {
kind,
bindings: BTreeMap::new(),
addresses: BTreeSet::new(),
onions: BTreeSet::new(),
domains: BTreeMap::new(),
hostname_info: BTreeMap::new(),
}
}
pub fn addresses(&self) -> impl Iterator<Item = &HostAddress> {
// TODO: handle primary
self.addresses.iter()
pub fn addresses<'a>(&'a self) -> impl Iterator<Item = HostAddress> + 'a {
self.onions
.iter()
.cloned()
.map(|address| HostAddress::Onion { address })
.chain(
self.domains
.iter()
.map(
|(address, DomainConfig { public, acme })| HostAddress::Domain {
address: address.clone(),
public: *public,
acme: acme.clone(),
},
),
)
}
}
@@ -104,12 +122,12 @@ pub fn host_for<'a>(
};
host_info(db, package_id)?.upsert(host_id, || {
let mut h = Host::new(host_kind);
h.addresses.insert(HostAddress::Onion {
address: tor_key
h.onions.insert(
tor_key
.or_not_found("generated tor key")?
.public()
.get_onion_address(),
});
);
Ok(h)
})
}
@@ -161,6 +179,10 @@ pub fn host<C: Context>() -> ParentHandler<C, HostParams> {
"address",
address::<C>().with_inherited(|HostParams { package }, _| package),
)
.subcommand(
"binding",
binding::<C>().with_inherited(|HostParams { package }, _| package),
)
}
pub async fn list_hosts(
@@ -178,122 +200,3 @@ pub async fn list_hosts(
.into_hosts()
.keys()
}
#[derive(Deserialize, Serialize, Parser)]
pub struct AddressApiParams {
host: HostId,
}
pub fn address<C: Context>() -> ParentHandler<C, AddressApiParams, PackageId> {
ParentHandler::<C, AddressApiParams, PackageId>::new()
.subcommand(
"add",
from_fn_async(add_address)
.with_inherited(|AddressApiParams { host }, package| (package, host))
.no_display()
.with_about("Add an address to this host")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove_address)
.with_inherited(|AddressApiParams { host }, package| (package, host))
.no_display()
.with_about("Remove an address from this host")
.with_call_remote::<CliContext>(),
)
.subcommand(
"list",
from_fn_async(list_addresses)
.with_inherited(|AddressApiParams { host }, package| (package, host))
.with_custom_display_fn(|_, res| {
for address in res {
println!("{address}")
}
Ok(())
})
.with_about("List addresses for this host")
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser)]
pub struct AddressParams {
pub address: HostAddress,
}
pub async fn add_address(
ctx: RpcContext,
AddressParams { address }: AddressParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
if let HostAddress::Onion { address } = address {
db.as_private()
.as_key_store()
.as_onion()
.get_key(&address)?;
}
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_addresses_mut()
.mutate(|a| Ok(a.insert(address)))
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
pub async fn remove_address(
ctx: RpcContext,
AddressParams { address }: AddressParams,
(package, host): (PackageId, HostId),
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package)
.or_not_found(&package)?
.as_hosts_mut()
.as_idx_mut(&host)
.or_not_found(&host)?
.as_addresses_mut()
.mutate(|a| Ok(a.remove(&address)))
})
.await?;
let service = ctx.services.get(&package).await;
let service_ref = service.as_ref().or_not_found(&package)?;
service_ref.update_host(host).await?;
Ok(())
}
pub async fn list_addresses(
ctx: RpcContext,
_: Empty,
(package, host): (PackageId, HostId),
) -> Result<BTreeSet<HostAddress>, Error> {
ctx.db
.peek()
.await
.into_public()
.into_package_data()
.into_idx(&package)
.or_not_found(&package)?
.into_hosts()
.into_idx(&host)
.or_not_found(&host)?
.into_addresses()
.de()
}

17
core/startos/src/net/mod.rs
View File

@@ -1,13 +1,13 @@
use rpc_toolkit::{Context, HandlerExt, ParentHandler};
pub mod acme;
pub mod dhcp;
pub mod dns;
pub mod forward;
pub mod host;
pub mod keys;
pub mod mdns;
pub mod net_controller;
pub mod network_interface;
pub mod service_interface;
pub mod ssl;
pub mod static_server;
@@ -17,20 +17,23 @@ pub mod vhost;
pub mod web_server;
pub mod wifi;
pub const PACKAGE_CERT_PATH: &str = "/var/lib/embassy/ssl";
pub fn net<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
"tor",
tor::tor::<C>().with_about("Tor commands such as list-services, logs, and reset"),
)
.subcommand(
"dhcp",
dhcp::dhcp::<C>().with_about("Command to update IP assigned from dhcp"),
)
.subcommand(
"acme",
acme::acme::<C>().with_about("Setup automatic clearnet certificate acquisition"),
)
.subcommand(
"network-interface",
network_interface::network_interface_api::<C>()
.with_about("View and edit network interface configurations"),
)
.subcommand(
"vhost",
vhost::vhost_api::<C>().with_about("Manage ssl virtual host proxy"),
)
}

554
core/startos/src/net/net_controller.rs
View File

@@ -5,6 +5,7 @@ use std::sync::{Arc, Weak};
use color_eyre::eyre::eyre;
use imbl::OrdMap;
use imbl_value::InternedString;
use ipnet::IpNet;
use models::{HostId, OptionExt, PackageId};
use torut::onion::{OnionAddressV3, TorSecretKeyV3};
use tracing::instrument;
@@ -15,11 +16,13 @@ use crate::hostname::Hostname;
use crate::net::dns::DnsController;
use crate::net::forward::LanPortForwardController;
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{AddSslOptions, BindId, BindOptions, LanInfo};
use crate::net::host::binding::{BindId, BindOptions};
use crate::net::host::{host_for, Host, HostKind, Hosts};
use crate::net::network_interface::NetworkInterfaceController;
use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname};
use crate::net::tor::TorController;
use crate::net::vhost::{AlpnInfo, VHostController};
use crate::net::utils::ipv6_is_local;
use crate::net::vhost::{AlpnInfo, TargetInfo, VHostController};
use crate::prelude::*;
use crate::util::serde::MaybeUtf8String;
use crate::HOST_IP;
@@ -28,6 +31,7 @@ pub struct PreInitNetController {
pub db: TypedPatchDb<Database>,
tor: TorController,
vhost: VHostController,
pub net_iface: Arc<NetworkInterfaceController>,
os_bindings: Vec<Arc<()>>,
server_hostnames: Vec<Option<InternedString>>,
}
@@ -40,10 +44,12 @@ impl PreInitNetController {
hostname: &Hostname,
os_tor_key: TorSecretKeyV3,
) -> Result<Self, Error> {
let net_iface = Arc::new(NetworkInterfaceController::new(db.clone()));
let mut res = Self {
db: db.clone(),
tor: TorController::new(tor_control, tor_socks),
vhost: VHostController::new(db),
vhost: VHostController::new(db, net_iface.clone()),
net_iface,
os_bindings: Vec::new(),
server_hostnames: Vec::new(),
};
@@ -56,11 +62,6 @@ impl PreInitNetController {
hostname: &Hostname,
tor_key: TorSecretKeyV3,
) -> Result<(), Error> {
let alpn = Err(AlpnInfo::Specified(vec![
MaybeUtf8String("http/1.1".into()),
MaybeUtf8String("h2".into()),
]));
self.server_hostnames = vec![
// LAN IP
None,
@@ -74,27 +75,29 @@ impl PreInitNetController {
Some(hostname.local_domain_name()),
];
let vhost_target = TargetInfo {
public: false,
acme: None,
addr: ([127, 0, 0, 1], 80).into(),
connect_ssl: Err(AlpnInfo::Specified(vec![
MaybeUtf8String("http/1.1".into()),
MaybeUtf8String("h2".into()),
])),
};
for hostname in self.server_hostnames.iter().cloned() {
self.os_bindings.push(
self.vhost
.add(hostname, 443, ([127, 0, 0, 1], 80).into(), alpn.clone())
.await?,
);
self.os_bindings
.push(self.vhost.add(hostname, 443, vhost_target.clone())?);
}
// Tor
self.os_bindings.push(
self.vhost
.add(
Some(InternedString::from_display(
&tor_key.public().get_onion_address(),
)),
443,
([127, 0, 0, 1], 80).into(),
alpn.clone(),
)
.await?,
);
self.os_bindings.push(self.vhost.add(
Some(InternedString::from_display(
&tor_key.public().get_onion_address(),
)),
443,
vhost_target,
)?);
self.os_bindings.extend(
self.tor
.add(
@@ -115,6 +118,7 @@ pub struct NetController {
db: TypedPatchDb<Database>,
pub(super) tor: TorController,
pub(super) vhost: VHostController,
pub net_iface: Arc<NetworkInterfaceController>,
pub(super) dns: DnsController,
pub(super) forward: LanPortForwardController,
pub(super) os_bindings: Vec<Arc<()>>,
@@ -127,6 +131,7 @@ impl NetController {
db,
tor,
vhost,
net_iface,
os_bindings,
server_hostnames,
}: PreInitNetController,
@@ -137,7 +142,8 @@ impl NetController {
tor,
vhost,
dns: DnsController::init(dns_bind).await?,
forward: LanPortForwardController::new(),
forward: LanPortForwardController::new(net_iface.subscribe()),
net_iface,
os_bindings,
server_hostnames,
};
@@ -169,15 +175,8 @@ impl NetController {
#[derive(Default, Debug)]
struct HostBinds {
lan: BTreeMap<
u16,
(
LanInfo,
Option<AddSslOptions>,
BTreeSet<InternedString>,
Vec<Arc<()>>,
),
>,
forwards: BTreeMap<u16, (SocketAddr, bool, Arc<()>)>,
vhosts: BTreeMap<(Option<InternedString>, u16), (TargetInfo, Arc<()>)>,
tor: BTreeMap<OnionAddressV3, (OrdMap<u16, SocketAddr>, Vec<Arc<()>>)>,
}
@@ -206,7 +205,7 @@ impl NetService {
internal_port: u16,
options: BindOptions,
) -> Result<(), Error> {
dbg!("bind", &kind, &id, internal_port, &options);
crate::dbg!("bind", &kind, &id, internal_port, &options);
let pkg_id = &self.id;
let host = self
.net_controller()?
@@ -263,134 +262,161 @@ impl NetService {
pub async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> {
let ctrl = self.net_controller()?;
let mut hostname_info = BTreeMap::new();
let mut forwards: BTreeMap<u16, (SocketAddr, bool)> = BTreeMap::new();
let mut vhosts: BTreeMap<(Option<InternedString>, u16), TargetInfo> = BTreeMap::new();
let mut tor: BTreeMap<OnionAddressV3, (TorSecretKeyV3, OrdMap<u16, SocketAddr>)> =
BTreeMap::new();
let mut hostname_info: BTreeMap<u16, Vec<HostnameInfo>> = BTreeMap::new();
let binds = self.binds.entry(id.clone()).or_default();
let peek = ctrl.db.peek().await;
// LAN
let server_info = peek.as_public().as_server_info();
let ip_info = server_info.as_ip_info().de()?;
let net_ifaces = server_info.as_network_interfaces().de()?;
let hostname = server_info.as_hostname().de()?;
for (port, bind) in &host.bindings {
if !bind.enabled {
continue;
}
let old_lan_bind = binds.lan.remove(port);
let lan_bind = old_lan_bind
.as_ref()
.filter(|(external, ssl, _, _)| {
ssl == &bind.options.add_ssl && bind.lan == *external
})
.cloned(); // only keep existing binding if relevant details match
if bind.lan.assigned_port.is_some() || bind.lan.assigned_ssl_port.is_some() {
let new_lan_bind = if let Some(b) = lan_bind {
b
} else {
let mut rcs = Vec::with_capacity(2 + host.addresses.len());
let mut hostnames = BTreeSet::new();
if let Some(ssl) = &bind.options.add_ssl {
let external = bind
.lan
.assigned_ssl_port
.or_not_found("assigned ssl port")?;
let target = (self.ip, *port).into();
let connect_ssl = if let Some(alpn) = ssl.alpn.clone() {
Err(alpn)
if bind.net.assigned_port.is_some() || bind.net.assigned_ssl_port.is_some() {
let mut hostnames = BTreeSet::new();
if let Some(ssl) = &bind.options.add_ssl {
let external = bind
.net
.assigned_ssl_port
.or_not_found("assigned ssl port")?;
let addr = (self.ip, *port).into();
let connect_ssl = if let Some(alpn) = ssl.alpn.clone() {
Err(alpn)
} else {
if bind.options.secure.as_ref().map_or(false, |s| s.ssl) {
Ok(())
} else {
if bind.options.secure.as_ref().map_or(false, |s| s.ssl) {
Ok(())
} else {
Err(AlpnInfo::Reflect)
}
};
for hostname in ctrl.server_hostnames.iter().cloned() {
rcs.push(
ctrl.vhost
.add(hostname, external, target, connect_ssl.clone())
.await?,
);
Err(AlpnInfo::Reflect)
}
for address in host.addresses() {
match address {
HostAddress::Onion { address } => {
let hostname = InternedString::from_display(address);
if hostnames.insert(hostname.clone()) {
rcs.push(
ctrl.vhost
.add(
Some(hostname),
external,
target,
connect_ssl.clone(),
)
.await?,
);
}
};
for hostname in ctrl.server_hostnames.iter().cloned() {
vhosts.insert(
(hostname, external),
TargetInfo {
public: bind.net.public,
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
for address in host.addresses() {
match address {
HostAddress::Onion { address } => {
let hostname = InternedString::from_display(&address);
if hostnames.insert(hostname.clone()) {
vhosts.insert(
(Some(hostname), external),
TargetInfo {
public: false,
acme: None,
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
HostAddress::Domain { address } => {
if hostnames.insert(address.clone()) {
let address = Some(address.clone());
rcs.push(
ctrl.vhost
.add(
address.clone(),
external,
target,
connect_ssl.clone(),
)
.await?,
);
if ssl.preferred_external_port == 443 {
rcs.push(
ctrl.vhost
.add(
address.clone(),
5443,
target,
connect_ssl.clone(),
)
.await?,
}
HostAddress::Domain {
address,
public,
acme,
} => {
if hostnames.insert(address.clone()) {
let address = Some(address.clone());
if ssl.preferred_external_port == 443 {
if public && bind.net.public {
vhosts.insert(
(address.clone(), 5443),
TargetInfo {
public: false,
acme: acme.clone(),
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
vhosts.insert(
(address.clone(), 443),
TargetInfo {
public: public && bind.net.public,
acme,
addr,
connect_ssl: connect_ssl.clone(),
},
);
} else {
vhosts.insert(
(address.clone(), external),
TargetInfo {
public: public && bind.net.public,
acme,
addr,
connect_ssl: connect_ssl.clone(),
},
);
}
}
}
}
}
if let Some(security) = bind.options.secure {
if bind.options.add_ssl.is_some() && security.ssl {
// doesn't make sense to have 2 listening ports, both with ssl
} else {
let external =
bind.lan.assigned_port.or_not_found("assigned lan port")?;
rcs.push(ctrl.forward.add(external, (self.ip, *port).into()).await?);
}
}
if let Some(security) = bind.options.secure {
if bind.options.add_ssl.is_some() && security.ssl {
// doesn't make sense to have 2 listening ports, both with ssl
} else {
let external = bind.net.assigned_port.or_not_found("assigned lan port")?;
forwards.insert(external, ((self.ip, *port).into(), bind.net.public));
}
(bind.lan, bind.options.add_ssl.clone(), hostnames, rcs)
};
}
let mut bind_hostname_info: Vec<HostnameInfo> =
hostname_info.remove(port).unwrap_or_default();
for (interface, ip_info) in &ip_info {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: false,
hostname: IpHostname::Local {
value: InternedString::from_display(&{
let hostname = &hostname;
lazy_format!("{hostname}.local")
}),
port: new_lan_bind.0.assigned_port,
ssl_port: new_lan_bind.0.assigned_ssl_port,
},
});
for (interface, public, ip_info) in
net_ifaces.iter().filter_map(|(interface, info)| {
if let Some(ip_info) = &info.ip_info {
Some((interface, info.public(), ip_info))
} else {
None
}
})
{
if !public {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: false,
hostname: IpHostname::Local {
value: InternedString::from_display(&{
let hostname = &hostname;
lazy_format!("{hostname}.local")
}),
port: bind.net.assigned_port,
ssl_port: bind.net.assigned_ssl_port,
},
});
}
for address in host.addresses() {
if let HostAddress::Domain { address } = address {
if let Some(ssl) = &new_lan_bind.1 {
if ssl.preferred_external_port == 443 {
if let HostAddress::Domain {
address,
public: domain_public,
..
} = address
{
if !public || (domain_public && bind.net.public) {
if bind
.options
.add_ssl
.as_ref()
.map_or(false, |ssl| ssl.preferred_external_port == 443)
{
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: false,
public: public && domain_public && bind.net.public, // TODO: check if port forward is active
hostname: IpHostname::Domain {
domain: address.clone(),
subdomain: None,
@@ -398,71 +424,65 @@ impl NetService {
ssl_port: Some(443),
},
});
} else {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public,
hostname: IpHostname::Domain {
domain: address.clone(),
subdomain: None,
port: bind.net.assigned_port,
ssl_port: bind.net.assigned_ssl_port,
},
});
}
}
}
}
if let Some(ipv4) = ip_info.ipv4 {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: false,
hostname: IpHostname::Ipv4 {
value: ipv4,
port: new_lan_bind.0.assigned_port,
ssl_port: new_lan_bind.0.assigned_ssl_port,
},
});
}
if let Some(ipv6) = ip_info.ipv6 {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: false,
hostname: IpHostname::Ipv6 {
value: ipv6,
port: new_lan_bind.0.assigned_port,
ssl_port: new_lan_bind.0.assigned_ssl_port,
},
});
if !public || bind.net.public {
if let Some(wan_ip) = ip_info.wan_ip.filter(|_| public) {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public,
hostname: IpHostname::Ipv4 {
value: wan_ip,
port: bind.net.assigned_port,
ssl_port: bind.net.assigned_ssl_port,
},
});
}
for ipnet in &ip_info.subnets {
match ipnet {
IpNet::V4(net) => {
if !public {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public,
hostname: IpHostname::Ipv4 {
value: net.addr(),
port: bind.net.assigned_port,
ssl_port: bind.net.assigned_ssl_port,
},
});
}
}
IpNet::V6(net) => {
bind_hostname_info.push(HostnameInfo::Ip {
network_interface_id: interface.clone(),
public: public && !ipv6_is_local(net.addr()),
hostname: IpHostname::Ipv6 {
value: net.addr(),
scope_id: ip_info.scope_id,
port: bind.net.assigned_port,
ssl_port: bind.net.assigned_ssl_port,
},
});
}
}
}
}
}
hostname_info.insert(*port, bind_hostname_info);
binds.lan.insert(*port, new_lan_bind);
}
if let Some((lan, _, hostnames, _)) = old_lan_bind {
if let Some(external) = lan.assigned_ssl_port {
for hostname in ctrl.server_hostnames.iter().cloned() {
ctrl.vhost.gc(hostname, external).await?;
}
for hostname in hostnames {
ctrl.vhost.gc(Some(hostname), external).await?;
}
}
if let Some(external) = lan.assigned_port {
ctrl.forward.gc(external).await?;
}
}
}
let mut removed = BTreeSet::new();
binds.lan.retain(|internal, (external, _, hostnames, _)| {
if host.bindings.get(internal).map_or(false, |b| b.enabled) {
true
} else {
removed.insert((*external, std::mem::take(hostnames)));
false
}
});
for (lan, hostnames) in removed {
if let Some(external) = lan.assigned_ssl_port {
for hostname in ctrl.server_hostnames.iter().cloned() {
ctrl.vhost.gc(hostname, external).await?;
}
for hostname in hostnames {
ctrl.vhost.gc(Some(hostname), external).await?;
}
}
if let Some(external) = lan.assigned_port {
ctrl.forward.gc(external).await?;
}
}
@@ -481,7 +501,7 @@ impl NetService {
SocketAddr::from((self.ip, *internal)),
);
if let (Some(ssl), Some(ssl_internal)) =
(&info.options.add_ssl, info.lan.assigned_ssl_port)
(&info.options.add_ssl, info.net.assigned_ssl_port)
{
tor_binds.insert(
ssl.preferred_external_port,
@@ -506,31 +526,13 @@ impl NetService {
}
}
let mut keep_tor_addrs = BTreeSet::new();
for tor_addr in host.addresses().filter_map(|a| {
if let HostAddress::Onion { address } = a {
Some(address)
} else {
None
}
}) {
keep_tor_addrs.insert(tor_addr);
let old_tor_bind = binds.tor.remove(tor_addr);
let tor_bind = old_tor_bind.filter(|(ports, _)| ports == &tor_binds);
let new_tor_bind = if let Some(tor_bind) = tor_bind {
tor_bind
} else {
let key = peek
.as_private()
.as_key_store()
.as_onion()
.get_key(tor_addr)?;
let rcs = ctrl
.tor
.add(key, tor_binds.clone().into_iter().collect())
.await?;
(tor_binds.clone(), rcs)
};
for tor_addr in host.onions.iter() {
let key = peek
.as_private()
.as_key_store()
.as_onion()
.get_key(tor_addr)?;
tor.insert(key.public().get_onion_address(), (key, tor_binds.clone()));
for (internal, ports) in &tor_hostname_ports {
let mut bind_hostname_info = hostname_info.remove(internal).unwrap_or_default();
bind_hostname_info.push(HostnameInfo::Onion {
@@ -542,16 +544,91 @@ impl NetService {
});
hostname_info.insert(*internal, bind_hostname_info);
}
binds.tor.insert(tor_addr.clone(), new_tor_bind);
}
for addr in binds.tor.keys() {
if !keep_tor_addrs.contains(addr) {
ctrl.tor.gc(Some(addr.clone()), None).await?;
let all = binds
.forwards
.keys()
.chain(forwards.keys())
.copied()
.collect::<BTreeSet<_>>();
for external in all {
let mut prev = binds.forwards.remove(&external);
if let Some((internal, public)) = forwards.remove(&external) {
prev = prev.filter(|(i, p, _)| i == &internal && *p == public);
binds.forwards.insert(
external,
if let Some(prev) = prev {
prev
} else {
(
internal,
public,
ctrl.forward.add(external, public, internal).await?,
)
},
);
}
}
ctrl.forward.gc().await?;
let all = binds
.vhosts
.keys()
.chain(vhosts.keys())
.cloned()
.collect::<BTreeSet<_>>();
for key in all {
let mut prev = binds.vhosts.remove(&key);
if let Some(target) = vhosts.remove(&key) {
prev = prev.filter(|(t, _)| t == &target);
binds.vhosts.insert(
key.clone(),
if let Some(prev) = prev {
prev
} else {
(target.clone(), ctrl.vhost.add(key.0, key.1, target)?)
},
);
} else {
if let Some((_, rc)) = prev {
drop(rc);
ctrl.vhost.gc(key.0, key.1);
}
}
}
self.net_controller()?
.db
let all = binds
.tor
.keys()
.chain(tor.keys())
.cloned()
.collect::<BTreeSet<_>>();
for onion in all {
let mut prev = binds.tor.remove(&onion);
if let Some((key, tor_binds)) = tor.remove(&onion) {
prev = prev.filter(|(b, _)| b == &tor_binds);
binds.tor.insert(
onion,
if let Some(prev) = prev {
prev
} else {
let rcs = ctrl
.tor
.add(key, tor_binds.iter().map(|(k, v)| (*k, *v)).collect())
.await?;
(tor_binds, rcs)
},
);
} else {
if let Some((_, rc)) = prev {
drop(rc);
ctrl.tor.gc(Some(onion), None).await?;
}
}
}
ctrl.db
.mutate(|db| {
host_for(db, &self.id, &id, host.kind)?
.as_hostname_info_mut()
@@ -579,29 +656,6 @@ impl NetService {
pub fn get_ip(&self) -> Ipv4Addr {
self.ip
}
pub fn get_lan_port(&self, host_id: HostId, internal_port: u16) -> Result<LanInfo, Error> {
let host_id_binds = self.binds.get_key_value(&host_id);
match host_id_binds {
Some((_, binds)) => {
if let Some((lan, _, _, _)) = binds.lan.get(&internal_port) {
Ok(*lan)
} else {
Err(Error::new(
eyre!(
"Internal Port {} not found in NetService binds",
internal_port
),
crate::ErrorKind::NotFound,
))
}
}
None => Err(Error::new(
eyre!("HostID {} not found in NetService binds", host_id),
crate::ErrorKind::NotFound,
)),
}
}
}
impl Drop for NetService {

1116
core/startos/src/net/network_interface.rs Normal file
View File

File diff suppressed because it is too large Load Diff

6
core/startos/src/net/service_interface.rs
View File

@@ -12,7 +12,8 @@ use ts_rs::TS;
#[serde(tag = "kind")]
pub enum HostnameInfo {
Ip {
network_interface_id: String,
#[ts(type = "string")]
network_interface_id: InternedString,
public: bool,
hostname: IpHostname,
},
@@ -43,6 +44,8 @@ pub enum IpHostname {
},
Ipv6 {
value: Ipv6Addr,
#[serde(default)]
scope_id: u32,
port: Option<u16>,
ssl_port: Option<u16>,
},
@@ -69,7 +72,6 @@ pub struct ServiceInterface {
pub id: ServiceInterfaceId,
pub name: String,
pub description: String,
pub has_primary: bool,
pub masked: bool,
pub address_info: AddressInfo,
#[serde(rename = "type")]

1
core/startos/src/net/ssl.rs
View File

@@ -17,7 +17,6 @@ use openssl::x509::{X509Builder, X509Extension, X509NameBuilder, X509};
use openssl::*;
use patch_db::HasModel;
use serde::{Deserialize, Serialize};
use tokio::time::Instant;
use tracing::instrument;
use crate::account::AccountInfo;

27
core/startos/src/net/static_server.rs
View File

@@ -8,15 +8,15 @@ use std::time::UNIX_EPOCH;
use async_compression::tokio::bufread::GzipEncoder;
use axum::body::Body;
use axum::extract::{self as x, Request};
use axum::response::Response;
use axum::routing::{any, get, post};
use axum::response::{Redirect, Response};
use axum::routing::{any, get};
use axum::Router;
use base64::display::Base64Display;
use digest::Digest;
use futures::future::ready;
use http::header::{
ACCEPT_ENCODING, ACCEPT_RANGES, CACHE_CONTROL, CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH,
CONTENT_RANGE, CONTENT_TYPE, ETAG, RANGE,
CONTENT_RANGE, CONTENT_TYPE, ETAG, HOST, RANGE,
};
use http::request::Parts as RequestParts;
use http::{HeaderValue, Method, StatusCode};
@@ -26,7 +26,6 @@ use new_mime_guess::MimeGuess;
use openssl::hash::MessageDigest;
use openssl::x509::X509;
use rpc_toolkit::{Context, HttpServer, Server};
use sqlx::query;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader};
use tokio_util::io::ReaderStream;
use url::Url;
@@ -47,7 +46,7 @@ use crate::s9pk::S9pk;
use crate::util::io::open_file;
use crate::util::net::SyncBody;
use crate::util::serde::BASE64;
use crate::{diagnostic_api, init_api, install_api, main_api, setup_api};
use crate::{diagnostic_api, init_api, install_api, main_api, setup_api, DATA_DIR};
const NOT_FOUND: &[u8] = b"Not Found";
const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
@@ -230,6 +229,20 @@ pub fn refresher() -> Router {
}))
}
pub fn redirecter() -> Router {
Router::new().fallback(get(|request: Request| async move {
Redirect::temporary(&format!(
"https://{}{}",
request
.headers()
.get(HOST)
.and_then(|s| s.to_str().ok())
.unwrap_or("localhost"),
request.uri()
))
}))
}
async fn proxy_request(ctx: RpcContext, request: Request, url: String) -> Result<Response, Error> {
if_authorized(&ctx, request, |mut request| async {
for header in PROXY_STRIP_HEADERS {
@@ -253,7 +266,7 @@ fn s9pk_router(ctx: RpcContext) -> Router {
let (parts, _) = request.into_parts();
match FileData::from_path(
&parts,
&ctx.datadir
&Path::new(DATA_DIR)
.join(PKG_ARCHIVE_DIR)
.join("installed")
.join(s9pk),
@@ -279,7 +292,7 @@ fn s9pk_router(ctx: RpcContext) -> Router {
let s9pk = S9pk::deserialize(
&MultiCursorFile::from(
open_file(
ctx.datadir
Path::new(DATA_DIR)
.join(PKG_ARCHIVE_DIR)
.join("installed")
.join(s9pk),

61
core/startos/src/net/utils.rs
View File

@@ -1,16 +1,25 @@
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6};
use std::path::Path;
use async_stream::try_stream;
use color_eyre::eyre::eyre;
use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use ipnet::{Ipv4Net, Ipv6Net};
use ipnet::{IpNet, Ipv4Net, Ipv6Net};
use nix::net::if_::if_nametoindex;
use tokio::net::{TcpListener, TcpStream};
use tokio::process::Command;
use crate::prelude::*;
use crate::util::Invoke;
use crate::Error;
pub fn ipv6_is_link_local(addr: Ipv6Addr) -> bool {
(addr.segments()[0] & 0xffc0) == 0xfe80
}
pub fn ipv6_is_local(addr: Ipv6Addr) -> bool {
addr.is_loopback() || (addr.segments()[0] & 0xfe00) == 0xfc00 || ipv6_is_link_local(addr)
}
fn parse_iface_ip(output: &str) -> Result<Vec<&str>, Error> {
let output = output.trim();
@@ -112,6 +121,52 @@ pub async fn find_eth_iface() -> Result<String, Error> {
))
}
pub async fn all_socket_addrs_for(port: u16) -> Result<Vec<SocketAddr>, Error> {
let mut res = Vec::new();
let raw = String::from_utf8(
Command::new("ip")
.arg("-o")
.arg("addr")
.arg("show")
.invoke(ErrorKind::ParseSysInfo)
.await?,
)?;
let err = |item: &str, lineno: usize, line: &str| {
Error::new(
eyre!("failed to parse ip info ({item}[line:{lineno}]) from {line:?}"),
ErrorKind::ParseSysInfo,
)
};
for (idx, line) in raw
.lines()
.map(|l| l.trim())
.enumerate()
.filter(|(_, l)| !l.is_empty())
{
let mut split = line.split_whitespace();
let _num = split.next();
let ifname = split.next().ok_or_else(|| err("ifname", idx, line))?;
let _kind = split.next();
let ipnet_str = split.next().ok_or_else(|| err("ipnet", idx, line))?;
let ipnet = ipnet_str
.parse::<IpNet>()
.with_ctx(|_| (ErrorKind::ParseSysInfo, err("ipnet", idx, ipnet_str)))?;
match ipnet.addr() {
IpAddr::V4(ip4) => res.push(SocketAddr::new(ip4.into(), port)),
IpAddr::V6(ip6) => res.push(SocketAddr::V6(SocketAddrV6::new(
ip6,
port,
0,
if_nametoindex(ifname)
.with_ctx(|_| (ErrorKind::ParseSysInfo, "reading scope_id"))?,
))),
}
}
Ok(res)
}
pub struct TcpListeners {
listeners: Vec<TcpListener>,
}

1053
core/startos/src/net/vhost.rs
View File

File diff suppressed because it is too large Load Diff

335
core/startos/src/net/web_server.rs
View File

@@ -1,134 +1,299 @@
use std::convert::Infallible;
use std::future::Future;
use std::net::SocketAddr;
use std::ops::Deref;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, RwLock};
use std::task::Poll;
use std::time::Duration;
use axum::extract::Request;
use axum::Router;
use axum_server::Handle;
use bytes::Bytes;
use futures::future::{ready, BoxFuture};
use futures::future::{BoxFuture, Either};
use futures::FutureExt;
use helpers::NonDetachingJoinHandle;
use hyper_util::rt::{TokioIo, TokioTimer};
use hyper_util::service::TowerToHyperService;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{oneshot, watch};
use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext};
use crate::net::network_interface::NetworkInterfaceListener;
use crate::net::static_server::{
diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, refresher,
diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, redirecter, refresher,
setup_ui_router,
};
use crate::prelude::*;
use crate::util::actor::background::BackgroundJobQueue;
#[derive(Clone)]
pub struct SwappableRouter(watch::Sender<Router>);
impl SwappableRouter {
pub fn new(router: Router) -> Self {
Self(watch::channel(router).0)
}
pub fn swap(&self, router: Router) {
let _ = self.0.send_replace(router);
}
pub struct Accepted {
pub https_redirect: bool,
pub stream: TcpStream,
}
pub struct SwappableRouterService {
router: watch::Receiver<Router>,
changed: Option<BoxFuture<'static, ()>>,
pub trait Accept {
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>>;
}
impl SwappableRouterService {
fn router(&self) -> Router {
self.router.borrow().clone()
}
fn changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> {
let mut changed = if let Some(changed) = self.changed.take() {
changed
} else {
let mut router = self.router.clone();
async move {
router.changed().await;
impl Accept for Vec<TcpListener> {
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
for listener in &*self {
if let Poll::Ready((stream, _)) = listener.poll_accept(cx)? {
return Poll::Ready(Ok(Accepted {
https_redirect: false,
stream,
}));
}
.boxed()
};
if changed.poll_unpin(cx).is_ready() {
return Poll::Ready(());
}
self.changed = Some(changed);
Poll::Pending
}
}
impl Clone for SwappableRouterService {
fn clone(&self) -> Self {
impl Accept for NetworkInterfaceListener {
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
NetworkInterfaceListener::poll_accept(self, cx, true).map(|res| {
res.map(|a| Accepted {
https_redirect: a.is_public,
stream: a.stream,
})
})
}
}
impl<A: Accept, B: Accept> Accept for Either<A, B> {
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
match self {
Either::Left(a) => a.poll_accept(cx),
Either::Right(b) => b.poll_accept(cx),
}
}
}
impl<A: Accept> Accept for Option<A> {
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
match self {
None => Poll::Pending,
Some(a) => a.poll_accept(cx),
}
}
}
#[pin_project::pin_project]
pub struct Acceptor<A: Accept> {
acceptor: (watch::Sender<A>, watch::Receiver<A>),
changed: Option<BoxFuture<'static, ()>>,
}
impl<A: Accept + Send + Sync + 'static> Acceptor<A> {
pub fn new(acceptor: A) -> Self {
Self {
router: self.router.clone(),
acceptor: watch::channel(acceptor),
changed: None,
}
}
}
impl<B> tower_service::Service<Request<B>> for SwappableRouterService
where
B: axum::body::HttpBody<Data = Bytes> + Send + 'static,
B::Error: Into<axum::BoxError>,
{
type Response = <Router as tower_service::Service<Request<B>>>::Response;
type Error = <Router as tower_service::Service<Request<B>>>::Error;
type Future = <Router as tower_service::Service<Request<B>>>::Future;
#[inline]
fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<(), Self::Error>> {
if self.changed(cx).is_ready() {
return Poll::Ready(Ok(()));
fn poll_changed(&mut self, cx: &mut std::task::Context<'_>) -> Poll<()> {
let mut changed = if let Some(changed) = self.changed.take() {
changed
} else {
let mut recv = self.acceptor.1.clone();
async move {
let _ = recv.changed().await;
}
.boxed()
};
let res = changed.poll_unpin(cx);
if res.is_pending() {
self.changed = Some(changed);
}
tower_service::Service::<Request<B>>::poll_ready(&mut self.router(), cx)
res
}
fn call(&mut self, req: Request<B>) -> Self::Future {
self.router().call(req)
fn poll_accept(&mut self, cx: &mut std::task::Context<'_>) -> Poll<Result<Accepted, Error>> {
let _ = self.poll_changed(cx);
let mut res = Poll::Pending;
self.acceptor.0.send_if_modified(|a| {
res = a.poll_accept(cx);
false
});
res
}
async fn accept(&mut self) -> Result<Accepted, Error> {
std::future::poll_fn(|cx| self.poll_accept(cx)).await
}
}
impl Acceptor<Vec<TcpListener>> {
pub async fn bind(listen: impl IntoIterator<Item = SocketAddr>) -> Result<Self, Error> {
Ok(Self::new(
futures::future::try_join_all(listen.into_iter().map(TcpListener::bind)).await?,
))
}
}
impl<T> tower_service::Service<T> for SwappableRouter {
type Response = SwappableRouterService;
type Error = Infallible;
type Future = futures::future::Ready<Result<Self::Response, Self::Error>>;
#[inline]
fn poll_ready(
&mut self,
_: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _: T) -> Self::Future {
ready(Ok(SwappableRouterService {
router: self.0.subscribe(),
changed: None,
}))
pub type UpgradableListener = Option<Either<Vec<TcpListener>, NetworkInterfaceListener>>;
impl Acceptor<UpgradableListener> {
pub async fn bind_upgradable(
listen: impl IntoIterator<Item = SocketAddr>,
) -> Result<Self, Error> {
Ok(Self::new(Some(Either::Left(
futures::future::try_join_all(listen.into_iter().map(TcpListener::bind)).await?,
))))
}
}
pub struct WebServer {
pub struct WebServerAcceptorSetter<A: Accept> {
acceptor: watch::Sender<A>,
}
impl<A: Accept, B: Accept> WebServerAcceptorSetter<Option<Either<A, B>>> {
pub fn try_upgrade<F: FnOnce(A) -> Result<B, Error>>(&self, f: F) -> Result<(), Error> {
let mut res = Ok(());
self.acceptor.send_modify(|a| {
*a = match a.take() {
Some(Either::Left(a)) => match f(a) {
Ok(b) => Some(Either::Right(b)),
Err(e) => {
res = Err(e);
None
}
},
x => x,
}
});
res
}
}
impl<A: Accept> Deref for WebServerAcceptorSetter<A> {
type Target = watch::Sender<A>;
fn deref(&self) -> &Self::Target {
&self.acceptor
}
}
pub struct WebServer<A: Accept> {
shutdown: oneshot::Sender<()>,
router: SwappableRouter,
router: watch::Sender<Option<Router>>,
acceptor: watch::Sender<A>,
thread: NonDetachingJoinHandle<()>,
}
impl WebServer {
pub fn new(bind: SocketAddr) -> Self {
let router = SwappableRouter::new(refresher());
let thread_router = router.clone();
impl<A: Accept + Send + Sync + 'static> WebServer<A> {
pub fn acceptor_setter(&self) -> WebServerAcceptorSetter<A> {
WebServerAcceptorSetter {
acceptor: self.acceptor.clone(),
}
}
pub fn new(mut acceptor: Acceptor<A>) -> Self {
let acceptor_send = acceptor.acceptor.0.clone();
let (router, service) = watch::channel::<Option<Router>>(None);
let (shutdown, shutdown_recv) = oneshot::channel();
let thread = NonDetachingJoinHandle::from(tokio::spawn(async move {
let handle = Handle::new();
let mut server = axum_server::bind(bind).handle(handle.clone());
server.http_builder().http1().preserve_header_case(true);
server.http_builder().http1().title_case_headers(true);
#[derive(Clone)]
struct QueueRunner {
queue: Arc<RwLock<Option<BackgroundJobQueue>>>,
}
impl<Fut> hyper::rt::Executor<Fut> for QueueRunner
where
Fut: Future + Send + 'static,
{
fn execute(&self, fut: Fut) {
if let Some(q) = &*self.queue.read().unwrap() {
q.add_job(fut);
} else {
tracing::warn!("job queued after shutdown");
}
}
}
if let (Err(e), _) = tokio::join!(server.serve(thread_router), async {
let _ = shutdown_recv.await;
handle.graceful_shutdown(Some(Duration::from_secs(0)));
}) {
tracing::error!("Spawning hyper server error: {}", e);
let accept = AtomicBool::new(true);
let queue_cell = Arc::new(RwLock::new(None));
let graceful = hyper_util::server::graceful::GracefulShutdown::new();
let mut server = hyper_util::server::conn::auto::Builder::new(QueueRunner {
queue: queue_cell.clone(),
});
server
.http1()
.timer(TokioTimer::new())
.title_case_headers(true)
.preserve_header_case(true)
.http2()
.timer(TokioTimer::new())
.enable_connect_protocol()
.keep_alive_interval(Duration::from_secs(60))
.keep_alive_timeout(Duration::from_secs(300));
let (queue, mut runner) = BackgroundJobQueue::new();
*queue_cell.write().unwrap() = Some(queue.clone());
let handler = async {
loop {
if let Err(e) = async {
let accepted = acceptor.accept().await?;
if accepted.https_redirect {
queue.add_job(
graceful.watch(
server
.serve_connection_with_upgrades(
TokioIo::new(accepted.stream),
TowerToHyperService::new(redirecter().into_service()),
)
.into_owned(),
),
);
} else {
let service = { service.borrow().clone() };
if let Some(service) = service {
queue.add_job(
graceful.watch(
server
.serve_connection_with_upgrades(
TokioIo::new(accepted.stream),
TowerToHyperService::new(service.into_service()),
)
.into_owned(),
),
);
} else {
queue.add_job(
graceful.watch(
server
.serve_connection_with_upgrades(
TokioIo::new(accepted.stream),
TowerToHyperService::new(
refresher().into_service(),
),
)
.into_owned(),
),
);
}
}
Ok::<_, Error>(())
}
.await
{
tracing::error!("Error accepting HTTP connection: {e}");
tracing::debug!("{e:?}");
}
}
}
.boxed();
tokio::select! {
_ = shutdown_recv => (),
_ = handler => (),
_ = &mut runner => (),
}
accept.store(false, std::sync::atomic::Ordering::SeqCst);
drop(queue);
drop(queue_cell.write().unwrap().take());
if !runner.is_empty() {
runner.await;
}
}));
Self {
shutdown,
router,
thread,
acceptor: acceptor_send,
}
}
@@ -138,7 +303,7 @@ impl WebServer {
}
pub fn serve_router(&mut self, router: Router) {
self.router.swap(router)
self.router.send_replace(Some(router));
}
pub fn serve_main(&mut self, ctx: RpcContext) {

21
core/startos/src/net/wifi.rs
View File

@@ -298,7 +298,7 @@ fn display_wifi_info(params: WithIoFormat<Empty>, info: WifiListInfo) {
let mut table_global = Table::new();
table_global.add_row(row![bc =>
"CONNECTED",
"SIGNAL_STRENGTH",
"SIGNAL STRENGTH",
"COUNTRY",
"ETHERNET",
]);
@@ -306,12 +306,12 @@ fn display_wifi_info(params: WithIoFormat<Empty>, info: WifiListInfo) {
&info
.connected
.as_ref()
.map_or("[N/A]".to_owned(), |c| c.0.clone()),
.map_or("N/A".to_owned(), |c| c.0.clone()),
&info
.connected
.as_ref()
.and_then(|x| info.ssids.get(x))
.map_or("[N/A]".to_owned(), |ss| format!("{}", ss.0)),
.map_or("N/A".to_owned(), |ss| format!("{}", ss.0)),
info.country.as_ref().map(|c| c.alpha2()).unwrap_or("00"),
&format!("{}", info.ethernet)
]);
@@ -897,32 +897,29 @@ impl TypedValueParser for CountryCodeParser {
}
#[instrument(skip_all)]
pub async fn synchronize_wpa_supplicant_conf<P: AsRef<Path>>(
pub async fn synchronize_network_manager<P: AsRef<Path>>(
main_datadir: P,
wifi: &mut WifiInfo,
) -> Result<(), Error> {
wifi.interface = find_wifi_iface().await?;
let Some(wifi_iface) = &wifi.interface else {
return Ok(());
};
let persistent = main_datadir.as_ref().join("system-connections");
tracing::debug!("persistent: {:?}", persistent);
// let supplicant = Path::new("/etc/wpa_supplicant.conf");
if tokio::fs::metadata(&persistent).await.is_err() {
tokio::fs::create_dir_all(&persistent).await?;
}
crate::disk::mount::util::bind(&persistent, "/etc/NetworkManager/system-connections", false)
.await?;
// if tokio::fs::metadata(&supplicant).await.is_err() {
// tokio::fs::write(&supplicant, include_str!("wpa_supplicant.conf.base")).await?;
// }
Command::new("systemctl")
.arg("restart")
.arg("NetworkManager")
.invoke(ErrorKind::Wifi)
.await?;
let Some(wifi_iface) = &wifi.interface else {
return Ok(());
};
Command::new("ifconfig")
.arg(wifi_iface)
.arg("up")

2
core/startos/src/os_install/gpt.rs
View File

@@ -50,7 +50,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
if part_info.guid.is_some() {
if entry.first_lba < if use_efi { 33759266 } else { 33570850 } {
return Err(Error::new(
eyre!("Not enough space before embassy data"),
eyre!("Not enough space before StartOS data"),
crate::ErrorKind::InvalidRequest,
));
}

17
core/startos/src/prelude.rs
View File

@@ -6,3 +6,20 @@ pub use tracing::instrument;
pub use crate::db::prelude::*;
pub use crate::ensure_code;
pub use crate::error::{Error, ErrorCollection, ErrorKind, ResultExt};
#[macro_export]
macro_rules! dbg {
() => {{
tracing::debug!("[{}:{}:{}]", file!(), line!(), column!());
}};
($e:expr) => {{
let e = $e;
tracing::debug!("[{}:{}:{}] {} = {e:?}", file!(), line!(), column!(), stringify!($e));
e
}};
($($e:expr),+) => {
($(
crate::dbg!($e)
),+)
}
}

1
core/startos/src/registry/context.rs
View File

@@ -19,7 +19,6 @@ use crate::context::config::{ContextConfig, CONFIG_PATH};
use crate::context::{CliContext, RpcContext};
use crate::prelude::*;
use crate::registry::auth::{SignatureHeader, AUTH_SIG_HEADER};
use crate::registry::device_info::{DeviceInfo, DEVICE_INFO_HEADER};
use crate::registry::signer::sign::AnySigningKey;
use crate::registry::RegistryDatabase;
use crate::rpc_continuations::RpcContinuations;

7
core/startos/src/registry/mod.rs
View File

@@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet};
use axum::Router;
use futures::future::ready;
use imbl_value::InternedString;
use models::DataUrl;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server};
use serde::{Deserialize, Serialize};
@@ -11,13 +10,13 @@ use ts_rs::TS;
use crate::context::CliContext;
use crate::middleware::cors::Cors;
use crate::net::static_server::{bad_request, not_found, server_error};
use crate::net::web_server::WebServer;
use crate::net::web_server::{Accept, WebServer};
use crate::prelude::*;
use crate::registry::auth::Auth;
use crate::registry::context::RegistryContext;
use crate::registry::device_info::DeviceInfoMiddleware;
use crate::registry::os::index::OsIndex;
use crate::registry::package::index::{Category, PackageIndex};
use crate::registry::package::index::PackageIndex;
use crate::registry::signer::SignerInfo;
use crate::rpc_continuations::Guid;
use crate::util::serde::HandlerExtSerde;
@@ -144,7 +143,7 @@ pub fn registry_router(ctx: RegistryContext) -> Router {
)
}
impl WebServer {
impl<A: Accept + Send + Sync + 'static> WebServer<A> {
pub fn serve_registry(&mut self, ctx: RegistryContext) {
self.serve_router(registry_router(ctx))
}

1
core/startos/src/registry/package/index.rs
View File

@@ -72,7 +72,6 @@ pub struct PackageVersionInfo {
pub icon: DataUrl<'static>,
pub description: Description,
pub release_notes: String,
#[ts(type = "string")]
pub git_hash: GitHash,
#[ts(type = "string")]
pub license: InternedString,

4
core/startos/src/registry/signer/commitment/merkle_archive.rs
View File

@@ -24,10 +24,10 @@ impl MerkleArchiveCommitment {
pub fn from_query(query: &str) -> Result<Option<Self>, Error> {
let mut root_sighash = None;
let mut root_maxsize = None;
for (k, v) in form_urlencoded::parse(dbg!(query).as_bytes()) {
for (k, v) in form_urlencoded::parse(query.as_bytes()) {
match &*k {
"rootSighash" => {
root_sighash = Some(dbg!(v).parse()?);
root_sighash = Some(v.parse()?);
}
"rootMaxsize" => {
root_maxsize = Some(v.parse()?);

29
core/startos/src/s9pk/git_hash.rs
View File

@@ -1,11 +1,13 @@
use std::path::Path;
use tokio::process::Command;
use ts_rs::TS;
use crate::prelude::*;
use crate::util::Invoke;
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, TS)]
#[ts(type = "string")]
pub struct GitHash(String);
impl GitHash {
@@ -31,6 +33,31 @@ impl GitHash {
}
Ok(GitHash(hash))
}
pub fn load_sync() -> Option<GitHash> {
let mut hash = String::from_utf8(
std::process::Command::new("git")
.arg("rev-parse")
.arg("HEAD")
.output()
.ok()?
.stdout,
)
.ok()?;
if !std::process::Command::new("git")
.arg("diff-index")
.arg("--quiet")
.arg("HEAD")
.arg("--")
.output()
.ok()?
.status
.success()
{
hash += "-modified";
}
Some(GitHash(hash))
}
}
impl AsRef<str> for GitHash {

5
core/startos/src/s9pk/v2/manifest.rs
View File

@@ -3,7 +3,6 @@ use std::path::Path;
use color_eyre::eyre::eyre;
use exver::{Version, VersionRange};
use helpers::const_true;
use imbl_value::InternedString;
pub use models::PackageId;
use models::{mime, ImageId, VolumeId};
@@ -62,8 +61,8 @@ pub struct Manifest {
pub dependencies: Dependencies,
#[serde(default)]
pub hardware_requirements: HardwareRequirements,
#[serde(default)]
#[ts(type = "string | null")]
#[ts(optional)]
#[serde(default = "GitHash::load_sync")]
pub git_hash: Option<GitHash>,
#[serde(default = "current_version")]
#[ts(type = "string")]

2
core/startos/src/service/effects/callbacks.rs
View File

@@ -294,7 +294,7 @@ impl CallbackHandler {
}
}
pub async fn call(mut self, args: Vector<Value>) -> Result<(), Error> {
dbg!(eyre!("callback fired: {}", self.handle.is_active()));
crate::dbg!(eyre!("callback fired: {}", self.handle.is_active()));
if let Some(seed) = self.seed.upgrade() {
seed.persistent_container
.callback(self.handle.take(), args)

4
core/startos/src/service/effects/dependency.rs
View File

@@ -17,11 +17,11 @@ use crate::db::model::package::{
use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::{FileSystem, MountType};
use crate::rpc_continuations::Guid;
use crate::service::effects::prelude::*;
use crate::status::health_check::NamedHealthCheckResult;
use crate::util::Invoke;
use crate::volume::data_dir;
use crate::DATA_DIR;
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[ts(export)]
@@ -55,7 +55,7 @@ pub async fn mount(
let context = context.deref()?;
let subpath = subpath.unwrap_or_default();
let subpath = subpath.strip_prefix("/").unwrap_or(&subpath);
let source = data_dir(&context.seed.ctx.datadir, &package_id, &volume_id).join(subpath);
let source = data_dir(DATA_DIR, &package_id, &volume_id).join(subpath);
if tokio::fs::metadata(&source).await.is_err() {
tokio::fs::create_dir_all(&source).await?;
}

4
core/startos/src/service/effects/mod.rs
View File

@@ -130,10 +130,6 @@ pub fn handler<C: Context>() -> ParentHandler<C> {
"get-host-info",
from_fn_async(net::host::get_host_info).no_cli(),
)
.subcommand(
"get-primary-url",
from_fn_async(net::host::get_primary_url).no_cli(),
)
.subcommand(
"get-container-ip",
from_fn_async(net::info::get_container_ip).no_cli(),

37
core/startos/src/service/effects/net/bind.rs
View File

@@ -1,6 +1,6 @@
use models::{HostId, PackageId};
use crate::net::host::binding::{BindId, BindOptions, LanInfo};
use crate::net::host::binding::{BindId, BindOptions, NetInfo};
use crate::net::host::HostKind;
use crate::service::effects::prelude::*;
@@ -53,15 +53,36 @@ pub struct GetServicePortForwardParams {
#[ts(optional)]
package_id: Option<PackageId>,
host_id: HostId,
internal_port: u32,
internal_port: u16,
}
pub async fn get_service_port_forward(
context: EffectContext,
data: GetServicePortForwardParams,
) -> Result<LanInfo, Error> {
let internal_port = data.internal_port as u16;
GetServicePortForwardParams {
package_id,
host_id,
internal_port,
}: GetServicePortForwardParams,
) -> Result<NetInfo, Error> {
let context = context.deref()?;
let net_service = context.seed.persistent_container.net_service.lock().await;
net_service.get_lan_port(data.host_id, internal_port)
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
Ok(context
.seed
.ctx
.db
.peek()
.await
.as_public()
.as_package_data()
.as_idx(&package_id)
.or_not_found(&package_id)?
.as_hosts()
.as_idx(&host_id)
.or_not_found(&host_id)?
.as_bindings()
.de()?
.get(&internal_port)
.or_not_found(lazy_format!("binding for port {internal_port}"))?
.net)
}

25
core/startos/src/service/effects/net/host.rs
View File

@@ -1,35 +1,10 @@
use models::{HostId, PackageId};
use crate::net::host::address::HostAddress;
use crate::net::host::Host;
use crate::service::effects::callbacks::CallbackHandler;
use crate::service::effects::prelude::*;
use crate::service::rpc::CallbackId;
#[derive(Debug, Clone, Serialize, Deserialize, TS)]
#[ts(export)]
#[serde(rename_all = "camelCase")]
pub struct GetPrimaryUrlParams {
#[ts(optional)]
package_id: Option<PackageId>,
host_id: HostId,
#[ts(optional)]
callback: Option<CallbackId>,
}
pub async fn get_primary_url(
context: EffectContext,
GetPrimaryUrlParams {
package_id,
host_id,
callback,
}: GetPrimaryUrlParams,
) -> Result<Option<HostAddress>, Error> {
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
Ok(None) // TODO
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]

3
core/startos/src/service/effects/net/interface.rs
View File

@@ -15,7 +15,6 @@ pub struct ExportServiceInterfaceParams {
id: ServiceInterfaceId,
name: String,
description: String,
has_primary: bool,
masked: bool,
address_info: AddressInfo,
r#type: ServiceInterfaceType,
@@ -26,7 +25,6 @@ pub async fn export_service_interface(
id,
name,
description,
has_primary,
masked,
address_info,
r#type,
@@ -39,7 +37,6 @@ pub async fn export_service_interface(
id: id.clone(),
name,
description,
has_primary,
masked,
address_info,
interface_type: r#type,

20
core/startos/src/service/effects/net/ssl.rs
View File

@@ -51,10 +51,16 @@ pub async fn get_ssl_certificate(
.iter()
.map(|(_, m)| m.as_hosts().as_entries())
.flatten_ok()
.map_ok(|(_, m)| m.as_addresses().de())
.map_ok(|(_, m)| {
Ok(m.as_onions()
.de()?
.iter()
.map(InternedString::from_display)
.chain(m.as_domains().keys()?)
.collect::<Vec<_>>())
})
.map(|a| a.and_then(|a| a))
.flatten_ok()
.map_ok(|a| InternedString::from_display(&a))
.try_collect::<_, BTreeSet<_>, _>()?;
for hostname in &hostnames {
if let Some(internal) = hostname
@@ -135,10 +141,16 @@ pub async fn get_ssl_key(
.into_iter()
.map(|m| m.as_hosts().as_entries())
.flatten_ok()
.map_ok(|(_, m)| m.as_addresses().de())
.map_ok(|(_, m)| {
Ok(m.as_onions()
.de()?
.iter()
.map(InternedString::from_display)
.chain(m.as_domains().keys()?)
.collect::<Vec<_>>())
})
.map(|a| a.and_then(|a| a))
.flatten_ok()
.map_ok(|a| InternedString::from_display(&a))
.try_collect::<_, BTreeSet<_>, _>()?;
for hostname in &hostnames {
if let Some(internal) = hostname

2
core/startos/src/service/effects/store.rs
View File

@@ -26,7 +26,7 @@ pub async fn get_store(
callback,
}: GetStoreParams,
) -> Result<Value, Error> {
dbg!(&callback);
crate::dbg!(&callback);
let context = context.deref()?;
let peeked = context.seed.ctx.db.peek().await;
let package_id = package_id.unwrap_or(context.seed.id.clone());

30
core/startos/src/service/mod.rs
View File

@@ -48,7 +48,7 @@ use crate::util::net::WebSocketExt;
use crate::util::serde::{NoOutput, Pem};
use crate::util::Never;
use crate::volume::data_dir;
use crate::CAP_1_KiB;
use crate::{CAP_1_KiB, DATA_DIR, PACKAGE_DATA};
pub mod action;
pub mod cli;
@@ -149,10 +149,10 @@ impl ServiceRef {
.values()
.flat_map(|h| h.bindings.values())
.flat_map(|b| {
b.lan
b.net
.assigned_port
.into_iter()
.chain(b.lan.assigned_ssl_port)
.chain(b.net.assigned_ssl_port)
}),
);
Ok(())
@@ -167,17 +167,18 @@ impl ServiceRef {
{
let state = pde.state_info.expect_removing()?;
for volume_id in &state.manifest.volumes {
let path = data_dir(&ctx.datadir, &state.manifest.id, volume_id);
let path = data_dir(DATA_DIR, &state.manifest.id, volume_id);
if tokio::fs::metadata(&path).await.is_ok() {
tokio::fs::remove_dir_all(&path).await?;
}
}
let logs_dir = ctx.datadir.join("logs").join(&state.manifest.id);
let logs_dir = Path::new(PACKAGE_DATA)
.join("logs")
.join(&state.manifest.id);
if tokio::fs::metadata(&logs_dir).await.is_ok() {
tokio::fs::remove_dir_all(&logs_dir).await?;
}
let archive_path = ctx
.datadir
let archive_path = Path::new(PACKAGE_DATA)
.join("archive")
.join("installed")
.join(&state.manifest.id);
@@ -278,7 +279,7 @@ impl Service {
let ctx = ctx.clone();
move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
for volume_id in &s9pk.as_manifest().volumes {
let path = data_dir(&ctx.datadir, &s9pk.as_manifest().id, volume_id);
let path = data_dir(DATA_DIR, &s9pk.as_manifest().id, volume_id);
if tokio::fs::metadata(&path).await.is_err() {
tokio::fs::create_dir_all(&path).await?;
}
@@ -291,7 +292,7 @@ impl Service {
Self::new(ctx, s9pk, start_stop).await.map(Some)
}
};
let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash
let s9pk_dir = Path::new(DATA_DIR).join(PKG_ARCHIVE_DIR).join("installed"); // TODO: make this based on hash
let s9pk_path = s9pk_dir.join(id).with_extension("s9pk");
let Some(entry) = ctx
.db
@@ -605,6 +606,7 @@ impl Service {
}
pub async fn update_host(&self, host_id: HostId) -> Result<(), Error> {
let mut service = self.seed.persistent_container.net_service.lock().await;
let host = self
.seed
.ctx
@@ -619,13 +621,7 @@ impl Service {
.as_idx(&host_id)
.or_not_found(&host_id)?
.de()?;
self.seed
.persistent_container
.net_service
.lock()
.await
.update(host_id, host)
.await
service.update(host_id, host).await
}
}
@@ -934,7 +930,6 @@ pub async fn attach(
.with_kind(ErrorKind::Network)?;
current_out = "stdout";
}
dbg!(&current_out);
ws.send(Message::Binary(out))
.await
.with_kind(ErrorKind::Network)?;
@@ -948,7 +943,6 @@ pub async fn attach(
.with_kind(ErrorKind::Network)?;
current_out = "stderr";
}
dbg!(&current_out);
ws.send(Message::Binary(err))
.await
.with_kind(ErrorKind::Network)?;

10
core/startos/src/service/persistent_container.rs
View File

@@ -39,7 +39,7 @@ use crate::util::io::create_file;
use crate::util::rpc_client::UnixRpcClient;
use crate::util::Invoke;
use crate::volume::data_dir;
use crate::ARCH;
use crate::{ARCH, DATA_DIR, PACKAGE_DATA};
const RPC_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
@@ -121,8 +121,8 @@ impl PersistentContainer {
.lxc_manager
.create(
Some(
&ctx.datadir
.join("package-data/logs")
&Path::new(PACKAGE_DATA)
.join("logs")
.join(&s9pk.as_manifest().id),
),
LxcConfig::default(),
@@ -157,7 +157,7 @@ impl PersistentContainer {
.await?;
let mount = MountGuard::mount(
&IdMapped::new(
Bind::new(data_dir(&ctx.datadir, &s9pk.as_manifest().id, volume)),
Bind::new(data_dir(DATA_DIR, &s9pk.as_manifest().id, volume)),
0,
100000,
65536,
@@ -452,7 +452,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn exit(mut self) -> Result<(), Error> {
if let Some(destroy) = self.destroy(false) {
dbg!(destroy.await)?;
destroy.await?;
}
tracing::info!("Service for {} exited", self.s9pk.as_manifest().id);

2
core/startos/src/service/rpc.rs
View File

@@ -155,7 +155,7 @@ impl serde::Serialize for Sandbox {
pub struct CallbackId(u64);
impl CallbackId {
pub fn register(self, container: &PersistentContainer) -> CallbackHandle {
dbg!(eyre!(
crate::dbg!(eyre!(
"callback {} registered for {}",
self.0,
container.s9pk.as_manifest().id

37
core/startos/src/service/service_actor.rs
View File

@@ -36,7 +36,41 @@ impl Actor for ServiceActor {
ServiceActorLoopNext::DontWait => (),
}
}
})
});
let seed = self.0.clone();
let mut ip_info = seed.ctx.net_controller.net_iface.subscribe();
jobs.add_job(async move {
loop {
if let Err(e) = async {
let mut service = seed.persistent_container.net_service.lock().await;
let hosts = seed
.ctx
.db
.peek()
.await
.as_public()
.as_package_data()
.as_idx(&seed.id)
.or_not_found(&seed.id)?
.as_hosts()
.de()?;
for (host_id, host) in hosts.0 {
service.update(host_id, host).await?;
}
Ok::<_, Error>(())
}
.await
{
tracing::error!("Error syncronizing net host after network change: {e}");
tracing::debug!("{e:?}");
}
if ip_info.changed().await.is_err() {
break;
};
}
});
}
}
@@ -92,7 +126,6 @@ async fn service_actor_loop(
..
} => MainStatus::Stopped,
};
let previous = i.as_status().de()?;
i.as_status_mut().ser(&main_status)?;
return Ok(previous
.major_changes(&main_status)

8
core/startos/src/service/service_map.rs
View File

@@ -1,3 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@@ -27,6 +28,7 @@ use crate::service::start_stop::StartStop;
use crate::service::{LoadDisposition, Service, ServiceRef};
use crate::status::MainStatus;
use crate::util::serde::Pem;
use crate::DATA_DIR;
pub type DownloadInstallFuture = BoxFuture<'static, Result<InstallFuture, Error>>;
pub type InstallFuture = BoxFuture<'static, Result<(), Error>>;
@@ -220,8 +222,7 @@ impl ServiceMap {
Ok(async move {
let (installed_path, sync_progress_task) = reload_guard
.handle(async {
let download_path = ctx
.datadir
let download_path = Path::new(DATA_DIR)
.join(PKG_ARCHIVE_DIR)
.join("downloading")
.join(&id)
@@ -251,8 +252,7 @@ impl ServiceMap {
file.sync_all().await?;
download_progress.complete();
let installed_path = ctx
.datadir
let installed_path = Path::new(DATA_DIR)
.join(PKG_ARCHIVE_DIR)
.join("installed")
.join(&id)

3
core/startos/src/service/transition/backup.rs
View File

@@ -15,6 +15,7 @@ use crate::service::ServiceActor;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{ConflictBuilder, Handler};
use crate::util::future::RemoteCancellable;
use crate::util::serde::NoOutput;
pub(in crate::service) struct Backup {
pub path: PathBuf,
@@ -48,7 +49,7 @@ impl Handler<Backup> for ServiceActor {
.mount_backup(path, ReadWrite)
.await?;
seed.persistent_container
.execute(id, ProcedureName::CreateBackup, Value::Null, None)
.execute::<NoOutput>(id, ProcedureName::CreateBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;

5
core/startos/src/service/transition/restore.rs
View File

@@ -11,6 +11,7 @@ use crate::service::ServiceActor;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{ConflictBuilder, Handler};
use crate::util::future::RemoteCancellable;
use crate::util::serde::NoOutput;
pub(in crate::service) struct Restore {
pub path: PathBuf,
@@ -38,7 +39,7 @@ impl Handler<Restore> for ServiceActor {
.mount_backup(path, ReadOnly)
.await?;
seed.persistent_container
.execute(id, ProcedureName::RestoreBackup, Value::Null, None)
.execute::<NoOutput>(id, ProcedureName::RestoreBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;
@@ -48,7 +49,7 @@ impl Handler<Restore> for ServiceActor {
Ok::<_, Error>(())
}
.map(|x| {
if let Err(err) = dbg!(x) {
if let Err(err) = x {
tracing::debug!("{:?}", err);
tracing::warn!("{}", err);
}

46
core/startos/src/setup.rs
View File

@@ -4,6 +4,7 @@ use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use const_format::formatcp;
use josekit::jwk::Jwk;
use patch_db::json_ptr::ROOT;
use rpc_toolkit::yajrc::RpcError;
@@ -38,7 +39,7 @@ use crate::rpc_continuations::Guid;
use crate::util::crypto::EncryptedWire;
use crate::util::io::{create_file, dir_copy, dir_size, Counter};
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt};
use crate::{Error, ErrorKind, ResultExt, DATA_DIR, MAIN_DATA, PACKAGE_DATA};
pub fn setup<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
@@ -80,7 +81,7 @@ async fn setup_init(
password: Option<String>,
init_phases: InitPhases,
) -> Result<(AccountInfo, PreInitNetController), Error> {
let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?;
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
let account = net_ctrl
.db
@@ -140,7 +141,7 @@ pub async fn attach(
disk_phase.start();
let requires_reboot = crate::disk::main::import(
&*disk_guid,
&setup_ctx.datadir,
DATA_DIR,
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
RepairStrategy::Aggressive
} else {
@@ -155,7 +156,7 @@ pub async fn attach(
.with_ctx(|_| (ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
}
if requires_reboot.0 {
crate::disk::main::export(&*disk_guid, &setup_ctx.datadir).await?;
crate::disk::main::export(&*disk_guid, DATA_DIR).await?;
return Err(Error::new(
eyre!(
"Errors were corrected with your disk, but the server must be restarted in order to proceed"
@@ -167,7 +168,7 @@ pub async fn attach(
let (account, net_ctrl) = setup_init(&setup_ctx, password, init_phases).await?;
let rpc_ctx = RpcContext::init(&setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?;
let rpc_ctx = RpcContext::init(&setup_ctx.webserver, &setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?;
Ok(((&account).try_into()?, rpc_ctx))
})?;
@@ -391,18 +392,13 @@ pub async fn execute_inner(
crate::disk::main::create(
&[start_os_logicalname],
&pvscan().await?,
&ctx.datadir,
DATA_DIR,
encryption_password,
)
.await?,
);
let _ = crate::disk::main::import(
&*guid,
&ctx.datadir,
RepairStrategy::Preen,
encryption_password,
)
.await?;
let _ = crate::disk::main::import(&*guid, DATA_DIR, RepairStrategy::Preen, encryption_password)
.await?;
disk_phase.complete();
let progress = SetupExecuteProgress {
@@ -456,9 +452,16 @@ async fn fresh_setup(
db.put(&ROOT, &Database::init(&account)?).await?;
drop(db);
let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?;
let InitResult { net_ctrl } = init(&ctx.webserver, &ctx.config, init_phases).await?;
let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?;
let rpc_ctx = RpcContext::init(
&ctx.webserver,
&ctx.config,
guid,
Some(net_ctrl),
rpc_ctx_phases,
)
.await?;
Ok(((&account).try_into()?, rpc_ctx))
}
@@ -513,10 +516,10 @@ async fn migrate(
)
.await?;
let main_transfer_args = ("/media/startos/migrate/main/", "/embassy-data/main/");
let main_transfer_args = ("/media/startos/migrate/main/", formatcp!("{MAIN_DATA}/"));
let package_data_transfer_args = (
"/media/startos/migrate/package-data/",
"/embassy-data/package-data/",
formatcp!("{PACKAGE_DATA}/"),
);
let tmpdir = Path::new(package_data_transfer_args.0).join("tmp");
@@ -571,7 +574,14 @@ async fn migrate(
let (account, net_ctrl) = setup_init(&ctx, Some(start_os_password), init_phases).await?;
let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?;
let rpc_ctx = RpcContext::init(
&ctx.webserver,
&ctx.config,
guid,
Some(net_ctrl),
rpc_ctx_phases,
)
.await?;
Ok(((&account).try_into()?, rpc_ctx))
}

8
core/startos/src/shutdown.rs
View File

@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::context::RpcContext;
@@ -7,7 +7,7 @@ use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH};
use crate::prelude::*;
use crate::sound::SHUTDOWN;
use crate::util::Invoke;
use crate::PLATFORM;
use crate::{DATA_DIR, PLATFORM};
#[derive(Debug, Clone)]
pub struct Shutdown {
@@ -87,7 +87,7 @@ pub async fn shutdown(ctx: RpcContext) -> Result<(), Error> {
.await?;
ctx.shutdown
.send(Some(Shutdown {
export_args: Some((ctx.disk_guid.clone(), ctx.datadir.clone())),
export_args: Some((ctx.disk_guid.clone(), Path::new(DATA_DIR).to_owned())),
restart: false,
}))
.map_err(|_| ())
@@ -107,7 +107,7 @@ pub async fn restart(ctx: RpcContext) -> Result<(), Error> {
.await?;
ctx.shutdown
.send(Some(Shutdown {
export_args: Some((ctx.disk_guid.clone(), ctx.datadir.clone())),
export_args: Some((ctx.disk_guid.clone(), Path::new(DATA_DIR).to_owned())),
restart: true,
}))
.map_err(|_| ())

2
core/startos/src/status/mod.rs
View File

@@ -80,7 +80,7 @@ impl MainStatus {
}
}
pub fn backing_up(self) -> Self {
pub fn backing_up(&self) -> Self {
MainStatus::BackingUp {
on_complete: if self.running() {
StartStop::Start

9
core/startos/src/system.rs
View File

@@ -30,6 +30,7 @@ use crate::util::cpupower::{get_available_governors, set_governor, Governor};
use crate::util::io::open_file;
use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
use crate::util::Invoke;
use crate::{MAIN_DATA, PACKAGE_DATA};
pub fn experimental<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
@@ -808,10 +809,10 @@ pub async fn get_mem_info() -> Result<MetricsMemory, Error> {
#[instrument(skip_all)]
async fn get_disk_info() -> Result<MetricsDisk, Error> {
let package_used_task = get_used("/embassy-data/package-data");
let package_available_task = get_available("/embassy-data/package-data");
let os_used_task = get_used("/embassy-data/main");
let os_available_task = get_available("/embassy-data/main");
let package_used_task = get_used(PACKAGE_DATA);
let package_available_task = get_available(PACKAGE_DATA);
let os_used_task = get_used(MAIN_DATA);
let os_available_task = get_available(MAIN_DATA);
let (package_used, package_available, os_used, os_available) = futures::try_join!(
package_used_task,

2
core/startos/src/update/mod.rs
View File

@@ -20,7 +20,7 @@ use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::efivarfs::{self, EfiVarFs};
use crate::disk::mount::filesystem::efivarfs::{ EfiVarFs};
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::MountType;
use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};

9
core/startos/src/util/actor/background.rs
View File

@@ -15,8 +15,13 @@ impl BackgroundJobQueue {
},
)
}
pub fn add_job(&self, fut: impl Future<Output = ()> + Send + 'static) {
let _ = self.0.send(fut.boxed());
pub fn add_job(&self, fut: impl Future + Send + 'static) {
let _ = self.0.send(
async {
fut.await;
}
.boxed(),
);
}
}

62
core/startos/src/util/future.rs
View File

@@ -1,11 +1,13 @@
use std::pin::Pin;
use std::task::{Context, Poll};
use futures::future::abortable;
use futures::stream::{AbortHandle, Abortable};
use futures::Future;
use futures::future::{abortable, pending, BoxFuture, FusedFuture};
use futures::stream::{AbortHandle, Abortable, BoxStream};
use futures::{Future, FutureExt, Stream, StreamExt};
use tokio::sync::watch;
use crate::prelude::*;
#[pin_project::pin_project(PinnedDrop)]
pub struct DropSignaling<F> {
#[pin]
@@ -102,6 +104,60 @@ impl CancellationHandle {
}
}
#[derive(Default)]
pub struct Until<'a> {
streams: Vec<BoxStream<'a, Result<(), Error>>>,
async_fns: Vec<Box<dyn FnMut() -> BoxFuture<'a, Result<(), Error>> + Send + 'a>>,
}
impl<'a> Until<'a> {
pub fn new() -> Self {
Self::default()
}
pub fn with_stream(
mut self,
stream: impl Stream<Item = Result<(), Error>> + Send + 'a,
) -> Self {
self.streams.push(stream.boxed());
self
}
pub fn with_async_fn<F, Fut>(mut self, mut f: F) -> Self
where
F: FnMut() -> Fut + Send + 'a,
Fut: Future<Output = Result<(), Error>> + FusedFuture + Send + 'a,
{
self.async_fns.push(Box::new(move || f().boxed()));
self
}
pub async fn run<Fut: Future<Output = Result<(), Error>> + Send>(
&mut self,
fut: Fut,
) -> Result<(), Error> {
let (res, _, _) = futures::future::select_all(
self.streams
.iter_mut()
.map(|s| {
async {
s.next().await.transpose()?.ok_or_else(|| {
Error::new(eyre!("stream is empty"), ErrorKind::Cancelled)
})
}
.boxed()
})
.chain(self.async_fns.iter_mut().map(|f| f()))
.chain([async {
fut.await?;
pending().await
}
.boxed()]),
)
.await;
res
}
}
#[tokio::test]
async fn test_cancellable() {
use std::sync::Arc;

38
core/startos/src/util/io.rs
View File

@@ -15,7 +15,7 @@ use futures::future::{BoxFuture, Fuse};
use futures::{AsyncSeek, FutureExt, Stream, TryStreamExt};
use helpers::NonDetachingJoinHandle;
use nix::unistd::{Gid, Uid};
use tokio::fs::File;
use tokio::fs::{File, OpenOptions};
use tokio::io::{
duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf,
};
@@ -460,18 +460,30 @@ impl<T> BackTrackingIO<T> {
}
}
}
pub fn rewind(&mut self) -> Vec<u8> {
pub fn rewind<'a>(&'a mut self) -> (Vec<u8>, &'a [u8]) {
match std::mem::take(&mut self.buffer) {
BTBuffer::Buffering { read, write } => {
self.buffer = BTBuffer::Rewound {
read: Cursor::new(read),
};
write
(
write,
match &self.buffer {
BTBuffer::Rewound { read } => read.get_ref(),
_ => unreachable!(),
},
)
}
BTBuffer::NotBuffering => Vec::new(),
BTBuffer::NotBuffering => (Vec::new(), &[]),
BTBuffer::Rewound { read } => {
self.buffer = BTBuffer::Rewound { read };
Vec::new()
(
Vec::new(),
match &self.buffer {
BTBuffer::Rewound { read } => read.get_ref(),
_ => unreachable!(),
},
)
}
}
}
@@ -529,7 +541,6 @@ impl<T: std::io::Read> std::io::Read for BackTrackingIO<T> {
}
BTBuffer::NotBuffering => self.io.read(buf),
BTBuffer::Rewound { read } => {
let mut ready = false;
if (read.position() as usize) < read.get_ref().len() {
let n = std::io::Read::read(read, buf)?;
if n != 0 {
@@ -923,6 +934,21 @@ pub async fn create_file(path: impl AsRef<Path>) -> Result<File, Error> {
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}")))
}
pub async fn append_file(path: impl AsRef<Path>) -> Result<File, Error> {
let path = path.as_ref();
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("mkdir -p {parent:?}")))?;
}
OpenOptions::new()
.create(true)
.append(true)
.open(path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("create {path:?}")))
}
pub async fn delete_file(path: impl AsRef<Path>) -> Result<(), Error> {
let path = path.as_ref();
tokio::fs::remove_file(path)

68
core/startos/src/util/logger.rs
View File

@@ -1,13 +1,62 @@
use std::io;
use std::fs::File;
use std::io::{self, Write};
use std::sync::{Arc, Mutex, MutexGuard};
use lazy_static::lazy_static;
use tracing::Subscriber;
use tracing_subscriber::fmt::MakeWriter;
use tracing_subscriber::util::SubscriberInitExt;
#[derive(Clone)]
pub struct EmbassyLogger {}
lazy_static! {
pub static ref LOGGER: StartOSLogger = StartOSLogger::init();
}
impl EmbassyLogger {
fn base_subscriber() -> impl Subscriber {
#[derive(Clone)]
pub struct StartOSLogger {
logfile: LogFile,
}
#[derive(Clone, Default)]
struct LogFile(Arc<Mutex<Option<File>>>);
impl<'a> MakeWriter<'a> for LogFile {
type Writer = Box<dyn Write + 'a>;
fn make_writer(&'a self) -> Self::Writer {
let f = self.0.lock().unwrap();
if f.is_some() {
struct TeeWriter<'a>(MutexGuard<'a, Option<File>>);
impl<'a> Write for TeeWriter<'a> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = if let Some(f) = &mut *self.0 {
f.write(buf)?
} else {
buf.len()
};
io::stderr().write_all(&buf[..n])?;
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
if let Some(f) = &mut *self.0 {
f.flush()?;
}
Ok(())
}
}
Box::new(TeeWriter(f))
} else {
drop(f);
Box::new(io::stderr())
}
}
}
impl StartOSLogger {
pub fn enable(&self) {}
pub fn set_logfile(&self, logfile: Option<File>) {
*self.logfile.0.lock().unwrap() = logfile;
}
fn base_subscriber(logfile: LogFile) -> impl Subscriber {
use tracing_error::ErrorLayer;
use tracing_subscriber::prelude::*;
use tracing_subscriber::{fmt, EnvFilter};
@@ -24,7 +73,7 @@ impl EmbassyLogger {
.add_directive("tokio=trace".parse().unwrap())
.add_directive("runtime=trace".parse().unwrap());
let fmt_layer = fmt::layer()
.with_writer(io::stderr)
.with_writer(logfile)
.with_line_number(true)
.with_file(true)
.with_target(true);
@@ -39,11 +88,12 @@ impl EmbassyLogger {
sub
}
pub fn init() -> Self {
Self::base_subscriber().init();
fn init() -> Self {
let logfile = LogFile::default();
Self::base_subscriber(logfile.clone()).init();
color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times"));
EmbassyLogger {}
StartOSLogger { logfile }
}
}

1
core/startos/src/util/rpc.rs
View File

@@ -3,7 +3,6 @@ use std::path::Path;
use clap::Parser;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use url::Url;
use crate::context::CliContext;
use crate::prelude::*;

6
core/startos/src/util/rpc_client.rs
View File

@@ -47,7 +47,7 @@ impl RpcClient {
let mut lines = BufReader::new(reader).lines();
while let Some(line) = lines.next_line().await.transpose() {
match line.map_err(Error::from).and_then(|l| {
serde_json::from_str::<RpcResponse>(dbg!(&l))
serde_json::from_str::<RpcResponse>(crate::dbg!(&l))
.with_kind(ErrorKind::Deserialization)
}) {
Ok(l) => {
@@ -114,7 +114,7 @@ impl RpcClient {
let (send, recv) = oneshot::channel();
w.lock().await.insert(id.clone(), send);
self.writer
.write_all((dbg!(serde_json::to_string(&request))? + "\n").as_bytes())
.write_all((crate::dbg!(serde_json::to_string(&request))? + "\n").as_bytes())
.await
.map_err(|e| {
let mut err = rpc_toolkit::yajrc::INTERNAL_ERROR.clone();
@@ -154,7 +154,7 @@ impl RpcClient {
params,
};
self.writer
.write_all((dbg!(serde_json::to_string(&request))? + "\n").as_bytes())
.write_all((crate::dbg!(serde_json::to_string(&request))? + "\n").as_bytes())
.await
.map_err(|e| {
let mut err = rpc_toolkit::yajrc::INTERNAL_ERROR.clone();

1
core/startos/src/util/sync.rs
View File

@@ -1,3 +1,4 @@
#[derive(Debug, Default)]
pub struct SyncMutex<T>(std::sync::Mutex<T>);
impl<T> SyncMutex<T> {
pub fn new(t: T) -> Self {

11
core/startos/src/version/mod.rs
View File

@@ -7,12 +7,10 @@ use futures::future::BoxFuture;
use futures::{Future, FutureExt};
use imbl::Vector;
use imbl_value::{to_value, InternedString};
use patch_db::json_ptr::{JsonPointer, ROOT};
use patch_db::json_ptr::{ ROOT};
use crate::context::RpcContext;
use crate::db::model::Database;
use crate::prelude::*;
use crate::progress::PhaseProgressTrackerHandle;
use crate::Error;
mod v0_3_5;
@@ -29,7 +27,9 @@ mod v0_3_6_alpha_7;
mod v0_3_6_alpha_8;
mod v0_3_6_alpha_9;
pub type Current = v0_3_6_alpha_9::Version; // VERSION_BUMP
mod v0_3_6_alpha_10;
pub type Current = v0_3_6_alpha_10::Version; // VERSION_BUMP
impl Current {
#[instrument(skip(self, db))]
@@ -108,6 +108,7 @@ enum Version {
V0_3_6_alpha_7(Wrapper<v0_3_6_alpha_7::Version>),
V0_3_6_alpha_8(Wrapper<v0_3_6_alpha_8::Version>),
V0_3_6_alpha_9(Wrapper<v0_3_6_alpha_9::Version>),
V0_3_6_alpha_10(Wrapper<v0_3_6_alpha_10::Version>),
Other(exver::Version),
}
@@ -141,6 +142,7 @@ impl Version {
Self::V0_3_6_alpha_7(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_8(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_9(v) => DynVersion(Box::new(v.0)),
Self::V0_3_6_alpha_10(v) => DynVersion(Box::new(v.0)),
Self::Other(v) => {
return Err(Error::new(
eyre!("unknown version {v}"),
@@ -166,6 +168,7 @@ impl Version {
Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_8(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_9(Wrapper(x)) => x.semver(),
Version::V0_3_6_alpha_10(Wrapper(x)) => x.semver(),
Version::Other(x) => x.clone(),
}
}

18
core/startos/src/version/v0_3_6_alpha_0.rs
View File

@@ -1,19 +1,16 @@
use std::collections::BTreeMap;
use std::future::Future;
use std::path::Path;
use chrono::{DateTime, Utc};
use const_format::formatcp;
use ed25519_dalek::SigningKey;
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::{json, InternedString};
use itertools::Itertools;
use models::PackageId;
use openssl::pkey::{PKey, Private};
use openssl::pkey::PKey;
use openssl::x509::X509;
use patch_db::ModelExt;
use sqlx::postgres::PgConnectOptions;
use sqlx::{PgPool, Row};
use ssh_key::Fingerprint;
use tokio::process::Command;
use torut::onion::TorSecretKeyV3;
@@ -23,15 +20,11 @@ use crate::account::AccountInfo;
use crate::auth::Sessions;
use crate::backup::target::cifs::CifsTargets;
use crate::context::RpcContext;
use crate::db::model::Database;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::util::unmount;
use crate::hostname::Hostname;
use crate::net::forward::AvailablePorts;
use crate::net::keys::KeyStore;
use crate::net::ssl::CertStore;
use crate::net::tor;
use crate::net::tor::OnionStore;
use crate::notifications::{Notification, Notifications};
use crate::prelude::*;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
@@ -39,6 +32,7 @@ use crate::ssh::{SshKeys, SshPubKey};
use crate::util::crypto::ed25519_expand_key;
use crate::util::serde::{Pem, PemEncoding};
use crate::util::Invoke;
use crate::{DATA_DIR, PACKAGE_DATA};
lazy_static::lazy_static! {
static ref V0_3_6_alpha_0: exver::Version = exver::Version::new(
@@ -191,7 +185,6 @@ async fn init_postgres(datadir: impl AsRef<Path>) -> Result<PgPool, Error> {
.run(&secret_store)
.await
.with_kind(crate::ErrorKind::Database)?;
dbg!("Init Postgres Done");
Ok(secret_store)
}
@@ -208,7 +201,7 @@ impl VersionT for Version {
&V0_3_0_COMPAT
}
async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
let pg = init_postgres("/embassy-data").await?;
let pg = init_postgres(DATA_DIR).await?;
let account = previous_account_info(&pg).await?;
let ssh_keys = previous_ssh_keys(&pg).await?;
@@ -315,7 +308,6 @@ impl VersionT for Version {
"private": private,
});
dbg!("Should be done with the up");
*db = next;
Ok(())
}
@@ -329,7 +321,7 @@ impl VersionT for Version {
#[instrument(skip(self, ctx))]
/// MUST be idempotent, and is run after *all* db migrations
async fn post_up(self, ctx: &RpcContext) -> Result<(), Error> {
let path = Path::new("/embassy-data/package-data/archive/");
let path = Path::new(formatcp!("{PACKAGE_DATA}/archive/"));
if !path.is_dir() {
return Err(Error::new(
eyre!(

94
core/startos/src/version/v0_3_6_alpha_10.rs Normal file
View File

@@ -0,0 +1,94 @@
use std::collections::{BTreeMap, BTreeSet};
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::InternedString;
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use super::v0_3_5::V0_3_0_COMPAT;
use super::{v0_3_6_alpha_9, VersionT};
use crate::net::host::address::DomainConfig;
use crate::prelude::*;
lazy_static::lazy_static! {
static ref V0_3_6_alpha_10: exver::Version = exver::Version::new(
[0, 3, 6],
[PreReleaseSegment::String("alpha".into()), 10.into()]
);
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "camelCase")]
#[serde(tag = "kind")]
enum HostAddress {
Onion { address: OnionAddressV3 },
Domain { address: InternedString },
}
#[derive(Clone, Copy, Debug, Default)]
pub struct Version;
impl VersionT for Version {
type Previous = v0_3_6_alpha_9::Version;
type PreUpRes = ();
async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
Ok(())
}
fn semver(self) -> exver::Version {
V0_3_6_alpha_10.clone()
}
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
for (_, package) in db["public"]["packageData"]
.as_object_mut()
.ok_or_else(|| {
Error::new(
eyre!("expected public.packageData to be an object"),
ErrorKind::Database,
)
})?
.iter_mut()
{
for (_, host) in package["hosts"]
.as_object_mut()
.ok_or_else(|| {
Error::new(
eyre!("expected public.packageData[id].hosts to be an object"),
ErrorKind::Database,
)
})?
.iter_mut()
{
let mut onions = BTreeSet::new();
let mut domains = BTreeMap::new();
let addresses = from_value::<BTreeSet<HostAddress>>(host["addresses"].clone())?;
for address in addresses {
match address {
HostAddress::Onion { address } => {
onions.insert(address);
}
HostAddress::Domain { address } => {
domains.insert(
address,
DomainConfig {
public: true,
acme: None,
},
);
}
}
}
host["onions"] = to_value(&onions)?;
host["domains"] = to_value(&domains)?;
}
}
Ok(())
}
fn down(self, _db: &mut Value) -> Result<(), Error> {
Ok(())
}
}

2
core/startos/src/version/v0_3_6_alpha_6.rs
View File

@@ -27,7 +27,7 @@ impl VersionT for Version {
async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
Ok(())
}
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
Ok(())
}
async fn post_up<'a>(self, ctx: &'a crate::context::RpcContext) -> Result<(), Error> {

2
core/startos/src/version/v0_3_6_alpha_7.rs
View File

@@ -1,5 +1,5 @@
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::{json, InOMap};
use imbl_value::json;
use tokio::process::Command;
use super::v0_3_5::V0_3_0_COMPAT;

5
core/startos/src/version/v0_3_6_alpha_8.rs
View File

@@ -1,3 +1,5 @@
use std::path::Path;
use exver::{PreReleaseSegment, VersionRange};
use tokio::fs::File;
@@ -12,6 +14,7 @@ use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::service::LoadDisposition;
use crate::util::io::create_file;
use crate::DATA_DIR;
lazy_static::lazy_static! {
static ref V0_3_6_alpha_8: exver::Version = exver::Version::new(
@@ -40,7 +43,7 @@ impl VersionT for Version {
Ok(())
}
async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> {
let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed");
let s9pk_dir = Path::new(DATA_DIR).join(PKG_ARCHIVE_DIR).join("installed");
if tokio::fs::metadata(&s9pk_dir).await.is_ok() {
let mut read_dir = tokio::fs::read_dir(&s9pk_dir).await?;

7
core/startos/src/volume.rs
View File

@@ -1,10 +1,9 @@
use std::path::{Path, PathBuf};
pub use helpers::script_dir;
use models::PackageId;
pub use models::VolumeId;
use models::{HostId, PackageId};
use crate::net::PACKAGE_CERT_PATH;
use crate::prelude::*;
use crate::util::VersionString;
@@ -36,7 +35,3 @@ pub fn asset_dir<P: AsRef<Path>>(
pub fn backup_dir(pkg_id: &PackageId) -> PathBuf {
Path::new(BACKUP_DIR).join(pkg_id).join("data")
}
pub fn cert_dir(pkg_id: &PackageId, host_id: &HostId) -> PathBuf {
Path::new(PACKAGE_CERT_PATH).join(pkg_id).join(host_id)
}