Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 20:14:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes for trixie and tor

Browse Source
This commit is contained in:
Aiden McClelland
2025-10-12 08:49:11 -06:00
parent 98f31d4891
commit a630ef9a54
42 changed files with 2715 additions and 1334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
core/startos/Cargo.toml
View File

@@ -14,7 +14,7 @@ keywords = [
name = "start-os"
readme = "README.md"
repository = "https://github.com/Start9Labs/start-os"
version = "0.4.0-alpha.11" # VERSION_BUMP
version = "0.4.0-alpha.12" # VERSION_BUMP
license = "MIT"
[lib]
@@ -42,16 +42,28 @@ name = "tunnelbox"
path = "src/main.rs"
[features]
arti = [
"arti-client",
"tor-rtcompat",
"tor-keymgr",
"tor-proto",
"tor-hscrypto",
"tor-llcrypto",
"tor-cell",
"tor-hsservice",
"safelog",
"models/arti",
]
cli = ["cli-startd", "cli-registry", "cli-tunnel"]
cli-container = ["procfs", "pty-process"]
cli-registry = []
cli-startd = []
cli-tunnel = []
default = ["cli", "startd", "registry", "cli-container", "tunnel"]
dev = ["backtrace-on-stack-overflow"]
dev = []
docker = []
registry = []
startd = ["mail-send"]
startd = []
test = []
tunnel = []
console = ["console-subscriber", "tokio/tracing"]
@@ -67,7 +79,7 @@ arti-client = { version = "0.33", features = [
"ephemeral-keystore",
"onion-service-client",
"onion-service-service",
], default-features = false, git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
], default-features = false, git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
aes = { version = "0.7.5", features = ["ctr"] }
async-acme = { version = "0.6.0", git = "https://github.com/dr-bonez/async-acme.git", features = [
"use_rustls",
@@ -98,6 +110,7 @@ console-subscriber = { version = "0.4.1", optional = true }
const_format = "0.2.34"
cookie = "0.18.0"
cookie_store = "0.21.0"
curve25519-dalek = "4.1.3"
der = { version = "0.7.9", features = ["derive", "pem"] }
digest = "0.10.7"
divrem = "1.0.0"
@@ -109,6 +122,7 @@ ed25519-dalek = { version = "2.2.0", features = [
"rand_core",
"digest",
"pkcs8",
"hazmat",
] }
ed25519-dalek-v1 = { package = "ed25519-dalek", version = "1" }
exver = { version = "0.2.0", git = "https://github.com/Start9Labs/exver-rs.git", features = [
@@ -156,6 +170,15 @@ jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
lazy_async_pool = "0.3.3"
lazy_format = "2.0"
lazy_static = "1.4.0"
lettre = { version = "0.11.18", default-features = false, features = [
"smtp-transport",
"pool",
"hostname",
"builder",
"tokio1-rustls",
"rustls-platform-verifier",
"aws-lc-rs",
] }
libc = "0.2.149"
log = "0.4.20"
mio = "1"
@@ -199,7 +222,7 @@ rpassword = "7.2.0"
rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "master" }
rust-argon2 = "2.0.0"
rustyline-async = "0.4.1"
safelog = { version = "0.4.8", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
safelog = { version = "0.4.8", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
semver = { version = "1.0.20", features = ["serde"] }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_cbor = { package = "ciborium", version = "0.2.1" }
@@ -214,7 +237,7 @@ shell-words = "1"
signal-hook = "0.3.17"
simple-logging = "2.0.2"
socket2 = { version = "0.6.0", features = ["all"] }
socks5-impl = { version = "0.7.2", features = ["server"] }
socks5-impl = { version = "0.7.2", features = ["server", "client"] }
sqlx = { version = "0.8.6", features = [
"runtime-tokio-rustls",
"postgres",
@@ -231,22 +254,23 @@ tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }
tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
tokio-tungstenite = { version = "0.26.2", features = ["native-tls", "url"] }
tokio-util = { version = "0.7.9", features = ["io"] }
tor-cell = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
tor-cell = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-hscrypto = { version = "0.33", features = [
"full",
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
tor-hsservice = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-hsservice = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-keymgr = { version = "0.33", features = [
"ephemeral-keystore",
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-llcrypto = { version = "0.33", features = [
"full",
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
tor-proto = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-proto = { version = "0.33", git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
tor-rtcompat = { version = "0.33", features = [
"tokio",
"rustls",
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit" }
], git = "https://github.com/Start9Labs/arti.git", branch = "patch/disable-exit", optional = true }
torut = "0.2.1"
tower-service = "0.3.3"
tracing = "0.1.39"
tracing-error = "0.2.0"
@@ -259,12 +283,9 @@ unix-named-pipe = "0.2.0"
url = { version = "2.4.1", features = ["serde"] }
urlencoding = "2.1.3"
uuid = { version = "1.4.1", features = ["v4"] }
x25519-dalek = "2.0.1"
x25519-dalek = { version = "2.0.1", features = ["static_secrets"] }
zbus = "5.1.1"
zeroize = "1.6.0"
mail-send = { git = "https://github.com/dr-bonez/mail-send.git", branch = "main", optional = true }
rustls = "0.23.20"
rustls-pki-types = { version = "1.10.1", features = ["alloc"] }
[profile.test]
opt-level = 3

17
core/startos/src/disk/mount/filesystem/idmapped.rs
View File

@@ -80,23 +80,6 @@ impl<Fs: FileSystem> FileSystem for IdMapped<Fs> {
}
Ok(())
}
async fn mount<P: AsRef<Path> + Send>(
&self,
mountpoint: P,
mount_type: MountType,
) -> Result<(), Error> {
self.pre_mount(mountpoint.as_ref()).await?;
Command::new("mount.next")
.args(
default_mount_command(self, mountpoint, mount_type)
.await?
.get_args(),
)
.invoke(ErrorKind::Filesystem)
.await?;
Ok(())
}
async fn source_hash(
&self,
) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {

94
core/startos/src/net/dns.rs
View File

@@ -33,7 +33,7 @@ use serde::{Deserialize, Serialize};
use tokio::net::{TcpListener, UdpSocket};
use tracing::instrument;
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::db::model::public::NetworkInterfaceInfo;
use crate::db::model::Database;
use crate::net::gateway::NetworkInterfaceWatcher;
@@ -66,7 +66,36 @@ pub fn dns_api<C: Context>() -> ParentHandler<C> {
"set-static",
from_fn_async(set_static_dns)
.no_display()
.with_about("Set static DNS servers"),
.with_about("Set static DNS servers")
.with_call_remote::<CliContext>(),
)
.subcommand(
"dump-table",
from_fn_async(dump_table)
.with_display_serializable()
.with_custom_display_fn(|HandlerArgs { params, .. }, res| {
use prettytable::*;
if let Some(format) = params.format {
return display_serializable(format, res);
}
let mut table = Table::new();
table.add_row(row![bc => "FQDN", "DESTINATION"]);
for (hostname, destination) in res {
if let Some(ip) = destination {
table.add_row(row![hostname, ip]);
} else {
table.add_row(row![hostname, "SELF"]);
}
}
table.print_tty(false)?;
Ok(())
})
.with_about("Dump address resolution table")
.with_call_remote::<CliContext>(),
)
}
@@ -142,6 +171,38 @@ pub async fn set_static_dns(
.result
}
pub async fn dump_table(
ctx: RpcContext,
) -> Result<BTreeMap<InternedString, Option<IpAddr>>, Error> {
Ok(ctx
.net_controller
.dns
.resolve
.upgrade()
.or_not_found("DnsController")?
.peek(|map| {
map.private_domains
.iter()
.map(|(d, _)| (d.clone(), None))
.chain(map.services.iter().filter_map(|(svc, ip)| {
ip.iter()
.find(|(_, rc)| rc.strong_count() > 0)
.map(|(ip, _)| {
(
svc.as_ref().map_or(
InternedString::from_static("startos"),
|svc| {
InternedString::from_display(&lazy_format!("{svc}.startos"))
},
),
Some(IpAddr::V4(*ip)),
)
})
}))
.collect()
}))
}
#[derive(Default)]
struct ResolveMap {
private_domains: BTreeMap<InternedString, Weak<()>>,
@@ -222,9 +283,9 @@ impl DnsClient {
});
loop {
if let Err::<(), Error>(e) = async {
let mut static_changed = db
let mut dns_changed = db
.subscribe(
"/public/serverInfo/network/dns/staticServers"
"/public/serverInfo/network/dns"
.parse::<JsonPointer>()
.with_kind(ErrorKind::Database)?,
)
@@ -275,7 +336,7 @@ impl DnsClient {
Client::new(stream, sender, None)
.await
.with_kind(ErrorKind::Network)?;
bg.insert(*addr, bg_thread.boxed());
bg.insert(*addr, bg_thread.fuse().boxed());
client
};
new.push((*addr, client));
@@ -286,7 +347,7 @@ impl DnsClient {
client.replace(new);
}
futures::future::select(
static_changed.recv().boxed(),
dns_changed.recv().boxed(),
futures::future::join(
futures::future::join_all(bg.values_mut()),
futures::future::pending::<()>(),
@@ -333,10 +394,20 @@ struct Resolver {
resolve: Arc<SyncRwLock<ResolveMap>>,
}
impl Resolver {
fn resolve(&self, name: &Name, src: IpAddr) -> Option<Vec<IpAddr>> {
fn resolve(&self, name: &Name, mut src: IpAddr) -> Option<Vec<IpAddr>> {
if name.zone_of(&*LOCALHOST) {
return Some(vec![Ipv4Addr::LOCALHOST.into(), Ipv6Addr::LOCALHOST.into()]);
}
src = match src {
IpAddr::V6(v6) => {
if let Some(v4) = v6.to_ipv4_mapped() {
IpAddr::V4(v4)
} else {
IpAddr::V6(v6)
}
}
a => a,
};
self.resolve.peek(|r| {
if r.private_domains
.get(&*name.to_lowercase().to_utf8().trim_end_matches('.'))
@@ -344,8 +415,11 @@ impl Resolver {
{
if let Some(res) = self.net_iface.peek(|i| {
i.values()
.chain([NetworkInterfaceInfo::lxc_bridge().1])
.flat_map(|i| i.ip_info.as_ref())
.chain([
NetworkInterfaceInfo::loopback().1,
NetworkInterfaceInfo::lxc_bridge().1,
])
.filter_map(|i| i.ip_info.as_ref())
.find(|i| i.subnets.iter().any(|s| s.contains(&src)))
.map(|ip_info| {
let mut res = ip_info.subnets.iter().collect::<Vec<_>>();
@@ -354,6 +428,8 @@ impl Resolver {
})
}) {
return Some(res);
} else {
tracing::warn!("Could not determine source interface of {src}");
}
}
if STARTOS.zone_of(name) || EMBASSY.zone_of(name) {

20
core/startos/src/net/host/address.rs
View File

@@ -357,15 +357,7 @@ pub async fn add_onion<Kind: HostApiKind>(
OnionParams { onion }: OnionParams,
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
.ok_or_else(|| {
Error::new(
eyre!("onion hostname must end in .onion"),
ErrorKind::InvalidOnionAddress,
)
})?
.parse::<OnionAddress>()?;
let onion = onion.parse::<OnionAddress>()?;
ctx.db
.mutate(|db| {
db.as_private().as_key_store().as_onion().get_key(&onion)?;
@@ -388,15 +380,7 @@ pub async fn remove_onion<Kind: HostApiKind>(
OnionParams { onion }: OnionParams,
inheritance: Kind::Inheritance,
) -> Result<(), Error> {
let onion = onion
.strip_suffix(".onion")
.ok_or_else(|| {
Error::new(
eyre!("onion hostname must end in .onion"),
ErrorKind::InvalidOnionAddress,
)
})?
.parse::<OnionAddress>()?;
let onion = onion.parse::<OnionAddress>()?;
ctx.db
.mutate(|db| {
Kind::host_for(&inheritance, db)?

2
core/startos/src/net/net_controller.rs
View File

@@ -688,7 +688,7 @@ impl NetServiceData {
.collect::<BTreeSet<_>>();
for onion in all {
let mut prev = binds.tor.remove(&onion);
if let Some((key, tor_binds)) = tor.remove(&onion) {
if let Some((key, tor_binds)) = tor.remove(&onion).filter(|(_, b)| !b.is_empty()) {
prev = prev.filter(|(b, _)| b == &tor_binds);
binds.tor.insert(
onion,

13
core/startos/src/net/tor.rs → core/startos/src/net/tor/arti.rs
View File

@@ -6,7 +6,7 @@ use std::sync::{Arc, Weak};
use std::time::{Duration, Instant};
use arti_client::config::onion_service::OnionServiceConfigBuilder;
use arti_client::{DataStream, TorClient, TorClientConfig};
use arti_client::{TorClient, TorClientConfig};
use base64::Engine;
use clap::Parser;
use color_eyre::eyre::eyre;
@@ -191,8 +191,7 @@ impl Model<OnionStore> {
Ok(key)
}
pub fn insert_key(&mut self, key: &TorSecretKey) -> Result<(), Error> {
self.insert(&key.onion_address(), &key)?;
Ok(())
self.insert(&key.onion_address(), &key)
}
pub fn get_key(&self, address: &OnionAddress) -> Result<TorSecretKey, Error> {
self.as_idx(address)
@@ -862,11 +861,11 @@ impl OnionService {
})))
}
pub fn proxy_all<Rcs: FromIterator<Arc<()>>>(
pub async fn proxy_all<Rcs: FromIterator<Arc<()>>>(
&self,
bindings: impl IntoIterator<Item = (u16, SocketAddr)>,
) -> Rcs {
self.0.bindings.mutate(|b| {
) -> Result<Rcs, Error> {
Ok(self.0.bindings.mutate(|b| {
bindings
.into_iter()
.map(|(port, target)| {
@@ -880,7 +879,7 @@ impl OnionService {
}
})
.collect()
})
}))
}
pub fn gc(&self) -> bool {

1034
core/startos/src/net/tor/ctor.rs Normal file
View File

File diff suppressed because it is too large Load Diff

10
core/startos/src/net/tor/mod.rs Normal file
View File

@@ -0,0 +1,10 @@
#[cfg(feature = "arti")]
mod arti;
#[cfg(not(feature = "arti"))]
mod ctor;
#[cfg(feature = "arti")]
pub use arti::{tor_api, OnionAddress, OnionStore, TorController, TorSecretKey};
#[cfg(not(feature = "arti"))]
pub use ctor::{tor_api, OnionAddress, OnionStore, TorController, TorSecretKey};

18
core/startos/src/registry/context.rs
View File

@@ -169,17 +169,23 @@ impl CallRemote<RegistryContext> for CliContext {
let url = if let Some(url) = self.registry_url.clone() {
url
} else if self.registry_hostname.is_some() {
format!(
let mut url: Url = format!(
"http://{}",
self.registry_listen.unwrap_or(DEFAULT_REGISTRY_LISTEN)
)
.parse()
.map_err(Error::from)?
.map_err(Error::from)?;
url.path_segments_mut()
.map_err(|_| Error::new(eyre!("cannot extend URL path"), ErrorKind::ParseUrl))?
.push("rpc")
.push("v0");
url
} else {
return Err(
Error::new(eyre!("`--registry` required"), ErrorKind::InvalidRequest).into(),
);
};
method = method.strip_prefix("registry.").unwrap_or(method);
let sig_context = self
.registry_hostname
@@ -203,7 +209,7 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
&self,
mut method: &str,
params: Value,
RegistryUrlParams { registry }: RegistryUrlParams,
RegistryUrlParams { mut registry }: RegistryUrlParams,
) -> Result<Value, RpcError> {
let mut headers = HeaderMap::new();
headers.insert(
@@ -211,6 +217,12 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
DeviceInfo::load(self).await?.to_header_value(),
);
registry
.path_segments_mut()
.map_err(|_| Error::new(eyre!("cannot extend URL path"), ErrorKind::ParseUrl))?
.push("rpc")
.push("v0");
method = method.strip_prefix("registry.").unwrap_or(method);
let sig_context = registry.host_str().map(InternedString::from);

90
core/startos/src/system.rs
View File

@@ -1,6 +1,5 @@
use std::collections::BTreeSet;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
@@ -10,8 +9,6 @@ use futures::{FutureExt, TryStreamExt};
use imbl::vector;
use imbl_value::InternedString;
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use rustls::RootCertStore;
use rustls_pki_types::CertificateDer;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use tokio::process::Command;
use tokio::sync::broadcast::Receiver;
@@ -498,7 +495,7 @@ pub struct MetricsFollowResponse {
#[command(rename_all = "kebab-case")]
pub struct MetricsFollowParams {
#[ts(skip)]
#[serde(rename = "__Auth_session")] // from Auth middleware
#[serde(rename = "__auth_session")] // from Auth middleware
session: Option<InternedString>,
}
@@ -1024,7 +1021,7 @@ pub struct TestSmtpParams {
#[arg(long)]
pub login: String,
#[arg(long)]
pub password: Option<String>,
pub password: String,
}
pub async fn test_smtp(
_: RpcContext,
@@ -1037,74 +1034,23 @@ pub async fn test_smtp(
password,
}: TestSmtpParams,
) -> Result<(), Error> {
#[cfg(feature = "mail-send")]
{
use mail_send::mail_builder::{self, MessageBuilder};
use mail_send::SmtpClientBuilder;
use rustls_pki_types::pem::PemObject;
use lettre::message::header::ContentType;
use lettre::transport::smtp::authentication::Credentials;
use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
let Some(pass_val) = password else {
return Err(Error::new(
eyre!("mail-send requires a password"),
ErrorKind::InvalidRequest,
));
};
let mut root_cert_store = RootCertStore::empty();
let pem = tokio::fs::read("/etc/ssl/certs/ca-certificates.crt").await?;
for cert in CertificateDer::pem_slice_iter(&pem) {
root_cert_store.add_parsable_certificates([cert.with_kind(ErrorKind::OpenSsl)?]);
}
let cfg = Arc::new(
rustls::ClientConfig::builder_with_provider(Arc::new(
rustls::crypto::ring::default_provider(),
))
.with_safe_default_protocol_versions()?
.with_root_certificates(root_cert_store)
.with_no_client_auth(),
);
let client = SmtpClientBuilder::new_with_tls_config(server, port, cfg)
.implicit_tls(false)
.credentials((login.split("@").next().unwrap().to_owned(), pass_val));
fn parse_address<'a>(addr: &'a str) -> mail_builder::headers::address::Address<'a> {
if addr.find("<").map_or(false, |start| {
addr.find(">").map_or(false, |end| start < end)
}) {
addr.split_once("<")
.map(|(name, addr)| (name.trim(), addr.strip_suffix(">").unwrap_or(addr)))
.unwrap()
.into()
} else {
addr.into()
}
}
let message = MessageBuilder::new()
.from(parse_address(&from))
.to(parse_address(&to))
.subject("StartOS Test Email")
.text_body("This is a test email sent from your StartOS Server");
client
.connect()
.await
.map_err(|e| {
Error::new(
eyre!("mail-send connection error: {:?}", e),
ErrorKind::Unknown,
)
})?
.send(message)
.await
.map_err(|e| Error::new(eyre!("mail-send send error: {:?}", e), ErrorKind::Unknown))?;
Ok(())
}
#[cfg(not(feature = "mail-send"))]
Err(Error::new(
eyre!("test-smtp requires mail-send feature to be enabled"),
ErrorKind::InvalidRequest,
))
AsyncSmtpTransport::<Tokio1Executor>::relay(&server)?
.credentials(Credentials::new(login, password))
.build()
.send(
Message::builder()
.from(from.parse()?)
.to(to.parse()?)
.subject("StartOS Test Email")
.header(ContentType::TEXT_PLAIN)
.body("This is a test email sent from your StartOS Server".to_owned())?,
)
.await?;
Ok(())
}
#[tokio::test]

24
core/startos/src/util/collections/mod.rs
View File

@@ -8,6 +8,30 @@ pub use eq_map::EqMap;
pub use eq_set::EqSet;
use imbl::OrdMap;
pub fn ordmap_retain<K: Ord + Clone, V: Clone, F: FnMut(&K, &mut V) -> bool>(
map: &mut OrdMap<K, V>,
mut f: F,
) {
let mut prev = None;
loop {
let next = if let Some(k) = prev.take() {
map.range((Bound::Excluded(k), Bound::Unbounded)).next()
} else {
map.get_min().map(|(k, v)| (k, v))
};
let Some((k, _)) = next else {
break;
};
let k = k.clone(); // hate that I have to do this but whatev
let v = map.get_mut(&k).unwrap();
if !f(&k, v) {
map.remove(&k);
}
prev = Some(k);
}
}
pub struct OrdMapIterMut<'a, K: 'a, V: 'a> {
map: *mut OrdMap<K, V>,
prev: Option<&'a K>,

15
core/startos/src/version/mod.rs
View File

@@ -51,8 +51,9 @@ mod v0_4_0_alpha_9;
mod v0_4_0_alpha_10;
mod v0_4_0_alpha_11;
mod v0_4_0_alpha_12;
pub type Current = v0_4_0_alpha_11::Version; // VERSION_BUMP
pub type Current = v0_4_0_alpha_12::Version; // VERSION_BUMP
impl Current {
#[instrument(skip(self, db))]
@@ -97,8 +98,8 @@ pub async fn post_init(
.as_server_info()
.as_post_init_migration_todos()
.de()?;
progress.start();
if !todos.is_empty() {
progress.set_total(todos.len() as u64);
while let Some((version, input)) = {
peek = ctx.db.peek().await;
peek.as_public()
@@ -121,7 +122,6 @@ pub async fn post_init(
})
.await
.result?;
progress += 1;
}
}
progress.complete();
@@ -166,7 +166,8 @@ enum Version {
V0_4_0_alpha_8(Wrapper<v0_4_0_alpha_8::Version>),
V0_4_0_alpha_9(Wrapper<v0_4_0_alpha_9::Version>),
V0_4_0_alpha_10(Wrapper<v0_4_0_alpha_10::Version>),
V0_4_0_alpha_11(Wrapper<v0_4_0_alpha_11::Version>), // VERSION_BUMP
V0_4_0_alpha_11(Wrapper<v0_4_0_alpha_11::Version>),
V0_4_0_alpha_12(Wrapper<v0_4_0_alpha_12::Version>), // VERSION_BUMP
Other(exver::Version),
}
@@ -220,7 +221,8 @@ impl Version {
Self::V0_4_0_alpha_8(v) => DynVersion(Box::new(v.0)),
Self::V0_4_0_alpha_9(v) => DynVersion(Box::new(v.0)),
Self::V0_4_0_alpha_10(v) => DynVersion(Box::new(v.0)),
Self::V0_4_0_alpha_11(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
Self::V0_4_0_alpha_11(v) => DynVersion(Box::new(v.0)),
Self::V0_4_0_alpha_12(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
Self::Other(v) => {
return Err(Error::new(
eyre!("unknown version {v}"),
@@ -266,7 +268,8 @@ impl Version {
Version::V0_4_0_alpha_8(Wrapper(x)) => x.semver(),
Version::V0_4_0_alpha_9(Wrapper(x)) => x.semver(),
Version::V0_4_0_alpha_10(Wrapper(x)) => x.semver(),
Version::V0_4_0_alpha_11(Wrapper(x)) => x.semver(), // VERSION_BUMP
Version::V0_4_0_alpha_11(Wrapper(x)) => x.semver(),
Version::V0_4_0_alpha_12(Wrapper(x)) => x.semver(), // VERSION_BUMP
Version::Other(x) => x.clone(),
}
}

255
core/startos/src/version/v0_3_6_alpha_0.rs
View File

@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::path::Path;
@@ -6,8 +6,8 @@ use chrono::{DateTime, Utc};
use const_format::formatcp;
use ed25519_dalek::SigningKey;
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::{InternedString, json};
use models::{PackageId, ReplayId};
use imbl_value::{json, InternedString};
use models::{HostId, Id, PackageId, ReplayId};
use openssl::pkey::PKey;
use openssl::x509::X509;
use sqlx::postgres::PgConnectOptions;
@@ -15,7 +15,7 @@ use sqlx::{PgPool, Row};
use tokio::process::Command;
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_3_5_2};
use super::{v0_3_5_2, VersionT};
use crate::account::AccountInfo;
use crate::auth::Sessions;
use crate::backup::target::cifs::CifsTargets;
@@ -24,15 +24,16 @@ use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::util::unmount;
use crate::hostname::Hostname;
use crate::net::forward::AvailablePorts;
use crate::net::host::Host;
use crate::net::keys::KeyStore;
use crate::net::tor::TorSecretKey;
use crate::net::tor::{OnionAddress, TorSecretKey};
use crate::notifications::Notifications;
use crate::prelude::*;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::ssh::{SshKeys, SshPubKey};
use crate::util::Invoke;
use crate::util::crypto::ed25519_expand_key;
use crate::util::serde::Pem;
use crate::util::Invoke;
use crate::{DATA_DIR, PACKAGE_DATA};
lazy_static::lazy_static! {
@@ -93,69 +94,6 @@ async fn init_postgres(datadir: impl AsRef<Path>) -> Result<PgPool, Error> {
crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
let pg_version_string = pg_version.to_string();
let pg_version_path = db_dir.join(&pg_version_string);
if exists
// maybe migrate
{
let incomplete_path = db_dir.join(format!("{pg_version}.migration.incomplete"));
if tokio::fs::metadata(&incomplete_path).await.is_ok() // previous migration was incomplete
&& tokio::fs::metadata(&pg_version_path).await.is_ok()
{
tokio::fs::remove_dir_all(&pg_version_path).await?;
}
if tokio::fs::metadata(&pg_version_path).await.is_err()
// need to migrate
{
let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
let conf_dir_tmp = {
let mut tmp = conf_dir.clone();
tmp.set_extension("tmp");
tmp
};
if tokio::fs::metadata(&conf_dir).await.is_ok() {
Command::new("mv")
.arg(&conf_dir)
.arg(&conf_dir_tmp)
.invoke(ErrorKind::Filesystem)
.await?;
}
let mut old_version = pg_version;
while old_version > 13
/* oldest pg version included in startos */
{
old_version -= 1;
let old_datadir = db_dir.join(old_version.to_string());
if tokio::fs::metadata(&old_datadir).await.is_ok() {
tokio::fs::File::create(&incomplete_path)
.await?
.sync_all()
.await?;
Command::new("pg_upgradecluster")
.arg(old_version.to_string())
.arg("main")
.invoke(crate::ErrorKind::Database)
.await?;
break;
}
}
if tokio::fs::metadata(&conf_dir).await.is_ok() {
if tokio::fs::metadata(&conf_dir).await.is_ok() {
tokio::fs::remove_dir_all(&conf_dir).await?;
}
Command::new("mv")
.arg(&conf_dir_tmp)
.arg(&conf_dir)
.invoke(ErrorKind::Filesystem)
.await?;
}
tokio::fs::remove_file(&incomplete_path).await?;
}
if tokio::fs::metadata(&incomplete_path).await.is_ok() {
unreachable!() // paranoia
}
}
Command::new("systemctl")
.arg("start")
.arg(format!("postgresql@{pg_version}-main.service"))
@@ -209,7 +147,12 @@ pub struct Version;
impl VersionT for Version {
type Previous = v0_3_5_2::Version;
type PreUpRes = (AccountInfo, SshKeys, CifsTargets);
type PreUpRes = (
AccountInfo,
SshKeys,
CifsTargets,
BTreeMap<PackageId, BTreeMap<HostId, TorSecretKey>>,
);
fn semver(self) -> exver::Version {
V0_3_6_alpha_0.clone()
}
@@ -224,9 +167,15 @@ impl VersionT for Version {
let cifs = previous_cifs(&pg).await?;
Ok((account, ssh_keys, cifs))
let tor_keys = previous_tor_keys(&pg).await?;
Ok((account, ssh_keys, cifs, tor_keys))
}
fn up(self, db: &mut Value, (account, ssh_keys, cifs): Self::PreUpRes) -> Result<Value, Error> {
fn up(
self,
db: &mut Value,
(account, ssh_keys, cifs, tor_keys): Self::PreUpRes,
) -> Result<Value, Error> {
let prev_package_data = db["package-data"].clone();
let wifi = json!({
@@ -259,12 +208,10 @@ impl VersionT for Version {
let tor_address: String = from_value(db["server-info"]["tor-address"].clone())?;
// Maybe we do this like the Public::init does
server_info["torAddress"] = json!(tor_address);
server_info["onionAddress"] = json!(
tor_address
.replace("https://", "")
.replace("http://", "")
.replace(".onion/", "")
);
server_info["onionAddress"] = json!(tor_address
.replace("https://", "")
.replace("http://", "")
.replace(".onion/", ""));
server_info["networkInterfaces"] = json!({});
server_info["statusInfo"] = status_info;
server_info["wifi"] = wifi;
@@ -288,9 +235,15 @@ impl VersionT for Version {
"ui": db["ui"],
});
let mut keystore = KeyStore::new(&account)?;
for key in tor_keys.values().flat_map(|v| v.values()) {
assert!(key.is_valid());
keystore.onion.insert(key.clone());
}
let private = {
let mut value = json!({});
value["keyStore"] = to_value(&KeyStore::new(&account)?)?;
value["keyStore"] = crate::dbg!(to_value(&keystore)?);
value["password"] = to_value(&account.password)?;
value["compatS9pkKey"] =
to_value(&crate::db::model::private::generate_developer_key())?;
@@ -373,6 +326,20 @@ impl VersionT for Version {
false
};
let onions = input[&*id]["installed"]["interface-addresses"]
.as_object()
.into_iter()
.flatten()
.filter_map(|(id, addrs)| {
addrs["tor-address"].as_str().map(|addr| {
Ok((
HostId::from(Id::try_from(id.clone())?),
addr.parse::<OnionAddress>()?,
))
})
})
.collect::<Result<BTreeMap<_, _>, Error>>()?;
if let Err(e) = async {
let package_s9pk = tokio::fs::File::open(path).await?;
let file = MultiCursorFile::open(&package_s9pk).await?;
@@ -390,19 +357,44 @@ impl VersionT for Version {
.await?
.await?;
if configured {
ctx.db
.mutate(|db| {
db.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&id)
.or_not_found(&id)?
let to_sync = ctx
.db
.mutate(|db| {
let mut to_sync = BTreeSet::new();
let package = db
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&id)
.or_not_found(&id)?;
if configured {
package
.as_tasks_mut()
.remove(&ReplayId::from("needs-config"))
})
.await
.result?;
.remove(&ReplayId::from("needs-config"))?;
}
for (id, onion) in onions {
package
.as_hosts_mut()
.upsert(&id, || Ok(Host::new()))?
.as_onions_mut()
.mutate(|o| {
o.clear();
o.insert(onion);
Ok(())
})?;
to_sync.insert(id);
}
Ok(to_sync)
})
.await
.result?;
if let Some(service) = &*ctx.services.get(&id).await {
for host_id in to_sync {
service.sync_host(host_id.clone()).await?;
}
}
Ok::<_, Error>(())
}
.await
@@ -470,14 +462,12 @@ async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<Accoun
.try_get::<Option<Vec<u8>>, _>("tor_key")
.with_ctx(|_| (ErrorKind::Database, "tor_key"))?
{
<[u8; 64]>::try_from(bytes)
.map_err(|e| {
Error::new(
eyre!("expected vec of len 64, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})
.with_ctx(|_| (ErrorKind::Database, "password.u8 64"))?
<[u8; 64]>::try_from(bytes).map_err(|e| {
Error::new(
eyre!("expected vec of len 64, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})?
} else {
ed25519_expand_key(
&<[u8; 32]>::try_from(
@@ -490,8 +480,7 @@ async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<Accoun
eyre!("expected vec of len 32, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})
.with_ctx(|_| (ErrorKind::Database, "password.u8 32"))?,
})?,
)
},
)?],
@@ -565,3 +554,69 @@ async fn previous_ssh_keys(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<SshKeys, E
};
Ok(ssh_keys)
}
#[tracing::instrument(skip_all)]
async fn previous_tor_keys(
pg: &sqlx::Pool<sqlx::Postgres>,
) -> Result<BTreeMap<PackageId, BTreeMap<HostId, TorSecretKey>>, Error> {
let mut res = BTreeMap::<PackageId, BTreeMap<HostId, TorSecretKey>>::new();
let net_key_query = sqlx::query(r#"SELECT * FROM network_keys"#)
.fetch_all(pg)
.await
.with_kind(ErrorKind::Database)?;
for row in net_key_query {
let package_id: PackageId = row
.try_get::<String, _>("package")
.with_ctx(|_| (ErrorKind::Database, "network_keys::package"))?
.parse()?;
let interface_id: HostId = row
.try_get::<String, _>("interface")
.with_ctx(|_| (ErrorKind::Database, "network_keys::interface"))?
.parse()?;
let key = TorSecretKey::from_bytes(ed25519_expand_key(
&<[u8; 32]>::try_from(
row.try_get::<Vec<u8>, _>("key")
.with_ctx(|_| (ErrorKind::Database, "network_keys::key"))?,
)
.map_err(|e| {
Error::new(
eyre!("expected vec of len 32, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})?,
))?;
res.entry(package_id).or_default().insert(interface_id, key);
}
let tor_key_query = sqlx::query(r#"SELECT * FROM tor"#)
.fetch_all(pg)
.await
.with_kind(ErrorKind::Database)?;
for row in tor_key_query {
let package_id: PackageId = row
.try_get::<String, _>("package")
.with_ctx(|_| (ErrorKind::Database, "tor::package"))?
.parse()?;
let interface_id: HostId = row
.try_get::<String, _>("interface")
.with_ctx(|_| (ErrorKind::Database, "tor::interface"))?
.parse()?;
let key = TorSecretKey::from_bytes(
<[u8; 64]>::try_from(
row.try_get::<Vec<u8>, _>("key")
.with_ctx(|_| (ErrorKind::Database, "tor::key"))?,
)
.map_err(|e| {
Error::new(
eyre!("expected vec of len 64, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})?,
)?;
res.entry(package_id).or_default().insert(interface_id, key);
}
Ok(res)
}

2
core/startos/src/version/v0_4_0_alpha_10.rs
View File

@@ -31,7 +31,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
let default_gateway = db["public"]["serverInfo"]["network"]["networkInterfaces"]
.as_object()

2
core/startos/src/version/v0_4_0_alpha_11.rs
View File

@@ -27,7 +27,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
Ok(Value::Null)
}

83
core/startos/src/version/v0_4_0_alpha_12.rs Normal file
View File

@@ -0,0 +1,83 @@
use std::collections::BTreeSet;
use exver::{PreReleaseSegment, VersionRange};
use imbl_value::InternedString;
use super::v0_3_5::V0_3_0_COMPAT;
use super::{v0_4_0_alpha_11, VersionT};
use crate::net::tor::TorSecretKey;
use crate::prelude::*;
lazy_static::lazy_static! {
static ref V0_4_0_alpha_12: exver::Version = exver::Version::new(
[0, 4, 0],
[PreReleaseSegment::String("alpha".into()), 12.into()]
);
}
#[derive(Clone, Copy, Debug, Default)]
pub struct Version;
impl VersionT for Version {
type Previous = v0_4_0_alpha_11::Version;
type PreUpRes = ();
async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
Ok(())
}
fn semver(self) -> exver::Version {
V0_4_0_alpha_12.clone()
}
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
let mut err = None;
let onion_store = db["private"]["keyStore"]["onion"]
.as_object_mut()
.or_not_found("private.keyStore.onion")?;
onion_store.retain(|o, v| match from_value::<TorSecretKey>(v.clone()) {
Ok(k) => k.is_valid() && &InternedString::from_display(&k.onion_address()) == o,
Err(e) => {
err = Some(e);
true
}
});
if let Some(e) = err {
return Err(e);
}
let allowed_addresses = onion_store.keys().cloned().collect::<BTreeSet<_>>();
let fix_host = |host: &mut Value| {
Ok::<_, Error>(
host["onions"]
.as_array_mut()
.or_not_found("host.onions")?
.retain(|addr| {
addr.as_str()
.map(|s| allowed_addresses.contains(s))
.unwrap_or(false)
}),
)
};
for (_, pde) in db["public"]["packageData"]
.as_object_mut()
.or_not_found("public.packageData")?
.iter_mut()
{
for (_, host) in pde["hosts"]
.as_object_mut()
.or_not_found("public.packageData[].hosts")?
.iter_mut()
{
fix_host(host)?;
}
}
fix_host(&mut db["public"]["serverInfo"]["network"]["host"])?;
Ok(Value::Null)
}
fn down(self, _db: &mut Value) -> Result<(), Error> {
Ok(())
}
}

4
core/startos/src/version/v0_4_0_alpha_4.rs
View File

@@ -1,7 +1,7 @@
use exver::{PreReleaseSegment, VersionRange};
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_3};
use super::{v0_4_0_alpha_3, VersionT};
use crate::context::RpcContext;
use crate::prelude::*;
use crate::util::io::create_file_mod;
@@ -29,7 +29,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
db["public"]["serverInfo"]
.as_object_mut()

4
core/startos/src/version/v0_4_0_alpha_5.rs
View File

@@ -1,7 +1,7 @@
use exver::{PreReleaseSegment, VersionRange};
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_4};
use super::{v0_4_0_alpha_4, VersionT};
use crate::prelude::*;
lazy_static::lazy_static! {
@@ -27,7 +27,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
Ok(Value::Null)
}

4
core/startos/src/version/v0_4_0_alpha_6.rs
View File

@@ -1,7 +1,7 @@
use exver::{PreReleaseSegment, VersionRange};
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_5};
use super::{v0_4_0_alpha_5, VersionT};
use crate::prelude::*;
lazy_static::lazy_static! {
@@ -27,7 +27,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
let ui = db["public"]["ui"]
.as_object_mut()

4
core/startos/src/version/v0_4_0_alpha_7.rs
View File

@@ -1,7 +1,7 @@
use exver::{PreReleaseSegment, VersionRange};
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_6};
use super::{v0_4_0_alpha_6, VersionT};
use crate::prelude::*;
lazy_static::lazy_static! {
@@ -27,7 +27,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
Ok(Value::Null)
}

4
core/startos/src/version/v0_4_0_alpha_8.rs
View File

@@ -1,7 +1,7 @@
use exver::{PreReleaseSegment, VersionRange};
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_7};
use super::{v0_4_0_alpha_7, VersionT};
use crate::prelude::*;
lazy_static::lazy_static! {
@@ -27,7 +27,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
Ok(Value::Null)
}

6
core/startos/src/version/v0_4_0_alpha_9.rs
View File

@@ -7,13 +7,13 @@ use imbl_value::{InOMap, InternedString};
use models::PackageId;
use super::v0_3_5::V0_3_0_COMPAT;
use super::{VersionT, v0_4_0_alpha_8};
use crate::DATA_DIR;
use super::{v0_4_0_alpha_8, VersionT};
use crate::context::RpcContext;
use crate::install::PKG_ARCHIVE_DIR;
use crate::prelude::*;
use crate::util::io::write_file_atomic;
use crate::volume::PKG_VOLUME_DIR;
use crate::DATA_DIR;
lazy_static::lazy_static! {
static ref V0_4_0_alpha_9: exver::Version = exver::Version::new(
@@ -38,7 +38,7 @@ impl VersionT for Version {
fn compat(self) -> &'static VersionRange {
&V0_3_0_COMPAT
}
#[instrument]
#[instrument(skip_all)]
fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
let mut res = InOMap::new();
let todos = db