Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-04-04 14:29:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/new registry (#2612)

Browse Source 
* wip

* overhaul boot process

* wip: new registry

* wip

* wip

* wip

* wip

* wip

* wip

* os registry complete

* ui fixes

* fixes

* fixes

* more fixes

* fix merkle archive
This commit is contained in:
Aiden McClelland
2024-05-06 10:20:44 -06:00
committed by GitHub
parent 8a38666105
commit 9b14d714ca
167 changed files with 6297 additions and 3190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/startos/src/net/dhcp.rs
View File

@@ -3,7 +3,7 @@ use std::net::IpAddr;
use clap::Parser;
use futures::TryStreamExt;
use rpc_toolkit::{from_fn_async, HandlerExt, ParentHandler};
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use ts_rs::TS;
@@ -53,12 +53,12 @@ pub async fn init_ips() -> Result<BTreeMap<String, IpInfo>, Error> {
}
// #[command(subcommands(update))]
pub fn dhcp() -> ParentHandler {
pub fn dhcp<C: Context>() -> ParentHandler<C> {
ParentHandler::new().subcommand(
"update",
from_fn_async::<_, _, (), Error, (RpcContext, UpdateParams)>(update)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser, TS)]

8
core/startos/src/net/mod.rs
View File

@@ -1,4 +1,4 @@
use rpc_toolkit::ParentHandler;
use rpc_toolkit::{Context, ParentHandler};
pub mod dhcp;
pub mod dns;
@@ -18,8 +18,8 @@ pub mod wifi;
pub const PACKAGE_CERT_PATH: &str = "/var/lib/embassy/ssl";
pub fn net() -> ParentHandler {
pub fn net<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand("tor", tor::tor())
.subcommand("dhcp", dhcp::dhcp())
.subcommand("tor", tor::tor::<C>())
.subcommand("dhcp", dhcp::dhcp::<C>())
}

28
core/startos/src/net/net_controller.rs
View File

@@ -6,12 +6,10 @@ use color_eyre::eyre::eyre;
use imbl::OrdMap;
use lazy_format::lazy_format;
use models::{HostId, OptionExt, PackageId};
use patch_db::PatchDb;
use tokio::sync::Mutex;
use torut::onion::{OnionAddressV3, TorSecretKeyV3};
use tracing::instrument;
use crate::db::prelude::PatchDbExt;
use crate::db::model::Database;
use crate::error::ErrorCollection;
use crate::hostname::Hostname;
use crate::net::dns::DnsController;
@@ -21,11 +19,12 @@ use crate::net::host::binding::{AddSslOptions, BindOptions};
use crate::net::host::{Host, HostKind};
use crate::net::tor::TorController;
use crate::net::vhost::{AlpnInfo, VHostController};
use crate::prelude::*;
use crate::util::serde::MaybeUtf8String;
use crate::{Error, HOST_IP};
use crate::HOST_IP;
pub struct NetController {
db: PatchDb,
db: TypedPatchDb<Database>,
pub(super) tor: TorController,
pub(super) vhost: VHostController,
pub(super) dns: DnsController,
@@ -36,7 +35,7 @@ pub struct NetController {
impl NetController {
#[instrument(skip_all)]
pub async fn init(
db: PatchDb,
db: TypedPatchDb<Database>,
tor_control: SocketAddr,
tor_socks: SocketAddr,
dns_bind: &[SocketAddr],
@@ -394,14 +393,23 @@ impl NetService {
pub fn get_ext_port(&self, host_id: HostId, internal_port: u16) -> Result<u16, Error> {
let host_id_binds = self.binds.get_key_value(&host_id);
match host_id_binds {
Some((id, binds)) => {
Some((_, binds)) => {
if let Some(ext_port_info) = binds.lan.get(&internal_port) {
Ok(ext_port_info.0)
} else {
Err(Error::new(eyre!("Internal Port {} not found in NetService binds", internal_port), crate::ErrorKind::NotFound))
Err(Error::new(
eyre!(
"Internal Port {} not found in NetService binds",
internal_port
),
crate::ErrorKind::NotFound,
))
}
},
None => Err(Error::new(eyre!("HostID {} not found in NetService binds", host_id), crate::ErrorKind::NotFound))
}
None => Err(Error::new(
eyre!("HostID {} not found in NetService binds", host_id),
crate::ErrorKind::NotFound,
)),
}
}
}

1
core/startos/src/net/ssl.rs
View File

@@ -17,7 +17,6 @@ use openssl::x509::{X509Builder, X509Extension, X509NameBuilder, X509};
use openssl::*;
use patch_db::HasModel;
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use tracing::instrument;
use crate::account::AccountInfo;

29
core/startos/src/net/static_server.rs
View File

@@ -24,18 +24,19 @@ use tokio::io::BufReader;
use tokio_util::io::ReaderStream;
use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext};
use crate::core::rpc_continuations::RequestGuid;
use crate::db::subscribe;
use crate::hostname::Hostname;
use crate::middleware::auth::{Auth, HasValidSession};
use crate::middleware::cors::Cors;
use crate::middleware::db::SyncDb;
use crate::middleware::diagnostic::DiagnosticMode;
use crate::rpc_continuations::RequestGuid;
use crate::{diagnostic_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt};
const NOT_FOUND: &[u8] = b"Not Found";
const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
const NOT_AUTHORIZED: &[u8] = b"Not Authorized";
const INTERNAL_SERVER_ERROR: &[u8] = b"Internal Server Error";
#[cfg(all(feature = "daemon", not(feature = "test")))]
const EMBEDDED_UIS: Dir<'_> =
@@ -112,7 +113,7 @@ pub fn main_ui_server_router(ctx: RpcContext) -> Router {
.route("/rpc/*path", {
let ctx = ctx.clone();
post(
Server::new(move || ready(Ok(ctx.clone())), main_api())
Server::new(move || ready(Ok(ctx.clone())), main_api::<RpcContext>())
.middleware(Cors::new())
.middleware(Auth::new())
.middleware(SyncDb::new()),
@@ -140,7 +141,7 @@ pub fn main_ui_server_router(ctx: RpcContext) -> Router {
tracing::debug!("No Guid Path");
bad_request()
}
Some(guid) => match ctx.get_ws_continuation_handler(&guid).await {
Some(guid) => match ctx.rpc_continuations.get_ws_handler(&guid).await {
Some(cont) => ws.on_upgrade(cont),
_ => not_found(),
},
@@ -163,7 +164,7 @@ pub fn main_ui_server_router(ctx: RpcContext) -> Router {
tracing::debug!("No Guid Path");
bad_request()
}
Some(guid) => match ctx.get_rest_continuation_handler(&guid).await {
Some(guid) => match ctx.rpc_continuations.get_rest_handler(&guid).await {
None => not_found(),
Some(cont) => cont(request).await.unwrap_or_else(server_error),
},
@@ -216,7 +217,7 @@ async fn if_authorized<
) -> Result<Response, Error> {
if let Err(e) = HasValidSession::from_header(parts.headers.get(http::header::COOKIE), ctx).await
{
un_authorized(e, parts.uri.path())
Ok(unauthorized(e, parts.uri.path()))
} else {
f().await
}
@@ -305,17 +306,17 @@ async fn main_start_os_ui(req: Request, ctx: RpcContext) -> Result<Response, Err
}
}
fn un_authorized(err: Error, path: &str) -> Result<Response, Error> {
pub fn unauthorized(err: Error, path: &str) -> Response {
tracing::warn!("unauthorized for {} @{:?}", err, path);
tracing::debug!("{:?}", err);
Ok(Response::builder()
Response::builder()
.status(StatusCode::UNAUTHORIZED)
.body(NOT_AUTHORIZED.into())
.unwrap())
.unwrap()
}
/// HTTP status code 404
fn not_found() -> Response {
pub fn not_found() -> Response {
Response::builder()
.status(StatusCode::NOT_FOUND)
.body(NOT_FOUND.into())
@@ -323,21 +324,23 @@ fn not_found() -> Response {
}
/// HTTP status code 405
fn method_not_allowed() -> Response {
pub fn method_not_allowed() -> Response {
Response::builder()
.status(StatusCode::METHOD_NOT_ALLOWED)
.body(METHOD_NOT_ALLOWED.into())
.unwrap()
}
fn server_error(err: Error) -> Response {
pub fn server_error(err: Error) -> Response {
tracing::error!("internal server error: {}", err);
tracing::debug!("{:?}", err);
Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(err.to_string().into())
.body(INTERNAL_SERVER_ERROR.into())
.unwrap()
}
fn bad_request() -> Response {
pub fn bad_request() -> Response {
Response::builder()
.status(StatusCode::BAD_REQUEST)
.body(Body::empty())

107
core/startos/src/net/tor.rs
View File

@@ -12,8 +12,7 @@ use helpers::NonDetachingJoinHandle;
use itertools::Itertools;
use lazy_static::lazy_static;
use regex::Regex;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, from_fn_async, AnyContext, Empty, HandlerExt, ParentHandler};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::net::TcpStream;
use tokio::process::Command;
@@ -25,10 +24,7 @@ use tracing::instrument;
use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::logs::{
cli_logs_generic_follow, cli_logs_generic_nofollow, fetch_logs, follow_logs, journalctl,
LogFollowResponse, LogResponse, LogSource,
};
use crate::logs::{journalctl, LogSource, LogsParams};
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
use crate::util::Invoke;
@@ -86,23 +82,27 @@ lazy_static! {
static ref PROGRESS_REGEX: Regex = Regex::new("PROGRESS=([0-9]+)").unwrap();
}
pub fn tor() -> ParentHandler {
pub fn tor<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
"list-services",
from_fn_async(list_services)
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|handle, result| {
.with_custom_display_fn(|handle, result| {
Ok(display_services(handle.params, result))
})
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
.subcommand("logs", logs())
.subcommand(
"logs",
from_fn_async(crate::logs::cli_logs::<RpcContext, Empty>).no_display(),
)
.subcommand(
"reset",
from_fn_async(reset)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser, TS)]
@@ -143,89 +143,10 @@ pub async fn list_services(ctx: RpcContext, _: Empty) -> Result<Vec<OnionAddress
ctx.net_controller.tor.list_services().await
}
#[derive(Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
pub struct LogsParams {
#[arg(short = 'l', long = "limit")]
#[ts(type = "number | null")]
limit: Option<usize>,
#[arg(short = 'c', long = "cursor")]
cursor: Option<String>,
#[arg(short = 'B', long = "before")]
#[serde(default)]
before: bool,
#[arg(short = 'f', long = "follow")]
#[serde(default)]
follow: bool,
}
pub fn logs() -> ParentHandler<LogsParams> {
ParentHandler::new()
.root_handler(
from_fn_async(cli_logs)
.no_display()
.with_inherited(|params, _| params),
)
.root_handler(
from_fn_async(logs_nofollow)
.with_inherited(|params, _| params)
.no_cli(),
)
.subcommand(
"follow",
from_fn_async(logs_follow)
.with_inherited(|params, _| params)
.no_cli(),
)
}
pub async fn cli_logs(
ctx: CliContext,
_: Empty,
LogsParams {
limit,
cursor,
before,
follow,
}: LogsParams,
) -> Result<(), RpcError> {
if follow {
if cursor.is_some() {
return Err(RpcError::from(Error::new(
eyre!("The argument '--cursor <cursor>' cannot be used with '--follow'"),
crate::ErrorKind::InvalidRequest,
)));
}
if before {
return Err(RpcError::from(Error::new(
eyre!("The argument '--before' cannot be used with '--follow'"),
crate::ErrorKind::InvalidRequest,
)));
}
cli_logs_generic_follow(ctx, "net.tor.logs.follow", None, limit).await
} else {
cli_logs_generic_nofollow(ctx, "net.tor.logs", None, limit, cursor, before).await
}
}
pub async fn logs_nofollow(
_: AnyContext,
_: Empty,
LogsParams {
limit,
cursor,
before,
..
}: LogsParams,
) -> Result<LogResponse, Error> {
fetch_logs(LogSource::Unit(SYSTEMD_UNIT), limit, cursor, before).await
}
pub async fn logs_follow(
ctx: RpcContext,
_: Empty,
LogsParams { limit, .. }: LogsParams,
) -> Result<LogFollowResponse, Error> {
follow_logs(ctx, LogSource::Unit(SYSTEMD_UNIT), limit).await
pub fn logs() -> ParentHandler<RpcContext, LogsParams> {
crate::logs::logs::<RpcContext, Empty>(|_: &RpcContext, _| async {
Ok(LogSource::Unit(SYSTEMD_UNIT))
})
}
fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), ConnError>> {

7
core/startos/src/net/vhost.rs
View File

@@ -19,6 +19,7 @@ use tokio_rustls::{LazyConfigAcceptor, TlsConnector};
use tracing::instrument;
use ts_rs::TS;
use crate::db::model::Database;
use crate::prelude::*;
use crate::util::io::{BackTrackingReader, TimeoutStream};
use crate::util::serde::MaybeUtf8String;
@@ -26,11 +27,11 @@ use crate::util::serde::MaybeUtf8String;
// not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
pub struct VHostController {
db: PatchDb,
db: TypedPatchDb<Database>,
servers: Mutex<BTreeMap<u16, VHostServer>>,
}
impl VHostController {
pub fn new(db: PatchDb) -> Self {
pub fn new(db: TypedPatchDb<Database>) -> Self {
Self {
db,
servers: Mutex::new(BTreeMap::new()),
@@ -100,7 +101,7 @@ struct VHostServer {
}
impl VHostServer {
#[instrument(skip_all)]
async fn new(port: u16, db: PatchDb) -> Result<Self, Error> {
async fn new(port: u16, db: TypedPatchDb<Database>) -> Result<Self, Error> {
// check if port allowed
let listener = TcpListener::bind(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port))
.await

58
core/startos/src/net/wifi.rs
View File

@@ -8,7 +8,7 @@ use clap::Parser;
use isocountry::CountryCode;
use lazy_static::lazy_static;
use regex::Regex;
use rpc_toolkit::{command, from_fn_async, AnyContext, Empty, HandlerExt, ParentHandler};
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::process::Command;
use tokio::sync::RwLock;
@@ -17,6 +17,7 @@ use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::db::model::public::WifiInfo;
use crate::db::model::Database;
use crate::net::utils::find_wifi_iface;
use crate::prelude::*;
use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
@@ -36,57 +37,55 @@ pub fn wifi_manager(ctx: &RpcContext) -> Result<&WifiManager, Error> {
}
}
pub fn wifi() -> ParentHandler {
pub fn wifi<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
"add",
from_fn_async(add)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
.subcommand(
"connect",
from_fn_async(connect)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
.subcommand(
"delete",
from_fn_async(delete)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
.subcommand(
"get",
from_fn_async(get)
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|handle, result| {
.with_custom_display_fn(|handle, result| {
Ok(display_wifi_info(handle.params, result))
})
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
.subcommand("country", country())
.subcommand("available", available())
.subcommand("country", country::<C>())
.subcommand("available", available::<C>())
}
pub fn available() -> ParentHandler {
pub fn available<C: Context>() -> ParentHandler<C> {
ParentHandler::new().subcommand(
"get",
from_fn_async(get_available)
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|handle, result| {
Ok(display_wifi_list(handle.params, result))
})
.with_remote_cli::<CliContext>(),
.with_custom_display_fn(|handle, result| Ok(display_wifi_list(handle.params, result)))
.with_call_remote::<CliContext>(),
)
}
pub fn country() -> ParentHandler {
pub fn country<C: Context>() -> ParentHandler<C> {
ParentHandler::new().subcommand(
"set",
from_fn_async(set_country)
.no_display()
.with_remote_cli::<CliContext>(),
.with_call_remote::<CliContext>(),
)
}
@@ -113,7 +112,7 @@ pub async fn add(ctx: RpcContext, AddParams { ssid, password }: AddParams) -> Re
));
}
async fn add_procedure(
db: PatchDb,
db: TypedPatchDb<Database>,
wifi_manager: WifiManager,
ssid: &Ssid,
password: &Psk,
@@ -170,7 +169,7 @@ pub async fn connect(ctx: RpcContext, SsidParams { ssid }: SsidParams) -> Result
));
}
async fn connect_procedure(
db: PatchDb,
db: TypedPatchDb<Database>,
wifi_manager: WifiManager,
ssid: &Ssid,
) -> Result<(), Error> {
@@ -718,7 +717,7 @@ impl WpaCli {
Ok(())
}
pub async fn save_config(&mut self, db: PatchDb) -> Result<(), Error> {
pub async fn save_config(&mut self, db: TypedPatchDb<Database>) -> Result<(), Error> {
let new_country = self.get_country_low().await?;
db.mutate(|d| {
d.as_public_mut()
@@ -758,7 +757,11 @@ impl WpaCli {
.collect())
}
#[instrument(skip_all)]
pub async fn select_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
pub async fn select_network(
&mut self,
db: TypedPatchDb<Database>,
ssid: &Ssid,
) -> Result<bool, Error> {
let m_id = self.check_active_network(ssid).await?;
match m_id {
None => Err(Error::new(
@@ -810,7 +813,11 @@ impl WpaCli {
}
}
#[instrument(skip_all)]
pub async fn remove_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
pub async fn remove_network(
&mut self,
db: TypedPatchDb<Database>,
ssid: &Ssid,
) -> Result<bool, Error> {
let found_networks = self.find_networks(ssid).await?;
if found_networks.is_empty() {
return Ok(true);
@@ -824,7 +831,7 @@ impl WpaCli {
#[instrument(skip_all)]
pub async fn set_add_network(
&mut self,
db: PatchDb,
db: TypedPatchDb<Database>,
ssid: &Ssid,
psk: &Psk,
) -> Result<(), Error> {
@@ -833,7 +840,12 @@ impl WpaCli {
Ok(())
}
#[instrument(skip_all)]
pub async fn add_network(&mut self, db: PatchDb, ssid: &Ssid, psk: &Psk) -> Result<(), Error> {
pub async fn add_network(
&mut self,
db: TypedPatchDb<Database>,
ssid: &Ssid,
psk: &Psk,
) -> Result<(), Error> {
self.add_network_low(ssid, psk).await?;
self.save_config(db).await?;
Ok(())