Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/fe new registry (#2647)

Browse Source 
* bugfixes

* update fe types

* implement new registry types in marketplace and ui

* fix marketplace types to have default params

* add alt implementation toggle

* merge cleanup

* more cleanup and notes

* fix build

* cleanup sync with next/minor

* add exver JS parser

* parse ValidExVer to string

* update types to interface

* add VersionRange and comparative functions

* Parse ExtendedVersion from string

* add conjunction, disjunction, and inversion logic

* consider flavor in satisfiedBy fn

* consider prerelease for ordering

* add compare fn for sorting

* rename fns for consistency

* refactoring

* update compare fn to return null if flavors don't match

* begin simplifying dependencies

* under construction

* wip

* add dependency metadata to CurrentDependencyInfo

* ditch inheritance for recursive VersionRange constructor. Recursive 'satisfiedBy' fn wip

* preprocess manifest

* misc fixes

* use sdk version as osVersion in manifest

* chore: Change the type to just validate and not generate all solutions.

* add publishedAt

* fix pegjs exports

* integrate exver into sdk

* misc fixes

* complete satisfiedBy fn

* refactor - use greaterThanOrEqual and lessThanOrEqual fns

* fix tests

* update dependency details

* update types

* remove interim types

* rename alt implementation to flavor

* cleanup os update

* format exver.ts

* add s9pk parsing endpoints

* fix build

* update to exver

* exver and bug fixes

* update static endpoints + cleanup

* cleanup

* update static proxy verification

* make mocks more robust; fix dep icon fallback; cleanup

* refactor alert versions and update fixtures

* registry bugfixes

* misc fixes

* cleanup unused

* convert patchdb ui seed to camelCase

* update otherVersions type

* change otherVersions: null to 'none'

* refactor and complete feature

* improve static endpoints

* fix install params

* mask systemd-networkd-wait-online

* fix static file fetching

* include non-matching versions in otherVersions

* convert release notes to modal and clean up displayExver

* alert for no other versions

* Fix ack-instructions casing

* fix indeterminate loader on service install

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Shadowy Super Coder <musashidisciple@proton.me>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>
This commit is contained in:
Lucy
2024-07-22 20:48:12 -04:00
committed by GitHub
parent 0fbb18b315
commit a535fc17c3
196 changed files with 7002 additions and 2162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/startos/src/backup/target/mod.rs
View File

@@ -8,6 +8,7 @@ use color_eyre::eyre::eyre;
use digest::generic_array::GenericArray;
use digest::OutputSizeUser;
use exver::Version;
use imbl_value::InternedString;
use models::PackageId;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
@@ -213,7 +214,7 @@ pub struct BackupInfo {
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PackageBackupInfo {
pub title: String,
pub title: InternedString,
pub version: VersionString,
pub os_version: Version,
pub timestamp: DateTime<Utc>,

85
core/startos/src/bins/start_init.rs
View File

@@ -141,6 +141,7 @@ async fn setup_or_init(
} else {
let init_ctx = InitContext::init(config).await?;
let handle = init_ctx.progress.clone();
let err_channel = init_ctx.error.clone();
let mut disk_phase = handle.add_phase("Opening data drive".into(), Some(10));
let init_phases = InitPhases::new(&handle);
@@ -148,47 +149,55 @@ async fn setup_or_init(
server.serve_init(init_ctx);
disk_phase.start();
let guid_string = tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
async {
disk_phase.start();
let guid_string = tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
.await?;
let disk_guid = Arc::new(String::from(guid_string.trim()));
let requires_reboot = crate::disk::main::import(
&**disk_guid,
config.datadir(),
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
RepairStrategy::Aggressive
} else {
RepairStrategy::Preen
},
if disk_guid.ends_with("_UNENC") {
None
} else {
Some(DEFAULT_PASSWORD)
},
)
.await?;
let disk_guid = Arc::new(String::from(guid_string.trim()));
let requires_reboot = crate::disk::main::import(
&**disk_guid,
config.datadir(),
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
RepairStrategy::Aggressive
} else {
RepairStrategy::Preen
},
if disk_guid.ends_with("_UNENC") {
None
} else {
Some(DEFAULT_PASSWORD)
},
)
.await?;
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
tokio::fs::remove_file(REPAIR_DISK_PATH)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
tokio::fs::remove_file(REPAIR_DISK_PATH)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
}
disk_phase.complete();
tracing::info!("Loaded Disk");
if requires_reboot.0 {
let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1));
reboot_phase.start();
return Ok(Err(Shutdown {
export_args: Some((disk_guid, config.datadir().to_owned())),
restart: true,
}));
}
let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?;
let rpc_ctx =
RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?;
Ok::<_, Error>(Ok((rpc_ctx, handle)))
}
disk_phase.complete();
tracing::info!("Loaded Disk");
if requires_reboot.0 {
let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1));
reboot_phase.start();
return Ok(Err(Shutdown {
export_args: Some((disk_guid, config.datadir().to_owned())),
restart: true,
}));
}
let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?;
let rpc_ctx = RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?;
Ok(Ok((rpc_ctx, handle)))
.await
.map_err(|e| {
err_channel.send_replace(Some(e.clone_output()));
e
})
}
}

3
core/startos/src/context/init.rs
View File

@@ -3,6 +3,7 @@ use std::sync::Arc;
use rpc_toolkit::Context;
use tokio::sync::broadcast::Sender;
use tokio::sync::watch;
use tracing::instrument;
use crate::context::config::ServerConfig;
@@ -12,6 +13,7 @@ use crate::Error;
pub struct InitContextSeed {
pub config: ServerConfig,
pub error: watch::Sender<Option<Error>>,
pub progress: FullProgressTracker,
pub shutdown: Sender<()>,
pub rpc_continuations: RpcContinuations,
@@ -25,6 +27,7 @@ impl InitContext {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
Ok(Self(Arc::new(InitContextSeed {
config: cfg.clone(),
error: watch::channel(None).0,
progress: FullProgressTracker::new(),
shutdown,
rpc_continuations: RpcContinuations::new(),

9
core/startos/src/db/model/package.rs
View File

@@ -372,14 +372,13 @@ impl Map for CurrentDependencies {
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
pub struct CurrentDependencyInfo {
#[ts(type = "string | null")]
pub title: Option<InternedString>,
pub icon: Option<DataUrl<'static>>,
#[serde(flatten)]
pub kind: CurrentDependencyKind,
pub title: String,
pub icon: DataUrl<'static>,
#[ts(type = "string")]
pub registry_url: Url,
#[ts(type = "string")]
pub version_spec: VersionRange,
pub version_range: VersionRange,
pub config_satisfied: bool,
}

13
core/startos/src/dependencies.rs
View File

@@ -2,18 +2,21 @@ use std::collections::BTreeMap;
use std::time::Duration;
use clap::Parser;
use imbl_value::InternedString;
use models::PackageId;
use patch_db::json_patch::merge;
use rpc_toolkit::{from_fn_async, Context, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use ts_rs::TS;
use url::Url;
use crate::config::{Config, ConfigSpec, ConfigureContext};
use crate::context::RpcContext;
use crate::db::model::package::CurrentDependencies;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::PathOrUrl;
use crate::Error;
pub fn dependency<C: Context>() -> ParentHandler<C> {
@@ -42,6 +45,16 @@ impl Map for Dependencies {
pub struct DepInfo {
pub description: Option<String>,
pub optional: bool,
pub s9pk: Option<PathOrUrl>,
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct DependencyMetadata {
#[ts(type = "string")]
pub title: InternedString,
}
#[derive(Deserialize, Serialize, Parser, TS)]

55
core/startos/src/init.rs
View File

@@ -554,33 +554,54 @@ pub struct InitProgressRes {
pub async fn init_progress(ctx: InitContext) -> Result<InitProgressRes, Error> {
let progress_tracker = ctx.progress.clone();
let progress = progress_tracker.snapshot();
let mut error = ctx.error.subscribe();
let guid = Guid::new();
ctx.rpc_continuations
.add(
guid.clone(),
RpcContinuation::ws(
|mut ws| async move {
if let Err(e) = async {
let mut stream = progress_tracker.stream(Some(Duration::from_millis(100)));
while let Some(progress) = stream.next().await {
ws.send(ws::Message::Text(
serde_json::to_string(&progress)
.with_kind(ErrorKind::Serialization)?,
))
.await
.with_kind(ErrorKind::Network)?;
if progress.overall.is_complete() {
break;
let res = tokio::try_join!(
async {
let mut stream =
progress_tracker.stream(Some(Duration::from_millis(100)));
while let Some(progress) = stream.next().await {
ws.send(ws::Message::Text(
serde_json::to_string(&progress)
.with_kind(ErrorKind::Serialization)?,
))
.await
.with_kind(ErrorKind::Network)?;
if progress.overall.is_complete() {
break;
}
}
Ok::<_, Error>(())
},
async {
if let Some(e) = error
.wait_for(|e| e.is_some())
.await
.ok()
.and_then(|e| e.as_ref().map(|e| e.clone_output()))
{
Err::<(), _>(e)
} else {
Ok(())
}
}
);
ws.normal_close("complete").await?;
Ok::<_, Error>(())
}
.await
if let Err(e) = ws
.close_result(res.map(|_| "complete").map_err(|e| {
tracing::error!("error in init progress websocket: {e}");
tracing::debug!("{e:?}");
e
}))
.await
{
tracing::error!("error in init progress websocket: {e}");
tracing::error!("error closing init progress websocket: {e}");
tracing::debug!("{e:?}");
}
},

1
core/startos/src/install/mod.rs
View File

@@ -111,6 +111,7 @@ impl std::fmt::Display for MinMax {
#[derive(Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct InstallParams {
#[ts(type = "string")]
registry: Url,

1
core/startos/src/middleware/db.rs
View File

@@ -7,7 +7,6 @@ use serde::Deserialize;
use crate::context::RpcContext;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Metadata {
#[serde(default)]
sync_db: bool,

459
core/startos/src/net/static_server.rs
View File

@@ -1,5 +1,8 @@
use std::cmp::min;
use std::future::Future;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::UNIX_EPOCH;
use async_compression::tokio::bufread::GzipEncoder;
@@ -8,36 +11,51 @@ use axum::extract::{self as x, Request};
use axum::response::Response;
use axum::routing::{any, get, post};
use axum::Router;
use base64::display::Base64Display;
use digest::Digest;
use futures::future::ready;
use http::header::ACCEPT_ENCODING;
use http::header::{
ACCEPT_ENCODING, ACCEPT_RANGES, CACHE_CONTROL, CONNECTION, CONTENT_ENCODING, CONTENT_LENGTH,
CONTENT_RANGE, CONTENT_TYPE, ETAG, RANGE,
};
use http::request::Parts as RequestParts;
use http::{Method, StatusCode};
use http::{HeaderValue, Method, StatusCode};
use imbl_value::InternedString;
use include_dir::Dir;
use new_mime_guess::MimeGuess;
use openssl::hash::MessageDigest;
use openssl::x509::X509;
use rpc_toolkit::{Context, HttpServer, Server};
use tokio::io::BufReader;
use sqlx::query;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader};
use tokio_util::io::ReaderStream;
use url::Url;
use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext};
use crate::hostname::Hostname;
use crate::install::PKG_ARCHIVE_DIR;
use crate::middleware::auth::{Auth, HasValidSession};
use crate::middleware::cors::Cors;
use crate::middleware::db::SyncDb;
use crate::prelude::*;
use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
use crate::rpc_continuations::{Guid, RpcContinuations};
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::util::io::open_file;
use crate::{
diagnostic_api, init_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt,
};
use crate::util::net::SyncBody;
use crate::util::serde::BASE64;
use crate::{diagnostic_api, init_api, install_api, main_api, setup_api};
const NOT_FOUND: &[u8] = b"Not Found";
const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed";
const NOT_AUTHORIZED: &[u8] = b"Not Authorized";
const INTERNAL_SERVER_ERROR: &[u8] = b"Internal Server Error";
const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "user-agent"];
#[cfg(all(feature = "daemon", not(feature = "test")))]
const EMBEDDED_UIS: Dir<'_> =
include_dir::include_dir!("$CARGO_MANIFEST_DIR/../../web/dist/static");
@@ -97,7 +115,7 @@ pub fn rpc_router<C: Context + Clone + AsRef<RpcContinuations>>(
fn serve_ui(req: Request, ui_mode: UiMode) -> Result<Response, Error> {
let (request_parts, _body) = req.into_parts();
match &request_parts.method {
&Method::GET => {
&Method::GET | &Method::HEAD => {
let uri_path = ui_mode.path(
request_parts
.uri
@@ -111,7 +129,7 @@ fn serve_ui(req: Request, ui_mode: UiMode) -> Result<Response, Error> {
.or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html")));
if let Some(file) = file {
FileData::from_embedded(&request_parts, file).into_response(&request_parts)
FileData::from_embedded(&request_parts, file)?.into_response(&request_parts)
} else {
Ok(not_found())
}
@@ -161,14 +179,35 @@ pub fn init_ui_router(ctx: InitContext) -> Router {
}
pub fn main_ui_router(ctx: RpcContext) -> Router {
rpc_router(
ctx.clone(),
rpc_router(ctx.clone(), {
let ctx = ctx.clone();
Server::new(move || ready(Ok(ctx.clone())), main_api::<RpcContext>())
.middleware(Cors::new())
.middleware(Auth::new())
.middleware(SyncDb::new()),
.middleware(SyncDb::new())
})
.route("/proxy/:url", {
let ctx = ctx.clone();
any(move |x::Path(url): x::Path<String>, request: Request| {
let ctx = ctx.clone();
async move {
proxy_request(ctx, request, url)
.await
.unwrap_or_else(server_error)
}
})
})
.nest("/s9pk", s9pk_router(ctx.clone()))
.route(
"/static/local-root-ca.crt",
get(move || {
let ctx = ctx.clone();
async move {
let account = ctx.account.read().await;
cert_send(&account.root_ca_cert, &account.hostname)
}
}),
)
// TODO: cert
.fallback(any(|request: Request| async move {
serve_ui(request, UiMode::Main).unwrap_or_else(server_error)
}))
@@ -179,29 +218,133 @@ pub fn refresher() -> Router {
let res = include_bytes!("./refresher.html");
FileData {
data: Body::from(&res[..]),
content_range: None,
e_tag: None,
encoding: None,
len: Some(res.len() as u64),
mime: Some("text/html".into()),
digest: None,
}
.into_response(&request.into_parts().0)
.unwrap_or_else(server_error)
}))
}
async fn proxy_request(ctx: RpcContext, request: Request, url: String) -> Result<Response, Error> {
if_authorized(&ctx, request, |mut request| async {
for header in PROXY_STRIP_HEADERS {
request.headers_mut().remove(*header);
}
*request.uri_mut() = url.parse()?;
let request = request.map(|b| reqwest::Body::wrap_stream(SyncBody::from(b)));
let response = ctx.client.execute(request.try_into()?).await?;
Ok(Response::from(response).map(|b| Body::new(b)))
})
.await
}
fn s9pk_router(ctx: RpcContext) -> Router {
Router::new()
.route("/installed/:s9pk", {
let ctx = ctx.clone();
any(
|x::Path(s9pk): x::Path<String>, request: Request| async move {
if_authorized(&ctx, request, |request| async {
let (parts, _) = request.into_parts();
match FileData::from_path(
&parts,
&ctx.datadir
.join(PKG_ARCHIVE_DIR)
.join("installed")
.join(s9pk),
)
.await?
{
Some(file) => file.into_response(&parts),
None => Ok(not_found()),
}
})
.await
.unwrap_or_else(server_error)
},
)
})
.route("/installed/:s9pk/*path", {
let ctx = ctx.clone();
any(
|x::Path(s9pk): x::Path<String>,
x::Path(path): x::Path<PathBuf>,
x::Query(commitment): x::Query<Option<MerkleArchiveCommitment>>,
request: Request| async move {
if_authorized(&ctx, request, |request| async {
let s9pk = S9pk::deserialize(
&MultiCursorFile::from(
open_file(
ctx.datadir
.join(PKG_ARCHIVE_DIR)
.join("installed")
.join(s9pk),
)
.await?,
),
commitment.as_ref(),
)
.await?;
let (parts, _) = request.into_parts();
match FileData::from_s9pk(&parts, &s9pk, &path).await? {
Some(file) => file.into_response(&parts),
None => Ok(not_found()),
}
})
.await
.unwrap_or_else(server_error)
},
)
})
.route(
"/proxy/:url/*path",
any(
|x::Path((url, path)): x::Path<(Url, PathBuf)>,
x::RawQuery(query): x::RawQuery,
request: Request| async move {
if_authorized(&ctx, request, |request| async {
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
query
.as_deref()
.map(MerkleArchiveCommitment::from_query)
.and_then(|a| a.transpose())
.transpose()?
.as_ref(),
)
.await?;
let (parts, _) = request.into_parts();
match FileData::from_s9pk(&parts, &s9pk, &path).await? {
Some(file) => file.into_response(&parts),
None => Ok(not_found()),
}
})
.await
.unwrap_or_else(server_error)
},
),
)
}
async fn if_authorized<
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Response, Error>> + Send + Sync,
F: FnOnce(Request) -> Fut,
Fut: Future<Output = Result<Response, Error>> + Send,
>(
ctx: &RpcContext,
parts: &RequestParts,
request: Request,
f: F,
) -> Result<Response, Error> {
if let Err(e) = HasValidSession::from_header(parts.headers.get(http::header::COOKIE), ctx).await
if let Err(e) =
HasValidSession::from_header(request.headers().get(http::header::COOKIE), ctx).await
{
Ok(unauthorized(e, parts.uri.path()))
Ok(unauthorized(e, request.uri().path()))
} else {
f().await
f(request).await
}
}
@@ -268,44 +411,117 @@ fn cert_send(cert: &X509, hostname: &Hostname) -> Result<Response, Error> {
.with_kind(ErrorKind::Network)
}
fn parse_range(header: &HeaderValue, len: u64) -> Result<(u64, u64, u64), Error> {
let r = header
.to_str()
.with_kind(ErrorKind::Network)?
.trim()
.strip_prefix("bytes=")
.ok_or_else(|| Error::new(eyre!("invalid range units"), ErrorKind::InvalidRequest))?;
if r.contains(",") {
return Err(Error::new(
eyre!("multi-range requests are unsupported"),
ErrorKind::InvalidRequest,
));
}
if let Some((start, end)) = r.split_once("-").map(|(s, e)| (s.trim(), e.trim())) {
Ok((
if start.is_empty() {
0u64
} else {
start.parse()?
},
if end.is_empty() {
len - 1
} else {
min(end.parse()?, len - 1)
},
len,
))
} else {
Ok((len - r.trim().parse::<u64>()?, len - 1, len))
}
}
struct FileData {
data: Body,
len: Option<u64>,
content_range: Option<(u64, u64, u64)>,
encoding: Option<&'static str>,
e_tag: Option<String>,
mime: Option<InternedString>,
digest: Option<(&'static str, Vec<u8>)>,
}
impl FileData {
fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self {
fn from_embedded(
req: &RequestParts,
file: &'static include_dir::File<'static>,
) -> Result<Self, Error> {
let path = file.path();
let (encoding, data) = req
.headers
.get_all(ACCEPT_ENCODING)
.into_iter()
.filter_map(|h| h.to_str().ok())
.flat_map(|s| s.split(","))
.filter_map(|s| s.split(";").next())
.map(|s| s.trim())
.fold((None, file.contents()), |acc, e| {
if let Some(file) = (e == "br")
.then_some(())
.and_then(|_| EMBEDDED_UIS.get_file(format!("{}.br", path.display())))
{
(Some("br"), file.contents())
} else if let Some(file) = (e == "gzip" && acc.0 != Some("br"))
.then_some(())
.and_then(|_| EMBEDDED_UIS.get_file(format!("{}.gz", path.display())))
{
(Some("gzip"), file.contents())
} else {
acc
}
});
let (encoding, data, len, content_range) = if let Some(range) = req.headers.get(RANGE) {
let data = file.contents();
let (start, end, size) = parse_range(range, data.len() as u64)?;
let encoding = req
.headers
.get_all(ACCEPT_ENCODING)
.into_iter()
.filter_map(|h| h.to_str().ok())
.flat_map(|s| s.split(","))
.filter_map(|s| s.split(";").next())
.map(|s| s.trim())
.any(|e| e == "gzip")
.then_some("gzip");
let data = if start > end {
&[]
} else {
&data[(start as usize)..=(end as usize)]
};
let (len, data) = if encoding == Some("gzip") {
(
None,
Body::from_stream(ReaderStream::new(GzipEncoder::new(Cursor::new(data)))),
)
} else {
(Some(data.len() as u64), Body::from(data))
};
(encoding, data, len, Some((start, end, size)))
} else {
let (encoding, data) = req
.headers
.get_all(ACCEPT_ENCODING)
.into_iter()
.filter_map(|h| h.to_str().ok())
.flat_map(|s| s.split(","))
.filter_map(|s| s.split(";").next())
.map(|s| s.trim())
.fold((None, file.contents()), |acc, e| {
if let Some(file) = (e == "br")
.then_some(())
.and_then(|_| EMBEDDED_UIS.get_file(format!("{}.br", path.display())))
{
(Some("br"), file.contents())
} else if let Some(file) = (e == "gzip" && acc.0 != Some("br"))
.then_some(())
.and_then(|_| EMBEDDED_UIS.get_file(format!("{}.gz", path.display())))
{
(Some("gzip"), file.contents())
} else {
acc
}
});
(encoding, Body::from(data), Some(data.len() as u64), None)
};
Self {
len: Some(data.len() as u64),
Ok(Self {
len,
encoding,
data: data.into(),
content_range,
data: if req.method == Method::HEAD {
Body::empty()
} else {
data
},
e_tag: file.metadata().map(|metadata| {
e_tag(
path,
@@ -323,11 +539,28 @@ impl FileData {
mime: MimeGuess::from_path(path)
.first()
.map(|m| m.essence_str().into()),
digest: None,
})
}
fn encode<R: AsyncRead + Send + 'static>(
encoding: &mut Option<&str>,
data: R,
len: u64,
) -> (Option<u64>, Body) {
if *encoding == Some("gzip") {
(
None,
Body::from_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(data)))),
)
} else {
*encoding = None;
(Some(len), Body::from_stream(ReaderStream::new(data)))
}
}
async fn from_path(req: &RequestParts, path: &Path) -> Result<Self, Error> {
let encoding = req
async fn from_path(req: &RequestParts, path: &Path) -> Result<Option<Self>, Error> {
let mut encoding = req
.headers
.get_all(ACCEPT_ENCODING)
.into_iter()
@@ -338,12 +571,23 @@ impl FileData {
.any(|e| e == "gzip")
.then_some("gzip");
let file = open_file(path).await?;
if tokio::fs::metadata(path).await.is_err() {
return Ok(None);
}
let mut file = open_file(path).await?;
let metadata = file
.metadata()
.await
.with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?;
let content_range = req
.headers
.get(RANGE)
.map(|r| parse_range(r, metadata.len()))
.transpose()?;
let e_tag = Some(e_tag(
path,
format!(
@@ -357,51 +601,123 @@ impl FileData {
.as_bytes(),
));
let (len, data) = if encoding == Some("gzip") {
(
None,
Body::from_stream(ReaderStream::new(GzipEncoder::new(BufReader::new(file)))),
)
let (len, data) = if let Some((start, end, _)) = content_range {
let len = end + 1 - start;
file.seek(std::io::SeekFrom::Start(start)).await?;
Self::encode(&mut encoding, file.take(len), len)
} else {
(
Some(metadata.len()),
Body::from_stream(ReaderStream::new(file)),
)
Self::encode(&mut encoding, file, metadata.len())
};
Ok(Self {
data,
Ok(Some(Self {
data: if req.method == Method::HEAD {
Body::empty()
} else {
data
},
len,
content_range,
encoding,
e_tag,
mime: MimeGuess::from_path(path)
.first()
.map(|m| m.essence_str().into()),
})
digest: None,
}))
}
async fn from_s9pk<S: FileSource>(
req: &RequestParts,
s9pk: &S9pk<S>,
path: &Path,
) -> Result<Option<Self>, Error> {
let mut encoding = req
.headers
.get_all(ACCEPT_ENCODING)
.into_iter()
.filter_map(|h| h.to_str().ok())
.flat_map(|s| s.split(","))
.filter_map(|s| s.split(";").next())
.map(|s| s.trim())
.any(|e| e == "gzip")
.then_some("gzip");
let Some(file) = s9pk.as_archive().contents().get_path(path) else {
return Ok(None);
};
let Some(contents) = file.as_file() else {
return Ok(None);
};
let (digest, len) = if let Some((hash, len)) = file.hash() {
(Some(("blake3", hash.as_bytes().to_vec())), len)
} else {
(None, contents.size().await?)
};
let content_range = req
.headers
.get(RANGE)
.map(|r| parse_range(r, len))
.transpose()?;
let (len, data) = if let Some((start, end, _)) = content_range {
let len = end + 1 - start;
Self::encode(&mut encoding, contents.slice(start, len).await?, len)
} else {
Self::encode(&mut encoding, contents.reader().await?.take(len), len)
};
Ok(Some(Self {
data: if req.method == Method::HEAD {
Body::empty()
} else {
data
},
len,
content_range,
encoding,
e_tag: None,
mime: MimeGuess::from_path(path)
.first()
.map(|m| m.essence_str().into()),
digest,
}))
}
fn into_response(self, req: &RequestParts) -> Result<Response, Error> {
let mut builder = Response::builder();
if let Some(mime) = self.mime {
builder = builder.header(http::header::CONTENT_TYPE, &*mime);
builder = builder.header(CONTENT_TYPE, &*mime);
}
if let Some(e_tag) = &self.e_tag {
builder = builder.header(http::header::ETAG, &**e_tag);
builder = builder
.header(ETAG, &**e_tag)
.header(CACHE_CONTROL, "public, max-age=21000000, immutable");
}
builder = builder.header(ACCEPT_RANGES, "bytes");
if let Some((start, end, size)) = self.content_range {
builder = builder
.header(CONTENT_RANGE, format!("bytes {start}-{end}/{size}"))
.status(StatusCode::PARTIAL_CONTENT);
}
if let Some((algorithm, digest)) = self.digest {
builder = builder.header(
"Repr-Digest",
format!("{algorithm}=:{}:", Base64Display::new(&digest, &BASE64)),
);
}
builder = builder.header(
http::header::CACHE_CONTROL,
"public, max-age=21000000, immutable",
);
if req
.headers
.get_all(http::header::CONNECTION)
.get_all(CONNECTION)
.iter()
.flat_map(|s| s.to_str().ok())
.flat_map(|s| s.split(","))
.any(|s| s.trim() == "keep-alive")
{
builder = builder.header(http::header::CONNECTION, "keep-alive");
builder = builder.header(CONNECTION, "keep-alive");
}
if self.e_tag.is_some()
@@ -411,14 +727,13 @@ impl FileData {
.and_then(|h| h.to_str().ok())
== self.e_tag.as_deref()
{
builder = builder.status(StatusCode::NOT_MODIFIED);
builder.body(Body::empty())
builder.status(StatusCode::NOT_MODIFIED).body(Body::empty())
} else {
if let Some(len) = self.len {
builder = builder.header(http::header::CONTENT_LENGTH, len);
builder = builder.header(CONTENT_LENGTH, len);
}
if let Some(encoding) = self.encoding {
builder = builder.header(http::header::CONTENT_ENCODING, encoding);
builder = builder.header(CONTENT_ENCODING, encoding);
}
builder.body(self.data)

25
core/startos/src/registry/admin.rs
View File

@@ -46,7 +46,7 @@ fn signers_api<C: Context>() -> ParentHandler<C> {
.with_metadata("admin", Value::Bool(true))
.no_cli(),
)
.subcommand("add", from_fn_async(cli_add_signer).no_display())
.subcommand("add", from_fn_async(cli_add_signer))
}
impl Model<BTreeMap<Guid, SignerInfo>> {
@@ -71,7 +71,7 @@ impl Model<BTreeMap<Guid, SignerInfo>> {
.ok_or_else(|| Error::new(eyre!("unknown signer"), ErrorKind::Authorization))
}
pub fn add_signer(&mut self, signer: &SignerInfo) -> Result<(), Error> {
pub fn add_signer(&mut self, signer: &SignerInfo) -> Result<Guid, Error> {
if let Some((guid, s)) = self
.as_entries()?
.into_iter()
@@ -89,7 +89,9 @@ impl Model<BTreeMap<Guid, SignerInfo>> {
ErrorKind::InvalidRequest,
));
}
self.insert(&Guid::new(), signer)
let id = Guid::new();
self.insert(&id, signer)?;
Ok(id)
}
}
@@ -122,7 +124,7 @@ pub fn display_signers<T>(params: WithIoFormat<T>, signers: BTreeMap<Guid, Signe
table.print_tty(false).unwrap();
}
pub async fn add_signer(ctx: RegistryContext, signer: SignerInfo) -> Result<(), Error> {
pub async fn add_signer(ctx: RegistryContext, signer: SignerInfo) -> Result<Guid, Error> {
ctx.db
.mutate(|db| db.as_index_mut().as_signers_mut().add_signer(&signer))
.await
@@ -155,7 +157,7 @@ pub async fn cli_add_signer(
},
..
}: HandlerArgs<CliContext, CliAddSignerParams>,
) -> Result<(), Error> {
) -> Result<Guid, Error> {
let signer = SignerInfo {
name,
contact,
@@ -165,15 +167,16 @@ pub async fn cli_add_signer(
TypedPatchDb::<RegistryDatabase>::load(PatchDb::open(database).await?)
.await?
.mutate(|db| db.as_index_mut().as_signers_mut().add_signer(&signer))
.await?;
.await
} else {
ctx.call_remote::<RegistryContext>(
&parent_method.into_iter().chain(method).join("."),
to_value(&signer)?,
from_value(
ctx.call_remote::<RegistryContext>(
&parent_method.into_iter().chain(method).join("."),
to_value(&signer)?,
)
.await?,
)
.await?;
}
Ok(())
}
#[derive(Debug, Deserialize, Serialize, TS)]

3
core/startos/src/registry/asset.rs
View File

@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWrite;
@@ -20,6 +21,8 @@ use crate::s9pk::S9pk;
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RegistryAsset<Commitment> {
#[ts(type = "string")]
pub published_at: DateTime<Utc>,
#[ts(type = "string")]
pub url: Url,
pub commitment: Commitment,

12
core/startos/src/registry/auth.rs
View File

@@ -27,7 +27,6 @@ use crate::util::serde::Base64;
pub const AUTH_SIG_HEADER: &str = "X-StartOS-Registry-Auth-Sig";
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Metadata {
#[serde(default)]
admin: bool,
@@ -75,9 +74,7 @@ pub struct RegistryAdminLogRecord {
pub key: AnyVerifyingKey,
}
#[derive(Serialize, Deserialize)]
pub struct SignatureHeader {
#[serde(flatten)]
pub commitment: RequestCommitment,
pub signer: AnyVerifyingKey,
pub signature: AnySignature,
@@ -93,14 +90,9 @@ impl SignatureHeader {
HeaderValue::from_str(url.query().unwrap_or_default()).unwrap()
}
pub fn from_header(header: &HeaderValue) -> Result<Self, Error> {
let url: Url = format!(
"http://localhost/?{}",
header.to_str().with_kind(ErrorKind::Utf8)?
)
.parse()?;
let query: BTreeMap<_, _> = url.query_pairs().collect();
let query: BTreeMap<_, _> = form_urlencoded::parse(header.as_bytes()).collect();
Ok(Self {
commitment: RequestCommitment::from_query(&url)?,
commitment: RequestCommitment::from_query(&header)?,
signer: query.get("signer").or_not_found("signer")?.parse()?,
signature: query.get("signature").or_not_found("signature")?.parse()?,
})

30
core/startos/src/registry/context.rs
View File

@@ -200,6 +200,19 @@ impl CallRemote<RegistryContext> for CliContext {
.send()
.await?;
if !res.status().is_success() {
let status = res.status();
let txt = res.text().await?;
let mut res = Err(Error::new(
eyre!("{}", status.canonical_reason().unwrap_or(status.as_str())),
ErrorKind::Network,
));
if !txt.is_empty() {
res = res.with_ctx(|_| (ErrorKind::Network, txt));
}
return res.map_err(From::from);
}
match res
.headers()
.get(CONTENT_TYPE)
@@ -210,7 +223,7 @@ impl CallRemote<RegistryContext> for CliContext {
.with_kind(ErrorKind::Deserialization)?
.result
}
_ => Err(Error::new(eyre!("missing content type"), ErrorKind::Network).into()),
_ => Err(Error::new(eyre!("unknown content type"), ErrorKind::Network).into()),
}
}
}
@@ -247,6 +260,19 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
.send()
.await?;
if !res.status().is_success() {
let status = res.status();
let txt = res.text().await?;
let mut res = Err(Error::new(
eyre!("{}", status.canonical_reason().unwrap_or(status.as_str())),
ErrorKind::Network,
));
if !txt.is_empty() {
res = res.with_ctx(|_| (ErrorKind::Network, txt));
}
return res.map_err(From::from);
}
match res
.headers()
.get(CONTENT_TYPE)
@@ -257,7 +283,7 @@ impl CallRemote<RegistryContext, RegistryUrlParams> for RpcContext {
.with_kind(ErrorKind::Deserialization)?
.result
}
_ => Err(Error::new(eyre!("missing content type"), ErrorKind::Network).into()),
_ => Err(Error::new(eyre!("unknown content type"), ErrorKind::Network).into()),
}
}
}

8
core/startos/src/registry/device_info.rs
View File

@@ -54,12 +54,7 @@ impl DeviceInfo {
HeaderValue::from_str(url.query().unwrap_or_default()).unwrap()
}
pub fn from_header_value(header: &HeaderValue) -> Result<Self, Error> {
let url: Url = format!(
"http://localhost/?{}",
header.to_str().with_kind(ErrorKind::ParseUrl)?
)
.parse()?;
let query: BTreeMap<_, _> = url.query_pairs().collect();
let query: BTreeMap<_, _> = form_urlencoded::parse(header.as_bytes()).collect();
Ok(Self {
os: OsInfo {
version: query
@@ -151,7 +146,6 @@ impl From<&RpcContext> for HardwareInfo {
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Metadata {
#[serde(default)]
get_device_info: bool,

29
core/startos/src/registry/mod.rs
View File

@@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
use axum::Router;
use futures::future::ready;
use imbl_value::InternedString;
use models::DataUrl;
use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server};
use serde::{Deserialize, Serialize};
@@ -16,7 +17,7 @@ use crate::registry::auth::Auth;
use crate::registry::context::RegistryContext;
use crate::registry::device_info::DeviceInfoMiddleware;
use crate::registry::os::index::OsIndex;
use crate::registry::package::index::PackageIndex;
use crate::registry::package::index::{Category, PackageIndex};
use crate::registry::signer::SignerInfo;
use crate::rpc_continuations::Guid;
use crate::util::serde::HandlerExtSerde;
@@ -45,6 +46,7 @@ impl RegistryDatabase {}
#[model = "Model<Self>"]
#[ts(export)]
pub struct FullIndex {
pub name: Option<String>,
pub icon: Option<DataUrl<'static>>,
pub package: PackageIndex,
pub os: OsIndex,
@@ -55,6 +57,25 @@ pub async fn get_full_index(ctx: RegistryContext) -> Result<FullIndex, Error> {
ctx.db.peek().await.into_index().de()
}
#[derive(Debug, Default, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RegistryInfo {
pub name: Option<String>,
pub icon: Option<DataUrl<'static>>,
#[ts(as = "BTreeMap::<String, Category>")]
pub categories: BTreeMap<InternedString, Category>,
}
pub async fn get_info(ctx: RegistryContext) -> Result<RegistryInfo, Error> {
let peek = ctx.db.peek().await.into_index();
Ok(RegistryInfo {
name: peek.as_name().de()?,
icon: peek.as_icon().de()?,
categories: peek.as_package().as_categories().de()?,
})
}
pub fn registry_api<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand(
@@ -63,6 +84,12 @@ pub fn registry_api<C: Context>() -> ParentHandler<C> {
.with_display_serializable()
.with_call_remote::<CliContext>(),
)
.subcommand(
"info",
from_fn_async(get_info)
.with_display_serializable()
.with_call_remote::<CliContext>(),
)
.subcommand("os", os::os_api::<C>())
.subcommand("package", package::package_api::<C>())
.subcommand("admin", admin::admin_api::<C>())

10
core/startos/src/registry/os/asset/add.rs
View File

@@ -2,6 +2,7 @@ use std::collections::{BTreeMap, HashMap};
use std::panic::UnwindSafe;
use std::path::PathBuf;
use chrono::Utc;
use clap::Parser;
use imbl_value::InternedString;
use itertools::Itertools;
@@ -12,7 +13,7 @@ use url::Url;
use crate::context::CliContext;
use crate::prelude::*;
use crate::progress::{FullProgressTracker};
use crate::progress::FullProgressTracker;
use crate::registry::asset::RegistryAsset;
use crate::registry::context::RegistryContext;
use crate::registry::os::index::OsVersionInfo;
@@ -33,19 +34,19 @@ pub fn add_api<C: Context>() -> ParentHandler<C> {
.subcommand(
"iso",
from_fn_async(add_iso)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"img",
from_fn_async(add_img)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"squashfs",
from_fn_async(add_squashfs)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
}
@@ -107,6 +108,7 @@ async fn add_asset(
)
.upsert(&platform, || {
Ok(RegistryAsset {
published_at: Utc::now(),
url,
commitment: commitment.clone(),
signatures: HashMap::new(),

6
core/startos/src/registry/os/asset/sign.rs
View File

@@ -30,19 +30,19 @@ pub fn sign_api<C: Context>() -> ParentHandler<C> {
.subcommand(
"iso",
from_fn_async(sign_iso)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"img",
from_fn_async(sign_img)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"squashfs",
from_fn_async(sign_squashfs)
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
}

8
core/startos/src/registry/os/version/mod.rs
View File

@@ -25,7 +25,7 @@ pub fn version_api<C: Context>() -> ParentHandler<C> {
"add",
from_fn_async(add_version)
.with_metadata("admin", Value::Bool(true))
.with_metadata("getSigner", Value::Bool(true))
.with_metadata("get_signer", Value::Bool(true))
.no_display()
.with_call_remote::<CliContext>(),
)
@@ -121,7 +121,7 @@ pub async fn remove_version(
#[command(rename_all = "kebab-case")]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct GetVersionParams {
pub struct GetOsVersionParams {
#[ts(type = "string | null")]
#[arg(long = "src")]
pub source: Option<VersionString>,
@@ -138,12 +138,12 @@ pub struct GetVersionParams {
pub async fn get_version(
ctx: RegistryContext,
GetVersionParams {
GetOsVersionParams {
source,
target,
server_id,
arch,
}: GetVersionParams,
}: GetOsVersionParams,
) -> Result<BTreeMap<VersionString, OsVersionInfo>, Error> {
if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, arch) {
let created_at = Utc::now();

20
core/startos/src/registry/package/add.rs
View File

@@ -11,13 +11,14 @@ use url::Url;
use crate::context::CliContext;
use crate::prelude::*;
use crate::progress::FullProgressTracker;
use crate::progress::{FullProgressTracker, ProgressTrackerWriter};
use crate::registry::context::RegistryContext;
use crate::registry::package::index::PackageVersionInfo;
use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
use crate::registry::signer::sign::ed25519::Ed25519;
use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::merkle_archive::source::ArchiveSource;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TrackingIO;
@@ -126,13 +127,16 @@ pub async fn cli_add_package(
sign_phase.complete();
verify_phase.start();
let mut src = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true)
let source = HttpSource::new(ctx.client.clone(), url.clone()).await?;
let len = source.size().await;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
if let Some(len) = len {
verify_phase.set_total(len);
}
let mut verify_writer = ProgressTrackerWriter::new(tokio::io::sink(), verify_phase);
src.serialize(&mut TrackingIO::new(0, &mut verify_writer), true)
.await?;
let (_, mut verify_phase) = verify_writer.into_inner();
verify_phase.complete();
index_phase.start();
@@ -140,7 +144,7 @@ pub async fn cli_add_package(
&parent_method.into_iter().chain(method).join("."),
imbl_value::json!({
"url": &url,
"signature": signature,
"signature": AnySignature::Ed25519(signature),
"commitment": commitment,
}),
)

81
core/startos/src/registry/package/get.rs
View File

@@ -21,6 +21,7 @@ use crate::util::VersionString;
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub enum PackageDetailLevel {
None,
Short,
Full,
}
@@ -50,7 +51,9 @@ pub struct GetPackageParams {
#[arg(skip)]
#[serde(rename = "__device_info")]
pub device_info: Option<DeviceInfo>,
pub other_versions: Option<PackageDetailLevel>,
#[serde(default)]
#[arg(default_value = "none")]
pub other_versions: PackageDetailLevel,
}
#[derive(Debug, Deserialize, Serialize, TS)]
@@ -126,7 +129,6 @@ fn get_matching_models<'a>(
db: &'a Model<PackageIndex>,
GetPackageParams {
id,
version,
source_version,
device_info,
..
@@ -148,22 +150,18 @@ fn get_matching_models<'a>(
.into_iter()
.map(|(v, info)| {
Ok::<_, Error>(
if version
if source_version.as_ref().map_or(Ok(true), |source_version| {
Ok::<_, Error>(
source_version.satisfies(
&info
.as_source_version()
.de()?
.unwrap_or(VersionRange::any()),
),
)
})? && device_info
.as_ref()
.map_or(true, |version| v.satisfies(version))
&& source_version.as_ref().map_or(Ok(true), |source_version| {
Ok::<_, Error>(
source_version.satisfies(
&info
.as_source_version()
.de()?
.unwrap_or(VersionRange::any()),
),
)
})?
&& device_info
.as_ref()
.map_or(Ok(true), |device_info| info.works_for_device(device_info))?
.map_or(Ok(true), |device_info| info.works_for_device(device_info))?
{
Some((k.clone(), ExtendedVersion::from(v), info))
} else {
@@ -187,24 +185,27 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
let mut other: BTreeMap<PackageId, BTreeMap<VersionString, &Model<PackageVersionInfo>>> =
Default::default();
for (id, version, info) in get_matching_models(&peek.as_index().as_package(), &params)? {
let mut package_best = best.remove(&id).unwrap_or_default();
let mut package_other = other.remove(&id).unwrap_or_default();
for worse_version in package_best
.keys()
.filter(|k| ***k < version)
.cloned()
.collect_vec()
let package_best = best.entry(id.clone()).or_default();
let package_other = other.entry(id.clone()).or_default();
if params
.version
.as_ref()
.map_or(true, |v| version.satisfies(v))
&& package_best.keys().all(|k| !(**k > version))
{
if let Some(info) = package_best.remove(&worse_version) {
package_other.insert(worse_version, info);
for worse_version in package_best
.keys()
.filter(|k| ***k < version)
.cloned()
.collect_vec()
{
if let Some(info) = package_best.remove(&worse_version) {
package_other.insert(worse_version, info);
}
}
}
if package_best.keys().all(|k| !(**k > version)) {
package_best.insert(version.into(), info);
}
best.insert(id.clone(), package_best);
if params.other_versions.is_some() {
other.insert(id.clone(), package_other);
} else {
package_other.insert(version.into(), info);
}
}
if let Some(id) = params.id {
@@ -224,12 +225,12 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
.try_collect()?;
let other = other.remove(&id).unwrap_or_default();
match params.other_versions {
None => to_value(&GetPackageResponse {
PackageDetailLevel::None => to_value(&GetPackageResponse {
categories,
best,
other_versions: None,
}),
Some(PackageDetailLevel::Short) => to_value(&GetPackageResponse {
PackageDetailLevel::Short => to_value(&GetPackageResponse {
categories,
best,
other_versions: Some(
@@ -239,7 +240,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
.try_collect()?,
),
}),
Some(PackageDetailLevel::Full) => to_value(&GetPackageResponseFull {
PackageDetailLevel::Full => to_value(&GetPackageResponseFull {
categories,
best,
other_versions: other
@@ -250,7 +251,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
}
} else {
match params.other_versions {
None => to_value(
PackageDetailLevel::None => to_value(
&best
.into_iter()
.map(|(id, best)| {
@@ -276,7 +277,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
})
.try_collect::<_, GetPackagesResponse, _>()?,
),
Some(PackageDetailLevel::Short) => to_value(
PackageDetailLevel::Short => to_value(
&best
.into_iter()
.map(|(id, best)| {
@@ -310,7 +311,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
})
.try_collect::<_, GetPackagesResponse, _>()?,
),
Some(PackageDetailLevel::Full) => to_value(
PackageDetailLevel::Full => to_value(
&best
.into_iter()
.map(|(id, best)| {
@@ -354,7 +355,7 @@ pub fn display_package_info(
}
if let Some(_) = params.rest.id {
if params.rest.other_versions == Some(PackageDetailLevel::Full) {
if params.rest.other_versions == PackageDetailLevel::Full {
for table in from_value::<GetPackageResponseFull>(info)?.tables() {
table.print_tty(false)?;
println!();
@@ -366,7 +367,7 @@ pub fn display_package_info(
}
}
} else {
if params.rest.other_versions == Some(PackageDetailLevel::Full) {
if params.rest.other_versions == PackageDetailLevel::Full {
for (_, package) in from_value::<GetPackagesResponseFull>(info)? {
for table in package.tables() {
table.print_tty(false)?;

46
core/startos/src/registry/package/index.rs
View File

@@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet};
use chrono::Utc;
use exver::{Version, VersionRange};
use imbl_value::InternedString;
use models::{DataUrl, PackageId, VersionString};
@@ -15,7 +16,7 @@ use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment
use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey};
use crate::rpc_continuations::Guid;
use crate::s9pk::git_hash::GitHash;
use crate::s9pk::manifest::{Description, HardwareRequirements};
use crate::s9pk::manifest::{Alerts, Description, HardwareRequirements};
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
@@ -49,12 +50,25 @@ pub struct Category {
pub description: Description,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct DependencyMetadata {
#[ts(type = "string | null")]
pub title: Option<InternedString>,
pub icon: Option<DataUrl<'static>>,
pub description: Option<String>,
pub optional: bool,
}
#[derive(Debug, Deserialize, Serialize, HasModel, TS)]
#[serde(rename_all = "camelCase")]
#[model = "Model<Self>"]
#[ts(export)]
pub struct PackageVersionInfo {
pub title: String,
#[ts(type = "string")]
pub title: InternedString,
pub icon: DataUrl<'static>,
pub description: Description,
pub release_notes: String,
@@ -70,6 +84,10 @@ pub struct PackageVersionInfo {
pub support_site: Url,
#[ts(type = "string")]
pub marketing_site: Url,
#[ts(type = "string | null")]
pub donation_url: Option<Url>,
pub alerts: Alerts,
pub dependency_metadata: BTreeMap<PackageId, DependencyMetadata>,
#[ts(type = "string")]
pub os_version: Version,
pub hardware_requirements: HardwareRequirements,
@@ -80,6 +98,19 @@ pub struct PackageVersionInfo {
impl PackageVersionInfo {
pub async fn from_s9pk<S: FileSource + Clone>(s9pk: &S9pk<S>, url: Url) -> Result<Self, Error> {
let manifest = s9pk.as_manifest();
let mut dependency_metadata = BTreeMap::new();
for (id, info) in &manifest.dependencies.0 {
let metadata = s9pk.dependency_metadata(id).await?;
dependency_metadata.insert(
id.clone(),
DependencyMetadata {
title: metadata.map(|m| m.title),
icon: s9pk.dependency_icon_data_url(id).await?,
description: info.description.clone(),
optional: info.optional,
},
);
}
Ok(Self {
title: manifest.title.clone(),
icon: s9pk.icon_data_url().await?,
@@ -91,10 +122,14 @@ impl PackageVersionInfo {
upstream_repo: manifest.upstream_repo.clone(),
support_site: manifest.support_site.clone(),
marketing_site: manifest.marketing_site.clone(),
donation_url: manifest.donation_url.clone(),
alerts: manifest.alerts.clone(),
dependency_metadata,
os_version: manifest.os_version.clone(),
hardware_requirements: manifest.hardware_requirements.clone(),
source_version: None, // TODO
s9pk: RegistryAsset {
published_at: Utc::now(),
url,
commitment: s9pk.as_archive().commitment().await?,
signatures: [(
@@ -114,8 +149,11 @@ impl PackageVersionInfo {
table.add_row(row![bc => &self.title]);
table.add_row(row![br -> "VERSION", AsRef::<str>::as_ref(version)]);
table.add_row(row![br -> "RELEASE NOTES", &self.release_notes]);
table.add_row(row![br -> "ABOUT", &self.description.short]);
table.add_row(row![br -> "DESCRIPTION", &self.description.long]);
table.add_row(row![br -> "ABOUT", &textwrap::wrap(&self.description.short, 80).join("\n")]);
table.add_row(row![
br -> "DESCRIPTION",
&textwrap::wrap(&self.description.long, 80).join("\n")
]);
table.add_row(row![br -> "GIT HASH", AsRef::<str>::as_ref(&self.git_hash)]);
table.add_row(row![br -> "LICENSE", &self.license]);
table.add_row(row![br -> "PACKAGE REPO", &self.wrapper_repo.to_string()]);

11
core/startos/src/registry/package/mod.rs
View File

@@ -16,14 +16,21 @@ pub fn package_api<C: Context>() -> ParentHandler<C> {
.with_display_serializable()
.with_call_remote::<CliContext>(),
)
.subcommand("add", from_fn_async(add::add_package).no_cli())
.subcommand(
"add",
from_fn_async(add::add_package)
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand("add", from_fn_async(add::cli_add_package).no_display())
.subcommand(
"get",
from_fn_async(get::get_package)
.with_metadata("get_device_info", Value::Bool(true))
.with_display_serializable()
.with_custom_display_fn(|handle, result| {
get::display_package_info(handle.params, result)
}),
})
.with_call_remote::<CliContext>(),
)
}

29
core/startos/src/registry/signer/commitment/merkle_archive.rs
View File

@@ -20,6 +20,35 @@ pub struct MerkleArchiveCommitment {
#[ts(type = "number")]
pub root_maxsize: u64,
}
impl MerkleArchiveCommitment {
pub fn from_query(query: &str) -> Result<Option<Self>, Error> {
let mut root_sighash = None;
let mut root_maxsize = None;
for (k, v) in form_urlencoded::parse(dbg!(query).as_bytes()) {
match &*k {
"rootSighash" => {
root_sighash = Some(dbg!(v).parse()?);
}
"rootMaxsize" => {
root_maxsize = Some(v.parse()?);
}
_ => (),
}
}
if root_sighash.is_some() || root_maxsize.is_some() {
Ok(Some(Self {
root_sighash: root_sighash
.or_not_found("rootSighash required if rootMaxsize specified")
.with_kind(ErrorKind::InvalidRequest)?,
root_maxsize: root_maxsize
.or_not_found("rootMaxsize required if rootSighash specified")
.with_kind(ErrorKind::InvalidRequest)?,
}))
} else {
Ok(None)
}
}
}
impl Digestable for MerkleArchiveCommitment {
fn update<D: Update>(&self, digest: &mut D) {
digest.update(&*self.root_sighash);

5
core/startos/src/registry/signer/commitment/request.rs
View File

@@ -5,6 +5,7 @@ use axum::body::Body;
use axum::extract::Request;
use digest::Update;
use futures::TryStreamExt;
use http::HeaderValue;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWrite;
use tokio_util::io::StreamReader;
@@ -37,8 +38,8 @@ impl RequestCommitment {
.append_pair("size", &self.size.to_string())
.append_pair("blake3", &self.blake3.to_string());
}
pub fn from_query(url: &Url) -> Result<Self, Error> {
let query: BTreeMap<_, _> = url.query_pairs().collect();
pub fn from_query(query: &HeaderValue) -> Result<Self, Error> {
let query: BTreeMap<_, _> = form_urlencoded::parse(query.as_bytes()).collect();
Ok(Self {
timestamp: query.get("timestamp").or_not_found("timestamp")?.parse()?,
nonce: query.get("nonce").or_not_found("nonce")?.parse()?,

48
core/startos/src/s9pk/merkle_archive/expected.rs
View File

@@ -1,4 +1,3 @@
use std::ffi::OsStr;
use std::path::Path;
@@ -7,16 +6,16 @@ use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::merkle_archive::Entry;
/// An object for tracking the files expected to be in an s9pk
/// An object for tracking the files expected to be in an s9pk
pub struct Expected<'a, T> {
keep: DirectoryContents<()>,
dir: &'a DirectoryContents<T>,
}
impl<'a, T> Expected<'a, T> {
pub fn new(dir: &'a DirectoryContents<T>,) -> Self {
pub fn new(dir: &'a DirectoryContents<T>) -> Self {
Self {
keep: DirectoryContents::new(),
dir
dir,
}
}
}
@@ -42,22 +41,23 @@ impl<'a, T: Clone> Expected<'a, T> {
path: impl AsRef<Path>,
mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
) -> Result<(), Error> {
let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
(
self.dir
.get_path(parent)
.and_then(|e| e.as_directory())
.ok_or_else(|| {
Error::new(
eyre!("directory {} missing from archive", parent.display()),
ErrorKind::ParseS9pk,
)
})?,
path.as_ref().strip_prefix(parent).unwrap(),
)
} else {
(self.dir, path.as_ref())
};
let (dir, stem) =
if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
(
self.dir
.get_path(parent)
.and_then(|e| e.as_directory())
.ok_or_else(|| {
Error::new(
eyre!("directory {} missing from archive", parent.display()),
ErrorKind::ParseS9pk,
)
})?,
path.as_ref().strip_prefix(parent).unwrap(),
)
} else {
(self.dir, path.as_ref())
};
let name = dir
.with_stem(&stem.as_os_str().to_string_lossy())
.filter(|(_, e)| e.as_file().is_some())
@@ -69,7 +69,7 @@ impl<'a, T: Clone> Expected<'a, T> {
),
ErrorKind::ParseS9pk,
)),
|acc, (name, _)|
|acc, (name, _)|
if valid_extension(Path::new(&*name).extension()) {
match acc {
Ok(_) => Err(Error::new(
@@ -96,8 +96,10 @@ impl<'a, T: Clone> Expected<'a, T> {
pub struct Filter(DirectoryContents<()>);
impl Filter {
pub fn keep_checked<T: FileSource + Clone>(&self, dir: &mut DirectoryContents<T>) -> Result<(), Error> {
pub fn keep_checked<T: FileSource + Clone>(
&self,
dir: &mut DirectoryContents<T>,
) -> Result<(), Error> {
dir.filter(|path| self.0.get_path(path).is_some())
}
}

4
core/startos/src/s9pk/merkle_archive/mod.rs
View File

@@ -233,6 +233,10 @@ impl<S> Entry<S> {
_ => None,
}
}
pub fn expect_file(&self) -> Result<&FileContents<S>, Error> {
self.as_file()
.ok_or_else(|| Error::new(eyre!("not a file"), ErrorKind::ParseS9pk))
}
pub fn as_directory(&self) -> Option<&DirectoryContents<S>> {
match self.as_contents() {
EntryContents::Directory(d) => Some(d),

52
core/startos/src/s9pk/merkle_archive/source/mod.rs
View File

@@ -1,3 +1,5 @@
use std::cmp::min;
use std::io::SeekFrom;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
@@ -6,7 +8,7 @@ use blake3::Hash;
use futures::future::BoxFuture;
use futures::{Future, FutureExt};
use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, Take};
use crate::prelude::*;
use crate::s9pk::merkle_archive::hash::VerifyingWriter;
@@ -17,8 +19,14 @@ pub mod multi_cursor_file;
pub trait FileSource: Send + Sync + Sized + 'static {
type Reader: AsyncRead + Unpin + Send;
type SliceReader: AsyncRead + Unpin + Send;
fn size(&self) -> impl Future<Output = Result<u64, Error>> + Send;
fn reader(&self) -> impl Future<Output = Result<Self::Reader, Error>> + Send;
fn slice(
&self,
position: u64,
size: u64,
) -> impl Future<Output = Result<Self::SliceReader, Error>> + Send;
fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
w: &mut W,
@@ -65,12 +73,16 @@ pub trait FileSource: Send + Sync + Sized + 'static {
impl<T: FileSource> FileSource for Arc<T> {
type Reader = T::Reader;
type SliceReader = T::SliceReader;
async fn size(&self) -> Result<u64, Error> {
self.deref().size().await
}
async fn reader(&self) -> Result<Self::Reader, Error> {
self.deref().reader().await
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
self.deref().slice(position, size).await
}
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
self.deref().copy(w).await
}
@@ -95,12 +107,16 @@ impl DynFileSource {
}
impl FileSource for DynFileSource {
type Reader = Box<dyn AsyncRead + Unpin + Send>;
type SliceReader = Box<dyn AsyncRead + Unpin + Send>;
async fn size(&self) -> Result<u64, Error> {
self.0.size().await
}
async fn reader(&self) -> Result<Self::Reader, Error> {
self.0.reader().await
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
self.0.slice(position, size).await
}
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
mut w: &mut W,
@@ -123,6 +139,11 @@ impl FileSource for DynFileSource {
trait DynableFileSource: Send + Sync + 'static {
async fn size(&self) -> Result<u64, Error>;
async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
async fn slice(
&self,
position: u64,
size: u64,
) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error>;
async fn copy_verify(
&self,
@@ -139,6 +160,13 @@ impl<T: FileSource> DynableFileSource for T {
async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
Ok(Box::new(FileSource::reader(self).await?))
}
async fn slice(
&self,
position: u64,
size: u64,
) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
Ok(Box::new(FileSource::slice(self, position, size).await?))
}
async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error> {
FileSource::copy(self, w).await
}
@@ -156,22 +184,34 @@ impl<T: FileSource> DynableFileSource for T {
impl FileSource for PathBuf {
type Reader = File;
type SliceReader = Take<Self::Reader>;
async fn size(&self) -> Result<u64, Error> {
Ok(tokio::fs::metadata(self).await?.len())
}
async fn reader(&self) -> Result<Self::Reader, Error> {
Ok(open_file(self).await?)
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
let mut r = FileSource::reader(self).await?;
r.seek(SeekFrom::Start(position)).await?;
Ok(r.take(size))
}
}
impl FileSource for Arc<[u8]> {
type Reader = std::io::Cursor<Self>;
type SliceReader = Take<Self::Reader>;
async fn size(&self) -> Result<u64, Error> {
Ok(self.len() as u64)
}
async fn reader(&self) -> Result<Self::Reader, Error> {
Ok(std::io::Cursor::new(self.clone()))
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
let mut r = FileSource::reader(self).await?;
r.seek(SeekFrom::Start(position)).await?;
Ok(r.take(size))
}
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
use tokio::io::AsyncWriteExt;
@@ -272,12 +312,18 @@ pub struct Section<S> {
}
impl<S: ArchiveSource> FileSource for Section<S> {
type Reader = S::FetchReader;
type SliceReader = S::FetchReader;
async fn size(&self) -> Result<u64, Error> {
Ok(self.size)
}
async fn reader(&self) -> Result<Self::Reader, Error> {
self.source.fetch(self.position, self.size).await
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
self.source
.fetch(self.position + position, min(size, self.size))
.await
}
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
self.source.copy_to(self.position, self.size, w).await
}
@@ -342,12 +388,16 @@ impl<S: FileSource> From<TmpSource<S>> for DynFileSource {
impl<S: FileSource> FileSource for TmpSource<S> {
type Reader = <S as FileSource>::Reader;
type SliceReader = <S as FileSource>::SliceReader;
async fn size(&self) -> Result<u64, Error> {
self.source.size().await
}
async fn reader(&self) -> Result<Self::Reader, Error> {
self.source.reader().await
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
self.source.slice(position, size).await
}
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
mut w: &mut W,

36
core/startos/src/s9pk/rpc.rs
View File

@@ -15,14 +15,36 @@ use crate::s9pk::v2::pack::ImageConfig;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::util::io::{create_file, open_file, TmpDir};
use crate::util::serde::{apply_expr, HandlerExtSerde};
use crate::util::Apply;
pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
pub fn s9pk() -> ParentHandler<CliContext> {
ParentHandler::new()
.subcommand("pack", from_fn_async(super::v2::pack::pack).no_display())
.subcommand(
"list-ingredients",
from_fn_async(super::v2::pack::list_ingredients).with_custom_display_fn(
|_, ingredients| {
ingredients
.into_iter()
.map(Some)
.apply(|i| itertools::intersperse(i, None))
.for_each(|i| {
if let Some(p) = i {
print!("{}", p.display())
} else {
print!(" ")
}
});
println!();
Ok(())
},
),
)
.subcommand("edit", edit())
.subcommand("inspect", inspect())
.subcommand("convert", from_fn_async(convert).no_display())
}
#[derive(Deserialize, Serialize, Parser)]
@@ -193,3 +215,17 @@ async fn inspect_manifest(
.await?;
Ok(s9pk.as_manifest().clone())
}
async fn convert(ctx: CliContext, S9pkPath { s9pk: s9pk_path }: S9pkPath) -> Result<(), Error> {
let mut s9pk = super::load(
MultiCursorFile::from(open_file(&s9pk_path).await?),
|| ctx.developer_key().cloned(),
None,
)
.await?;
let tmp_path = s9pk_path.with_extension("s9pk.tmp");
s9pk.serialize(&mut create_file(&tmp_path).await?, true)
.await?;
tokio::fs::rename(tmp_path, s9pk_path).await?;
Ok(())
}

6
core/startos/src/s9pk/v2/compat.rs
View File

@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::sync::Arc;
@@ -199,8 +199,9 @@ impl From<ManifestV1> for Manifest {
let default_url = value.upstream_repo.clone();
Self {
id: value.id,
title: value.title,
title: value.title.into(),
version: ExtendedVersion::from(value.version).into(),
satisfies: BTreeSet::new(),
release_notes: value.release_notes,
license: value.license.into(),
wrapper_repo: value.wrapper_repo,
@@ -233,6 +234,7 @@ impl From<ManifestV1> for Manifest {
DepInfo {
description: value.description,
optional: !value.requirement.required(),
s9pk: None,
},
)
})

15
core/startos/src/s9pk/v2/manifest.rs
View File

@@ -31,8 +31,10 @@ fn current_version() -> Version {
#[ts(export)]
pub struct Manifest {
pub id: PackageId,
pub title: String,
#[ts(type = "string")]
pub title: InternedString,
pub version: VersionString,
pub satisfies: BTreeSet<VersionString>,
pub release_notes: String,
#[ts(type = "string")]
pub license: InternedString, // type of license
@@ -81,6 +83,15 @@ impl Manifest {
expected.check_file("LICENSE.md")?;
expected.check_file("instructions.md")?;
expected.check_file("javascript.squashfs")?;
for (dependency, _) in &self.dependencies.0 {
let dep_path = Path::new("dependencies").join(dependency);
let _ = expected.check_file(dep_path.join("metadata.json"));
let _ = expected.check_stem(dep_path.join("icon"), |ext| {
ext.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |mime| mime.starts_with("image/"))
});
}
for assets in &self.assets {
expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?;
}
@@ -148,7 +159,7 @@ impl Manifest {
#[ts(export)]
pub struct HardwareRequirements {
#[serde(default)]
#[ts(type = "{ [key: string]: string }")] // TODO more specific key
#[ts(type = "{ device?: string, processor?: string }")]
pub device: BTreeMap<String, Regex>,
#[ts(type = "number | null")]
pub ram: Option<u64>,

108
core/startos/src/s9pk/v2/mod.rs
View File

@@ -6,10 +6,10 @@ use imbl_value::InternedString;
use models::{mime, DataUrl, PackageId};
use tokio::fs::File;
use crate::dependencies::DependencyMetadata;
use crate::prelude::*;
use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::file_contents::FileContents;
use crate::s9pk::merkle_archive::sink::Sink;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{
@@ -18,6 +18,7 @@ use crate::s9pk::merkle_archive::source::{
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::v2::pack::{ImageSource, PackSource};
use crate::util::io::{open_file, TmpDir};
use crate::util::serde::IoFormat;
const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
@@ -33,6 +34,10 @@ pub mod pack;
├── icon.<ext>
├── LICENSE.md
├── instructions.md
├── dependencies
│ └── <id>
│ ├── metadata.json
│ └── icon.<ext>
├── javascript.squashfs
├── assets
│ └── <id>.squashfs (xN)
@@ -52,9 +57,10 @@ fn priority(s: &str) -> Option<usize> {
a if Path::new(a).file_stem() == Some(OsStr::new("icon")) => Some(1),
"LICENSE.md" => Some(2),
"instructions.md" => Some(3),
"javascript.squashfs" => Some(4),
"assets" => Some(5),
"images" => Some(6),
"dependencies" => Some(4),
"javascript.squashfs" => Some(5),
"assets" => Some(6),
"images" => Some(7),
_ => None,
}
}
@@ -101,22 +107,16 @@ impl<S: FileSource + Clone> S9pk<S> {
filter.keep_checked(self.archive.contents_mut())
}
pub async fn icon(&self) -> Result<(InternedString, FileContents<S>), Error> {
pub async fn icon(&self) -> Result<(InternedString, Entry<S>), Error> {
let mut best_icon = None;
for (path, icon) in self
.archive
.contents()
.with_stem("icon")
.filter(|(p, _)| {
Path::new(&*p)
.extension()
.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |e| e.starts_with("image/"))
})
.filter_map(|(k, v)| v.into_file().map(|f| (k, f)))
{
let size = icon.size().await?;
for (path, icon) in self.archive.contents().with_stem("icon").filter(|(p, v)| {
Path::new(&*p)
.extension()
.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
}) {
let size = icon.expect_file()?.size().await?;
best_icon = match best_icon {
Some((s, a)) if s >= size => Some((s, a)),
_ => Some((size, (path, icon))),
@@ -134,7 +134,75 @@ impl<S: FileSource + Clone> S9pk<S> {
.and_then(|e| e.to_str())
.and_then(mime)
.unwrap_or("image/png");
DataUrl::from_reader(mime, contents.reader().await?, Some(contents.size().await?)).await
Ok(DataUrl::from_vec(
mime,
contents.expect_file()?.to_vec(contents.hash()).await?,
))
}
pub async fn dependency_icon(
&self,
id: &PackageId,
) -> Result<Option<(InternedString, Entry<S>)>, Error> {
let mut best_icon = None;
for (path, icon) in self
.archive
.contents()
.get_path(Path::new("dependencies").join(id))
.and_then(|p| p.as_directory())
.into_iter()
.flat_map(|d| {
d.with_stem("icon").filter(|(p, v)| {
Path::new(&*p)
.extension()
.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
})
})
{
let size = icon.expect_file()?.size().await?;
best_icon = match best_icon {
Some((s, a)) if s >= size => Some((s, a)),
_ => Some((size, (path, icon))),
};
}
Ok(best_icon.map(|(_, a)| a))
}
pub async fn dependency_icon_data_url(
&self,
id: &PackageId,
) -> Result<Option<DataUrl<'static>>, Error> {
let Some((name, contents)) = self.dependency_icon(id).await? else {
return Ok(None);
};
let mime = Path::new(&*name)
.extension()
.and_then(|e| e.to_str())
.and_then(mime)
.unwrap_or("image/png");
Ok(Some(DataUrl::from_vec(
mime,
contents.expect_file()?.to_vec(contents.hash()).await?,
)))
}
pub async fn dependency_metadata(
&self,
id: &PackageId,
) -> Result<Option<DependencyMetadata>, Error> {
if let Some(entry) = self
.archive
.contents()
.get_path(Path::new("dependencies").join(id).join("metadata.json"))
{
Ok(Some(IoFormat::Json.from_slice(
&entry.expect_file()?.to_vec(entry.hash()).await?,
)?))
} else {
Ok(None)
}
}
pub async fn serialize<W: Sink>(&mut self, w: &mut W, verify: bool) -> Result<(), Error> {

224
core/startos/src/s9pk/v2/pack.rs
View File

@@ -1,6 +1,4 @@
use std::collections::BTreeSet;
use std::ffi::OsStr;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -10,25 +8,29 @@ use futures::{FutureExt, TryStreamExt};
use imbl_value::InternedString;
use models::{ImageId, PackageId, VersionString};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncRead;
use tokio::process::Command;
use tokio::sync::OnceCell;
use tokio_stream::wrappers::ReadDirStream;
use tracing::{debug, warn};
use ts_rs::TS;
use crate::context::CliContext;
use crate::dependencies::DependencyMetadata;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{
into_dyn_read, ArchiveSource, DynFileSource, FileSource, TmpSource,
into_dyn_read, ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource,
};
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::{create_file, open_file, TmpDir};
use crate::util::Invoke;
use crate::util::serde::IoFormat;
use crate::util::{new_guid, Invoke, PathOrUrl};
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
@@ -83,7 +85,8 @@ pub enum PackSource {
Squashfs(Arc<SqfsDir>),
}
impl FileSource for PackSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
type Reader = DynRead;
type SliceReader = DynRead;
async fn size(&self) -> Result<u64, Error> {
match self {
Self::Buffered(a) => Ok(a.len() as u64),
@@ -102,11 +105,23 @@ impl FileSource for PackSource {
}
async fn reader(&self) -> Result<Self::Reader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
Self::File(f) => Ok(into_dyn_read(open_file(f).await?)),
Self::Buffered(a) => Ok(into_dyn_read(FileSource::reader(a).await?)),
Self::File(f) => Ok(into_dyn_read(FileSource::reader(f).await?)),
Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
}
}
async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(FileSource::slice(a, position, size).await?)),
Self::File(f) => Ok(into_dyn_read(FileSource::slice(f, position, size).await?)),
Self::Squashfs(dir) => dir
.file()
.await?
.fetch(position, size)
.await
.map(into_dyn_read),
}
}
}
impl From<PackSource> for DynFileSource {
fn from(value: PackSource) -> Self {
@@ -150,24 +165,71 @@ impl PackParams {
if let Some(icon) = &self.icon {
Ok(icon.clone())
} else {
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
Err(e) => Ok({
let path = x.path();
if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
Ok(path)
} else {
Err(e)
}
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
.try_filter(|x| {
ready(
x.path()
.file_stem()
.map_or(false, |s| s.eq_ignore_ascii_case("icon")),
)
})
}}).await?
.map_err(Error::from)
.try_fold(
Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
|acc, x| async move {
match acc {
Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
Err(e) => Ok({
let path = x.path();
if path
.file_stem()
.map_or(false, |s| s.eq_ignore_ascii_case("icon"))
{
Ok(path)
} else {
Err(e)
}
}),
}
},
)
.await?
}
}
fn license(&self) -> PathBuf {
self.license
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("LICENSE.md"))
async fn license(&self) -> Result<PathBuf, Error> {
if let Some(license) = &self.license {
Ok(license.clone())
} else {
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
.try_filter(|x| {
ready(
x.path()
.file_stem()
.map_or(false, |s| s.eq_ignore_ascii_case("license")),
)
})
.map_err(Error::from)
.try_fold(
Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
|acc, x| async move {
match acc {
Ok(_) => Err(Error::new(eyre!("multiple licenses found in working directory, please specify which to use with `--license`"), ErrorKind::InvalidRequest)),
Err(e) => Ok({
let path = x.path();
if path
.file_stem()
.map_or(false, |s| s.eq_ignore_ascii_case("license"))
{
Ok(path)
} else {
Err(e)
}
}),
}
},
)
.await?
}
}
fn instructions(&self) -> PathBuf {
self.instructions
@@ -282,6 +344,15 @@ pub enum ImageSource {
DockerTag(String),
}
impl ImageSource {
pub fn ingredients(&self) -> Vec<PathBuf> {
match self {
Self::Packed => Vec::new(),
Self::DockerBuild { dockerfile, .. } => {
vec![dockerfile.clone().unwrap_or_else(|| "Dockerfile".into())]
}
Self::DockerTag(_) => Vec::new(),
}
}
#[instrument(skip_all)]
pub fn load<'a, S: From<TmpSource<PackSource>> + FileSource + Clone>(
&'a self,
@@ -320,7 +391,7 @@ impl ImageSource {
format!("--platform=linux/{arch}")
};
// docker buildx build ${path} -o type=image,name=start9/${id}
let tag = format!("start9/{id}/{image_id}:{version}");
let tag = format!("start9/{id}/{image_id}:{}", new_guid());
Command::new(CONTAINER_TOOL)
.arg("build")
.arg(workdir)
@@ -501,7 +572,7 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
"LICENSE.md".into(),
Entry::file(TmpSource::new(
tmp_dir.clone(),
PackSource::File(params.license()),
PackSource::File(params.license().await?),
)),
);
files.insert(
@@ -541,6 +612,54 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
s9pk.load_images(tmp_dir.clone()).await?;
let mut to_insert = Vec::new();
for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
if let Some(s9pk) = dependency.s9pk.take() {
let s9pk = match s9pk {
PathOrUrl::Path(path) => {
S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
.await?
.into_dyn()
}
PathOrUrl::Url(url) => {
if url.scheme() == "http" || url.scheme() == "https" {
S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
None,
)
.await?
.into_dyn()
} else {
return Err(Error::new(
eyre!("unknown scheme: {}", url.scheme()),
ErrorKind::InvalidRequest,
));
}
}
};
let dep_path = Path::new("dependencies").join(id);
to_insert.push((
dep_path.join("metadata.json"),
Entry::file(PackSource::Buffered(
IoFormat::Json
.to_vec(&DependencyMetadata {
title: s9pk.as_manifest().title.clone(),
})?
.into(),
)),
));
let icon = s9pk.icon().await?;
to_insert.push((
dep_path.join(&*icon.0),
Entry::file(PackSource::Buffered(
icon.1.expect_file()?.to_vec(icon.1.hash()).await?.into(),
)),
));
} else {
warn!("no s9pk specified for {id}, leaving metadata empty");
}
}
s9pk.validate_and_filter(None)?;
s9pk.serialize(
@@ -555,3 +674,58 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
Ok(())
}
#[instrument(skip_all)]
pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<PathBuf>, Error> {
let js_path = params.javascript().join("index.js");
let manifest: Manifest = match async {
serde_json::from_slice(
&Command::new("node")
.arg("-e")
.arg(format!(
"console.log(JSON.stringify(require('{}').manifest))",
js_path.display()
))
.invoke(ErrorKind::Javascript)
.await?,
)
.with_kind(ErrorKind::Deserialization)
}
.await
{
Ok(m) => m,
Err(e) => {
warn!("failed to load manifest: {e}");
debug!("{e:?}");
return Ok(vec![
js_path,
params.icon().await?,
params.license().await?,
params.instructions(),
]);
}
};
let mut ingredients = vec![
js_path,
params.icon().await?,
params.license().await?,
params.instructions(),
];
for (_, dependency) in manifest.dependencies.0 {
if let Some(PathOrUrl::Path(p)) = dependency.s9pk {
ingredients.push(p);
}
}
let assets_dir = params.assets();
for assets in manifest.assets {
ingredients.push(assets_dir.join(assets));
}
for image in manifest.images.values() {
ingredients.extend(image.source.ingredients());
}
Ok(ingredients)
}

156
core/startos/src/service/service_effect_handler.rs
View File

@@ -1147,18 +1147,14 @@ enum DependencyRequirement {
#[ts(type = "string[]")]
health_checks: BTreeSet<HealthCheckId>,
#[ts(type = "string")]
version_spec: VersionRange,
#[ts(type = "string")]
registry_url: Url,
version_range: VersionRange,
},
#[serde(rename_all = "camelCase")]
Exists {
#[ts(type = "string")]
id: PackageId,
#[ts(type = "string")]
version_spec: VersionRange,
#[ts(type = "string")]
registry_url: Url,
version_range: VersionRange,
},
}
// filebrowser:exists,bitcoind:running:foo+bar+baz
@@ -1168,8 +1164,7 @@ impl FromStr for DependencyRequirement {
match s.split_once(':') {
Some((id, "e")) | Some((id, "exists")) => Ok(Self::Exists {
id: id.parse()?,
registry_url: "".parse()?, // TODO
version_spec: "*".parse()?, // TODO
version_range: "*".parse()?, // TODO
}),
Some((id, rest)) => {
let health_checks = match rest.split_once(':') {
@@ -1192,15 +1187,13 @@ impl FromStr for DependencyRequirement {
Ok(Self::Running {
id: id.parse()?,
health_checks,
registry_url: "".parse()?, // TODO
version_spec: "*".parse()?, // TODO
version_range: "*".parse()?, // TODO
})
}
None => Ok(Self::Running {
id: s.parse()?,
health_checks: BTreeSet::new(),
registry_url: "".parse()?, // TODO
version_spec: "*".parse()?, // TODO
version_range: "*".parse()?, // TODO
}),
}
}
@@ -1234,59 +1227,20 @@ async fn set_dependencies(
let mut deps = BTreeMap::new();
for dependency in dependencies {
let (dep_id, kind, registry_url, version_spec) = match dependency {
DependencyRequirement::Exists {
id,
registry_url,
version_spec,
} => (
id,
CurrentDependencyKind::Exists,
registry_url,
version_spec,
),
let (dep_id, kind, version_range) = match dependency {
DependencyRequirement::Exists { id, version_range } => {
(id, CurrentDependencyKind::Exists, version_range)
}
DependencyRequirement::Running {
id,
health_checks,
registry_url,
version_spec,
version_range,
} => (
id,
CurrentDependencyKind::Running { health_checks },
registry_url,
version_spec,
version_range,
),
};
let (icon, title) = match async {
let remote_s9pk = S9pk::deserialize(
&Arc::new(
HttpSource::new(
context.seed.ctx.client.clone(),
registry_url
.join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?,
)
.await?,
),
None, // TODO
)
.await?;
let icon = remote_s9pk.icon_data_url().await?;
Ok::<_, Error>((icon, remote_s9pk.as_manifest().title.clone()))
}
.await
{
Ok(a) => a,
Err(e) => {
tracing::error!("Error fetching remote s9pk: {e}");
tracing::debug!("{e:?}");
(
DataUrl::from_slice("image/png", include_bytes!("../install/package-icon.png")),
dep_id.to_string(),
)
}
};
let config_satisfied =
if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await {
context
@@ -1300,17 +1254,25 @@ async fn set_dependencies(
} else {
true
};
deps.insert(
dep_id,
CurrentDependencyInfo {
kind,
registry_url,
version_spec,
icon,
title,
config_satisfied,
},
);
let info = CurrentDependencyInfo {
title: context
.seed
.persistent_container
.s9pk
.dependency_metadata(&dep_id)
.await?
.map(|m| m.title),
icon: context
.seed
.persistent_container
.s9pk
.dependency_icon_data_url(&dep_id)
.await?,
kind,
version_range,
config_satisfied,
};
deps.insert(dep_id, info);
}
context
.seed
@@ -1343,23 +1305,19 @@ async fn get_dependencies(context: EffectContext) -> Result<Vec<DependencyRequir
.into_iter()
.map(|(id, current_dependency_info)| {
let CurrentDependencyInfo {
registry_url,
version_spec,
version_range,
kind,
..
} = current_dependency_info;
Ok::<_, Error>(match kind {
CurrentDependencyKind::Exists => DependencyRequirement::Exists {
id,
registry_url,
version_spec,
},
CurrentDependencyKind::Exists => {
DependencyRequirement::Exists { id, version_range }
}
CurrentDependencyKind::Running { health_checks } => {
DependencyRequirement::Running {
id,
health_checks,
version_spec,
registry_url,
version_range,
}
}
})
@@ -1381,7 +1339,8 @@ struct CheckDependenciesResult {
package_id: PackageId,
is_installed: bool,
is_running: bool,
health_checks: Vec<HealthCheckResult>,
config_satisfied: bool,
health_checks: BTreeMap<HealthCheckId, HealthCheckResult>,
#[ts(type = "string | null")]
version: Option<exver::ExtendedVersion>,
}
@@ -1415,24 +1374,27 @@ async fn check_dependencies(
package_id,
is_installed: false,
is_running: false,
health_checks: vec![],
config_satisfied: false,
health_checks: Default::default(),
version: None,
});
continue;
};
let installed_version = package
.as_state_info()
.as_manifest(ManifestPreference::New)
.as_version()
.de()?
.into_version();
let manifest = package.as_state_info().as_manifest(ManifestPreference::New);
let installed_version = manifest.as_version().de()?.into_version();
let satisfies = manifest.as_satisfies().de()?;
let version = Some(installed_version.clone());
if !installed_version.satisfies(&dependency_info.version_spec) {
if ![installed_version]
.into_iter()
.chain(satisfies.into_iter().map(|v| v.into_version()))
.any(|v| v.satisfies(&dependency_info.version_range))
{
results.push(CheckDependenciesResult {
package_id,
is_installed: false,
is_running: false,
health_checks: vec![],
config_satisfied: false,
health_checks: Default::default(),
version,
});
continue;
@@ -1444,17 +1406,23 @@ async fn check_dependencies(
} else {
false
};
let health_checks = status
.health()
.cloned()
.unwrap_or_default()
.into_iter()
.map(|(_, val)| val)
.collect();
let health_checks =
if let CurrentDependencyKind::Running { health_checks } = &dependency_info.kind {
status
.health()
.cloned()
.unwrap_or_default()
.into_iter()
.filter(|(id, _)| health_checks.contains(id))
.collect()
} else {
Default::default()
};
results.push(CheckDependenciesResult {
package_id,
is_installed,
is_running,
config_satisfied: dependency_info.config_satisfied,
health_checks,
version,
});

51
core/startos/src/util/mod.rs
View File

@@ -1,13 +1,16 @@
use std::collections::{BTreeMap, VecDeque};
use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::process::Stdio;
use std::str::FromStr;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use ::serde::{Deserialize, Serialize};
use async_trait::async_trait;
use color_eyre::eyre::{self, eyre};
use fd_lock_rs::FdLock;
@@ -24,9 +27,12 @@ use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
use tracing::instrument;
use ts_rs::TS;
use url::Url;
use crate::shutdown::Shutdown;
use crate::util::io::create_file;
use crate::util::serde::{deserialize_from_str, serialize_display};
use crate::{Error, ErrorKind, ResultExt as _};
pub mod actor;
pub mod clap;
@@ -648,3 +654,48 @@ pub fn new_guid() -> InternedString {
&buf,
))
}
#[derive(Debug, Clone, TS)]
#[ts(type = "string")]
pub enum PathOrUrl {
Path(PathBuf),
Url(Url),
}
impl FromStr for PathOrUrl {
type Err = <PathBuf as FromStr>::Err;
fn from_str(s: &str) -> Result<Self, Self::Err> {
if let Ok(url) = s.parse::<Url>() {
if url.scheme() == "file" {
Ok(Self::Path(url.path().parse()?))
} else {
Ok(Self::Url(url))
}
} else {
Ok(Self::Path(s.parse()?))
}
}
}
impl fmt::Display for PathOrUrl {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Path(p) => write!(f, "file://{}", p.display()),
Self::Url(u) => write!(f, "{u}"),
}
}
}
impl<'de> Deserialize<'de> for PathOrUrl {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: ::serde::Deserializer<'de>,
{
deserialize_from_str(deserializer)
}
}
impl Serialize for PathOrUrl {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: ::serde::Serializer,
{
serialize_display(self, serializer)
}
}

51
core/startos/src/util/net.rs
View File

@@ -1,19 +1,25 @@
use core::fmt;
use std::borrow::Cow;
use std::sync::Mutex;
use axum::extract::ws::{self, CloseFrame};
use futures::Future;
use futures::{Future, Stream, StreamExt};
use crate::prelude::*;
pub trait WebSocketExt {
fn normal_close(
self,
msg: impl Into<Cow<'static, str>>,
) -> impl Future<Output = Result<(), Error>>;
msg: impl Into<Cow<'static, str>> + Send,
) -> impl Future<Output = Result<(), Error>> + Send;
fn close_result(
self,
result: Result<impl Into<Cow<'static, str>> + Send, impl fmt::Display + Send>,
) -> impl Future<Output = Result<(), Error>> + Send;
}
impl WebSocketExt for ws::WebSocket {
async fn normal_close(mut self, msg: impl Into<Cow<'static, str>>) -> Result<(), Error> {
async fn normal_close(mut self, msg: impl Into<Cow<'static, str>> + Send) -> Result<(), Error> {
self.send(ws::Message::Close(Some(CloseFrame {
code: 1000,
reason: msg.into(),
@@ -21,4 +27,41 @@ impl WebSocketExt for ws::WebSocket {
.await
.with_kind(ErrorKind::Network)
}
async fn close_result(
mut self,
result: Result<impl Into<Cow<'static, str>> + Send, impl fmt::Display + Send>,
) -> Result<(), Error> {
match result {
Ok(msg) => self
.send(ws::Message::Close(Some(CloseFrame {
code: 1000,
reason: msg.into(),
})))
.await
.with_kind(ErrorKind::Network),
Err(e) => self
.send(ws::Message::Close(Some(CloseFrame {
code: 1011,
reason: e.to_string().into(),
})))
.await
.with_kind(ErrorKind::Network),
}
}
}
pub struct SyncBody(Mutex<axum::body::BodyDataStream>);
impl From<axum::body::Body> for SyncBody {
fn from(value: axum::body::Body) -> Self {
SyncBody(Mutex::new(value.into_data_stream()))
}
}
impl Stream for SyncBody {
type Item = <axum::body::BodyDataStream as Stream>::Item;
fn poll_next(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
self.0.lock().unwrap().poll_next_unpin(cx)
}
}

31
core/startos/src/util/rpc.rs
View File

@@ -12,7 +12,7 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::ArchiveSource;
use crate::util::io::{open_file, ParallelBlake3Writer};
use crate::util::serde::Base16;
use crate::util::Apply;
use crate::util::{Apply, PathOrUrl};
use crate::CAP_10_MiB;
pub fn util<C: Context>() -> ParentHandler<C> {
@@ -45,21 +45,20 @@ pub async fn b3sum(
}
b3sum_source(file).await
}
if let Ok(url) = file.parse::<Url>() {
if url.scheme() == "file" {
b3sum_file(url.path(), allow_mmap).await
} else if url.scheme() == "http" || url.scheme() == "https" {
HttpSource::new(ctx.client.clone(), url)
.await?
.apply(b3sum_source)
.await
} else {
return Err(Error::new(
eyre!("unknown scheme: {}", url.scheme()),
ErrorKind::InvalidRequest,
));
match file.parse::<PathOrUrl>()? {
PathOrUrl::Path(path) => b3sum_file(path, allow_mmap).await,
PathOrUrl::Url(url) => {
if url.scheme() == "http" || url.scheme() == "https" {
HttpSource::new(ctx.client.clone(), url)
.await?
.apply(b3sum_source)
.await
} else {
Err(Error::new(
eyre!("unknown scheme: {}", url.scheme()),
ErrorKind::InvalidRequest,
))
}
}
} else {
b3sum_file(file, allow_mmap).await
}
}

18
core/startos/src/util/serde.rs
View File

@@ -1,8 +1,10 @@
use std::any::Any;
use std::collections::VecDeque;
use std::marker::PhantomData;
use std::ops::Deref;
use std::str::FromStr;
use base64::Engine;
use clap::builder::ValueParserFactory;
use clap::{ArgMatches, CommandFactory, FromArgMatches};
use color_eyre::eyre::eyre;
@@ -37,7 +39,11 @@ pub fn deserialize_from_str<
{
type Value = T;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a parsable string")
write!(
formatter,
"a string that can be parsed as a {}",
std::any::type_name::<T>()
)
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
@@ -988,18 +994,24 @@ impl<T: AsRef<[u8]>> Serialize for Base32<T> {
}
}
pub const BASE64: base64::engine::GeneralPurpose = base64::engine::GeneralPurpose::new(
&base64::alphabet::STANDARD,
base64::engine::GeneralPurposeConfig::new(),
);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, TS)]
#[ts(type = "string", concrete(T = Vec<u8>))]
pub struct Base64<T>(pub T);
impl<T: AsRef<[u8]>> std::fmt::Display for Base64<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&base64::encode(self.0.as_ref()))
f.write_str(&BASE64.encode(self.0.as_ref()))
}
}
impl<T: TryFrom<Vec<u8>>> FromStr for Base64<T> {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
base64::decode(&s)
BASE64
.decode(&s)
.with_kind(ErrorKind::Deserialization)?
.apply(TryFrom::try_from)
.map(Self)