From fd7c2fbe9343eafee33f818e8a4650ce639fe0f2 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Fri, 31 May 2024 12:13:23 -0600 Subject: [PATCH] Feature/registry package index (#2623) * include system images in compat s9pk * wip * wip * update types * wip * fix signature serialization * Add SignatureHeader conversions * finish display impl for get --------- Co-authored-by: Shadowy Super Coder --- Makefile | 3 +- core/Cargo.lock | 13 + core/helpers/src/script_dir.rs | 8 +- core/models/src/errors.rs | 2 + core/models/src/id/image.rs | 4 +- core/models/src/version.rs | 44 +- core/startos/Cargo.toml | 1 + core/startos/src/backup/target/mod.rs | 8 +- core/startos/src/context/cli.rs | 6 +- core/startos/src/db/model/public.rs | 7 +- core/startos/src/disk/util.rs | 4 +- core/startos/src/install/mod.rs | 34 +- core/startos/src/lib.rs | 4 +- core/startos/src/logs.rs | 6 +- core/startos/src/lxc/mod.rs | 10 +- core/startos/src/net/static_server.rs | 6 +- core/startos/src/registry/admin.rs | 36 +- core/startos/src/registry/asset.rs | 44 +- core/startos/src/registry/auth.rs | 109 ++-- core/startos/src/registry/context.rs | 54 +- core/startos/src/registry/device_info.rs | 199 ++++++++ core/startos/src/registry/mod.rs | 28 +- core/startos/src/registry/os/asset/add.rs | 253 ++++------ core/startos/src/registry/os/asset/get.rs | 36 +- core/startos/src/registry/os/asset/sign.rs | 76 ++- core/startos/src/registry/os/index.rs | 23 +- core/startos/src/registry/os/version/mod.rs | 23 +- .../startos/src/registry/os/version/signer.rs | 21 +- core/startos/src/registry/package/add.rs | 170 +++++++ core/startos/src/registry/package/get.rs | 387 ++++++++++++++ core/startos/src/registry/package/index.rs | 163 ++++++ core/startos/src/registry/package/mod.rs | 29 ++ core/startos/src/registry/signer.rs | 477 ------------------ .../src/registry/signer/commitment/blake3.rs | 50 ++ .../signer/commitment/merkle_archive.rs | 98 ++++ .../src/registry/signer/commitment/mod.rs | 25 + .../src/registry/signer/commitment/request.rs | 102 ++++ core/startos/src/registry/signer/mod.rs | 154 ++++++ .../src/registry/signer/sign/ed25519.rs | 34 ++ core/startos/src/registry/signer/sign/mod.rs | 348 +++++++++++++ core/startos/src/rpc_continuations.rs | 37 +- core/startos/src/s9pk/{v1 => }/git_hash.rs | 35 +- .../s9pk/merkle_archive/directory_contents.rs | 15 +- .../src/s9pk/merkle_archive/file_contents.rs | 10 +- core/startos/src/s9pk/merkle_archive/hash.rs | 7 +- core/startos/src/s9pk/merkle_archive/mod.rs | 147 ++++-- core/startos/src/s9pk/merkle_archive/sink.rs | 54 +- .../src/s9pk/merkle_archive/source/http.rs | 163 ++++-- .../src/s9pk/merkle_archive/source/mod.rs | 74 ++- .../source/multi_cursor_file.rs | 121 +++-- core/startos/src/s9pk/merkle_archive/test.rs | 16 +- .../src/s9pk/merkle_archive/write_queue.rs | 2 + core/startos/src/s9pk/mod.rs | 1 + core/startos/src/s9pk/rpc.rs | 4 +- core/startos/src/s9pk/v1/manifest.rs | 164 +++++- core/startos/src/s9pk/v1/mod.rs | 1 - core/startos/src/s9pk/v1/reader.rs | 6 +- core/startos/src/s9pk/v2/compat.rs | 81 +-- core/startos/src/s9pk/v2/manifest.rs | 26 +- core/startos/src/s9pk/v2/mod.rs | 16 +- core/startos/src/service/mod.rs | 8 +- .../src/service/service_effect_handler.rs | 15 +- core/startos/src/service/service_map.rs | 2 +- core/startos/src/update/mod.rs | 28 +- core/startos/src/upload.rs | 68 ++- core/startos/src/util/io.rs | 118 ++++- core/startos/src/util/lshw.rs | 7 +- core/startos/src/util/mod.rs | 2 +- core/startos/src/util/rpc.rs | 60 +-- core/startos/src/util/serde.rs | 41 +- core/startos/src/version/mod.rs | 6 +- core/startos/src/volume.rs | 4 +- sdk/lib/osBindings/AcceptSigners.ts | 4 +- sdk/lib/osBindings/AddAdminParams.ts | 3 +- sdk/lib/osBindings/AddAssetParams.ts | 13 +- sdk/lib/osBindings/AddVersionParams.ts | 3 +- .../{SignerKey.ts => AnySignature.ts} | 3 +- sdk/lib/osBindings/AnySigningKey.ts | 3 + sdk/lib/osBindings/AnyVerifyingKey.ts | 3 + sdk/lib/osBindings/Blake3Commitment.ts | 4 + sdk/lib/osBindings/Category.ts | 4 + sdk/lib/osBindings/FullIndex.ts | 10 +- sdk/lib/osBindings/GetOsAssetParams.ts | 3 +- sdk/lib/osBindings/GetPackageParams.ts | 10 + sdk/lib/osBindings/GetPackageResponse.ts | 9 + sdk/lib/osBindings/GetPackageResponseFull.ts | 8 + sdk/lib/osBindings/{Pem.ts => Guid.ts} | 2 +- sdk/lib/osBindings/HardwareRequirements.ts | 2 +- .../osBindings/ListVersionSignersParams.ts | 3 +- sdk/lib/osBindings/Manifest.ts | 6 +- ...ignature.ts => MerkleArchiveCommitment.ts} | 9 +- sdk/lib/osBindings/OsIndex.ts | 3 +- sdk/lib/osBindings/OsVersionInfo.ts | 10 +- sdk/lib/osBindings/PackageDetailLevel.ts | 3 + sdk/lib/osBindings/PackageIndex.ts | 9 + sdk/lib/osBindings/PackageInfo.ts | 9 + sdk/lib/osBindings/PackageInfoShort.ts | 3 + sdk/lib/osBindings/PackageVersionInfo.ts | 25 + sdk/lib/osBindings/RegistryAsset.ts | 9 +- sdk/lib/osBindings/RemoveVersionParams.ts | 3 +- ...1SignatureInfo.ts => RequestCommitment.ts} | 10 +- sdk/lib/osBindings/ServerInfo.ts | 3 +- sdk/lib/osBindings/SetMainStatus.ts | 4 +- sdk/lib/osBindings/SetMainStatusStatus.ts | 3 + sdk/lib/osBindings/SignAssetParams.ts | 7 +- sdk/lib/osBindings/Signature.ts | 4 - sdk/lib/osBindings/SignatureInfo.ts | 7 - sdk/lib/osBindings/SignerInfo.ts | 4 +- sdk/lib/osBindings/Version.ts | 3 + sdk/lib/osBindings/VersionSignerParams.ts | 4 +- sdk/lib/osBindings/index.ts | 24 +- .../show/additional/additional.component.html | 11 - .../ui/src/app/services/api/api.fixures.ts | 3 - 113 files changed, 3265 insertions(+), 1436 deletions(-) create mode 100644 core/startos/src/registry/device_info.rs create mode 100644 core/startos/src/registry/package/add.rs create mode 100644 core/startos/src/registry/package/get.rs create mode 100644 core/startos/src/registry/package/index.rs create mode 100644 core/startos/src/registry/package/mod.rs delete mode 100644 core/startos/src/registry/signer.rs create mode 100644 core/startos/src/registry/signer/commitment/blake3.rs create mode 100644 core/startos/src/registry/signer/commitment/merkle_archive.rs create mode 100644 core/startos/src/registry/signer/commitment/mod.rs create mode 100644 core/startos/src/registry/signer/commitment/request.rs create mode 100644 core/startos/src/registry/signer/mod.rs create mode 100644 core/startos/src/registry/signer/sign/ed25519.rs create mode 100644 core/startos/src/registry/signer/sign/mod.rs rename core/startos/src/s9pk/{v1 => }/git_hash.rs (52%) rename sdk/lib/osBindings/{SignerKey.ts => AnySignature.ts} (55%) create mode 100644 sdk/lib/osBindings/AnySigningKey.ts create mode 100644 sdk/lib/osBindings/AnyVerifyingKey.ts create mode 100644 sdk/lib/osBindings/Blake3Commitment.ts create mode 100644 sdk/lib/osBindings/Category.ts create mode 100644 sdk/lib/osBindings/GetPackageParams.ts create mode 100644 sdk/lib/osBindings/GetPackageResponse.ts create mode 100644 sdk/lib/osBindings/GetPackageResponseFull.ts rename sdk/lib/osBindings/{Pem.ts => Guid.ts} (80%) rename sdk/lib/osBindings/{Blake3Ed25519Signature.ts => MerkleArchiveCommitment.ts} (52%) create mode 100644 sdk/lib/osBindings/PackageDetailLevel.ts create mode 100644 sdk/lib/osBindings/PackageIndex.ts create mode 100644 sdk/lib/osBindings/PackageInfo.ts create mode 100644 sdk/lib/osBindings/PackageInfoShort.ts create mode 100644 sdk/lib/osBindings/PackageVersionInfo.ts rename sdk/lib/osBindings/{Blake3Ed2551SignatureInfo.ts => RequestCommitment.ts} (51%) create mode 100644 sdk/lib/osBindings/SetMainStatusStatus.ts delete mode 100644 sdk/lib/osBindings/Signature.ts delete mode 100644 sdk/lib/osBindings/SignatureInfo.ts create mode 100644 sdk/lib/osBindings/Version.ts diff --git a/Makefile b/Makefile index 1dffd3629..ff5b9c4ad 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,6 @@ install: $(ALL_TARGETS) $(call mkdir,$(DESTDIR)/usr/lib/startos/system-images) $(call cp,system-images/compat/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/compat.tar) $(call cp,system-images/utils/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/utils.tar) - $(call cp,system-images/binfmt/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/binfmt.tar) $(call cp,firmware/$(PLATFORM),$(DESTDIR)/usr/lib/startos/firmware) @@ -184,7 +183,7 @@ container-runtime/node_modules: container-runtime/package.json container-runtime npm --prefix container-runtime ci touch container-runtime/node_modules -sdk/lib/osBindings: $(shell core/startos/bindings) +sdk/lib/osBindings: core/startos/bindings mkdir -p sdk/lib/osBindings ls core/startos/bindings/*.ts | sed 's/core\/startos\/bindings\/\([^.]*\)\.ts/export { \1 } from ".\/\1";/g' > core/startos/bindings/index.ts npm --prefix sdk exec -- prettier --config ./sdk/package.json -w ./core/startos/bindings/*.ts diff --git a/core/Cargo.lock b/core/Cargo.lock index 920390d65..4ae02e98e 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1153,10 +1153,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", + "der_derive", "pem-rfc7468", "zeroize", ] +[[package]] +name = "der_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "deranged" version = "0.3.11" @@ -4603,6 +4615,7 @@ dependencies = [ "cookie 0.18.1", "cookie_store", "current_platform", + "der", "digest 0.10.7", "divrem", "ed25519 2.2.3", diff --git a/core/helpers/src/script_dir.rs b/core/helpers/src/script_dir.rs index d90051899..5cedd419f 100644 --- a/core/helpers/src/script_dir.rs +++ b/core/helpers/src/script_dir.rs @@ -1,10 +1,14 @@ use std::path::{Path, PathBuf}; -use models::{PackageId, Version}; +use models::{PackageId, VersionString}; pub const PKG_SCRIPT_DIR: &str = "package-data/scripts"; -pub fn script_dir>(datadir: P, pkg_id: &PackageId, version: &Version) -> PathBuf { +pub fn script_dir>( + datadir: P, + pkg_id: &PackageId, + version: &VersionString, +) -> PathBuf { datadir .as_ref() .join(&*PKG_SCRIPT_DIR) diff --git a/core/models/src/errors.rs b/core/models/src/errors.rs index 340fd15ca..95416ec80 100644 --- a/core/models/src/errors.rs +++ b/core/models/src/errors.rs @@ -89,6 +89,7 @@ pub enum ErrorKind { Timeout = 71, Lxc = 72, Cancelled = 73, + Git = 74, } impl ErrorKind { pub fn as_str(&self) -> &'static str { @@ -167,6 +168,7 @@ impl ErrorKind { Timeout => "Timeout Error", Lxc => "LXC Error", Cancelled => "Cancelled", + Git => "Git Error", } } } diff --git a/core/models/src/id/image.rs b/core/models/src/id/image.rs index bbb0a601e..69a04f880 100644 --- a/core/models/src/id/image.rs +++ b/core/models/src/id/image.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use serde::{Deserialize, Deserializer, Serialize}; use ts_rs::TS; -use crate::{Id, InvalidId, PackageId, Version}; +use crate::{Id, InvalidId, PackageId, VersionString}; #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, TS)] #[ts(type = "string")] @@ -21,7 +21,7 @@ impl std::fmt::Display for ImageId { } } impl ImageId { - pub fn for_package(&self, pkg_id: &PackageId, pkg_version: Option<&Version>) -> String { + pub fn for_package(&self, pkg_id: &PackageId, pkg_version: Option<&VersionString>) -> String { format!( "start9/{}/{}:{}", pkg_id, diff --git a/core/models/src/version.rs b/core/models/src/version.rs index 012f362aa..48871e3a1 100644 --- a/core/models/src/version.rs +++ b/core/models/src/version.rs @@ -6,12 +6,12 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use ts_rs::TS; #[derive(Debug, Clone, TS)] -#[ts(type = "string")] -pub struct Version { +#[ts(type = "string", rename = "Version")] +pub struct VersionString { version: emver::Version, string: String, } -impl Version { +impl VersionString { pub fn as_str(&self) -> &str { self.string.as_str() } @@ -19,76 +19,76 @@ impl Version { self.version } } -impl std::fmt::Display for Version { +impl std::fmt::Display for VersionString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.string) } } -impl std::str::FromStr for Version { +impl std::str::FromStr for VersionString { type Err = ::Err; fn from_str(s: &str) -> Result { - Ok(Version { + Ok(VersionString { string: s.to_owned(), version: s.parse()?, }) } } -impl From for Version { +impl From for VersionString { fn from(v: emver::Version) -> Self { - Version { + VersionString { string: v.to_string(), version: v, } } } -impl From for emver::Version { - fn from(v: Version) -> Self { +impl From for emver::Version { + fn from(v: VersionString) -> Self { v.version } } -impl Default for Version { +impl Default for VersionString { fn default() -> Self { Self::from(emver::Version::default()) } } -impl Deref for Version { +impl Deref for VersionString { type Target = emver::Version; fn deref(&self) -> &Self::Target { &self.version } } -impl AsRef for Version { +impl AsRef for VersionString { fn as_ref(&self) -> &emver::Version { &self.version } } -impl AsRef for Version { +impl AsRef for VersionString { fn as_ref(&self) -> &str { self.as_str() } } -impl PartialEq for Version { - fn eq(&self, other: &Version) -> bool { +impl PartialEq for VersionString { + fn eq(&self, other: &VersionString) -> bool { self.version.eq(&other.version) } } -impl Eq for Version {} -impl PartialOrd for Version { +impl Eq for VersionString {} +impl PartialOrd for VersionString { fn partial_cmp(&self, other: &Self) -> Option { self.version.partial_cmp(&other.version) } } -impl Ord for Version { +impl Ord for VersionString { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.version.cmp(&other.version) } } -impl Hash for Version { +impl Hash for VersionString { fn hash(&self, state: &mut H) { self.version.hash(state) } } -impl<'de> Deserialize<'de> for Version { +impl<'de> Deserialize<'de> for VersionString { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -98,7 +98,7 @@ impl<'de> Deserialize<'de> for Version { Ok(Self { string, version }) } } -impl Serialize for Version { +impl Serialize for VersionString { fn serialize(&self, serializer: S) -> Result where S: Serializer, diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index d88b51f96..bd064a167 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -73,6 +73,7 @@ console-subscriber = { version = "0.2", optional = true } cookie = "0.18.0" cookie_store = "0.20.0" current_platform = "0.2.0" +der = { version = "0.7.9", features = ["derive", "pem"] } digest = "0.10.7" divrem = "1.0.0" ed25519 = { version = "2.2.3", features = ["pkcs8", "pem", "alloc"] } diff --git a/core/startos/src/backup/target/mod.rs b/core/startos/src/backup/target/mod.rs index 4d00a2501..2c7a28d94 100644 --- a/core/startos/src/backup/target/mod.rs +++ b/core/startos/src/backup/target/mod.rs @@ -29,7 +29,7 @@ use crate::util::clap::FromStrParser; use crate::util::serde::{ deserialize_from_str, display_serializable, serialize_display, HandlerExtSerde, WithIoFormat, }; -use crate::util::Version; +use crate::util::VersionString; pub mod cifs; @@ -194,7 +194,7 @@ pub async fn list(ctx: RpcContext) -> Result>, pub package_backups: BTreeMap, } @@ -203,8 +203,8 @@ pub struct BackupInfo { #[serde(rename_all = "camelCase")] pub struct PackageBackupInfo { pub title: String, - pub version: Version, - pub os_version: Version, + pub version: VersionString, + pub os_version: VersionString, pub timestamp: DateTime, } diff --git a/core/startos/src/context/cli.rs b/core/startos/src/context/cli.rs index 166ab3fd7..014457b67 100644 --- a/core/startos/src/context/cli.rs +++ b/core/startos/src/context/cli.rs @@ -21,7 +21,7 @@ use crate::context::config::{local_config_path, ClientConfig}; use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; use crate::prelude::*; -use crate::rpc_continuations::RequestGuid; +use crate::rpc_continuations::Guid; #[derive(Debug)] pub struct CliContextSeed { @@ -164,7 +164,7 @@ impl CliContext { pub async fn ws_continuation( &self, - guid: RequestGuid, + guid: Guid, ) -> Result>, Error> { let mut url = self.base_url.clone(); let ws_scheme = match url.scheme() { @@ -194,7 +194,7 @@ impl CliContext { pub async fn rest_continuation( &self, - guid: RequestGuid, + guid: Guid, body: reqwest::Body, headers: reqwest::header::HeaderMap, ) -> Result { diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index fe85056f9..e5257f2a4 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -21,7 +21,7 @@ use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr}; use crate::prelude::*; use crate::progress::FullProgress; use crate::util::cpupower::Governor; -use crate::util::Version; +use crate::util::VersionString; use crate::version::{Current, VersionT}; use crate::{ARCH, PLATFORM}; @@ -109,8 +109,7 @@ pub struct ServerInfo { pub platform: InternedString, pub id: String, pub hostname: String, - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, #[ts(type = "string | null")] pub last_backup: Option>, #[ts(type = "string")] @@ -136,7 +135,7 @@ pub struct ServerInfo { #[serde(default)] pub zram: bool, pub governor: Option, - pub smtp: Option + pub smtp: Option, } #[derive(Debug, Deserialize, Serialize, HasModel, TS)] diff --git a/core/startos/src/disk/util.rs b/core/startos/src/disk/util.rs index b0bc00a5d..a98c52418 100644 --- a/core/startos/src/disk/util.rs +++ b/core/startos/src/disk/util.rs @@ -20,7 +20,7 @@ use super::mount::guard::TmpMountGuard; use crate::disk::mount::guard::GenericMountGuard; use crate::disk::OsPartitionInfo; use crate::util::serde::IoFormat; -use crate::util::{Invoke, Version}; +use crate::util::{Invoke, VersionString}; use crate::{Error, ResultExt as _}; #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -56,7 +56,7 @@ pub struct PartitionInfo { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct EmbassyOsRecoveryInfo { - pub version: Version, + pub version: VersionString, pub full: bool, pub password_hash: Option, pub wrapped_key: Option, diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index f5cff36d9..c4e4452b7 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use clap::builder::ValueParserFactory; @@ -22,7 +23,7 @@ use crate::context::{CliContext, RpcContext}; use crate::db::model::package::{ManifestPreference, PackageState, PackageStateMatchModelRef}; use crate::prelude::*; use crate::progress::{FullProgress, PhasedProgressBar}; -use crate::rpc_continuations::{RequestGuid, RpcContinuation}; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::s9pk::manifest::PackageId; use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::S9pk; @@ -139,15 +140,18 @@ pub async fn install( let registry = registry.unwrap_or_else(|| crate::DEFAULT_MARKETPLACE.parse().unwrap()); let version_priority = version_priority.unwrap_or_default(); let s9pk = S9pk::deserialize( - &HttpSource::new( - ctx.client.clone(), - format!( - "{}/package/v0/{}.s9pk?spec={}&version-priority={}", - registry, id, version, version_priority, + &Arc::new( + HttpSource::new( + ctx.client.clone(), + format!( + "{}/package/v0/{}.s9pk?spec={}&version-priority={}", + registry, id, version, version_priority, + ) + .parse()?, ) - .parse()?, - ) - .await?, + .await?, + ), + None, // TODO true, ) .await?; @@ -170,8 +174,8 @@ pub async fn install( #[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct SideloadResponse { - pub upload: RequestGuid, - pub progress: RequestGuid, + pub upload: Guid, + pub progress: Guid, } #[instrument(skip_all)] @@ -179,7 +183,7 @@ pub async fn sideload(ctx: RpcContext) -> Result { let (upload, file) = upload(&ctx).await?; let (id_send, id_recv) = oneshot::channel(); let (err_send, err_recv) = oneshot::channel(); - let progress = RequestGuid::new(); + let progress = Guid::new(); let db = ctx.db.clone(); let mut sub = db .subscribe( @@ -256,7 +260,11 @@ pub async fn sideload(ctx: RpcContext) -> Result { .await; tokio::spawn(async move { if let Err(e) = async { - let s9pk = S9pk::deserialize(&file, true).await?; + let s9pk = S9pk::deserialize( + &file, None, // TODO + true, + ) + .await?; let _ = id_send.send(s9pk.as_manifest().id.clone()); ctx.services .install(ctx.clone(), s9pk, None::) diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index 3152fcab3..d60a2db24 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -1,6 +1,8 @@ pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com"; // pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com"; -pub const BUFFER_SIZE: usize = 1024; +pub const CAP_1_KiB: usize = 1024; +pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; +pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; pub const HOST_IP: [u8; 4] = [172, 18, 0, 1]; pub const TARGET: &str = current_platform::CURRENT_PLATFORM; lazy_static::lazy_static! { diff --git a/core/startos/src/logs.rs b/core/startos/src/logs.rs index 340b04b8b..bb1198451 100644 --- a/core/startos/src/logs.rs +++ b/core/startos/src/logs.rs @@ -26,7 +26,7 @@ use crate::context::{CliContext, RpcContext}; use crate::error::ResultExt; use crate::lxc::ContainerId; use crate::prelude::*; -use crate::rpc_continuations::{RequestGuid, RpcContinuation, RpcContinuations}; +use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations}; use crate::util::serde::Reversible; use crate::util::Invoke; @@ -118,7 +118,7 @@ pub struct LogResponse { #[serde(rename_all = "camelCase")] pub struct LogFollowResponse { start_cursor: Option, - guid: RequestGuid, + guid: Guid, } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] @@ -581,7 +581,7 @@ pub async fn follow_logs>( first_entry = Some(entry); } - let guid = RequestGuid::new(); + let guid = Guid::new(); ctx.as_ref() .add( guid.clone(), diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index 084364c4a..f64ecebe7 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -33,7 +33,7 @@ use crate::disk::mount::filesystem::{MountType, ReadWrite}; use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard}; use crate::disk::mount::util::unmount; use crate::prelude::*; -use crate::rpc_continuations::{RequestGuid, RpcContinuation}; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::util::clap::FromStrParser; use crate::util::rpc_client::UnixRpcClient; use crate::util::{new_guid, Invoke}; @@ -433,7 +433,7 @@ pub struct ConnectParams { pub async fn connect_rpc( ctx: RpcContext, ConnectParams { guid }: ConnectParams, -) -> Result { +) -> Result { connect( &ctx, ctx.dev.lxc.lock().await.get(&guid).ok_or_else(|| { @@ -443,11 +443,11 @@ pub async fn connect_rpc( .await } -pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result { +pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result { use axum::extract::ws::Message; let rpc = container.connect_rpc(Some(Duration::from_secs(30))).await?; - let guid = RequestGuid::new(); + let guid = Guid::new(); ctx.rpc_continuations .add( guid.clone(), @@ -504,7 +504,7 @@ pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result Result<(), Error> { +pub async fn connect_cli(ctx: &CliContext, guid: Guid) -> Result<(), Error> { use futures::SinkExt; use tokio_tungstenite::tungstenite::Message; diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index 85737598b..fff1731ce 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -30,7 +30,7 @@ use crate::middleware::auth::{Auth, HasValidSession}; use crate::middleware::cors::Cors; use crate::middleware::db::SyncDb; use crate::middleware::diagnostic::DiagnosticMode; -use crate::rpc_continuations::RequestGuid; +use crate::rpc_continuations::Guid; use crate::{diagnostic_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt}; const NOT_FOUND: &[u8] = b"Not Found"; @@ -136,7 +136,7 @@ pub fn main_ui_server_router(ctx: RpcContext) -> Router { let ctx = ctx.clone(); move |x::Path(path): x::Path, ws: axum::extract::ws::WebSocketUpgrade| async move { - match RequestGuid::from(&path) { + match Guid::from(&path) { None => { tracing::debug!("No Guid Path"); bad_request() @@ -159,7 +159,7 @@ pub fn main_ui_server_router(ctx: RpcContext) -> Router { .path() .strip_prefix("/rest/rpc/") .unwrap_or_default(); - match RequestGuid::from(&path) { + match Guid::from(&path) { None => { tracing::debug!("No Guid Path"); bad_request() diff --git a/core/startos/src/registry/admin.rs b/core/startos/src/registry/admin.rs index b09fa05aa..cd795e5cd 100644 --- a/core/startos/src/registry/admin.rs +++ b/core/startos/src/registry/admin.rs @@ -10,10 +10,11 @@ use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; use crate::registry::context::RegistryContext; -use crate::registry::signer::{ContactInfo, SignerInfo, SignerKey}; +use crate::registry::signer::sign::AnyVerifyingKey; +use crate::registry::signer::{ContactInfo, SignerInfo}; use crate::registry::RegistryDatabase; -use crate::rpc_continuations::RequestGuid; -use crate::util::serde::{display_serializable, HandlerExtSerde, Pem, WithIoFormat}; +use crate::rpc_continuations::Guid; +use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat}; pub fn admin_api() -> ParentHandler { ParentHandler::new() @@ -48,8 +49,8 @@ fn signers_api() -> ParentHandler { .subcommand("add", from_fn_async(cli_add_signer).no_display()) } -impl Model> { - pub fn get_signer(&self, key: &SignerKey) -> Result { +impl Model> { + pub fn get_signer(&self, key: &AnyVerifyingKey) -> Result { self.as_entries()? .into_iter() .map(|(guid, s)| Ok::<_, Error>((guid, s.as_keys().de()?))) @@ -60,7 +61,7 @@ impl Model> { .ok_or_else(|| Error::new(eyre!("unknown signer"), ErrorKind::Authorization)) } - pub fn get_signer_info(&self, key: &SignerKey) -> Result<(RequestGuid, SignerInfo), Error> { + pub fn get_signer_info(&self, key: &AnyVerifyingKey) -> Result<(Guid, SignerInfo), Error> { self.as_entries()? .into_iter() .map(|(guid, s)| Ok::<_, Error>((guid, s.de()?))) @@ -88,17 +89,15 @@ impl Model> { ErrorKind::InvalidRequest, )); } - self.insert(&RequestGuid::new(), signer) + self.insert(&Guid::new(), signer) } } -pub async fn list_signers( - ctx: RegistryContext, -) -> Result, Error> { +pub async fn list_signers(ctx: RegistryContext) -> Result, Error> { ctx.db.peek().await.into_index().into_signers().de() } -pub fn display_signers(params: WithIoFormat, signers: BTreeMap) { +pub fn display_signers(params: WithIoFormat, signers: BTreeMap) { use prettytable::*; if let Some(format) = params.format { @@ -137,8 +136,8 @@ pub struct CliAddSignerParams { pub name: String, #[arg(long = "contact", short = 'c')] pub contact: Vec, - #[arg(long = "ed25519-key")] - pub ed25519_keys: Vec>, + #[arg(long = "key")] + pub keys: Vec, pub database: Option, } @@ -151,7 +150,7 @@ pub async fn cli_add_signer( CliAddSignerParams { name, contact, - ed25519_keys, + keys, database, }, .. @@ -160,7 +159,7 @@ pub async fn cli_add_signer( let signer = SignerInfo { name, contact, - keys: ed25519_keys.into_iter().map(SignerKey::Ed25519).collect(), + keys: keys.into_iter().collect(), }; if let Some(database) = database { TypedPatchDb::::load(PatchDb::open(database).await?) @@ -181,8 +180,7 @@ pub async fn cli_add_signer( #[serde(rename_all = "camelCase")] #[ts(export)] pub struct AddAdminParams { - #[ts(type = "string")] - pub signer: RequestGuid, + pub signer: Guid, } pub async fn add_admin( @@ -206,7 +204,7 @@ pub async fn add_admin( #[command(rename_all = "kebab-case")] #[serde(rename_all = "camelCase")] pub struct CliAddAdminParams { - pub signer: RequestGuid, + pub signer: Guid, pub database: Option, } @@ -242,7 +240,7 @@ pub async fn cli_add_admin( Ok(()) } -pub async fn list_admins(ctx: RegistryContext) -> Result, Error> { +pub async fn list_admins(ctx: RegistryContext) -> Result, Error> { let db = ctx.db.peek().await; let admins = db.as_admins().de()?; Ok(db diff --git a/core/startos/src/registry/asset.rs b/core/startos/src/registry/asset.rs index c9acced45..ea37b2309 100644 --- a/core/startos/src/registry/asset.rs +++ b/core/startos/src/registry/asset.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use reqwest::Client; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWrite; @@ -5,32 +7,48 @@ use ts_rs::TS; use url::Url; use crate::prelude::*; -use crate::registry::signer::{AcceptSigners, FileValidator, SignatureInfo}; +use crate::registry::signer::commitment::{Commitment, Digestable}; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey}; +use crate::registry::signer::AcceptSigners; +use crate::s9pk::merkle_archive::source::http::HttpSource; -#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[derive(Debug, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -#[model = "Model"] #[ts(export)] -pub struct RegistryAsset { +pub struct RegistryAsset { #[ts(type = "string")] pub url: Url, - pub signature_info: SignatureInfo, + pub commitment: Commitment, + pub signatures: HashMap, } -impl AsRef for RegistryAsset { - fn as_ref(&self) -> &RegistryAsset { - self +impl RegistryAsset { + pub fn all_signers(&self) -> AcceptSigners { + AcceptSigners::All( + self.signatures + .keys() + .cloned() + .map(AcceptSigners::Signer) + .collect(), + ) } } -impl RegistryAsset { - pub fn validate(&self, accept: AcceptSigners) -> Result { - self.signature_info.validate(accept) +impl RegistryAsset { + pub fn validate(&self, context: &str, mut accept: AcceptSigners) -> Result<&Commitment, Error> { + for (signer, signature) in &self.signatures { + accept.process_signature(signer, &self.commitment, context, signature)?; + } + accept.try_accept()?; + Ok(&self.commitment) } +} +impl Commitment<&'a HttpSource>> RegistryAsset { pub async fn download( &self, client: Client, dst: &mut (impl AsyncWrite + Unpin + Send + ?Sized), - validator: &FileValidator, ) -> Result<(), Error> { - validator.download(self.url.clone(), client, dst).await + self.commitment + .copy_to(&HttpSource::new(client, self.url.clone()).await?, dst) + .await } } diff --git a/core/startos/src/registry/auth.rs b/core/startos/src/registry/auth.rs index 187750870..741090b4d 100644 --- a/core/startos/src/registry/auth.rs +++ b/core/startos/src/registry/auth.rs @@ -6,19 +6,23 @@ use axum::body::Body; use axum::extract::Request; use axum::response::Response; use chrono::Utc; -use http_body_util::BodyExt; +use http::HeaderValue; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{Middleware, RpcRequest, RpcResponse}; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha512}; use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; use ts_rs::TS; +use url::Url; use crate::prelude::*; use crate::registry::context::RegistryContext; -use crate::registry::signer::SignerKey; -use crate::util::serde::{Base64, Pem}; +use crate::registry::signer::commitment::request::RequestCommitment; +use crate::registry::signer::commitment::Commitment; +use crate::registry::signer::sign::{ + AnySignature, AnySigningKey, AnyVerifyingKey, SignatureScheme, +}; +use crate::util::serde::Base64; pub const AUTH_SIG_HEADER: &str = "X-StartOS-Registry-Auth-Sig"; @@ -34,7 +38,7 @@ pub struct Metadata { #[derive(Clone)] pub struct Auth { nonce_cache: Arc>>, // for replay protection - signer: Option>, + signer: Option>, } impl Auth { pub fn new() -> Self { @@ -68,41 +72,57 @@ pub struct RegistryAdminLogRecord { pub name: String, #[ts(type = "{ id: string | number | null; method: string; params: any }")] pub request: RpcRequest, - pub key: SignerKey, + pub key: AnyVerifyingKey, } #[derive(Serialize, Deserialize)] pub struct SignatureHeader { - pub timestamp: i64, - pub nonce: u64, #[serde(flatten)] - pub signer: SignerKey, - pub signature: Base64<[u8; 64]>, + pub commitment: RequestCommitment, + pub signer: AnyVerifyingKey, + pub signature: AnySignature, } impl SignatureHeader { - pub fn sign_ed25519( - key: &ed25519_dalek::SigningKey, - body: &[u8], - context: &str, - ) -> Result { + pub fn to_header(&self) -> HeaderValue { + let mut url: Url = "http://localhost".parse().unwrap(); + self.commitment.append_query(&mut url); + url.query_pairs_mut() + .append_pair("signer", &self.signer.to_string()); + url.query_pairs_mut() + .append_pair("signature", &self.signature.to_string()); + HeaderValue::from_str(url.query().unwrap_or_default()).unwrap() + } + pub fn from_header(header: &HeaderValue) -> Result { + let url: Url = format!( + "http://localhost/?{}", + header.to_str().with_kind(ErrorKind::Utf8)? + ) + .parse()?; + let query: BTreeMap<_, _> = url.query_pairs().collect(); + Ok(Self { + commitment: RequestCommitment::from_query(&url)?, + signer: query.get("signer").or_not_found("signer")?.parse()?, + signature: query.get("signature").or_not_found("signature")?.parse()?, + }) + } + pub fn sign(signer: &AnySigningKey, body: &[u8], context: &str) -> Result { let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs() as i64) .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1); let nonce = rand::random(); - let signer = SignerKey::Ed25519(Pem(key.verifying_key())); - let mut hasher = Sha512::new(); - hasher.update(&i64::to_be_bytes(timestamp)); - hasher.update(&u64::to_be_bytes(nonce)); - hasher.update(body); - let signature = Base64( - key.sign_prehashed(hasher, Some(context.as_bytes()))? - .to_bytes(), - ); - Ok(Self { + let commitment = RequestCommitment { timestamp, nonce, - signer, + size: body.len() as u64, + blake3: Base64(*blake3::hash(body).as_bytes()), + }; + let signature = signer + .scheme() + .sign_commitment(&signer, &commitment, context)?; + Ok(Self { + commitment, + signer: signer.verifying_key(), signature, }) } @@ -120,43 +140,40 @@ impl Middleware for Auth { async { let request = request; let SignatureHeader { - timestamp, - nonce, + commitment, signer, signature, - } = serde_urlencoded::from_str( + } = SignatureHeader::from_header( request .headers() .get(AUTH_SIG_HEADER) .or_not_found("missing X-StartOS-Registry-Auth-Sig") - .with_kind(ErrorKind::InvalidRequest)? - .to_str() - .with_kind(ErrorKind::Utf8)?, - ) - .with_kind(ErrorKind::Deserialization)?; + .with_kind(ErrorKind::InvalidRequest)?, + )?; + + signer.scheme().verify_commitment( + &signer, + &commitment, + &ctx.hostname, + &signature, + )?; + let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs() as i64) .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1); - if (now - timestamp).abs() > 30 { + if (now - commitment.timestamp).abs() > 30 { return Err(Error::new( eyre!("timestamp not within 30s of now"), ErrorKind::InvalidSignature, )); } - self.handle_nonce(nonce).await?; - let body = std::mem::replace(request.body_mut(), Body::empty()) - .collect() - .await - .with_kind(ErrorKind::Network)? - .to_bytes(); - let mut verifier = signer.verifier(); - verifier.update(&i64::to_be_bytes(timestamp)); - verifier.update(&u64::to_be_bytes(nonce)); - verifier.update(&body); + self.handle_nonce(commitment.nonce).await?; + + let mut body = Vec::with_capacity(commitment.size as usize); + commitment.copy_to(request, &mut body).await?; *request.body_mut() = Body::from(body); - verifier.verify(&*signature, &ctx.hostname)?; Ok(signer) } .await diff --git a/core/startos/src/registry/context.rs b/core/startos/src/registry/context.rs index dac833434..99d60307b 100644 --- a/core/startos/src/registry/context.rs +++ b/core/startos/src/registry/context.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use clap::Parser; use imbl_value::InternedString; use patch_db::PatchDb; +use reqwest::{Client, Proxy}; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{CallRemote, Context, Empty}; use serde::{Deserialize, Serialize}; @@ -17,9 +18,10 @@ use crate::context::config::{ContextConfig, CONFIG_PATH}; use crate::context::{CliContext, RpcContext}; use crate::prelude::*; use crate::registry::auth::{SignatureHeader, AUTH_SIG_HEADER}; +use crate::registry::device_info::{DeviceInfo, DEVICE_INFO_HEADER}; +use crate::registry::signer::sign::AnySigningKey; use crate::registry::RegistryDatabase; use crate::rpc_continuations::RpcContinuations; -use crate::version::VersionT; #[derive(Debug, Clone, Default, Deserialize, Serialize, Parser)] #[serde(rename_all = "kebab-case")] @@ -31,6 +33,8 @@ pub struct RegistryConfig { pub listen: Option, #[arg(short = 'h', long = "hostname")] pub hostname: InternedString, + #[arg(short = 'p', long = "proxy")] + pub tor_proxy: Option, #[arg(short = 'd', long = "datadir")] pub datadir: Option, } @@ -58,6 +62,7 @@ pub struct RegistryContextSeed { pub db: TypedPatchDb, pub datadir: PathBuf, pub rpc_continuations: RpcContinuations, + pub client: Client, pub shutdown: Sender<()>, } @@ -81,6 +86,11 @@ impl RegistryContext { || async { Ok(Default::default()) }, ) .await?; + let tor_proxy_url = config + .tor_proxy + .clone() + .map(Ok) + .unwrap_or_else(|| "socks5h://localhost:9050".parse())?; Ok(Self(Arc::new(RegistryContextSeed { hostname: config.hostname.clone(), listen: config @@ -89,6 +99,16 @@ impl RegistryContext { db, datadir, rpc_continuations: RpcContinuations::new(), + client: Client::builder() + .proxy(Proxy::custom(move |url| { + if url.host_str().map_or(false, |h| h.ends_with(".onion")) { + Some(tor_proxy_url.clone()) + } else { + None + } + })) + .build() + .with_kind(crate::ErrorKind::ParseUrl)?, shutdown, }))) } @@ -145,12 +165,11 @@ impl CallRemote for CliContext { .header(CONTENT_LENGTH, body.len()) .header( AUTH_SIG_HEADER, - serde_urlencoded::to_string(&SignatureHeader::sign_ed25519( - self.developer_key()?, + SignatureHeader::sign( + &AnySigningKey::Ed25519(self.developer_key()?.clone()), &body, &host, - )?) - .with_kind(ErrorKind::Serialization)?, + )?.to_header(), ) .body(body) .send() @@ -171,29 +190,6 @@ impl CallRemote for CliContext { } } -fn hardware_header(ctx: &RpcContext) -> String { - let mut url: Url = "http://localhost".parse().unwrap(); - url.query_pairs_mut() - .append_pair( - "os.version", - &crate::version::Current::new().semver().to_string(), - ) - .append_pair( - "os.compat", - &crate::version::Current::new().compat().to_string(), - ) - .append_pair("os.arch", &*crate::PLATFORM) - .append_pair("hardware.arch", &*crate::ARCH) - .append_pair("hardware.ram", &ctx.hardware.ram.to_string()); - - for hw in &ctx.hardware.devices { - url.query_pairs_mut() - .append_pair(&format!("hardware.device.{}", hw.class()), hw.product()); - } - - url.query().unwrap_or_default().to_string() -} - impl CallRemote for RpcContext { async fn call_remote( &self, @@ -221,7 +217,7 @@ impl CallRemote for RpcContext { .header(CONTENT_TYPE, "application/json") .header(ACCEPT, "application/json") .header(CONTENT_LENGTH, body.len()) - .header("X-StartOS-Hardware", &hardware_header(self)) + .header(DEVICE_INFO_HEADER, DeviceInfo::from(self).to_header_value()) .body(body) .send() .await?; diff --git a/core/startos/src/registry/device_info.rs b/core/startos/src/registry/device_info.rs new file mode 100644 index 000000000..7da5bd8b9 --- /dev/null +++ b/core/startos/src/registry/device_info.rs @@ -0,0 +1,199 @@ +use std::collections::BTreeMap; +use std::convert::identity; +use std::ops::Deref; + +use axum::extract::Request; +use axum::response::Response; +use emver::{Version, VersionRange}; +use http::HeaderValue; +use imbl_value::InternedString; +use rpc_toolkit::{Middleware, RpcRequest, RpcResponse}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use url::Url; + +use crate::context::RpcContext; +use crate::prelude::*; +use crate::registry::context::RegistryContext; +use crate::util::VersionString; +use crate::version::VersionT; + +pub const DEVICE_INFO_HEADER: &str = "X-StartOS-Device-Info"; + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct DeviceInfo { + pub os: OsInfo, + pub hardware: HardwareInfo, +} +impl From<&RpcContext> for DeviceInfo { + fn from(value: &RpcContext) -> Self { + Self { + os: OsInfo::from(value), + hardware: HardwareInfo::from(value), + } + } +} +impl DeviceInfo { + pub fn to_header_value(&self) -> HeaderValue { + let mut url: Url = "http://localhost".parse().unwrap(); + url.query_pairs_mut() + .append_pair("os.version", &self.os.version.to_string()) + .append_pair("os.compat", &self.os.compat.to_string()) + .append_pair("os.platform", &*self.os.platform) + .append_pair("hardware.arch", &*self.hardware.arch) + .append_pair("hardware.ram", &self.hardware.ram.to_string()); + + for (class, products) in &self.hardware.devices { + for product in products { + url.query_pairs_mut() + .append_pair(&format!("hardware.device.{}", class), product); + } + } + + HeaderValue::from_str(url.query().unwrap_or_default()).unwrap() + } + pub fn from_header_value(header: &HeaderValue) -> Result { + let url: Url = format!( + "http://localhost/?{}", + header.to_str().with_kind(ErrorKind::ParseUrl)? + ) + .parse()?; + let query: BTreeMap<_, _> = url.query_pairs().collect(); + Ok(Self { + os: OsInfo { + version: query + .get("os.version") + .or_not_found("os.version")? + .parse()?, + compat: query.get("os.compat").or_not_found("os.compat")?.parse()?, + platform: query + .get("os.platform") + .or_not_found("os.platform")? + .deref() + .into(), + }, + hardware: HardwareInfo { + arch: query + .get("hardware.arch") + .or_not_found("hardware.arch")? + .parse()?, + ram: query + .get("hardware.ram") + .or_not_found("hardware.ram")? + .parse()?, + devices: identity(query) + .split_off("hardware.device.") + .into_iter() + .filter_map(|(k, v)| { + k.strip_prefix("hardware.device.") + .map(|k| (k.into(), v.into_owned())) + }) + .fold(BTreeMap::new(), |mut acc, (k, v)| { + let mut devs = acc.remove(&k).unwrap_or_default(); + devs.push(v); + acc.insert(k, devs); + acc + }), + }, + }) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct OsInfo { + #[ts(as = "VersionString")] + pub version: Version, + #[ts(type = "string")] + pub compat: VersionRange, + #[ts(type = "string")] + pub platform: InternedString, +} +impl From<&RpcContext> for OsInfo { + fn from(_: &RpcContext) -> Self { + Self { + version: crate::version::Current::new().semver(), + compat: crate::version::Current::new().compat().clone(), + platform: InternedString::intern(&*crate::PLATFORM), + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct HardwareInfo { + #[ts(type = "string")] + pub arch: InternedString, + #[ts(type = "number")] + pub ram: u64, + #[ts(as = "BTreeMap::>")] + pub devices: BTreeMap>, +} + +impl From<&RpcContext> for HardwareInfo { + fn from(value: &RpcContext) -> Self { + Self { + arch: InternedString::intern(&**crate::ARCH), + ram: value.hardware.ram, + devices: value + .hardware + .devices + .iter() + .fold(BTreeMap::new(), |mut acc, dev| { + let mut devs = acc.remove(dev.class()).unwrap_or_default(); + devs.push(dev.product().to_owned()); + acc.insert(dev.class().into(), devs); + acc + }), + } + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Metadata { + #[serde(default)] + get_device_info: bool, +} + +#[derive(Clone)] +pub struct DeviceInfoMiddleware { + device_info: Option, +} +impl DeviceInfoMiddleware { + pub fn new() -> Self { + Self { device_info: None } + } +} + +impl Middleware for DeviceInfoMiddleware { + type Metadata = Metadata; + async fn process_http_request( + &mut self, + _: &RegistryContext, + request: &mut Request, + ) -> Result<(), Response> { + self.device_info = request.headers_mut().remove(DEVICE_INFO_HEADER); + Ok(()) + } + async fn process_rpc_request( + &mut self, + _: &RegistryContext, + metadata: Self::Metadata, + request: &mut RpcRequest, + ) -> Result<(), RpcResponse> { + async move { + if metadata.get_device_info { + if let Some(device_info) = &self.device_info { + request.params["__device_info"] = + to_value(&DeviceInfo::from_header_value(device_info)?)?; + } + } + + Ok::<_, Error>(()) + } + .await + .map_err(|e| RpcResponse::from_result(Err(e))) + } +} diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index c063e5823..656edf337 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -3,20 +3,23 @@ use std::net::SocketAddr; use axum::Router; use futures::future::ready; +use models::DataUrl; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler, Server}; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::context::{CliContext}; +use crate::context::CliContext; use crate::middleware::cors::Cors; use crate::net::static_server::{bad_request, not_found, server_error}; use crate::net::web_server::WebServer; use crate::prelude::*; use crate::registry::auth::Auth; -use crate::registry::context::{RegistryContext}; +use crate::registry::context::RegistryContext; +use crate::registry::device_info::DeviceInfoMiddleware; use crate::registry::os::index::OsIndex; +use crate::registry::package::index::PackageIndex; use crate::registry::signer::SignerInfo; -use crate::rpc_continuations::RequestGuid; +use crate::rpc_continuations::Guid; use crate::util::serde::HandlerExtSerde; pub mod admin; @@ -24,26 +27,29 @@ pub mod asset; pub mod auth; pub mod context; pub mod db; +pub mod device_info; pub mod os; +pub mod package; pub mod signer; #[derive(Debug, Default, Deserialize, Serialize, HasModel)] #[serde(rename_all = "camelCase")] #[model = "Model"] pub struct RegistryDatabase { - pub admins: BTreeSet, + pub admins: BTreeSet, pub index: FullIndex, } +impl RegistryDatabase {} #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] #[ts(export)] pub struct FullIndex { - // pub package: PackageIndex, + pub icon: Option>, + pub package: PackageIndex, pub os: OsIndex, - #[ts(as = "BTreeMap::")] - pub signers: BTreeMap, + pub signers: BTreeMap, } pub async fn get_full_index(ctx: RegistryContext) -> Result { @@ -59,6 +65,7 @@ pub fn registry_api() -> ParentHandler { .with_call_remote::(), ) .subcommand("os", os::os_api::()) + .subcommand("package", package::package_api::()) .subcommand("admin", admin::admin_api::()) .subcommand("db", db::db_api::()) } @@ -72,7 +79,8 @@ pub fn registry_server_router(ctx: RegistryContext) -> Router { post( Server::new(move || ready(Ok(ctx.clone())), registry_api()) .middleware(Cors::new()) - .middleware(Auth::new()), + .middleware(Auth::new()) + .middleware(DeviceInfoMiddleware::new()), ) }) .route( @@ -81,7 +89,7 @@ pub fn registry_server_router(ctx: RegistryContext) -> Router { let ctx = ctx.clone(); move |x::Path(path): x::Path, ws: axum::extract::ws::WebSocketUpgrade| async move { - match RequestGuid::from(&path) { + match Guid::from(&path) { None => { tracing::debug!("No Guid Path"); bad_request() @@ -104,7 +112,7 @@ pub fn registry_server_router(ctx: RegistryContext) -> Router { .path() .strip_prefix("/rest/rpc/") .unwrap_or_default(); - match RequestGuid::from(&path) { + match Guid::from(&path) { None => { tracing::debug!("No Guid Path"); bad_request() diff --git a/core/startos/src/registry/os/asset/add.rs b/core/startos/src/registry/os/asset/add.rs index d2c20e711..6ca495547 100644 --- a/core/startos/src/registry/os/asset/add.rs +++ b/core/startos/src/registry/os/asset/add.rs @@ -1,17 +1,13 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::panic::UnwindSafe; use std::path::PathBuf; -use std::time::Duration; -use axum::response::Response; use clap::Parser; -use futures::{FutureExt, TryStreamExt}; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha512}; use ts_rs::TS; use url::Url; @@ -22,10 +18,15 @@ use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; use crate::registry::os::SIG_CONTEXT; -use crate::registry::signer::{Blake3Ed25519Signature, Signature, SignatureInfo, SignerKey}; -use crate::rpc_continuations::{RequestGuid, RpcContinuation}; +use crate::registry::signer::commitment::blake3::Blake3Commitment; +use crate::registry::signer::sign::ed25519::Ed25519; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme}; +use crate::s9pk::merkle_archive::hash::VerifyingWriter; +use crate::s9pk::merkle_archive::source::http::HttpSource; +use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::ArchiveSource; -use crate::util::{Apply, Version}; +use crate::util::serde::Base64; +use crate::util::VersionString; pub fn add_api() -> ParentHandler { ParentHandler::new() @@ -53,39 +54,37 @@ pub fn add_api() -> ParentHandler { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct AddAssetParams { - #[ts(type = "string")] - pub url: Url, - pub signature: Signature, - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, #[ts(type = "string")] pub platform: InternedString, - #[serde(default)] - pub upload: bool, + #[ts(type = "string")] + pub url: Url, #[serde(rename = "__auth_signer")] - pub signer: SignerKey, + #[ts(skip)] + pub signer: AnyVerifyingKey, + pub signature: AnySignature, + pub commitment: Blake3Commitment, } async fn add_asset( ctx: RegistryContext, AddAssetParams { - url, - signature, version, platform, - upload, + url, signer, + signature, + commitment, }: AddAssetParams, - accessor: impl FnOnce(&mut Model) -> &mut Model> + accessor: impl FnOnce( + &mut Model, + ) -> &mut Model>> + UnwindSafe + Send, -) -> Result, Error> { - ensure_code!( - signature.signer() == signer, - ErrorKind::InvalidSignature, - "asset signature does not match request signer" - ); - +) -> Result<(), Error> { + signer + .scheme() + .verify_commitment(&signer, &commitment, SIG_CONTEXT, &signature)?; ctx.db .mutate(|db| { let signer_guid = db.as_index().as_signers().get_signer(&signer)?; @@ -95,7 +94,7 @@ async fn add_asset( .as_versions() .as_idx(&version) .or_not_found(&version)? - .as_signers() + .as_authorized() .de()? .contains(&signer_guid) { @@ -109,11 +108,21 @@ async fn add_asset( .upsert(&platform, || { Ok(RegistryAsset { url, - signature_info: SignatureInfo::new(SIG_CONTEXT), + commitment: commitment.clone(), + signatures: HashMap::new(), }) })? - .as_signature_info_mut() - .mutate(|s| s.add_sig(&signature))?; + .mutate(|s| { + if s.commitment != commitment { + Err(Error::new( + eyre!("commitment does not match"), + ErrorKind::InvalidSignature, + )) + } else { + s.signatures.insert(signer, signature); + Ok(()) + } + })?; Ok(()) } else { Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization)) @@ -121,80 +130,18 @@ async fn add_asset( }) .await?; - let guid = if upload { - let guid = RequestGuid::new(); - let auth_guid = guid.clone(); - let signer = signature.signer(); - let hostname = ctx.hostname.clone(); - ctx.rpc_continuations - .add( - guid.clone(), - RpcContinuation::rest( - Box::new(|req| { - async move { - Ok( - if async move { - let auth_sig = base64::decode( - req.headers().get("X-StartOS-Registry-Auth-Sig")?, - ) - .ok()?; - signer - .verify_message( - auth_guid.as_ref().as_bytes(), - &auth_sig, - &hostname, - ) - .ok()?; - - Some(()) - } - .await - .is_some() - { - Response::builder() - .status(200) - .body(axum::body::Body::empty()) - .with_kind(ErrorKind::Network)? - } else { - Response::builder() - .status(401) - .body(axum::body::Body::empty()) - .with_kind(ErrorKind::Network)? - }, - ) - } - .boxed() - }), - Duration::from_secs(30), - ), - ) - .await; - Some(guid) - } else { - None - }; - - Ok(guid) + Ok(()) } -pub async fn add_iso( - ctx: RegistryContext, - params: AddAssetParams, -) -> Result, Error> { +pub async fn add_iso(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> { add_asset(ctx, params, |m| m.as_iso_mut()).await } -pub async fn add_img( - ctx: RegistryContext, - params: AddAssetParams, -) -> Result, Error> { +pub async fn add_img(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> { add_asset(ctx, params, |m| m.as_img_mut()).await } -pub async fn add_squashfs( - ctx: RegistryContext, - params: AddAssetParams, -) -> Result, Error> { +pub async fn add_squashfs(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> { add_asset(ctx, params, |m| m.as_squashfs_mut()).await } @@ -205,11 +152,9 @@ pub struct CliAddAssetParams { #[arg(short = 'p', long = "platform")] pub platform: InternedString, #[arg(short = 'v', long = "version")] - pub version: Version, + pub version: VersionString, pub file: PathBuf, pub url: Url, - #[arg(short = 'u', long = "upload")] - pub upload: bool, } pub async fn cli_add_asset( @@ -223,7 +168,6 @@ pub async fn cli_add_asset( version, file: path, url, - upload, }, .. }: HandlerArgs, @@ -240,21 +184,18 @@ pub async fn cli_add_asset( } }; - let file = tokio::fs::File::open(&path).await?.into(); + let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); let mut progress = FullProgressTracker::new(); let progress_handle = progress.handle(); let mut sign_phase = progress_handle.add_phase(InternedString::intern("Signing File"), Some(10)); + let mut verify_phase = + progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); let mut index_phase = progress_handle.add_phase( InternedString::intern("Adding File to Registry Index"), Some(1), ); - let mut upload_phase = if upload { - Some(progress_handle.add_phase(InternedString::intern("Uploading File"), Some(100))) - } else { - None - }; let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", path.display())); @@ -270,70 +211,46 @@ pub async fn cli_add_asset( .into(); sign_phase.start(); - let blake3_sig = - Blake3Ed25519Signature::sign_file(ctx.developer_key()?, &file, SIG_CONTEXT).await?; - let size = blake3_sig.size; - let signature = Signature::Blake3Ed25519(blake3_sig); + let blake3 = file.blake3_mmap().await?; + let size = file + .size() + .await + .ok_or_else(|| Error::new(eyre!("failed to read file metadata"), ErrorKind::Filesystem))?; + let commitment = Blake3Commitment { + hash: Base64(*blake3.as_bytes()), + size, + }; + let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?; sign_phase.complete(); - index_phase.start(); - let add_res = from_value::>( - ctx.call_remote::( - &parent_method - .into_iter() - .chain(method) - .chain([ext]) - .join("."), - imbl_value::json!({ - "platform": platform, - "version": version, - "url": &url, - "signature": signature, - "upload": upload, - }), - ) - .await?, - )?; - index_phase.complete(); + verify_phase.start(); + let src = HttpSource::new(ctx.client.clone(), url.clone()).await?; + let mut writer = verify_phase.writer(VerifyingWriter::new( + tokio::io::sink(), + Some((blake3::Hash::from_bytes(*commitment.hash), commitment.size)), + )); + src.copy_all_to(&mut writer).await?; + let (verifier, mut verify_phase) = writer.into_inner(); + verifier.verify().await?; + verify_phase.complete(); - if let Some(guid) = add_res { - upload_phase.as_mut().map(|p| p.start()); - upload_phase.as_mut().map(|p| p.set_total(size)); - let reg_url = ctx.registry_url.as_ref().or_not_found("--registry")?; - ctx.client - .post(url) - .header("X-StartOS-Registry-Token", guid.as_ref()) - .header( - "X-StartOS-Registry-Auth-Sig", - base64::encode( - ctx.developer_key()? - .sign_prehashed( - Sha512::new_with_prefix(guid.as_ref().as_bytes()), - Some( - reg_url - .host() - .or_not_found("registry hostname")? - .to_string() - .as_bytes(), - ), - )? - .to_bytes(), - ), - ) - .body(reqwest::Body::wrap_stream( - tokio_util::io::ReaderStream::new(file.fetch(0, size).await?).inspect_ok( - move |b| { - upload_phase - .as_mut() - .map(|p| *p += b.len() as u64) - .apply(|_| ()) - }, - ), - )) - .send() - .await?; - // upload_phase.as_mut().map(|p| p.complete()); - } + index_phase.start(); + ctx.call_remote::( + &parent_method + .into_iter() + .chain(method) + .chain([ext]) + .join("."), + imbl_value::json!({ + "platform": platform, + "version": version, + "url": &url, + "signature": signature, + "commitment": commitment, + }), + ) + .await?; + index_phase.complete(); progress_handle.complete(); diff --git a/core/startos/src/registry/os/asset/get.rs b/core/startos/src/registry/os/asset/get.rs index 52e95bc98..e099a50cf 100644 --- a/core/startos/src/registry/os/asset/get.rs +++ b/core/startos/src/registry/os/asset/get.rs @@ -16,8 +16,11 @@ use crate::progress::{FullProgressTracker, PhasedProgressBar}; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; +use crate::registry::os::SIG_CONTEXT; +use crate::registry::signer::commitment::blake3::Blake3Commitment; +use crate::registry::signer::commitment::Commitment; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; -use crate::util::Version; +use crate::util::VersionString; pub fn get_api() -> ParentHandler { ParentHandler::new() @@ -33,8 +36,7 @@ pub fn get_api() -> ParentHandler { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct GetOsAssetParams { - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, #[ts(type = "string")] pub platform: InternedString, } @@ -42,10 +44,12 @@ pub struct GetOsAssetParams { async fn get_os_asset( ctx: RegistryContext, GetOsAssetParams { version, platform }: GetOsAssetParams, - accessor: impl FnOnce(&Model) -> &Model> + accessor: impl FnOnce( + &Model, + ) -> &Model>> + UnwindSafe + Send, -) -> Result { +) -> Result, Error> { accessor( ctx.db .peek() @@ -64,21 +68,21 @@ async fn get_os_asset( pub async fn get_iso( ctx: RegistryContext, params: GetOsAssetParams, -) -> Result { +) -> Result, Error> { get_os_asset(ctx, params, |info| info.as_iso()).await } pub async fn get_img( ctx: RegistryContext, params: GetOsAssetParams, -) -> Result { +) -> Result, Error> { get_os_asset(ctx, params, |info| info.as_img()).await } pub async fn get_squashfs( ctx: RegistryContext, params: GetOsAssetParams, -) -> Result { +) -> Result, Error> { get_os_asset(ctx, params, |info| info.as_squashfs()).await } @@ -86,7 +90,7 @@ pub async fn get_squashfs( #[command(rename_all = "kebab-case")] #[serde(rename_all = "camelCase")] pub struct CliGetOsAssetParams { - pub version: Version, + pub version: VersionString, pub platform: InternedString, #[arg(long = "download", short = 'd')] pub download: Option, @@ -112,8 +116,8 @@ async fn cli_get_os_asset( }, .. }: HandlerArgs, -) -> Result { - let res = from_value::( +) -> Result, Error> { + let res = from_value::>( ctx.call_remote::( &parent_method.into_iter().chain(method).join("."), json!({ @@ -124,7 +128,7 @@ async fn cli_get_os_asset( .await?, )?; - let validator = res.validate(res.signature_info.all_signers())?; + res.validate(SIG_CONTEXT, res.all_signers())?; if let Some(download) = download { let mut file = AtomicFile::new(&download, None::<&Path>) @@ -135,7 +139,7 @@ async fn cli_get_os_asset( let progress_handle = progress.handle(); let mut download_phase = progress_handle.add_phase(InternedString::intern("Downloading File"), Some(100)); - download_phase.set_total(validator.size()?); + download_phase.set_total(res.commitment.size); let reverify_phase = if reverify { Some(progress_handle.add_phase(InternedString::intern("Reverifying File"), Some(10))) } else { @@ -157,7 +161,7 @@ async fn cli_get_os_asset( download_phase.start(); let mut download_writer = download_phase.writer(&mut *file); - res.download(ctx.client.clone(), &mut download_writer, &validator) + res.download(ctx.client.clone(), &mut download_writer) .await?; let (_, mut download_phase) = download_writer.into_inner(); file.save().await.with_kind(ErrorKind::Filesystem)?; @@ -165,8 +169,8 @@ async fn cli_get_os_asset( if let Some(mut reverify_phase) = reverify_phase { reverify_phase.start(); - validator - .validate_file(&MultiCursorFile::from( + res.commitment + .check(&MultiCursorFile::from( tokio::fs::File::open(download).await?, )) .await?; diff --git a/core/startos/src/registry/os/asset/sign.rs b/core/startos/src/registry/os/asset/sign.rs index e19a82899..0cb657bef 100644 --- a/core/startos/src/registry/os/asset/sign.rs +++ b/core/startos/src/registry/os/asset/sign.rs @@ -17,25 +17,47 @@ use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; use crate::registry::os::SIG_CONTEXT; -use crate::registry::signer::{Blake3Ed25519Signature, Signature}; -use crate::util::Version; +use crate::registry::signer::commitment::blake3::Blake3Commitment; +use crate::registry::signer::sign::ed25519::Ed25519; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme}; +use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; +use crate::s9pk::merkle_archive::source::ArchiveSource; +use crate::util::serde::Base64; +use crate::util::VersionString; pub fn sign_api() -> ParentHandler { ParentHandler::new() - .subcommand("iso", from_fn_async(sign_iso).no_cli()) - .subcommand("img", from_fn_async(sign_img).no_cli()) - .subcommand("squashfs", from_fn_async(sign_squashfs).no_cli()) + .subcommand( + "iso", + from_fn_async(sign_iso) + .with_metadata("getSigner", Value::Bool(true)) + .no_cli(), + ) + .subcommand( + "img", + from_fn_async(sign_img) + .with_metadata("getSigner", Value::Bool(true)) + .no_cli(), + ) + .subcommand( + "squashfs", + from_fn_async(sign_squashfs) + .with_metadata("getSigner", Value::Bool(true)) + .no_cli(), + ) } #[derive(Debug, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] pub struct SignAssetParams { - #[ts(type = "string")] - version: Version, + version: VersionString, #[ts(type = "string")] platform: InternedString, - signature: Signature, + #[ts(skip)] + #[serde(rename = "__auth_signer")] + signer: AnyVerifyingKey, + signature: AnySignature, } async fn sign_asset( @@ -43,22 +65,25 @@ async fn sign_asset( SignAssetParams { version, platform, + signer, signature, }: SignAssetParams, - accessor: impl FnOnce(&mut Model) -> &mut Model> + accessor: impl FnOnce( + &mut Model, + ) -> &mut Model>> + UnwindSafe + Send, ) -> Result<(), Error> { ctx.db .mutate(|db| { - let guid = db.as_index().as_signers().get_signer(&signature.signer())?; + let guid = db.as_index().as_signers().get_signer(&signer)?; if !db .as_index() .as_os() .as_versions() .as_idx(&version) .or_not_found(&version)? - .as_signers() + .as_authorized() .de()? .contains(&guid) { @@ -77,8 +102,16 @@ async fn sign_asset( ) .as_idx_mut(&platform) .or_not_found(&platform)? - .as_signature_info_mut() - .mutate(|s| s.add_sig(&signature))?; + .mutate(|s| { + signer.scheme().verify_commitment( + &signer, + &s.commitment, + SIG_CONTEXT, + &signature, + )?; + s.signatures.insert(signer, signature); + Ok(()) + })?; Ok(()) }) @@ -104,7 +137,7 @@ pub struct CliSignAssetParams { #[arg(short = 'p', long = "platform")] pub platform: InternedString, #[arg(short = 'v', long = "version")] - pub version: Version, + pub version: VersionString, pub file: PathBuf, } @@ -134,7 +167,7 @@ pub async fn cli_sign_asset( } }; - let file = tokio::fs::File::open(&path).await?.into(); + let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); let mut progress = FullProgressTracker::new(); let progress_handle = progress.handle(); @@ -159,9 +192,16 @@ pub async fn cli_sign_asset( .into(); sign_phase.start(); - let blake3_sig = - Blake3Ed25519Signature::sign_file(ctx.developer_key()?, &file, SIG_CONTEXT).await?; - let signature = Signature::Blake3Ed25519(blake3_sig); + let blake3 = file.blake3_mmap().await?; + let size = file + .size() + .await + .ok_or_else(|| Error::new(eyre!("failed to read file metadata"), ErrorKind::Filesystem))?; + let commitment = Blake3Commitment { + hash: Base64(*blake3.as_bytes()), + size, + }; + let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?; sign_phase.complete(); index_phase.start(); diff --git a/core/startos/src/registry/os/index.rs b/core/startos/src/registry/os/index.rs index 186a7e95e..3ee75bc6a 100644 --- a/core/startos/src/registry/os/index.rs +++ b/core/startos/src/registry/os/index.rs @@ -8,16 +8,16 @@ use ts_rs::TS; use crate::prelude::*; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; -use crate::rpc_continuations::RequestGuid; -use crate::util::Version; +use crate::registry::signer::commitment::blake3::Blake3Commitment; +use crate::rpc_continuations::Guid; +use crate::util::VersionString; #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] #[ts(export)] pub struct OsIndex { - #[ts(as = "BTreeMap::")] - pub versions: BTreeMap, + pub versions: BTreeMap, } #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] @@ -29,14 +29,13 @@ pub struct OsVersionInfo { pub release_notes: String, #[ts(type = "string")] pub source_version: VersionRange, - #[ts(type = "string[]")] - pub signers: BTreeSet, - #[ts(as = "BTreeMap::")] - pub iso: BTreeMap, // platform (i.e. x86_64-nonfree) -> asset - #[ts(as = "BTreeMap::")] - pub squashfs: BTreeMap, // platform (i.e. x86_64-nonfree) -> asset - #[ts(as = "BTreeMap::")] - pub img: BTreeMap, // platform (i.e. raspberrypi) -> asset + pub authorized: BTreeSet, + #[ts(as = "BTreeMap::>")] + pub iso: BTreeMap>, // platform (i.e. x86_64-nonfree) -> asset + #[ts(as = "BTreeMap::>")] + pub squashfs: BTreeMap>, // platform (i.e. x86_64-nonfree) -> asset + #[ts(as = "BTreeMap::>")] + pub img: BTreeMap>, // platform (i.e. raspberrypi) -> asset } pub async fn get_os_index(ctx: RegistryContext) -> Result { diff --git a/core/startos/src/registry/os/version/mod.rs b/core/startos/src/registry/os/version/mod.rs index 5af407c5c..5bf926e5e 100644 --- a/core/startos/src/registry/os/version/mod.rs +++ b/core/startos/src/registry/os/version/mod.rs @@ -11,9 +11,9 @@ use crate::context::CliContext; use crate::prelude::*; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; -use crate::registry::signer::SignerKey; +use crate::registry::signer::sign::AnyVerifyingKey; use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat}; -use crate::util::Version; +use crate::util::VersionString; pub mod signer; @@ -51,8 +51,7 @@ pub fn version_api() -> ParentHandler { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct AddVersionParams { - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, pub headline: String, pub release_notes: String, #[ts(type = "string")] @@ -60,7 +59,7 @@ pub struct AddVersionParams { #[arg(skip)] #[ts(skip)] #[serde(rename = "__auth_signer")] - pub signer: Option, + pub signer: Option, } pub async fn add_version( @@ -86,7 +85,7 @@ pub async fn add_version( i.headline = headline; i.release_notes = release_notes; i.source_version = source_version; - i.signers.extend(signer); + i.authorized.extend(signer); Ok(()) }) }) @@ -98,8 +97,7 @@ pub async fn add_version( #[serde(rename_all = "camelCase")] #[ts(export)] pub struct RemoveVersionParams { - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, } pub async fn remove_version( @@ -124,7 +122,7 @@ pub async fn remove_version( pub struct GetVersionParams { #[ts(type = "string | null")] #[arg(long = "src")] - pub source: Option, + pub source: Option, #[ts(type = "string | null")] #[arg(long = "target")] pub target: Option, @@ -133,7 +131,7 @@ pub struct GetVersionParams { pub async fn get_version( ctx: RegistryContext, GetVersionParams { source, target }: GetVersionParams, -) -> Result, Error> { +) -> Result, Error> { let target = target.unwrap_or(VersionRange::Any); ctx.db .peek() @@ -153,7 +151,10 @@ pub async fn get_version( .collect() } -pub fn display_version_info(params: WithIoFormat, info: BTreeMap) { +pub fn display_version_info( + params: WithIoFormat, + info: BTreeMap, +) { use prettytable::*; if let Some(format) = params.format { diff --git a/core/startos/src/registry/os/version/signer.rs b/core/startos/src/registry/os/version/signer.rs index 01173ad18..bb15860aa 100644 --- a/core/startos/src/registry/os/version/signer.rs +++ b/core/startos/src/registry/os/version/signer.rs @@ -10,9 +10,9 @@ use crate::prelude::*; use crate::registry::admin::display_signers; use crate::registry::context::RegistryContext; use crate::registry::signer::SignerInfo; -use crate::rpc_continuations::RequestGuid; +use crate::rpc_continuations::Guid; use crate::util::serde::HandlerExtSerde; -use crate::util::Version; +use crate::util::VersionString; pub fn signer_api() -> ParentHandler { ParentHandler::new() @@ -44,10 +44,8 @@ pub fn signer_api() -> ParentHandler { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct VersionSignerParams { - #[ts(type = "string")] - pub version: Version, - #[ts(type = "string")] - pub signer: RequestGuid, + pub version: VersionString, + pub signer: Guid, } pub async fn add_version_signer( @@ -67,7 +65,7 @@ pub async fn add_version_signer( .as_versions_mut() .as_idx_mut(&version) .or_not_found(&version)? - .as_signers_mut() + .as_authorized_mut() .mutate(|s| Ok(s.insert(signer)))?; Ok(()) @@ -87,7 +85,7 @@ pub async fn remove_version_signer( .as_versions_mut() .as_idx_mut(&version) .or_not_found(&version)? - .as_signers_mut() + .as_authorized_mut() .mutate(|s| Ok(s.remove(&signer)))? { return Err(Error::new( @@ -106,21 +104,20 @@ pub async fn remove_version_signer( #[serde(rename_all = "camelCase")] #[ts(export)] pub struct ListVersionSignersParams { - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, } pub async fn list_version_signers( ctx: RegistryContext, ListVersionSignersParams { version }: ListVersionSignersParams, -) -> Result, Error> { +) -> Result, Error> { let db = ctx.db.peek().await; db.as_index() .as_os() .as_versions() .as_idx(&version) .or_not_found(&version)? - .as_signers() + .as_authorized() .de()? .into_iter() .filter_map(|guid| { diff --git a/core/startos/src/registry/package/add.rs b/core/startos/src/registry/package/add.rs new file mode 100644 index 000000000..6a1050b99 --- /dev/null +++ b/core/startos/src/registry/package/add.rs @@ -0,0 +1,170 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use clap::Parser; +use helpers::NonDetachingJoinHandle; +use imbl_value::InternedString; +use itertools::Itertools; +use rpc_toolkit::HandlerArgs; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use url::Url; + +use crate::context::CliContext; +use crate::prelude::*; +use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::registry::context::RegistryContext; +use crate::registry::package::index::PackageVersionInfo; +use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; +use crate::registry::signer::sign::ed25519::Ed25519; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme}; +use crate::s9pk::merkle_archive::source::http::HttpSource; +use crate::s9pk::v2::SIG_CONTEXT; +use crate::s9pk::S9pk; +use crate::util::io::TrackingIO; + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct AddPackageParams { + #[ts(type = "string")] + pub url: Url, + #[ts(skip)] + #[serde(rename = "__auth_signer")] + pub uploader: AnyVerifyingKey, + pub commitment: MerkleArchiveCommitment, + pub signature: AnySignature, +} + +pub async fn add_package( + ctx: RegistryContext, + AddPackageParams { + url, + uploader, + commitment, + signature, + }: AddPackageParams, +) -> Result<(), Error> { + uploader + .scheme() + .verify_commitment(&uploader, &commitment, SIG_CONTEXT, &signature)?; + let peek = ctx.db.peek().await; + let uploader_guid = peek.as_index().as_signers().get_signer(&uploader)?; + let s9pk = S9pk::deserialize( + &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?), + Some(&commitment), + false, + ) + .await?; + + let manifest = s9pk.as_manifest(); + + let mut info = PackageVersionInfo::from_s9pk(&s9pk, url).await?; + if !info.s9pk.signatures.contains_key(&uploader) { + info.s9pk.signatures.insert(uploader.clone(), signature); + } + + ctx.db + .mutate(|db| { + if db.as_admins().de()?.contains(&uploader_guid) + || db + .as_index() + .as_package() + .as_packages() + .as_idx(&manifest.id) + .or_not_found(&manifest.id)? + .as_authorized() + .de()? + .contains(&uploader_guid) + { + let package = db + .as_index_mut() + .as_package_mut() + .as_packages_mut() + .upsert(&manifest.id, || Ok(Default::default()))?; + package.as_versions_mut().insert(&manifest.version, &info)?; + + Ok(()) + } else { + Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization)) + } + }) + .await +} + +#[derive(Debug, Deserialize, Serialize, Parser)] +#[command(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] +pub struct CliAddPackageParams { + pub file: PathBuf, + pub url: Url, +} + +pub async fn cli_add_package( + HandlerArgs { + context: ctx, + parent_method, + method, + params: CliAddPackageParams { file, url }, + .. + }: HandlerArgs, +) -> Result<(), Error> { + let s9pk = S9pk::open(&file, None, false).await?; + + let mut progress = FullProgressTracker::new(); + let progress_handle = progress.handle(); + let mut sign_phase = progress_handle.add_phase(InternedString::intern("Signing File"), Some(1)); + let mut verify_phase = + progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); + let mut index_phase = progress_handle.add_phase( + InternedString::intern("Adding File to Registry Index"), + Some(1), + ); + + let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { + let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", file.display())); + loop { + let snap = progress.snapshot(); + bar.update(&snap); + if snap.overall.is_complete() { + break; + } + progress.changed().await + } + }) + .into(); + + sign_phase.start(); + let commitment = s9pk.as_archive().commitment().await?; + let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?; + sign_phase.complete(); + + verify_phase.start(); + let mut src = S9pk::deserialize( + &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?), + Some(&commitment), + false, + ) + .await?; + src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true) + .await?; + verify_phase.complete(); + + index_phase.start(); + ctx.call_remote::( + &parent_method.into_iter().chain(method).join("."), + imbl_value::json!({ + "url": &url, + "signature": signature, + "commitment": commitment, + }), + ) + .await?; + index_phase.complete(); + + progress_handle.complete(); + + progress_task.await.with_kind(ErrorKind::Unknown)?; + + Ok(()) +} diff --git a/core/startos/src/registry/package/get.rs b/core/startos/src/registry/package/get.rs new file mode 100644 index 000000000..835192361 --- /dev/null +++ b/core/startos/src/registry/package/get.rs @@ -0,0 +1,387 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use clap::{Parser, ValueEnum}; +use emver::{Version, VersionRange}; +use imbl_value::InternedString; +use itertools::Itertools; +use models::PackageId; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use crate::prelude::*; +use crate::registry::context::RegistryContext; +use crate::registry::device_info::DeviceInfo; +use crate::registry::package::index::{PackageIndex, PackageVersionInfo}; +use crate::util::serde::{display_serializable, WithIoFormat}; +use crate::util::VersionString; + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS, ValueEnum, +)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub enum PackageDetailLevel { + Short, + Full, +} +impl Default for PackageDetailLevel { + fn default() -> Self { + Self::Short + } +} + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct PackageInfoShort { + pub release_notes: String, +} + +#[derive(Debug, Deserialize, Serialize, TS, Parser)] +#[serde(rename_all = "camelCase")] +#[command(rename_all = "kebab-case")] +#[ts(export)] +pub struct GetPackageParams { + pub id: Option, + #[ts(type = "string | null")] + pub version: Option, + #[ts(type = "string | null")] + pub source_version: Option, + #[ts(skip)] + #[arg(skip)] + #[serde(rename = "__device_info")] + pub device_info: Option, + pub other_versions: Option, +} + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct GetPackageResponse { + #[ts(type = "string[]")] + pub categories: BTreeSet, + pub best: BTreeMap, + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub other_versions: Option>, +} +impl GetPackageResponse { + pub fn tables(&self) -> Vec { + use prettytable::*; + + let mut res = Vec::with_capacity(self.best.len()); + + for (version, info) in &self.best { + let mut table = info.table(version); + + let lesser_versions: BTreeMap<_, _> = self + .other_versions + .as_ref() + .into_iter() + .flatten() + .filter(|(v, _)| ***v < **version) + .collect(); + + if !lesser_versions.is_empty() { + table.add_row(row![bc => "OLDER VERSIONS"]); + table.add_row(row![bc => "VERSION", "RELEASE NOTES"]); + for (version, info) in lesser_versions { + table.add_row(row![AsRef::::as_ref(version), &info.release_notes]); + } + } + + res.push(table); + } + + res + } +} + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct GetPackageResponseFull { + #[ts(type = "string[]")] + pub categories: BTreeSet, + pub best: BTreeMap, + pub other_versions: BTreeMap, +} +impl GetPackageResponseFull { + pub fn tables(&self) -> Vec { + let mut res = Vec::with_capacity(self.best.len()); + + let all: BTreeMap<_, _> = self.best.iter().chain(self.other_versions.iter()).collect(); + + for (version, info) in all { + res.push(info.table(version)); + } + + res + } +} + +pub type GetPackagesResponse = BTreeMap; +pub type GetPackagesResponseFull = BTreeMap; + +fn get_matching_models<'a>( + db: &'a Model, + GetPackageParams { + id, + version, + source_version, + device_info, + .. + }: &GetPackageParams, +) -> Result)>, Error> { + if let Some(id) = id { + if let Some(pkg) = db.as_packages().as_idx(id) { + vec![(id.clone(), pkg)] + } else { + vec![] + } + } else { + db.as_packages().as_entries()? + } + .iter() + .map(|(k, v)| { + Ok(v.as_versions() + .as_entries()? + .into_iter() + .map(|(v, info)| { + Ok::<_, Error>( + if version + .as_ref() + .map_or(true, |version| v.satisfies(version)) + && source_version.as_ref().map_or(Ok(true), |source_version| { + Ok::<_, Error>( + source_version.satisfies( + &info + .as_source_version() + .de()? + .unwrap_or(VersionRange::any()), + ), + ) + })? + && device_info + .as_ref() + .map_or(Ok(true), |device_info| info.works_for_device(device_info))? + { + Some((k.clone(), Version::from(v), info)) + } else { + None + }, + ) + }) + .flatten_ok()) + }) + .flatten_ok() + .map(|res| res.and_then(|a| a)) + .collect() +} + +pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Result { + use patch_db::ModelExt; + + let peek = ctx.db.peek().await; + let mut best: BTreeMap>> = + Default::default(); + let mut other: BTreeMap>> = + Default::default(); + for (id, version, info) in get_matching_models(&peek.as_index().as_package(), ¶ms)? { + let mut package_best = best.remove(&id).unwrap_or_default(); + let mut package_other = other.remove(&id).unwrap_or_default(); + for worse_version in package_best + .keys() + .filter(|k| ***k < version) + .cloned() + .collect_vec() + { + if let Some(info) = package_best.remove(&worse_version) { + package_other.insert(worse_version, info); + } + } + if package_best.keys().all(|k| !(**k > version)) { + package_best.insert(version.into(), info); + } + best.insert(id.clone(), package_best); + if params.other_versions.is_some() { + other.insert(id.clone(), package_other); + } + } + if let Some(id) = params.id { + let categories = peek + .as_index() + .as_package() + .as_packages() + .as_idx(&id) + .map(|p| p.as_categories().de()) + .transpose()? + .unwrap_or_default(); + let best = best + .remove(&id) + .unwrap_or_default() + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?; + let other = other.remove(&id).unwrap_or_default(); + match params.other_versions { + None => to_value(&GetPackageResponse { + categories, + best, + other_versions: None, + }), + Some(PackageDetailLevel::Short) => to_value(&GetPackageResponse { + categories, + best, + other_versions: Some( + other + .into_iter() + .map(|(k, v)| from_value(v.as_value().clone()).map(|v| (k, v))) + .try_collect()?, + ), + }), + Some(PackageDetailLevel::Full) => to_value(&GetPackageResponseFull { + categories, + best, + other_versions: other + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?, + }), + } + } else { + match params.other_versions { + None => to_value( + &best + .into_iter() + .map(|(id, best)| { + let categories = peek + .as_index() + .as_package() + .as_packages() + .as_idx(&id) + .map(|p| p.as_categories().de()) + .transpose()? + .unwrap_or_default(); + Ok::<_, Error>(( + id, + GetPackageResponse { + categories, + best: best + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?, + other_versions: None, + }, + )) + }) + .try_collect::<_, GetPackagesResponse, _>()?, + ), + Some(PackageDetailLevel::Short) => to_value( + &best + .into_iter() + .map(|(id, best)| { + let categories = peek + .as_index() + .as_package() + .as_packages() + .as_idx(&id) + .map(|p| p.as_categories().de()) + .transpose()? + .unwrap_or_default(); + let other = other.remove(&id).unwrap_or_default(); + Ok::<_, Error>(( + id, + GetPackageResponse { + categories, + best: best + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?, + other_versions: Some( + other + .into_iter() + .map(|(k, v)| { + from_value(v.as_value().clone()).map(|v| (k, v)) + }) + .try_collect()?, + ), + }, + )) + }) + .try_collect::<_, GetPackagesResponse, _>()?, + ), + Some(PackageDetailLevel::Full) => to_value( + &best + .into_iter() + .map(|(id, best)| { + let categories = peek + .as_index() + .as_package() + .as_packages() + .as_idx(&id) + .map(|p| p.as_categories().de()) + .transpose()? + .unwrap_or_default(); + let other = other.remove(&id).unwrap_or_default(); + Ok::<_, Error>(( + id, + GetPackageResponseFull { + categories, + best: best + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?, + other_versions: other + .into_iter() + .map(|(k, v)| v.de().map(|v| (k, v))) + .try_collect()?, + }, + )) + }) + .try_collect::<_, GetPackagesResponseFull, _>()?, + ), + } + } +} + +pub fn display_package_info( + params: WithIoFormat, + info: Value, +) -> Result<(), Error> { + if let Some(format) = params.format { + display_serializable(format, info); + return Ok(()); + } + + if let Some(_) = params.rest.id { + if params.rest.other_versions == Some(PackageDetailLevel::Full) { + for table in from_value::(info)?.tables() { + table.print_tty(false)?; + println!(); + } + } else { + for table in from_value::(info)?.tables() { + table.print_tty(false)?; + println!(); + } + } + } else { + if params.rest.other_versions == Some(PackageDetailLevel::Full) { + for (_, package) in from_value::(info)? { + for table in package.tables() { + table.print_tty(false)?; + println!(); + } + } + } else { + for (_, package) in from_value::(info)? { + for table in package.tables() { + table.print_tty(false)?; + println!(); + } + } + } + } + Ok(()) +} diff --git a/core/startos/src/registry/package/index.rs b/core/startos/src/registry/package/index.rs new file mode 100644 index 000000000..0e6969fa5 --- /dev/null +++ b/core/startos/src/registry/package/index.rs @@ -0,0 +1,163 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use emver::{Version, VersionRange}; +use imbl_value::InternedString; +use models::{DataUrl, PackageId, VersionString}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use url::Url; + +use crate::prelude::*; +use crate::registry::asset::RegistryAsset; +use crate::registry::context::RegistryContext; +use crate::registry::device_info::DeviceInfo; +use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey}; +use crate::rpc_continuations::Guid; +use crate::s9pk::git_hash::GitHash; +use crate::s9pk::manifest::{Description, HardwareRequirements}; +use crate::s9pk::merkle_archive::source::FileSource; +use crate::s9pk::S9pk; + +#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct PackageIndex { + #[ts(as = "BTreeMap::")] + pub categories: BTreeMap, + pub packages: BTreeMap, +} + +#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct PackageInfo { + pub authorized: BTreeSet, + pub versions: BTreeMap, + #[ts(type = "string[]")] + pub categories: BTreeSet, +} + +#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct Category { + pub name: String, + pub description: Description, +} + +#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct PackageVersionInfo { + pub title: String, + pub icon: DataUrl<'static>, + pub description: Description, + pub release_notes: String, + #[ts(type = "string")] + pub git_hash: GitHash, + #[ts(type = "string")] + pub license: InternedString, + #[ts(type = "string")] + pub wrapper_repo: Url, + #[ts(type = "string")] + pub upstream_repo: Url, + #[ts(type = "string")] + pub support_site: Url, + #[ts(type = "string")] + pub marketing_site: Url, + pub os_version: VersionString, + pub hardware_requirements: HardwareRequirements, + #[ts(type = "string | null")] + pub source_version: Option, + pub s9pk: RegistryAsset, +} +impl PackageVersionInfo { + pub async fn from_s9pk(s9pk: &S9pk, url: Url) -> Result { + let manifest = s9pk.as_manifest(); + Ok(Self { + title: manifest.title.clone(), + icon: s9pk.icon_data_url().await?, + description: manifest.description.clone(), + release_notes: manifest.release_notes.clone(), + git_hash: manifest.git_hash.clone().or_not_found("git hash")?, + license: manifest.license.clone(), + wrapper_repo: manifest.wrapper_repo.clone(), + upstream_repo: manifest.upstream_repo.clone(), + support_site: manifest.support_site.clone(), + marketing_site: manifest.marketing_site.clone(), + os_version: manifest.os_version.clone(), + hardware_requirements: manifest.hardware_requirements.clone(), + source_version: None, // TODO + s9pk: RegistryAsset { + url, + commitment: s9pk.as_archive().commitment().await?, + signatures: [( + AnyVerifyingKey::Ed25519(s9pk.as_archive().signer()), + AnySignature::Ed25519(s9pk.as_archive().signature().await?), + )] + .into_iter() + .collect(), + }, + }) + } + pub fn table(&self, version: &VersionString) -> prettytable::Table { + use prettytable::*; + + let mut table = Table::new(); + + table.add_row(row![bc => &self.title]); + table.add_row(row![br -> "VERSION", AsRef::::as_ref(version)]); + table.add_row(row![br -> "RELEASE NOTES", &self.release_notes]); + table.add_row(row![br -> "ABOUT", &self.description.short]); + table.add_row(row![br -> "DESCRIPTION", &self.description.long]); + table.add_row(row![br -> "GIT HASH", AsRef::::as_ref(&self.git_hash)]); + table.add_row(row![br -> "LICENSE", &self.license]); + table.add_row(row![br -> "PACKAGE REPO", &self.wrapper_repo.to_string()]); + table.add_row(row![br -> "SERVICE REPO", &self.upstream_repo.to_string()]); + table.add_row(row![br -> "WEBSITE", &self.marketing_site.to_string()]); + table.add_row(row![br -> "SUPPORT", &self.support_site.to_string()]); + + table + } +} +impl Model { + pub fn works_for_device(&self, device_info: &DeviceInfo) -> Result { + if !self.as_os_version().de()?.satisfies(&device_info.os.compat) { + return Ok(false); + } + let hw = self.as_hardware_requirements().de()?; + if let Some(arch) = hw.arch { + if !arch.contains(&device_info.hardware.arch) { + return Ok(false); + } + } + if let Some(ram) = hw.ram { + if device_info.hardware.ram < ram { + return Ok(false); + } + } + for (class, regex) in hw.device { + if !device_info + .hardware + .devices + .get(&*class) + .unwrap_or(&Vec::new()) + .iter() + .any(|product| regex.as_ref().is_match(product)) + { + return Ok(false); + } + } + + Ok(true) + } +} + +pub async fn get_package_index(ctx: RegistryContext) -> Result { + ctx.db.peek().await.into_index().into_package().de() +} diff --git a/core/startos/src/registry/package/mod.rs b/core/startos/src/registry/package/mod.rs new file mode 100644 index 000000000..ac09afbb1 --- /dev/null +++ b/core/startos/src/registry/package/mod.rs @@ -0,0 +1,29 @@ +use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; + +use crate::context::CliContext; +use crate::prelude::*; +use crate::util::serde::HandlerExtSerde; + +pub mod add; +pub mod get; +pub mod index; + +pub fn package_api() -> ParentHandler { + ParentHandler::new() + .subcommand( + "index", + from_fn_async(index::get_package_index) + .with_display_serializable() + .with_call_remote::(), + ) + .subcommand("add", from_fn_async(add::add_package).no_cli()) + .subcommand("add", from_fn_async(add::cli_add_package).no_display()) + .subcommand( + "get", + from_fn_async(get::get_package) + .with_display_serializable() + .with_custom_display_fn(|handle, result| { + get::display_package_info(handle.params, result) + }), + ) +} diff --git a/core/startos/src/registry/signer.rs b/core/startos/src/registry/signer.rs deleted file mode 100644 index bf5374d75..000000000 --- a/core/startos/src/registry/signer.rs +++ /dev/null @@ -1,477 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::str::FromStr; - -use clap::builder::ValueParserFactory; -use imbl_value::InternedString; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha512}; -use tokio::io::AsyncWrite; -use ts_rs::TS; -use url::Url; - -use crate::prelude::*; -use crate::s9pk::merkle_archive::source::http::HttpSource; -use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; -use crate::s9pk::merkle_archive::source::{ArchiveSource, FileSource}; -use crate::util::clap::FromStrParser; -use crate::util::serde::{Base64, Pem}; - -#[derive(Debug, Deserialize, Serialize, HasModel, TS)] -#[serde(rename_all = "camelCase")] -#[model = "Model"] -#[ts(export)] -pub struct SignerInfo { - pub name: String, - pub contact: Vec, - pub keys: HashSet, -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -#[serde(tag = "alg", content = "pubkey")] -pub enum SignerKey { - Ed25519(Pem), -} -impl SignerKey { - pub fn verifier(&self) -> Verifier { - match self { - Self::Ed25519(k) => Verifier::Ed25519(*k, Sha512::new()), - } - } - pub fn verify_message( - &self, - message: &[u8], - signature: &[u8], - context: &str, - ) -> Result<(), Error> { - let mut v = self.verifier(); - v.update(message); - v.verify(signature, context) - } -} -impl std::fmt::Display for SignerKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Ed25519(k) => write!(f, "{k}"), - } - } -} - -pub enum Verifier { - Ed25519(Pem, Sha512), -} -impl Verifier { - pub fn update(&mut self, data: &[u8]) { - match self { - Self::Ed25519(_, h) => h.update(data), - } - } - pub fn verify(self, signature: &[u8], context: &str) -> Result<(), Error> { - match self { - Self::Ed25519(k, h) => k.verify_prehashed_strict( - h, - Some(context.as_bytes()), - &ed25519_dalek::Signature::from_slice(signature)?, - )?, - } - Ok(()) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -// TODO: better types -pub enum ContactInfo { - Email(String), - Matrix(String), - Website(#[ts(type = "string")] Url), -} -impl std::fmt::Display for ContactInfo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Email(e) => write!(f, "mailto:{e}"), - Self::Matrix(m) => write!(f, "https://matrix.to/#/{m}"), - Self::Website(w) => write!(f, "{w}"), - } - } -} -impl FromStr for ContactInfo { - type Err = Error; - fn from_str(s: &str) -> Result { - Ok(if let Some(s) = s.strip_prefix("mailto:") { - Self::Email(s.to_owned()) - } else if let Some(s) = s.strip_prefix("https://matrix.to/#/") { - Self::Matrix(s.to_owned()) - } else { - Self::Website(s.parse()?) - }) - } -} -impl ValueParserFactory for ContactInfo { - type Parser = FromStrParser; - fn value_parser() -> Self::Parser { - Self::Parser::new() - } -} - -#[derive(Debug, Deserialize, Serialize, HasModel, TS)] -#[serde(rename_all = "kebab-case")] -#[model = "Model"] -#[ts(export)] -pub struct SignatureInfo { - #[ts(type = "string")] - pub context: InternedString, - pub blake3_ed255i9: Option, -} -impl SignatureInfo { - pub fn new(context: &str) -> Self { - Self { - context: context.into(), - blake3_ed255i9: None, - } - } - pub fn validate(&self, accept: AcceptSigners) -> Result { - FileValidator::from_signatures(self.signatures(), accept, &self.context) - } - pub fn all_signers(&self) -> AcceptSigners { - AcceptSigners::All( - self.signatures() - .map(|s| AcceptSigners::Signer(s.signer())) - .collect(), - ) - .flatten() - } - pub fn signatures(&self) -> impl Iterator + '_ { - self.blake3_ed255i9.iter().flat_map(|info| { - info.signatures - .iter() - .map(|(k, s)| (k.clone(), *s)) - .map(|(pubkey, signature)| { - Signature::Blake3Ed25519(Blake3Ed25519Signature { - hash: info.hash, - size: info.size, - pubkey, - signature, - }) - }) - }) - } - pub fn add_sig(&mut self, signature: &Signature) -> Result<(), Error> { - signature.validate(&self.context)?; - match signature { - Signature::Blake3Ed25519(s) => { - if self - .blake3_ed255i9 - .as_ref() - .map_or(true, |info| info.hash == s.hash) - { - let new = if let Some(mut info) = self.blake3_ed255i9.take() { - info.signatures.insert(s.pubkey, s.signature); - info - } else { - s.info() - }; - self.blake3_ed255i9 = Some(new); - Ok(()) - } else { - Err(Error::new( - eyre!("hash sum mismatch"), - ErrorKind::InvalidSignature, - )) - } - } - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub enum AcceptSigners { - #[serde(skip)] - Accepted(Signature), - Signer(SignerKey), - Any(Vec), - All(Vec), -} -impl AcceptSigners { - const fn null() -> Self { - Self::Any(Vec::new()) - } - pub fn flatten(self) -> Self { - match self { - Self::Any(mut s) | Self::All(mut s) if s.len() == 1 => s.swap_remove(0).flatten(), - s => s, - } - } - pub fn accepted(&self) -> bool { - match self { - Self::Accepted(_) => true, - Self::All(s) => s.iter().all(|s| s.accepted()), - _ => false, - } - } - pub fn try_accept( - self, - context: &str, - ) -> Box> + Send + Sync + '_> { - match self { - Self::Accepted(s) => Box::new(std::iter::once(s).map(|s| { - s.validate(context)?; - Ok(s) - })), - Self::All(s) => Box::new(s.into_iter().flat_map(|s| s.try_accept(context))), - _ => Box::new(std::iter::once(Err(Error::new( - eyre!("signer(s) not accepted"), - ErrorKind::InvalidSignature, - )))), - } - } - pub fn process_signature(&mut self, sig: &Signature) { - let new = match std::mem::replace(self, Self::null()) { - Self::Accepted(s) => Self::Accepted(s), - Self::Signer(s) => { - if s == sig.signer() { - Self::Accepted(sig.clone()) - } else { - Self::Signer(s) - } - } - Self::All(mut s) => { - s.iter_mut().for_each(|s| s.process_signature(sig)); - - Self::All(s) - } - Self::Any(mut s) => { - if let Some(s) = s - .iter_mut() - .map(|s| { - s.process_signature(sig); - s - }) - .filter(|s| s.accepted()) - .next() - { - std::mem::replace(s, Self::null()) - } else { - Self::Any(s) - } - } - }; - *self = new; - } -} - -#[must_use] -pub struct FileValidator { - blake3: Option, - size: Option, -} -impl FileValidator { - fn add_blake3(&mut self, hash: [u8; 32], size: u64) -> Result<(), Error> { - if let Some(h) = self.blake3 { - ensure_code!(h == hash, ErrorKind::InvalidSignature, "hash sum mismatch"); - } - self.blake3 = Some(blake3::Hash::from_bytes(hash)); - if let Some(s) = self.size { - ensure_code!(s == size, ErrorKind::InvalidSignature, "file size mismatch"); - } - self.size = Some(size); - Ok(()) - } - pub fn blake3(&self) -> Result { - if let Some(hash) = self.blake3 { - Ok(hash) - } else { - Err(Error::new( - eyre!("no BLAKE3 signatures found"), - ErrorKind::InvalidSignature, - )) - } - } - pub fn size(&self) -> Result { - if let Some(size) = self.size { - Ok(size) - } else { - Err(Error::new( - eyre!("no signatures found"), - ErrorKind::InvalidSignature, - )) - } - } - pub fn from_signatures( - signatures: impl IntoIterator, - mut accept: AcceptSigners, - context: &str, - ) -> Result { - let mut res = Self { - blake3: None, - size: None, - }; - for signature in signatures { - accept.process_signature(&signature); - } - for signature in accept.try_accept(context) { - match signature? { - Signature::Blake3Ed25519(s) => res.add_blake3(*s.hash, s.size)?, - } - } - - Ok(res) - } - pub async fn download( - &self, - url: Url, - client: Client, - dst: &mut (impl AsyncWrite + Unpin + Send + ?Sized), - ) -> Result<(), Error> { - let src = HttpSource::new(client, url).await?; - let (Some(hash), Some(size)) = (self.blake3, self.size) else { - return Err(Error::new( - eyre!("no BLAKE3 signatures found"), - ErrorKind::InvalidSignature, - )); - }; - src.section(0, size) - .copy_verify(dst, Some((hash, size))) - .await?; - - Ok(()) - } - pub async fn validate_file(&self, file: &MultiCursorFile) -> Result<(), Error> { - ensure_code!( - file.size().await == Some(self.size()?), - ErrorKind::InvalidSignature, - "file size mismatch" - ); - ensure_code!( - file.blake3_mmap().await? == self.blake3()?, - ErrorKind::InvalidSignature, - "hash sum mismatch" - ); - Ok(()) - } -} - -#[derive(Debug, Deserialize, Serialize, HasModel, TS)] -#[serde(rename_all = "camelCase")] -#[model = "Model"] -#[ts(export)] -pub struct Blake3Ed2551SignatureInfo { - pub hash: Base64<[u8; 32]>, - pub size: u64, - pub signatures: HashMap, Base64<[u8; 64]>>, -} -impl Blake3Ed2551SignatureInfo { - pub fn validate(&self, context: &str) -> Result>, Error> { - self.signatures - .iter() - .map(|(k, s)| { - let sig = Blake3Ed25519Signature { - hash: self.hash, - size: self.size, - pubkey: k.clone(), - signature: *s, - }; - sig.validate(context)?; - Ok(sig.pubkey) - }) - .collect() - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub enum Signature { - Blake3Ed25519(Blake3Ed25519Signature), -} -impl Signature { - pub fn validate(&self, context: &str) -> Result<(), Error> { - match self { - Self::Blake3Ed25519(a) => a.validate(context), - } - } - pub fn signer(&self) -> SignerKey { - match self { - Self::Blake3Ed25519(s) => SignerKey::Ed25519(s.pubkey.clone()), - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub struct Blake3Ed25519Signature { - pub hash: Base64<[u8; 32]>, - pub size: u64, - pub pubkey: Pem, - // ed25519-sig(sha512(blake3(file) + len_u64_be(file))) - pub signature: Base64<[u8; 64]>, -} -impl Blake3Ed25519Signature { - pub async fn sign_file( - key: &ed25519_dalek::SigningKey, - file: &MultiCursorFile, - context: &str, - ) -> Result { - let size = file - .size() - .await - .ok_or_else(|| Error::new(eyre!("failed to get file size"), ErrorKind::Filesystem))?; - let hash = file.blake3_mmap().await?; - let signature = key.sign_prehashed( - Sha512::new_with_prefix(hash.as_bytes()).chain_update(u64::to_be_bytes(size)), - Some(context.as_bytes()), - )?; - Ok(Self { - hash: Base64(*hash.as_bytes()), - size, - pubkey: Pem::new(key.verifying_key()), - signature: Base64(signature.to_bytes()), - }) - } - - pub fn validate(&self, context: &str) -> Result<(), Error> { - let sig = ed25519_dalek::Signature::from_bytes(&*self.signature); - self.pubkey.verify_prehashed_strict( - Sha512::new_with_prefix(*self.hash).chain_update(u64::to_be_bytes(self.size)), - Some(context.as_bytes()), - &sig, - )?; - Ok(()) - } - - pub async fn check_file(&self, file: &MultiCursorFile) -> Result<(), Error> { - let size = file - .size() - .await - .ok_or_else(|| Error::new(eyre!("failed to get file size"), ErrorKind::Filesystem))?; - if self.size != size { - return Err(Error::new( - eyre!("incorrect file size: expected {} got {}", self.size, size), - ErrorKind::InvalidSignature, - )); - } - let hash = file.blake3_mmap().await?; - if &*self.hash != hash.as_bytes() { - return Err(Error::new( - eyre!("hash sum mismatch"), - ErrorKind::InvalidSignature, - )); - } - Ok(()) - } - - pub fn info(&self) -> Blake3Ed2551SignatureInfo { - Blake3Ed2551SignatureInfo { - hash: self.hash, - size: self.size, - signatures: [(self.pubkey, self.signature)].into_iter().collect(), - } - } -} diff --git a/core/startos/src/registry/signer/commitment/blake3.rs b/core/startos/src/registry/signer/commitment/blake3.rs new file mode 100644 index 000000000..d99e68c16 --- /dev/null +++ b/core/startos/src/registry/signer/commitment/blake3.rs @@ -0,0 +1,50 @@ +use blake3::Hash; +use digest::Update; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWrite; +use ts_rs::TS; + +use crate::prelude::*; +use crate::registry::signer::commitment::{Commitment, Digestable}; +use crate::s9pk::merkle_archive::hash::VerifyingWriter; +use crate::s9pk::merkle_archive::source::ArchiveSource; +use crate::util::io::{ParallelBlake3Writer, TrackingIO}; +use crate::util::serde::Base64; +use crate::CAP_10_MiB; + +#[derive(Clone, Debug, Deserialize, Serialize, HasModel, PartialEq, Eq, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct Blake3Commitment { + pub hash: Base64<[u8; 32]>, + #[ts(type = "number")] + pub size: u64, +} +impl Digestable for Blake3Commitment { + fn update(&self, digest: &mut D) { + digest.update(&*self.hash); + digest.update(&u64::to_be_bytes(self.size)); + } +} +impl<'a, Resource: ArchiveSource> Commitment<&'a Resource> for Blake3Commitment { + async fn create(resource: &'a Resource) -> Result { + let mut hasher = TrackingIO::new(0, ParallelBlake3Writer::new(CAP_10_MiB)); + resource.copy_all_to(&mut hasher).await?; + Ok(Self { + size: hasher.position(), + hash: Base64(*hasher.into_inner().finalize().await?.as_bytes()), + }) + } + async fn copy_to( + &self, + resource: &'a Resource, + writer: W, + ) -> Result<(), Error> { + let mut hasher = + VerifyingWriter::new(writer, Some((Hash::from_bytes(*self.hash), self.size))); + resource.copy_to(0, self.size, &mut hasher).await?; + hasher.verify().await?; + Ok(()) + } +} diff --git a/core/startos/src/registry/signer/commitment/merkle_archive.rs b/core/startos/src/registry/signer/commitment/merkle_archive.rs new file mode 100644 index 000000000..0b61734b4 --- /dev/null +++ b/core/startos/src/registry/signer/commitment/merkle_archive.rs @@ -0,0 +1,98 @@ +use digest::Update; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWrite; +use ts_rs::TS; + +use crate::prelude::*; +use crate::registry::signer::commitment::{Commitment, Digestable}; +use crate::s9pk::merkle_archive::source::FileSource; +use crate::s9pk::merkle_archive::MerkleArchive; +use crate::s9pk::S9pk; +use crate::util::io::TrackingIO; +use crate::util::serde::Base64; + +#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct MerkleArchiveCommitment { + pub root_sighash: Base64<[u8; 32]>, + #[ts(type = "number")] + pub root_maxsize: u64, +} +impl Digestable for MerkleArchiveCommitment { + fn update(&self, digest: &mut D) { + digest.update(&*self.root_sighash); + digest.update(&u64::to_be_bytes(self.root_maxsize)); + } +} +impl<'a, S: FileSource + Clone> Commitment<&'a MerkleArchive> for MerkleArchiveCommitment { + async fn create(resource: &'a MerkleArchive) -> Result { + resource.commitment().await + } + async fn check(&self, resource: &'a MerkleArchive) -> Result<(), Error> { + let MerkleArchiveCommitment { + root_sighash, + root_maxsize, + } = resource.commitment().await?; + if root_sighash != self.root_sighash { + return Err(Error::new( + eyre!("merkle root mismatch"), + ErrorKind::InvalidSignature, + )); + } + if root_maxsize > self.root_maxsize { + return Err(Error::new( + eyre!("merkle root directory max size too large"), + ErrorKind::InvalidSignature, + )); + } + Ok(()) + } + async fn copy_to( + &self, + resource: &'a MerkleArchive, + writer: W, + ) -> Result<(), Error> { + self.check(resource).await?; + resource + .serialize(&mut TrackingIO::new(0, writer), true) + .await + } +} + +impl<'a, S: FileSource + Clone> Commitment<&'a S9pk> for MerkleArchiveCommitment { + async fn create(resource: &'a S9pk) -> Result { + resource.as_archive().commitment().await + } + async fn check(&self, resource: &'a S9pk) -> Result<(), Error> { + let MerkleArchiveCommitment { + root_sighash, + root_maxsize, + } = resource.as_archive().commitment().await?; + if root_sighash != self.root_sighash { + return Err(Error::new( + eyre!("merkle root mismatch"), + ErrorKind::InvalidSignature, + )); + } + if root_maxsize > self.root_maxsize { + return Err(Error::new( + eyre!("merkle root directory max size too large"), + ErrorKind::InvalidSignature, + )); + } + Ok(()) + } + async fn copy_to( + &self, + resource: &'a S9pk, + writer: W, + ) -> Result<(), Error> { + self.check(resource).await?; + resource + .clone() + .serialize(&mut TrackingIO::new(0, writer), true) + .await + } +} diff --git a/core/startos/src/registry/signer/commitment/mod.rs b/core/startos/src/registry/signer/commitment/mod.rs new file mode 100644 index 000000000..b85e02a4e --- /dev/null +++ b/core/startos/src/registry/signer/commitment/mod.rs @@ -0,0 +1,25 @@ +use digest::Update; +use futures::Future; +use tokio::io::AsyncWrite; + +use crate::prelude::*; + +pub mod blake3; +pub mod merkle_archive; +pub mod request; + +pub trait Digestable { + fn update(&self, digest: &mut D); +} + +pub trait Commitment: Sized + Digestable { + fn create(resource: Resource) -> impl Future> + Send; + fn copy_to( + &self, + resource: Resource, + writer: W, + ) -> impl Future> + Send; + fn check(&self, resource: Resource) -> impl Future> + Send { + self.copy_to(resource, tokio::io::sink()) + } +} diff --git a/core/startos/src/registry/signer/commitment/request.rs b/core/startos/src/registry/signer/commitment/request.rs new file mode 100644 index 000000000..62d59163f --- /dev/null +++ b/core/startos/src/registry/signer/commitment/request.rs @@ -0,0 +1,102 @@ +use std::time::{SystemTime, UNIX_EPOCH}; +use std::collections::BTreeMap; + +use axum::body::Body; +use axum::extract::Request; +use digest::Update; +use futures::TryStreamExt; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWrite; +use tokio_util::io::StreamReader; +use ts_rs::TS; +use url::Url; + +use crate::prelude::*; +use crate::registry::signer::commitment::{Commitment, Digestable}; +use crate::s9pk::merkle_archive::hash::VerifyingWriter; +use crate::util::serde::Base64; + +#[derive(Clone, Debug, Deserialize, Serialize, HasModel, PartialEq, Eq, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct RequestCommitment { + #[ts(type = "number")] + pub timestamp: i64, + #[ts(type = "number")] + pub nonce: u64, + #[ts(type = "number")] + pub size: u64, + pub blake3: Base64<[u8; 32]>, +} +impl RequestCommitment { + pub fn append_query(&self, url: &mut Url) { + url.query_pairs_mut() + .append_pair("timestamp", &self.timestamp.to_string()) + .append_pair("nonce", &self.nonce.to_string()) + .append_pair("size", &self.size.to_string()) + .append_pair("blake3", &self.blake3.to_string()); + } + pub fn from_query(url: &Url) -> Result { + let query: BTreeMap<_, _> = url.query_pairs().collect(); + Ok(Self { + timestamp: query.get("timestamp").or_not_found("timestamp")?.parse()?, + nonce: query.get("nonce").or_not_found("nonce")?.parse()?, + size: query.get("size").or_not_found("size")?.parse()?, + blake3: query.get("blake3").or_not_found("blake3")?.parse()?, + }) + } +} +impl Digestable for RequestCommitment { + fn update(&self, digest: &mut D) { + digest.update(&i64::to_be_bytes(self.timestamp)); + digest.update(&u64::to_be_bytes(self.nonce)); + digest.update(&u64::to_be_bytes(self.size)); + digest.update(&*self.blake3); + } +} +impl<'a> Commitment<&'a mut Request> for RequestCommitment { + async fn create(resource: &'a mut Request) -> Result { + use http_body_util::BodyExt; + + let body = std::mem::replace(resource.body_mut(), Body::empty()) + .collect() + .await + .with_kind(ErrorKind::Network)? + .to_bytes(); + let res = Self { + timestamp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1), + nonce: rand::random(), + size: body.len() as u64, + blake3: Base64(*blake3::hash(&*body).as_bytes()), + }; + *resource.body_mut() = Body::from(body); + Ok(res) + } + async fn copy_to( + &self, + resource: &'a mut Request, + writer: W, + ) -> Result<(), Error> { + use tokio::io::AsyncReadExt; + + let mut body = StreamReader::new( + std::mem::replace(resource.body_mut(), Body::empty()) + .into_data_stream() + .map_err(std::io::Error::other), + ) + .take(self.size); + + let mut writer = VerifyingWriter::new( + writer, + Some((blake3::Hash::from_bytes(*self.blake3), self.size)), + ); + tokio::io::copy(&mut body, &mut writer).await?; + writer.verify().await?; + + Ok(()) + } +} diff --git a/core/startos/src/registry/signer/mod.rs b/core/startos/src/registry/signer/mod.rs new file mode 100644 index 000000000..99b23b88e --- /dev/null +++ b/core/startos/src/registry/signer/mod.rs @@ -0,0 +1,154 @@ +use std::collections::HashSet; +use std::str::FromStr; + +use clap::builder::ValueParserFactory; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use url::Url; + +use crate::prelude::*; +use crate::registry::signer::commitment::Digestable; +use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme}; +use crate::util::clap::FromStrParser; + +pub mod commitment; +pub mod sign; + +#[derive(Debug, Deserialize, Serialize, HasModel, TS)] +#[serde(rename_all = "camelCase")] +#[model = "Model"] +#[ts(export)] +pub struct SignerInfo { + pub name: String, + pub contact: Vec, + pub keys: HashSet, +} + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +// TODO: better types +pub enum ContactInfo { + Email(String), + Matrix(String), + Website(#[ts(type = "string")] Url), +} +impl std::fmt::Display for ContactInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Email(e) => write!(f, "mailto:{e}"), + Self::Matrix(m) => write!(f, "https://matrix.to/#/{m}"), + Self::Website(w) => write!(f, "{w}"), + } + } +} +impl FromStr for ContactInfo { + type Err = Error; + fn from_str(s: &str) -> Result { + Ok(if let Some(s) = s.strip_prefix("mailto:") { + Self::Email(s.to_owned()) + } else if let Some(s) = s.strip_prefix("https://matrix.to/#/") { + Self::Matrix(s.to_owned()) + } else { + Self::Website(s.parse()?) + }) + } +} +impl ValueParserFactory for ContactInfo { + type Parser = FromStrParser; + fn value_parser() -> Self::Parser { + Self::Parser::new() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub enum AcceptSigners { + #[serde(skip)] + Accepted, + Signer(AnyVerifyingKey), + Any(Vec), + All(Vec), +} +impl AcceptSigners { + const fn null() -> Self { + Self::Any(Vec::new()) + } + pub fn flatten(self) -> Self { + match self { + Self::Any(mut s) | Self::All(mut s) if s.len() == 1 => s.swap_remove(0).flatten(), + s => s, + } + } + pub fn accepted(&self) -> bool { + match self { + Self::Accepted => true, + _ => false, + } + } + pub fn try_accept(self) -> Result<(), Error> { + if self.accepted() { + Ok(()) + } else { + Err(Error::new( + eyre!("signer(s) not accepted"), + ErrorKind::InvalidSignature, + )) + } + } + pub fn process_signature( + &mut self, + signer: &AnyVerifyingKey, + commitment: &impl Digestable, + context: &str, + signature: &AnySignature, + ) -> Result<(), Error> { + let mut res = Ok(()); + let new = match std::mem::replace(self, Self::null()) { + Self::Accepted => Self::Accepted, + Self::Signer(s) => { + if &s == signer { + res = signer + .scheme() + .verify_commitment(signer, commitment, context, signature); + Self::Accepted + } else { + Self::Signer(s) + } + } + Self::All(mut s) => { + res = s + .iter_mut() + .map(|s| s.process_signature(signer, commitment, context, signature)) + .collect(); + if s.iter().all(|s| s.accepted()) { + Self::Accepted + } else { + Self::All(s) + } + } + Self::Any(mut s) => { + match s + .iter_mut() + .map(|s| { + s.process_signature(signer, commitment, context, signature)?; + Ok(s) + }) + .filter_ok(|s| s.accepted()) + .next() + { + Some(Ok(s)) => std::mem::replace(s, Self::null()), + Some(Err(e)) => { + res = Err(e); + Self::Any(s) + } + None => Self::Any(s), + } + } + }; + *self = new; + res + } +} diff --git a/core/startos/src/registry/signer/sign/ed25519.rs b/core/startos/src/registry/signer/sign/ed25519.rs new file mode 100644 index 000000000..3ec4c136e --- /dev/null +++ b/core/startos/src/registry/signer/sign/ed25519.rs @@ -0,0 +1,34 @@ +use ed25519_dalek::{Signature, SigningKey, VerifyingKey}; +use sha2::Sha512; + +use crate::prelude::*; +use crate::registry::signer::sign::SignatureScheme; + +pub struct Ed25519; +impl SignatureScheme for Ed25519 { + type SigningKey = SigningKey; + type VerifyingKey = VerifyingKey; + type Signature = Signature; + type Digest = Sha512; + fn new_digest(&self) -> Self::Digest { + ::new() + } + fn sign( + &self, + key: &Self::SigningKey, + digest: Self::Digest, + context: &str, + ) -> Result { + Ok(key.sign_prehashed(digest, Some(context.as_bytes()))?) + } + fn verify( + &self, + key: &Self::VerifyingKey, + digest: Self::Digest, + context: &str, + signature: &Self::Signature, + ) -> Result<(), Error> { + key.verify_prehashed_strict(digest, Some(context.as_bytes()), signature)?; + Ok(()) + } +} diff --git a/core/startos/src/registry/signer/sign/mod.rs b/core/startos/src/registry/signer/sign/mod.rs new file mode 100644 index 000000000..50576a198 --- /dev/null +++ b/core/startos/src/registry/signer/sign/mod.rs @@ -0,0 +1,348 @@ +use std::fmt::Display; +use std::str::FromStr; + +use ::ed25519::pkcs8::BitStringRef; +use clap::builder::ValueParserFactory; +use der::referenced::OwnedToRef; +use der::{Decode, Encode}; +use pkcs8::der::AnyRef; +use pkcs8::{PrivateKeyInfo, SubjectPublicKeyInfo}; +use serde::{Deserialize, Serialize}; +use sha2::Sha512; +use ts_rs::TS; + +use crate::prelude::*; +use crate::registry::signer::commitment::Digestable; +use crate::registry::signer::sign::ed25519::Ed25519; +use crate::util::clap::FromStrParser; +use crate::util::serde::{deserialize_from_str, serialize_display}; + +pub mod ed25519; + +pub trait SignatureScheme { + type SigningKey; + type VerifyingKey; + type Signature; + type Digest: digest::Update; + fn new_digest(&self) -> Self::Digest; + fn sign( + &self, + key: &Self::SigningKey, + digest: Self::Digest, + context: &str, + ) -> Result; + fn sign_commitment( + &self, + key: &Self::SigningKey, + commitment: &C, + context: &str, + ) -> Result { + let mut digest = self.new_digest(); + commitment.update(&mut digest); + self.sign(key, digest, context) + } + fn verify( + &self, + key: &Self::VerifyingKey, + digest: Self::Digest, + context: &str, + signature: &Self::Signature, + ) -> Result<(), Error>; + fn verify_commitment( + &self, + key: &Self::VerifyingKey, + commitment: &C, + context: &str, + signature: &Self::Signature, + ) -> Result<(), Error> { + let mut digest = self.new_digest(); + commitment.update(&mut digest); + self.verify(key, digest, context, signature) + } +} + +pub enum AnyScheme { + Ed25519(Ed25519), +} +impl From for AnyScheme { + fn from(value: Ed25519) -> Self { + Self::Ed25519(value) + } +} +impl SignatureScheme for AnyScheme { + type SigningKey = AnySigningKey; + type VerifyingKey = AnyVerifyingKey; + type Signature = AnySignature; + type Digest = AnyDigest; + fn new_digest(&self) -> Self::Digest { + match self { + Self::Ed25519(s) => AnyDigest::Sha512(s.new_digest()), + } + } + fn sign( + &self, + key: &Self::SigningKey, + digest: Self::Digest, + context: &str, + ) -> Result { + match (self, key, digest) { + (Self::Ed25519(s), AnySigningKey::Ed25519(key), AnyDigest::Sha512(digest)) => { + Ok(AnySignature::Ed25519(s.sign(key, digest, context)?)) + } + _ => Err(Error::new( + eyre!("mismatched signature algorithm"), + ErrorKind::InvalidSignature, + )), + } + } + fn verify( + &self, + key: &Self::VerifyingKey, + digest: Self::Digest, + context: &str, + signature: &Self::Signature, + ) -> Result<(), Error> { + match (self, key, digest, signature) { + ( + Self::Ed25519(s), + AnyVerifyingKey::Ed25519(key), + AnyDigest::Sha512(digest), + AnySignature::Ed25519(signature), + ) => s.verify(key, digest, context, signature), + _ => Err(Error::new( + eyre!("mismatched signature algorithm"), + ErrorKind::InvalidSignature, + )), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, TS)] +#[ts(export, type = "string")] +pub enum AnySigningKey { + Ed25519(::SigningKey), +} +impl AnySigningKey { + pub fn scheme(&self) -> AnyScheme { + match self { + Self::Ed25519(_) => AnyScheme::Ed25519(Ed25519), + } + } + pub fn verifying_key(&self) -> AnyVerifyingKey { + match self { + Self::Ed25519(k) => AnyVerifyingKey::Ed25519(k.into()), + } + } +} +impl<'a> TryFrom> for AnySigningKey { + type Error = pkcs8::Error; + fn try_from(value: PrivateKeyInfo<'a>) -> Result { + if value.algorithm == ed25519_dalek::pkcs8::ALGORITHM_ID { + Ok(Self::Ed25519(ed25519_dalek::SigningKey::try_from(value)?)) + } else { + Err(pkcs8::spki::Error::OidUnknown { + oid: value.algorithm.oid, + } + .into()) + } + } +} +impl pkcs8::EncodePrivateKey for AnySigningKey { + fn to_pkcs8_der(&self) -> pkcs8::Result { + match self { + Self::Ed25519(s) => s.to_pkcs8_der(), + } + } +} +impl FromStr for AnySigningKey { + type Err = Error; + fn from_str(s: &str) -> Result { + use pkcs8::DecodePrivateKey; + Self::from_pkcs8_pem(s).with_kind(ErrorKind::Deserialization) + } +} +impl Display for AnySigningKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use pkcs8::EncodePrivateKey; + f.write_str( + &self + .to_pkcs8_pem(pkcs8::LineEnding::LF) + .map_err(|_| std::fmt::Error)?, + ) + } +} +impl<'de> Deserialize<'de> for AnySigningKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserialize_from_str(deserializer) + } +} +impl Serialize for AnySigningKey { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_display(self, serializer) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, TS)] +#[ts(export, type = "string")] +pub enum AnyVerifyingKey { + Ed25519(::VerifyingKey), +} +impl AnyVerifyingKey { + pub fn scheme(&self) -> AnyScheme { + match self { + Self::Ed25519(_) => AnyScheme::Ed25519(Ed25519), + } + } +} +impl<'a> TryFrom, BitStringRef<'a>>> for AnyVerifyingKey { + type Error = pkcs8::spki::Error; + fn try_from( + value: SubjectPublicKeyInfo, BitStringRef<'a>>, + ) -> Result { + if value.algorithm == ed25519_dalek::pkcs8::ALGORITHM_ID { + Ok(Self::Ed25519(ed25519_dalek::VerifyingKey::try_from(value)?)) + } else { + Err(pkcs8::spki::Error::OidUnknown { + oid: value.algorithm.oid, + }) + } + } +} +impl pkcs8::EncodePublicKey for AnyVerifyingKey { + fn to_public_key_der(&self) -> pkcs8::spki::Result { + match self { + Self::Ed25519(s) => s.to_public_key_der(), + } + } +} +impl FromStr for AnyVerifyingKey { + type Err = Error; + fn from_str(s: &str) -> Result { + use pkcs8::DecodePublicKey; + Self::from_public_key_pem(s).with_kind(ErrorKind::Deserialization) + } +} +impl Display for AnyVerifyingKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use pkcs8::EncodePublicKey; + f.write_str( + &self + .to_public_key_pem(pkcs8::LineEnding::LF) + .map_err(|_| std::fmt::Error)?, + ) + } +} +impl<'de> Deserialize<'de> for AnyVerifyingKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserialize_from_str(deserializer) + } +} +impl Serialize for AnyVerifyingKey { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_display(self, serializer) + } +} +impl ValueParserFactory for AnyVerifyingKey { + type Parser = FromStrParser; + fn value_parser() -> Self::Parser { + Self::Parser::new() + } +} + +#[derive(Clone, Debug)] +pub enum AnyDigest { + Sha512(Sha512), +} +impl digest::Update for AnyDigest { + fn update(&mut self, data: &[u8]) { + match self { + Self::Sha512(d) => digest::Update::update(d, data), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, TS)] +#[ts(export, type = "string")] +pub enum AnySignature { + Ed25519(::Signature), +} +impl FromStr for AnySignature { + type Err = Error; + fn from_str(s: &str) -> Result { + use der::DecodePem; + + #[derive(der::Sequence)] + struct AnySignatureDer { + alg: pkcs8::spki::AlgorithmIdentifierOwned, + sig: der::asn1::OctetString, + } + impl der::pem::PemLabel for AnySignatureDer { + const PEM_LABEL: &'static str = "SIGNATURE"; + } + + let der = AnySignatureDer::from_pem(s.as_bytes()).with_kind(ErrorKind::Deserialization)?; + if der.alg.oid == ed25519_dalek::pkcs8::ALGORITHM_ID.oid + && der.alg.parameters.owned_to_ref() == ed25519_dalek::pkcs8::ALGORITHM_ID.parameters + { + Ok(Self::Ed25519( + ed25519_dalek::Signature::from_slice(der.sig.as_bytes()) + .with_kind(ErrorKind::Deserialization)?, + )) + } else { + Err(pkcs8::spki::Error::OidUnknown { oid: der.alg.oid }) + .with_kind(ErrorKind::Deserialization) + } + } +} +impl Display for AnySignature { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use der::EncodePem; + + #[derive(der::Sequence)] + struct AnySignatureDer<'a> { + alg: pkcs8::AlgorithmIdentifierRef<'a>, + sig: der::asn1::OctetString, + } + impl<'a> der::pem::PemLabel for AnySignatureDer<'a> { + const PEM_LABEL: &'static str = "SIGNATURE"; + } + f.write_str( + &match self { + Self::Ed25519(s) => AnySignatureDer { + alg: ed25519_dalek::pkcs8::ALGORITHM_ID, + sig: der::asn1::OctetString::new(s.to_bytes()).map_err(|_| std::fmt::Error)?, + }, + } + .to_pem(der::pem::LineEnding::LF) + .map_err(|_| std::fmt::Error)?, + ) + } +} +impl<'de> Deserialize<'de> for AnySignature { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserialize_from_str(deserializer) + } +} +impl Serialize for AnySignature { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_display(self, serializer) + } +} diff --git a/core/startos/src/rpc_continuations.rs b/core/startos/src/rpc_continuations.rs index 04b88ea8e..ce8bf43fd 100644 --- a/core/startos/src/rpc_continuations.rs +++ b/core/startos/src/rpc_continuations.rs @@ -10,20 +10,24 @@ use futures::future::BoxFuture; use helpers::TimedResource; use imbl_value::InternedString; use tokio::sync::Mutex; +use ts_rs::TS; #[allow(unused_imports)] use crate::prelude::*; use crate::util::clap::FromStrParser; use crate::util::new_guid; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] -pub struct RequestGuid(InternedString); -impl RequestGuid { +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, TS, +)] +#[ts(type = "string")] +pub struct Guid(InternedString); +impl Guid { pub fn new() -> Self { Self(new_guid()) } - pub fn from(r: &str) -> Option { + pub fn from(r: &str) -> Option { if r.len() != 32 { return None; } @@ -32,21 +36,21 @@ impl RequestGuid { return None; } } - Some(RequestGuid(InternedString::intern(r))) + Some(Guid(InternedString::intern(r))) } } -impl AsRef for RequestGuid { +impl AsRef for Guid { fn as_ref(&self) -> &str { self.0.as_ref() } } -impl FromStr for RequestGuid { +impl FromStr for Guid { type Err = Error; fn from_str(s: &str) -> Result { Self::from(s).ok_or_else(|| Error::new(eyre!("invalid guid"), ErrorKind::Deserialization)) } } -impl ValueParserFactory for RequestGuid { +impl ValueParserFactory for Guid { type Parser = FromStrParser; fn value_parser() -> Self::Parser { Self::Parser::new() @@ -55,13 +59,10 @@ impl ValueParserFactory for RequestGuid { #[test] fn parse_guid() { - println!( - "{:?}", - RequestGuid::from(&format!("{}", RequestGuid::new())) - ) + println!("{:?}", Guid::from(&format!("{}", Guid::new()))) } -impl std::fmt::Display for RequestGuid { +impl std::fmt::Display for Guid { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } @@ -91,7 +92,7 @@ impl RpcContinuation { } } -pub struct RpcContinuations(Mutex>); +pub struct RpcContinuations(Mutex>); impl RpcContinuations { pub fn new() -> Self { RpcContinuations(Mutex::new(BTreeMap::new())) @@ -112,12 +113,12 @@ impl RpcContinuations { } #[instrument(skip_all)] - pub async fn add(&self, guid: RequestGuid, handler: RpcContinuation) { + pub async fn add(&self, guid: Guid, handler: RpcContinuation) { self.clean().await; self.0.lock().await.insert(guid, handler); } - pub async fn get_ws_handler(&self, guid: &RequestGuid) -> Option { + pub async fn get_ws_handler(&self, guid: &Guid) -> Option { let mut continuations = self.0.lock().await; if !matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) { return None; @@ -128,8 +129,8 @@ impl RpcContinuations { x.get().await } - pub async fn get_rest_handler(&self, guid: &RequestGuid) -> Option { - let mut continuations: tokio::sync::MutexGuard<'_, BTreeMap> = + pub async fn get_rest_handler(&self, guid: &Guid) -> Option { + let mut continuations: tokio::sync::MutexGuard<'_, BTreeMap> = self.0.lock().await; if !matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) { return None; diff --git a/core/startos/src/s9pk/v1/git_hash.rs b/core/startos/src/s9pk/git_hash.rs similarity index 52% rename from core/startos/src/s9pk/v1/git_hash.rs rename to core/startos/src/s9pk/git_hash.rs index b2990a111..02f83bf4a 100644 --- a/core/startos/src/s9pk/v1/git_hash.rs +++ b/core/startos/src/s9pk/git_hash.rs @@ -1,24 +1,35 @@ use std::path::Path; -use crate::Error; +use tokio::process::Command; + +use crate::prelude::*; +use crate::util::Invoke; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct GitHash(String); impl GitHash { pub async fn from_path(path: impl AsRef) -> Result { - let hash = tokio::process::Command::new("git") - .args(["describe", "--always", "--abbrev=40", "--dirty=-modified"]) - .current_dir(path) - .output() - .await?; - if !hash.status.success() { - return Err(Error::new( - color_eyre::eyre::eyre!("Could not get hash: {}", String::from_utf8(hash.stderr)?), - crate::ErrorKind::Filesystem, - )); + let mut hash = String::from_utf8( + Command::new("git") + .arg("rev-parse") + .arg("HEAD") + .current_dir(&path) + .invoke(ErrorKind::Git) + .await?, + )?; + if Command::new("git") + .arg("diff-index") + .arg("--quiet") + .arg("HEAD") + .arg("--") + .invoke(ErrorKind::Git) + .await + .is_err() + { + hash += "-modified"; } - Ok(GitHash(String::from_utf8(hash.stdout)?)) + Ok(GitHash(hash)) } } diff --git a/core/startos/src/s9pk/merkle_archive/directory_contents.rs b/core/startos/src/s9pk/merkle_archive/directory_contents.rs index ebc46a60b..77c3582d9 100644 --- a/core/startos/src/s9pk/merkle_archive/directory_contents.rs +++ b/core/startos/src/s9pk/merkle_archive/directory_contents.rs @@ -12,11 +12,12 @@ use itertools::Itertools; use tokio::io::AsyncRead; use crate::prelude::*; -use crate::s9pk::merkle_archive::sink::{Sink, TrackingWriter}; +use crate::s9pk::merkle_archive::sink::Sink; use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section}; use crate::s9pk::merkle_archive::write_queue::WriteQueue; use crate::s9pk::merkle_archive::{varint, Entry, EntryContents}; -use crate::util::io::ParallelBlake3Writer; +use crate::util::io::{ParallelBlake3Writer, TrackingIO}; +use crate::CAP_10_MiB; #[derive(Clone)] pub struct DirectoryContents { @@ -151,7 +152,7 @@ impl DirectoryContents { Ok(()) } } -impl DirectoryContents> { +impl DirectoryContents> { #[instrument(skip_all)] pub fn deserialize<'a>( source: &'a S, @@ -181,7 +182,7 @@ impl DirectoryContents> { let mut entries = OrdMap::new(); for _ in 0..len { let name = varint::deserialize_varstring(&mut toc_reader).await?; - let entry = Entry::deserialize(source, &mut toc_reader).await?; + let entry = Entry::deserialize(source.clone(), &mut toc_reader).await?; entries.insert(name.into(), entry); } @@ -202,7 +203,7 @@ impl DirectoryContents> { .boxed() } } -impl DirectoryContents { +impl DirectoryContents { pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> { for k in self.keys().cloned().collect::>() { let path = Path::new(&*k); @@ -239,8 +240,7 @@ impl DirectoryContents { #[instrument(skip_all)] pub fn sighash<'a>(&'a self) -> BoxFuture<'a, Result> { async move { - let mut hasher = - TrackingWriter::new(0, ParallelBlake3Writer::new(super::hash::BUFFER_CAPACITY)); + let mut hasher = TrackingIO::new(0, ParallelBlake3Writer::new(CAP_10_MiB)); let mut sig_contents = OrdMap::new(); for (name, entry) in &**self { sig_contents.insert(name.clone(), entry.to_missing().await?); @@ -280,6 +280,7 @@ impl DirectoryContents { Ok(()) } + pub fn into_dyn(self) -> DirectoryContents { DirectoryContents { contents: self diff --git a/core/startos/src/s9pk/merkle_archive/file_contents.rs b/core/startos/src/s9pk/merkle_archive/file_contents.rs index a2d19ed88..c34193e31 100644 --- a/core/startos/src/s9pk/merkle_archive/file_contents.rs +++ b/core/startos/src/s9pk/merkle_archive/file_contents.rs @@ -2,9 +2,10 @@ use blake3::Hash; use tokio::io::AsyncRead; use crate::prelude::*; -use crate::s9pk::merkle_archive::sink::{Sink, TrackingWriter}; +use crate::s9pk::merkle_archive::sink::Sink; use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section}; -use crate::util::io::ParallelBlake3Writer; +use crate::util::io::{ParallelBlake3Writer, TrackingIO}; +use crate::CAP_10_MiB; #[derive(Debug, Clone)] pub struct FileContents(S); @@ -19,7 +20,7 @@ impl FileContents { impl FileContents> { #[instrument(skip_all)] pub async fn deserialize( - source: &S, + source: S, header: &mut (impl AsyncRead + Unpin + Send), size: u64, ) -> Result { @@ -34,8 +35,7 @@ impl FileContents> { } impl FileContents { pub async fn hash(&self) -> Result<(Hash, u64), Error> { - let mut hasher = - TrackingWriter::new(0, ParallelBlake3Writer::new(super::hash::BUFFER_CAPACITY)); + let mut hasher = TrackingIO::new(0, ParallelBlake3Writer::new(CAP_10_MiB)); self.serialize_body(&mut hasher, None).await?; let size = hasher.position(); let hash = hasher.into_inner().finalize().await?; diff --git a/core/startos/src/s9pk/merkle_archive/hash.rs b/core/startos/src/s9pk/merkle_archive/hash.rs index fc635c435..c7ad470e4 100644 --- a/core/startos/src/s9pk/merkle_archive/hash.rs +++ b/core/startos/src/s9pk/merkle_archive/hash.rs @@ -6,8 +6,7 @@ use tokio_util::either::Either; use crate::prelude::*; use crate::util::io::{ParallelBlake3Writer, TeeWriter}; - -pub const BUFFER_CAPACITY: usize = 10 * 1024 * 1024; // 10MiB +use crate::CAP_10_MiB; #[pin_project::pin_project] pub struct VerifyingWriter { @@ -21,8 +20,8 @@ impl VerifyingWriter { writer: if verify.is_some() { Either::Left(TeeWriter::new( w, - ParallelBlake3Writer::new(BUFFER_CAPACITY), - BUFFER_CAPACITY, + ParallelBlake3Writer::new(CAP_10_MiB), + CAP_10_MiB, )) } else { Either::Right(w) diff --git a/core/startos/src/s9pk/merkle_archive/mod.rs b/core/startos/src/s9pk/merkle_archive/mod.rs index 3863ceeaf..1c0d6b786 100644 --- a/core/startos/src/s9pk/merkle_archive/mod.rs +++ b/core/startos/src/s9pk/merkle_archive/mod.rs @@ -7,11 +7,16 @@ use sha2::{Digest, Sha512}; use tokio::io::AsyncRead; use crate::prelude::*; +use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; +use crate::registry::signer::sign::ed25519::Ed25519; +use crate::registry::signer::sign::SignatureScheme; use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; use crate::s9pk::merkle_archive::file_contents::FileContents; use crate::s9pk::merkle_archive::sink::Sink; use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section}; use crate::s9pk::merkle_archive::write_queue::WriteQueue; +use crate::util::serde::Base64; +use crate::CAP_1_MiB; pub mod directory_contents; pub mod file_contents; @@ -70,12 +75,13 @@ impl MerkleArchive { self.contents.sort_by(sort_by) } } -impl MerkleArchive> { +impl MerkleArchive> { #[instrument(skip_all)] pub async fn deserialize( source: &S, context: &str, header: &mut (impl AsyncRead + Unpin + Send), + commitment: Option<&MerkleArchiveCommitment>, ) -> Result { use tokio::io::AsyncReadExt; @@ -101,6 +107,32 @@ impl MerkleArchive> { &signature, )?; + if let Some(MerkleArchiveCommitment { + root_sighash, + root_maxsize, + }) = commitment + { + if sighash.as_bytes() != &**root_sighash { + return Err(Error::new( + eyre!("merkle root mismatch"), + ErrorKind::InvalidSignature, + )); + } + if max_size > *root_maxsize { + return Err(Error::new( + eyre!("merkle root directory max size too large"), + ErrorKind::InvalidSignature, + )); + } + } else { + if max_size > CAP_1_MiB as u64 { + return Err(Error::new( + eyre!("merkle root directory max size over 1MiB, cancelling download in case of DOS attack"), + ErrorKind::InvalidSignature, + )); + } + } + let contents = DirectoryContents::deserialize(source, header, (sighash, max_size)).await?; Ok(Self { @@ -109,38 +141,50 @@ impl MerkleArchive> { }) } } -impl MerkleArchive { +impl MerkleArchive { pub async fn update_hashes(&mut self, only_missing: bool) -> Result<(), Error> { self.contents.update_hashes(only_missing).await } pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> { self.contents.filter(filter) } + pub async fn commitment(&self) -> Result { + let root_maxsize = match self.signer { + Signer::Signed(_, _, s, _) => s, + _ => self.contents.toc_size(), + }; + let root_sighash = self.contents.sighash().await?; + Ok(MerkleArchiveCommitment { + root_sighash: Base64(*root_sighash.as_bytes()), + root_maxsize, + }) + } + pub async fn signature(&self) -> Result { + match &self.signer { + Signer::Signed(_, s, _, _) => Ok(*s), + Signer::Signer(k, context) => { + Ed25519.sign_commitment(k, &self.commitment().await?, context) + } + } + } #[instrument(skip_all)] pub async fn serialize(&self, w: &mut W, verify: bool) -> Result<(), Error> { use tokio::io::AsyncWriteExt; - let sighash = self.contents.sighash().await?; - let size = self.contents.toc_size(); + let commitment = self.commitment().await?; - let (pubkey, signature, max_size) = match &self.signer { - Signer::Signed(pubkey, signature, max_size, _) => (*pubkey, *signature, *max_size), - Signer::Signer(s, context) => ( - s.into(), - ed25519_dalek::SigningKey::sign_prehashed( - s, - Sha512::new_with_prefix(sighash.as_bytes()) - .chain_update(&u64::to_be_bytes(size)), - Some(context.as_bytes()), - )?, - size, - ), + let (pubkey, signature) = match &self.signer { + Signer::Signed(pubkey, signature, _, _) => (*pubkey, *signature), + Signer::Signer(s, context) => { + (s.into(), Ed25519.sign_commitment(s, &commitment, context)?) + } }; w.write_all(pubkey.as_bytes()).await?; w.write_all(&signature.to_bytes()).await?; - w.write_all(sighash.as_bytes()).await?; - w.write_all(&u64::to_be_bytes(max_size)).await?; + w.write_all(&*commitment.root_sighash).await?; + w.write_all(&u64::to_be_bytes(commitment.root_maxsize)) + .await?; let mut next_pos = w.current_position().await?; next_pos += DirectoryContents::::header_size(); self.contents.serialize_header(next_pos, w).await?; @@ -216,11 +260,10 @@ impl Entry { + self.contents.header_size() } } -impl Entry {} -impl Entry> { +impl Entry> { #[instrument(skip_all)] pub async fn deserialize( - source: &S, + source: S, header: &mut (impl AsyncRead + Unpin + Send), ) -> Result { use tokio::io::AsyncReadExt; @@ -241,24 +284,19 @@ impl Entry> { }) } } -impl Entry { +impl Entry { pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> { if let EntryContents::Directory(d) = &mut self.contents { d.filter(filter)?; } Ok(()) } - pub async fn read_file_to_vec(&self) -> Result, Error> { - match self.as_contents() { - EntryContents::File(f) => Ok(f.to_vec(self.hash).await?), - EntryContents::Directory(_) => Err(Error::new( - eyre!("expected file, found directory"), - ErrorKind::ParseS9pk, - )), - EntryContents::Missing => { - Err(Error::new(eyre!("entry is missing"), ErrorKind::ParseS9pk)) - } + pub async fn update_hash(&mut self, only_missing: bool) -> Result<(), Error> { + if let EntryContents::Directory(d) = &mut self.contents { + d.update_hashes(only_missing).await?; } + self.hash = Some(self.contents.hash().await?); + Ok(()) } pub async fn to_missing(&self) -> Result { let hash = if let Some(hash) = self.hash { @@ -271,13 +309,6 @@ impl Entry { contents: EntryContents::Missing, }) } - pub async fn update_hash(&mut self, only_missing: bool) -> Result<(), Error> { - if let EntryContents::Directory(d) = &mut self.contents { - d.update_hashes(only_missing).await?; - } - self.hash = Some(self.contents.hash().await?); - Ok(()) - } #[instrument(skip_all)] pub async fn serialize_header( &self, @@ -302,6 +333,20 @@ impl Entry { } } } +impl Entry { + pub async fn read_file_to_vec(&self) -> Result, Error> { + match self.as_contents() { + EntryContents::File(f) => Ok(f.to_vec(self.hash).await?), + EntryContents::Directory(_) => Err(Error::new( + eyre!("expected file, found directory"), + ErrorKind::ParseS9pk, + )), + EntryContents::Missing => { + Err(Error::new(eyre!("entry is missing"), ErrorKind::ParseS9pk)) + } + } + } +} #[derive(Debug, Clone)] pub enum EntryContents { @@ -329,10 +374,10 @@ impl EntryContents { matches!(self, &EntryContents::Directory(_)) } } -impl EntryContents> { +impl EntryContents> { #[instrument(skip_all)] pub async fn deserialize( - source: &S, + source: S, header: &mut (impl AsyncRead + Unpin + Send), (hash, size): (Hash, u64), ) -> Result { @@ -346,7 +391,7 @@ impl EntryContents> { FileContents::deserialize(source, header, size).await?, )), 2 => Ok(Self::Directory( - DirectoryContents::deserialize(source, header, (hash, size)).await?, + DirectoryContents::deserialize(&source, header, (hash, size)).await?, )), id => Err(Error::new( eyre!("Unknown type id {id} found in MerkleArchive"), @@ -355,7 +400,7 @@ impl EntryContents> { } } } -impl EntryContents { +impl EntryContents { pub async fn hash(&self) -> Result<(Hash, u64), Error> { match self { Self::Missing => Err(Error::new( @@ -366,6 +411,15 @@ impl EntryContents { Self::Directory(d) => Ok((d.sighash().await?, d.toc_size())), } } + pub fn into_dyn(self) -> EntryContents { + match self { + Self::Missing => EntryContents::Missing, + Self::File(f) => EntryContents::File(f.into_dyn()), + Self::Directory(d) => EntryContents::Directory(d.into_dyn()), + } + } +} +impl EntryContents { #[instrument(skip_all)] pub async fn serialize_header( &self, @@ -381,11 +435,4 @@ impl EntryContents { Self::Directory(d) => Some(d.serialize_header(position, w).await?), }) } - pub fn into_dyn(self) -> EntryContents { - match self { - Self::Missing => EntryContents::Missing, - Self::File(f) => EntryContents::File(f.into_dyn()), - Self::Directory(d) => EntryContents::Directory(d.into_dyn()), - } - } } diff --git a/core/startos/src/s9pk/merkle_archive/sink.rs b/core/startos/src/s9pk/merkle_archive/sink.rs index ec1431b8e..5357eb2d6 100644 --- a/core/startos/src/s9pk/merkle_archive/sink.rs +++ b/core/startos/src/s9pk/merkle_archive/sink.rs @@ -1,6 +1,7 @@ use tokio::io::{AsyncSeek, AsyncWrite}; use crate::prelude::*; +use crate::util::io::TrackingIO; #[async_trait::async_trait] pub trait Sink: AsyncWrite + Unpin + Send { @@ -17,57 +18,8 @@ impl Sink for S { } #[async_trait::async_trait] -impl Sink for TrackingWriter { +impl Sink for TrackingIO { async fn current_position(&mut self) -> Result { - Ok(self.position) - } -} - -#[pin_project::pin_project] -pub struct TrackingWriter { - position: u64, - #[pin] - writer: W, -} -impl TrackingWriter { - pub fn new(start: u64, w: W) -> Self { - Self { - position: start, - writer: w, - } - } - pub fn position(&self) -> u64 { - self.position - } - pub fn into_inner(self) -> W { - self.writer - } -} -impl AsyncWrite for TrackingWriter { - fn poll_write( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &[u8], - ) -> std::task::Poll> { - let this = self.project(); - match this.writer.poll_write(cx, buf) { - std::task::Poll::Ready(Ok(written)) => { - *this.position += written as u64; - std::task::Poll::Ready(Ok(written)) - } - a => a, - } - } - fn poll_flush( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.project().writer.poll_flush(cx) - } - fn poll_shutdown( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.project().writer.poll_shutdown(cx) + Ok(self.position()) } } diff --git a/core/startos/src/s9pk/merkle_archive/source/http.rs b/core/startos/src/s9pk/merkle_archive/source/http.rs index 738c480ed..fda9d32ed 100644 --- a/core/startos/src/s9pk/merkle_archive/source/http.rs +++ b/core/startos/src/s9pk/merkle_archive/source/http.rs @@ -1,23 +1,25 @@ +use std::collections::BTreeSet; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + use bytes::Bytes; -use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use reqwest::header::{ACCEPT_RANGES, CONTENT_LENGTH, RANGE}; use reqwest::{Client, Url}; -use tokio::io::{AsyncRead, AsyncReadExt, Take}; +use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf, Take}; use tokio_util::io::StreamReader; use crate::prelude::*; use crate::s9pk::merkle_archive::source::ArchiveSource; +use crate::util::io::TrackingIO; +use crate::util::Apply; -#[derive(Clone)] pub struct HttpSource { url: Url, client: Client, size: Option, - range_support: Result< - (), - (), // Arc>> - >, + range_support: Result<(), Arc>>>>, } impl HttpSource { pub async fn new(client: Client, url: Url) -> Result { @@ -32,7 +34,8 @@ impl HttpSource { .headers() .get(ACCEPT_RANGES) .and_then(|s| s.to_str().ok()) - == Some("bytes"); + == Some("bytes") + && false; let size = head .headers() .get(CONTENT_LENGTH) @@ -45,53 +48,141 @@ impl HttpSource { range_support: if range_support { Ok(()) } else { - Err(()) // Err(Arc::new(Mutex::new(None))) + Err(Arc::new(Mutex::new(BTreeSet::new()))) }, }) } } impl ArchiveSource for HttpSource { - type Reader = Take; + type Reader = HttpReader; async fn size(&self) -> Option { self.size } + async fn fetch_all(&self) -> Result { + Ok(StreamReader::new( + self.client + .get(self.url.clone()) + .send() + .await + .with_kind(ErrorKind::Network)? + .error_for_status() + .with_kind(ErrorKind::Network)? + .bytes_stream() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) + .apply(boxed), + )) + } async fn fetch(&self, position: u64, size: u64) -> Result { - match self.range_support { - Ok(_) => Ok(HttpReader::Range(StreamReader::new(if size > 0 { - self.client - .get(self.url.clone()) - .header(RANGE, format!("bytes={}-{}", position, position + size - 1)) - .send() - .await - .with_kind(ErrorKind::Network)? - .error_for_status() - .with_kind(ErrorKind::Network)? - .bytes_stream() - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) - .boxed() - } else { - futures::stream::empty().boxed() - })) - .take(size)), - _ => todo!(), + match &self.range_support { + Ok(_) => Ok(HttpReader::Range( + StreamReader::new(if size > 0 { + self.client + .get(self.url.clone()) + .header(RANGE, format!("bytes={}-{}", position, position + size - 1)) + .send() + .await + .with_kind(ErrorKind::Network)? + .error_for_status() + .with_kind(ErrorKind::Network)? + .bytes_stream() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) + .apply(boxed) + } else { + futures::stream::empty().apply(boxed) + }) + .take(size), + )), + Err(pool) => { + fn get_reader_for( + pool: &Arc>>>, + position: u64, + ) -> Option> { + let mut lock = pool.lock().unwrap(); + let pos = lock.range(..position).last()?.position(); + lock.take(&pos) + } + let reader = get_reader_for(pool, position); + let mut reader = if let Some(reader) = reader { + reader + } else { + TrackingIO::new( + 0, + StreamReader::new( + self.client + .get(self.url.clone()) + .send() + .await + .with_kind(ErrorKind::Network)? + .error_for_status() + .with_kind(ErrorKind::Network)? + .bytes_stream() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) + .apply(boxed), + ), + ) + }; + if reader.position() < position { + let to_skip = position - reader.position(); + tokio::io::copy(&mut (&mut reader).take(to_skip), &mut tokio::io::sink()) + .await?; + } + Ok(HttpReader::Rangeless { + pool: pool.clone(), + reader: Some(reader.take(size)), + }) + } } } } -#[pin_project::pin_project(project = HttpReaderProj)] +type BoxStream<'a, T> = Pin + Send + Sync + 'a>>; +fn boxed<'a, T>(stream: impl Stream + Send + Sync + 'a) -> BoxStream<'a, T> { + Box::pin(stream) +} +type HttpBodyReader = StreamReader>, Bytes>; + +#[pin_project::pin_project(project = HttpReaderProj, PinnedDrop)] pub enum HttpReader { - Range(#[pin] StreamReader>, Bytes>), - // Rangeless(#[pin] RangelessReader), + Range(#[pin] Take), + Rangeless { + pool: Arc>>>, + #[pin] + reader: Option>>, + }, } impl AsyncRead for HttpReader { fn poll_read( - self: std::pin::Pin<&mut Self>, + self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { + buf: &mut ReadBuf<'_>, + ) -> Poll> { match self.project() { HttpReaderProj::Range(r) => r.poll_read(cx, buf), - // HttpReaderProj::Rangeless(r) => r.poll_read(cx, buf), + HttpReaderProj::Rangeless { mut reader, .. } => { + let mut finished = false; + if let Some(reader) = reader.as_mut().as_pin_mut() { + let start = buf.filled().len(); + futures::ready!(reader.poll_read(cx, buf)?); + finished = start == buf.filled().len(); + } + if finished { + reader.take(); + } + Poll::Ready(Ok(())) + } + } + } +} +#[pin_project::pinned_drop] +impl PinnedDrop for HttpReader { + fn drop(self: Pin<&mut Self>) { + match self.project() { + HttpReaderProj::Range(_) => (), + HttpReaderProj::Rangeless { pool, mut reader } => { + if let Some(reader) = reader.take() { + pool.lock().unwrap().insert(reader.into_inner()); + } + } } } } diff --git a/core/startos/src/s9pk/merkle_archive/source/mod.rs b/core/startos/src/s9pk/merkle_archive/source/mod.rs index 6e3b1e584..f6922d109 100644 --- a/core/startos/src/s9pk/merkle_archive/source/mod.rs +++ b/core/startos/src/s9pk/merkle_archive/source/mod.rs @@ -1,3 +1,4 @@ +use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; @@ -13,7 +14,7 @@ use crate::s9pk::merkle_archive::hash::VerifyingWriter; pub mod http; pub mod multi_cursor_file; -pub trait FileSource: Clone + Send + Sync + Sized + 'static { +pub trait FileSource: Send + Sync + Sized + 'static { type Reader: AsyncRead + Unpin + Send; fn size(&self) -> impl Future> + Send; fn reader(&self) -> impl Future> + Send; @@ -61,6 +62,29 @@ pub trait FileSource: Clone + Send + Sync + Sized + 'static { } } +impl FileSource for Arc { + type Reader = T::Reader; + async fn size(&self) -> Result { + self.deref().size().await + } + async fn reader(&self) -> Result { + self.deref().reader().await + } + async fn copy(&self, w: &mut W) -> Result<(), Error> { + self.deref().copy(w).await + } + async fn copy_verify( + &self, + w: &mut W, + verify: Option<(Hash, u64)>, + ) -> Result<(), Error> { + self.deref().copy_verify(w, verify).await + } + async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result, Error> { + self.deref().to_vec(verify).await + } +} + #[derive(Clone)] pub struct DynFileSource(Arc); impl DynFileSource { @@ -155,16 +179,28 @@ impl FileSource for Arc<[u8]> { } } -pub trait ArchiveSource: Clone + Send + Sync + Sized + 'static { +pub trait ArchiveSource: Send + Sync + Sized + 'static { type Reader: AsyncRead + Unpin + Send; fn size(&self) -> impl Future> + Send { async { None } } + fn fetch_all( + &self, + ) -> impl Future> + Send; fn fetch( &self, position: u64, size: u64, ) -> impl Future> + Send; + fn copy_all_to( + &self, + w: &mut W, + ) -> impl Future> + Send { + async move { + tokio::io::copy(&mut self.fetch_all().await?, w).await?; + Ok(()) + } + } fn copy_to( &self, position: u64, @@ -176,17 +212,47 @@ pub trait ArchiveSource: Clone + Send + Sync + Sized + 'static { Ok(()) } } - fn section(&self, position: u64, size: u64) -> Section { + fn section(self, position: u64, size: u64) -> Section { Section { - source: self.clone(), + source: self, position, size, } } } +impl ArchiveSource for Arc { + type Reader = T::Reader; + async fn size(&self) -> Option { + self.deref().size().await + } + async fn fetch_all(&self) -> Result { + self.deref().fetch_all().await + } + async fn fetch(&self, position: u64, size: u64) -> Result { + self.deref().fetch(position, size).await + } + async fn copy_all_to( + &self, + w: &mut W, + ) -> Result<(), Error> { + self.deref().copy_all_to(w).await + } + async fn copy_to( + &self, + position: u64, + size: u64, + w: &mut W, + ) -> Result<(), Error> { + self.deref().copy_to(position, size, w).await + } +} + impl ArchiveSource for Arc<[u8]> { type Reader = tokio::io::Take>; + async fn fetch_all(&self) -> Result { + Ok(std::io::Cursor::new(self.clone())) + } async fn fetch(&self, position: u64, size: u64) -> Result { use tokio::io::AsyncReadExt; diff --git a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs index 00188559a..9cc162f0e 100644 --- a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs +++ b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs @@ -1,18 +1,35 @@ use std::io::SeekFrom; use std::os::fd::{AsRawFd, RawFd}; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::sync::Arc; +use std::task::Poll; use tokio::fs::File; -use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf, Take}; use tokio::sync::{Mutex, OwnedMutexGuard}; use crate::disk::mount::filesystem::loop_dev::LoopDev; use crate::prelude::*; use crate::s9pk::merkle_archive::source::{ArchiveSource, Section}; -fn path_from_fd(fd: RawFd) -> PathBuf { - Path::new("/proc/self/fd").join(fd.to_string()) +fn path_from_fd(fd: RawFd) -> Result { + #[cfg(target_os = "linux")] + let path = Path::new("/proc/self/fd").join(fd.to_string()); + #[cfg(target_os = "macos")] // here be dragons + let path = unsafe { + let mut buf = [0u8; libc::PATH_MAX as usize]; + if libc::fcntl(fd, libc::F_GETPATH, buf.as_mut_ptr().cast::()) == -1 { + return Err(std::io::Error::last_os_error().into()); + } + Path::new( + &*std::ffi::CStr::from_bytes_until_nul(&buf) + .with_kind(ErrorKind::Utf8)? + .to_string_lossy(), + ) + .to_owned() + }; + Ok(path) } #[derive(Clone)] @@ -21,18 +38,26 @@ pub struct MultiCursorFile { file: Arc>, } impl MultiCursorFile { - fn path(&self) -> PathBuf { + fn path(&self) -> Result { path_from_fd(self.fd) } pub async fn open(fd: &impl AsRawFd) -> Result { - let fd = fd.as_raw_fd(); - Ok(Self { - fd, - file: Arc::new(Mutex::new(File::open(path_from_fd(fd)).await?)), - }) + let f = File::open(path_from_fd(fd.as_raw_fd())?).await?; + Ok(Self::from(f)) + } + pub async fn cursor(&self) -> Result { + Ok(FileCursor( + if let Ok(file) = self.file.clone().try_lock_owned() { + file + } else { + Arc::new(Mutex::new(File::open(self.path()?).await?)) + .try_lock_owned() + .expect("freshly created") + }, + )) } pub async fn blake3_mmap(&self) -> Result { - let path = self.path(); + let path = self.path()?; tokio::task::spawn_blocking(move || { let mut hasher = blake3::Hasher::new(); hasher.update_mmap_rayon(path)?; @@ -52,76 +77,44 @@ impl From for MultiCursorFile { } #[pin_project::pin_project] -pub struct FileSectionReader { - #[pin] - file: OwnedMutexGuard, - remaining: u64, -} -impl AsyncRead for FileSectionReader { +pub struct FileCursor(#[pin] OwnedMutexGuard); +impl AsyncRead for FileCursor { fn poll_read( - self: std::pin::Pin<&mut Self>, + self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { + buf: &mut ReadBuf<'_>, + ) -> Poll> { let this = self.project(); - if *this.remaining == 0 { - return std::task::Poll::Ready(Ok(())); - } - let before = buf.filled().len() as u64; - let res = std::pin::Pin::new(&mut (&mut **this.file.get_mut()).take(*this.remaining)) - .poll_read(cx, buf); - *this.remaining = this - .remaining - .saturating_sub(buf.filled().len() as u64 - before); - res + Pin::new(&mut (&mut **this.0.get_mut())).poll_read(cx, buf) } } impl ArchiveSource for MultiCursorFile { - type Reader = FileSectionReader; + type Reader = Take; async fn size(&self) -> Option { - tokio::fs::metadata(self.path()).await.ok().map(|m| m.len()) + tokio::fs::metadata(self.path().ok()?) + .await + .ok() + .map(|m| m.len()) + } + async fn fetch_all(&self) -> Result { + use tokio::io::AsyncSeekExt; + + let mut file = self.cursor().await?; + file.0.seek(SeekFrom::Start(0)).await?; + Ok(file) } async fn fetch(&self, position: u64, size: u64) -> Result { use tokio::io::AsyncSeekExt; - let mut file = if let Ok(file) = self.file.clone().try_lock_owned() { - file - } else { - #[cfg(target_os = "linux")] - let file = File::open(self.path()).await?; - #[cfg(target_os = "macos")] // here be dragons - let file = unsafe { - let mut buf = [0u8; libc::PATH_MAX as usize]; - if libc::fcntl( - self.fd, - libc::F_GETPATH, - buf.as_mut_ptr().cast::(), - ) == -1 - { - return Err(std::io::Error::last_os_error().into()); - } - File::open( - &*std::ffi::CStr::from_bytes_until_nul(&buf) - .with_kind(ErrorKind::Utf8)? - .to_string_lossy(), - ) - .await? - }; - Arc::new(Mutex::new(file)) - .try_lock_owned() - .expect("freshly created") - }; - file.seek(SeekFrom::Start(position)).await?; - Ok(Self::Reader { - file, - remaining: size, - }) + let mut file = self.cursor().await?; + file.0.seek(SeekFrom::Start(position)).await?; + Ok(file.take(size)) } } impl From<&Section> for LoopDev { fn from(value: &Section) -> Self { - LoopDev::new(value.source.path(), value.position, value.size) + LoopDev::new(value.source.path().unwrap(), value.position, value.size) } } diff --git a/core/startos/src/s9pk/merkle_archive/test.rs b/core/startos/src/s9pk/merkle_archive/test.rs index e902aafd9..861f3b04c 100644 --- a/core/startos/src/s9pk/merkle_archive/test.rs +++ b/core/startos/src/s9pk/merkle_archive/test.rs @@ -8,9 +8,9 @@ use ed25519_dalek::SigningKey; use crate::prelude::*; use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; use crate::s9pk::merkle_archive::file_contents::FileContents; -use crate::s9pk::merkle_archive::sink::TrackingWriter; use crate::s9pk::merkle_archive::source::FileSource; use crate::s9pk::merkle_archive::{Entry, EntryContents, MerkleArchive}; +use crate::util::io::TrackingIO; /// Creates a MerkleArchive (a1) with the provided files at the provided paths. NOTE: later files can overwrite previous files/directories at the same path /// Tests: @@ -60,10 +60,15 @@ fn test(files: Vec<(PathBuf, String)>) -> Result<(), Error> { .block_on(async move { a1.update_hashes(true).await?; let mut s1 = Vec::new(); - a1.serialize(&mut TrackingWriter::new(0, &mut s1), true) - .await?; + a1.serialize(&mut TrackingIO::new(0, &mut s1), true).await?; let s1: Arc<[u8]> = s1.into(); - let a2 = MerkleArchive::deserialize(&s1, "test", &mut Cursor::new(s1.clone())).await?; + let a2 = MerkleArchive::deserialize( + &s1, + "test", + &mut Cursor::new(s1.clone()), + Some(&a1.commitment().await?), + ) + .await?; for (path, content) in check_set { match a2 @@ -88,8 +93,7 @@ fn test(files: Vec<(PathBuf, String)>) -> Result<(), Error> { } let mut s2 = Vec::new(); - a2.serialize(&mut TrackingWriter::new(0, &mut s2), true) - .await?; + a2.serialize(&mut TrackingIO::new(0, &mut s2), true).await?; let s2: Arc<[u8]> = s2.into(); ensure_code!(s1 == s2, ErrorKind::Pack, "s1 does not match s2"); diff --git a/core/startos/src/s9pk/merkle_archive/write_queue.rs b/core/startos/src/s9pk/merkle_archive/write_queue.rs index 9496d5e83..4e1bb3a73 100644 --- a/core/startos/src/s9pk/merkle_archive/write_queue.rs +++ b/core/startos/src/s9pk/merkle_archive/write_queue.rs @@ -30,6 +30,8 @@ impl<'a, S: FileSource> WriteQueue<'a, S> { self.queue.push_back(entry); Ok(res) } +} +impl<'a, S: FileSource + Clone> WriteQueue<'a, S> { pub async fn serialize(&mut self, w: &mut W, verify: bool) -> Result<(), Error> { loop { let Some(next) = self.queue.pop_front() else { diff --git a/core/startos/src/s9pk/mod.rs b/core/startos/src/s9pk/mod.rs index 83924293a..fcf9379a0 100644 --- a/core/startos/src/s9pk/mod.rs +++ b/core/startos/src/s9pk/mod.rs @@ -1,3 +1,4 @@ +pub mod git_hash; pub mod merkle_archive; pub mod rpc; pub mod v1; diff --git a/core/startos/src/s9pk/rpc.rs b/core/startos/src/s9pk/rpc.rs index ca149c3e3..89cfc9b5a 100644 --- a/core/startos/src/s9pk/rpc.rs +++ b/core/startos/src/s9pk/rpc.rs @@ -172,14 +172,14 @@ async fn add_image( .join(&arch) .join(&id) .with_extension("env"), - Entry::file(DynFileSource::new(Arc::from(Vec::from(env)))), + Entry::file(DynFileSource::new(Arc::<[u8]>::from(Vec::from(env)))), )?; archive.contents_mut().insert_path( Path::new("images") .join(&arch) .join(&id) .with_extension("json"), - Entry::file(DynFileSource::new(Arc::from( + Entry::file(DynFileSource::new(Arc::<[u8]>::from( serde_json::to_vec(&serde_json::json!({ "workdir": workdir })) diff --git a/core/startos/src/s9pk/v1/manifest.rs b/core/startos/src/s9pk/v1/manifest.rs index 264843766..845d8f773 100644 --- a/core/startos/src/s9pk/v1/manifest.rs +++ b/core/startos/src/s9pk/v1/manifest.rs @@ -3,35 +3,38 @@ use std::path::{Path, PathBuf}; use emver::VersionRange; use imbl_value::InOMap; +use indexmap::IndexMap; pub use models::PackageId; -use models::VolumeId; +use models::{ActionId, HealthCheckId, ImageId, VolumeId}; use serde::{Deserialize, Serialize}; use url::Url; -use super::git_hash::GitHash; use crate::prelude::*; +use crate::s9pk::git_hash::GitHash; use crate::s9pk::manifest::{Alerts, Description, HardwareRequirements}; -use crate::util::Version; +use crate::util::serde::{Duration, IoFormat}; +use crate::util::VersionString; use crate::version::{Current, VersionT}; -fn current_version() -> Version { +fn current_version() -> VersionString { Current::new().semver().into() } -#[derive(Clone, Debug, Deserialize, Serialize, HasModel)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -#[model = "Model"] pub struct Manifest { #[serde(default = "current_version")] - pub eos_version: Version, + pub eos_version: VersionString, pub id: PackageId, #[serde(default)] pub git_hash: Option, + pub title: String, + pub version: VersionString, + pub description: Description, #[serde(default)] pub assets: Assets, - pub title: String, - pub version: Version, - pub description: Description, + #[serde(default)] + pub build: Option>, pub release_notes: String, pub license: String, // type of license pub wrapper_repo: Url, @@ -41,10 +44,23 @@ pub struct Manifest { pub donation_url: Option, #[serde(default)] pub alerts: Alerts, + pub main: PackageProcedure, + pub health_checks: HealthChecks, + pub config: Option, + pub properties: Option, pub volumes: BTreeMap, + // #[serde(default)] + // pub interfaces: Interfaces, + // #[serde(default)] + pub backup: BackupActions, + #[serde(default)] + pub migrations: Migrations, + #[serde(default)] + pub actions: BTreeMap, + // #[serde(default)] + // pub permissions: Permissions, #[serde(default)] pub dependencies: BTreeMap, - pub config: Option>, #[serde(default)] pub replaces: Vec, @@ -53,6 +69,123 @@ pub struct Manifest { pub hardware_requirements: HardwareRequirements, } +impl Manifest { + pub fn package_procedures(&self) -> impl Iterator { + use std::iter::once; + let main = once(&self.main); + let cfg_get = self.config.as_ref().map(|a| &a.get).into_iter(); + let cfg_set = self.config.as_ref().map(|a| &a.set).into_iter(); + let props = self.properties.iter(); + let backups = vec![&self.backup.create, &self.backup.restore].into_iter(); + let migrations = self + .migrations + .to + .values() + .chain(self.migrations.from.values()); + let actions = self.actions.values().map(|a| &a.implementation); + main.chain(cfg_get) + .chain(cfg_set) + .chain(props) + .chain(backups) + .chain(migrations) + .chain(actions) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, HasModel)] +#[serde(rename_all = "kebab-case")] +#[serde(tag = "type")] +#[model = "Model"] +pub enum PackageProcedure { + Docker(DockerProcedure), + Script(Value), +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct DockerProcedure { + pub image: ImageId, + #[serde(default)] + pub system: bool, + pub entrypoint: String, + #[serde(default)] + pub args: Vec, + #[serde(default)] + pub inject: bool, + #[serde(default)] + pub mounts: BTreeMap, + #[serde(default)] + pub io_format: Option, + #[serde(default)] + pub sigterm_timeout: Option, + #[serde(default)] + pub shm_size_mb: Option, // TODO: use postfix sizing? like 1k vs 1m vs 1g + #[serde(default)] + pub gpu_acceleration: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct HealthChecks(pub BTreeMap); + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct HealthCheck { + pub name: String, + pub success_message: Option, + #[serde(flatten)] + implementation: PackageProcedure, + pub timeout: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct ConfigActions { + pub get: PackageProcedure, + pub set: PackageProcedure, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct BackupActions { + pub create: PackageProcedure, + pub restore: PackageProcedure, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Migrations { + pub from: IndexMap, + pub to: IndexMap, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Action { + pub name: String, + pub description: String, + #[serde(default)] + pub warning: Option, + pub implementation: PackageProcedure, + // pub allowed_statuses: Vec, + // #[serde(default)] + // pub input_spec: ConfigSpec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct DepInfo { + pub version: VersionRange, + pub requirement: DependencyRequirement, + pub description: Option, + #[serde(default)] + pub config: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct DependencyConfig { + check: PackageProcedure, + auto_configure: PackageProcedure, +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] #[serde(tag = "type")] @@ -67,15 +200,6 @@ impl DependencyRequirement { } } -#[derive(Clone, Debug, Deserialize, Serialize, HasModel)] -#[serde(rename_all = "kebab-case")] -#[model = "Model"] -pub struct DepInfo { - pub version: VersionRange, - pub requirement: DependencyRequirement, - pub description: Option, -} - #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Assets { diff --git a/core/startos/src/s9pk/v1/mod.rs b/core/startos/src/s9pk/v1/mod.rs index 173921a75..9910d0adb 100644 --- a/core/startos/src/s9pk/v1/mod.rs +++ b/core/startos/src/s9pk/v1/mod.rs @@ -6,7 +6,6 @@ use ts_rs::TS; pub mod builder; pub mod docker; -pub mod git_hash; pub mod header; pub mod manifest; pub mod reader; diff --git a/core/startos/src/s9pk/v1/reader.rs b/core/startos/src/s9pk/v1/reader.rs index 4288ac0b0..7437fdaa9 100644 --- a/core/startos/src/s9pk/v1/reader.rs +++ b/core/startos/src/s9pk/v1/reader.rs @@ -20,7 +20,7 @@ use super::header::{FileSection, Header, TableOfContents}; use super::SIG_CONTEXT; use crate::prelude::*; use crate::s9pk::v1::docker::DockerReader; -use crate::util::Version; +use crate::util::VersionString; #[pin_project::pin_project] #[derive(Debug)] @@ -83,11 +83,11 @@ impl<'a, R: AsyncSeek + Unpin> AsyncSeek for ReadHandle<'a, R> { pub struct ImageTag { pub package_id: PackageId, pub image_id: ImageId, - pub version: Version, + pub version: VersionString, } impl ImageTag { #[instrument(skip_all)] - pub fn validate(&self, id: &PackageId, version: &Version) -> Result<(), Error> { + pub fn validate(&self, id: &PackageId, version: &VersionString) -> Result<(), Error> { if id != &self.package_id { return Err(Error::new( eyre!( diff --git a/core/startos/src/s9pk/v2/compat.rs b/core/startos/src/s9pk/v2/compat.rs index 109ea43b4..5d4ad2f44 100644 --- a/core/startos/src/s9pk/v2/compat.rs +++ b/core/startos/src/s9pk/v2/compat.rs @@ -1,9 +1,10 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::io::Cursor; use std::path::{Path, PathBuf}; use std::sync::Arc; use itertools::Itertools; +use models::ImageId; use tokio::fs::File; use tokio::io::{AsyncRead, AsyncSeek, AsyncWriteExt}; use tokio::process::Command; @@ -16,7 +17,7 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::{FileSource, Section}; use crate::s9pk::merkle_archive::{Entry, MerkleArchive}; use crate::s9pk::rpc::SKIP_ENV; -use crate::s9pk::v1::manifest::Manifest as ManifestV1; +use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure}; use crate::s9pk::v1::reader::S9pkReader; use crate::s9pk::v2::{S9pk, SIG_CONTEXT}; use crate::util::io::TmpDir; @@ -72,6 +73,17 @@ impl S9pk> { let manifest = from_value::(manifest_raw.clone())?; let mut new_manifest = Manifest::from(manifest.clone()); + let images: BTreeMap = manifest + .package_procedures() + .filter_map(|p| { + if let PackageProcedure::Docker(p) = p { + Some((p.image.clone(), p.system)) + } else { + None + } + }) + .collect(); + // LICENSE.md let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into(); archive.insert_path( @@ -109,61 +121,14 @@ impl S9pk> { .input(Some(&mut reader.docker_images(&arch).await?)) .invoke(ErrorKind::Docker) .await?; - #[derive(serde::Deserialize)] - #[serde(rename_all = "PascalCase")] - struct DockerImagesOut { - repository: Option, - tag: Option, - #[serde(default)] - names: Vec, - } - for image in { - #[cfg(feature = "docker")] - let images = std::str::from_utf8( - &Command::new(CONTAINER_TOOL) - .arg("images") - .arg("--format=json") - .invoke(ErrorKind::Docker) - .await?, - )? - .lines() - .map(|l| serde_json::from_str::(l)) - .collect::, _>>() - .with_kind(ErrorKind::Deserialization)? - .into_iter(); - #[cfg(not(feature = "docker"))] - let images = serde_json::from_slice::>( - &Command::new(CONTAINER_TOOL) - .arg("images") - .arg("--format=json") - .invoke(ErrorKind::Docker) - .await?, - ) - .with_kind(ErrorKind::Deserialization)? - .into_iter(); - images - } - .flat_map(|i| { - if let (Some(repository), Some(tag)) = (i.repository, i.tag) { - vec![format!("{repository}:{tag}")] + for (image, system) in &images { + new_manifest.images.insert(image.clone()); + let sqfs_path = images_dir.join(image).with_extension("squashfs"); + let image_name = if *system { + format!("start9/{}:latest", image) } else { - i.names - .into_iter() - .filter_map(|i| i.strip_prefix("docker.io/").map(|s| s.to_owned())) - .collect() - } - }) - .filter_map(|i| { - i.strip_suffix(&format!(":{}", manifest.version)) - .map(|s| s.to_owned()) - }) - .filter_map(|i| { - i.strip_prefix(&format!("start9/{}/", manifest.id)) - .map(|s| s.to_owned()) - }) { - new_manifest.images.insert(image.parse()?); - let sqfs_path = images_dir.join(&image).with_extension("squashfs"); - let image_name = format!("start9/{}/{}:{}", manifest.id, image, manifest.version); + format!("start9/{}/{}:{}", manifest.id, image, manifest.version) + }; let id = String::from_utf8( Command::new(CONTAINER_TOOL) .arg("create") @@ -323,6 +288,7 @@ impl S9pk> { Ok(S9pk::deserialize( &MultiCursorFile::from(File::open(destination.as_ref()).await?), + None, false, ) .await?) @@ -337,8 +303,7 @@ impl From for Manifest { title: value.title, version: value.version, release_notes: value.release_notes, - license: value.license, - replaces: value.replaces, + license: value.license.into(), wrapper_repo: value.wrapper_repo, upstream_repo: value.upstream_repo, support_site: value.support_site.unwrap_or_else(|| default_url.clone()), diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs index ee2358b13..b5ada621b 100644 --- a/core/startos/src/s9pk/v2/manifest.rs +++ b/core/startos/src/s9pk/v2/manifest.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use color_eyre::eyre::eyre; use helpers::const_true; +use imbl_value::InternedString; pub use models::PackageId; use models::{ImageId, VolumeId}; use serde::{Deserialize, Serialize}; @@ -10,12 +11,12 @@ use url::Url; use crate::dependencies::Dependencies; use crate::prelude::*; -use crate::s9pk::v1::git_hash::GitHash; +use crate::s9pk::git_hash::GitHash; use crate::util::serde::Regex; -use crate::util::Version; +use crate::util::VersionString; use crate::version::{Current, VersionT}; -fn current_version() -> Version { +fn current_version() -> VersionString { Current::new().semver().into() } @@ -26,12 +27,10 @@ fn current_version() -> Version { pub struct Manifest { pub id: PackageId, pub title: String, - #[ts(type = "string")] - pub version: Version, + pub version: VersionString, pub release_notes: String, - pub license: String, // type of license - #[serde(default)] - pub replaces: Vec, + #[ts(type = "string")] + pub license: InternedString, // type of license #[ts(type = "string")] pub wrapper_repo: Url, #[ts(type = "string")] @@ -56,8 +55,7 @@ pub struct Manifest { #[ts(type = "string | null")] pub git_hash: Option, #[serde(default = "current_version")] - #[ts(type = "string")] - pub os_version: Version, + pub os_version: VersionString, #[serde(default = "const_true")] pub has_config: bool, } @@ -68,9 +66,11 @@ pub struct Manifest { pub struct HardwareRequirements { #[serde(default)] #[ts(type = "{ [key: string]: string }")] - device: BTreeMap, - ram: Option, - pub arch: Option>, + pub device: BTreeMap, + #[ts(type = "number | null")] + pub ram: Option, + #[ts(type = "string[] | null")] + pub arch: Option>, } #[derive(Clone, Debug, Deserialize, Serialize, TS)] diff --git a/core/startos/src/s9pk/v2/mod.rs b/core/startos/src/s9pk/v2/mod.rs index 9452b7a72..e206c8553 100644 --- a/core/startos/src/s9pk/v2/mod.rs +++ b/core/startos/src/s9pk/v2/mod.rs @@ -7,6 +7,7 @@ use models::{mime, DataUrl, PackageId}; use tokio::fs::File; use crate::prelude::*; +use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; use crate::s9pk::manifest::Manifest; use crate::s9pk::merkle_archive::file_contents::FileContents; use crate::s9pk::merkle_archive::sink::Sink; @@ -96,7 +97,7 @@ impl S9pk { } } -impl S9pk { +impl S9pk { pub async fn new(archive: MerkleArchive, size: Option) -> Result { let manifest = extract_manifest(&archive).await?; Ok(Self { @@ -173,9 +174,13 @@ impl S9pk { } } -impl S9pk> { +impl S9pk> { #[instrument(skip_all)] - pub async fn deserialize(source: &S, apply_filter: bool) -> Result { + pub async fn deserialize( + source: &S, + commitment: Option<&MerkleArchiveCommitment>, + apply_filter: bool, + ) -> Result { use tokio::io::AsyncReadExt; let mut header = source @@ -193,7 +198,8 @@ impl S9pk> { "Invalid Magic or Unexpected Version" ); - let mut archive = MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header).await?; + let mut archive = + MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?; if apply_filter { archive.filter(filter)?; @@ -211,7 +217,7 @@ impl S9pk> { } impl S9pk { pub async fn from_file(file: File, apply_filter: bool) -> Result { - Self::deserialize(&MultiCursorFile::from(file), apply_filter).await + Self::deserialize(&MultiCursorFile::from(file), None, apply_filter).await } pub async fn open( path: impl AsRef, diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index ec8566133..e6ccdd131 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -22,7 +22,7 @@ use crate::install::PKG_ARCHIVE_DIR; use crate::lxc::ContainerId; use crate::prelude::*; use crate::progress::{NamedProgress, Progress}; -use crate::rpc_continuations::RequestGuid; +use crate::rpc_continuations::Guid; use crate::s9pk::S9pk; use crate::service::service_map::InstallProgressHandles; use crate::service::transition::TransitionKind; @@ -254,7 +254,7 @@ impl Service { pub async fn install( ctx: RpcContext, s9pk: S9pk, - src_version: Option, + src_version: Option, progress: Option, ) -> Result { let manifest = s9pk.as_manifest().clone(); @@ -339,7 +339,7 @@ impl Service { Ok(()) } - pub async fn uninstall(self, target_version: Option) -> Result<(), Error> { + pub async fn uninstall(self, target_version: Option) -> Result<(), Error> { self.seed .persistent_container .execute(ProcedureName::Uninit, to_value(&target_version)?, None) // TODO timeout @@ -513,7 +513,7 @@ pub struct ConnectParams { pub async fn connect_rpc( ctx: RpcContext, ConnectParams { id }: ConnectParams, -) -> Result { +) -> Result { let id_ref = &id; crate::lxc::connect( &ctx, diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 44d17da30..31c5a8026 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -1336,12 +1336,15 @@ async fn set_dependencies( }; let (icon, title) = match async { let remote_s9pk = S9pk::deserialize( - &HttpSource::new( - ctx.ctx.client.clone(), - registry_url - .join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?, - ) - .await?, + &Arc::new( + HttpSource::new( + ctx.ctx.client.clone(), + registry_url + .join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?, + ) + .await?, + ), + None, // TODO true, ) .await?; diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index 02b0538ec..ba9188f32 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -89,7 +89,7 @@ impl ServiceMap { } #[instrument(skip_all)] - pub async fn install( + pub async fn install( &self, ctx: RpcContext, mut s9pk: S9pk, diff --git a/core/startos/src/update/mod.rs b/core/startos/src/update/mod.rs index 5bcf8445d..be908e776 100644 --- a/core/startos/src/update/mod.rs +++ b/core/startos/src/update/mod.rs @@ -26,8 +26,10 @@ use crate::progress::{ use crate::registry::asset::RegistryAsset; use crate::registry::context::{RegistryContext, RegistryUrlParams}; use crate::registry::os::index::OsVersionInfo; -use crate::registry::signer::FileValidator; -use crate::rpc_continuations::{RequestGuid, RpcContinuation}; +use crate::registry::os::SIG_CONTEXT; +use crate::registry::signer::commitment::blake3::Blake3Commitment; +use crate::registry::signer::commitment::Commitment; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, @@ -54,7 +56,7 @@ pub struct UpdateSystemRes { #[ts(type = "string | null")] target: Option, #[ts(type = "string | null")] - progress: Option, + progress: Option, } /// An user/ daemon would call this to update the system to the latest version and do the updates available, @@ -83,7 +85,7 @@ pub async fn update_system( let target = maybe_do_update(ctx.clone(), registry, target.unwrap_or(VersionRange::Any)).await?; let progress = if progress && target.is_some() { - let guid = RequestGuid::new(); + let guid = Guid::new(); ctx.clone() .rpc_continuations .add( @@ -246,12 +248,12 @@ async fn maybe_do_update( )); } - let validator = asset.validate(asset.signature_info.all_signers())?; + asset.validate(SIG_CONTEXT, asset.all_signers())?; let mut progress = FullProgressTracker::new(); let progress_handle = progress.handle(); let mut download_phase = progress_handle.add_phase("Downloading File".into(), Some(100)); - download_phase.set_total(validator.size()?); + download_phase.set_total(asset.commitment.size); let reverify_phase = progress_handle.add_phase("Reverifying File".into(), Some(10)); let sync_boot_phase = progress_handle.add_phase("Syncing Boot Files".into(), Some(1)); let finalize_phase = progress_handle.add_phase("Finalizing Update".into(), Some(1)); @@ -300,7 +302,6 @@ async fn maybe_do_update( tokio::spawn(async move { let res = do_update( ctx.clone(), - validator, asset, UpdateProgressHandles { progress_handle, @@ -382,8 +383,7 @@ struct UpdateProgressHandles { #[instrument(skip_all)] async fn do_update( ctx: RpcContext, - validator: FileValidator, - asset: RegistryAsset, + asset: RegistryAsset, UpdateProgressHandles { progress_handle, mut download_phase, @@ -394,21 +394,23 @@ async fn do_update( ) -> Result<(), Error> { download_phase.start(); let path = Path::new("/media/startos/images") - .join(hex::encode(&validator.blake3()?.as_bytes()[..16])) + .join(hex::encode(&asset.commitment.hash[..16])) .with_extension("rootfs"); let mut dst = AtomicFile::new(&path, None::<&Path>) .await .with_kind(ErrorKind::Filesystem)?; let mut download_writer = download_phase.writer(&mut *dst); asset - .download(ctx.client.clone(), &mut download_writer, &validator) + .download(ctx.client.clone(), &mut download_writer) .await?; let (_, mut download_phase) = download_writer.into_inner(); + dst.sync_all().await?; download_phase.complete(); reverify_phase.start(); - validator - .validate_file(&MultiCursorFile::open(&*dst).await?) + asset + .commitment + .check(&MultiCursorFile::open(&*dst).await?) .await?; dst.save().await.with_kind(ErrorKind::Filesystem)?; reverify_phase.complete(); diff --git a/core/startos/src/upload.rs b/core/startos/src/upload.rs index 65494b1e4..f28d81799 100644 --- a/core/startos/src/upload.rs +++ b/core/startos/src/upload.rs @@ -9,18 +9,18 @@ use futures::{FutureExt, StreamExt}; use http::header::CONTENT_LENGTH; use http::StatusCode; use tokio::fs::File; -use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::sync::watch; use crate::context::RpcContext; use crate::prelude::*; -use crate::rpc_continuations::{RequestGuid, RpcContinuation}; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::ArchiveSource; use crate::util::io::TmpDir; -pub async fn upload(ctx: &RpcContext) -> Result<(RequestGuid, UploadingFile), Error> { - let guid = RequestGuid::new(); +pub async fn upload(ctx: &RpcContext) -> Result<(Guid, UploadingFile), Error> { + let guid = Guid::new(); let (mut handle, file) = UploadingFile::new().await?; ctx.rpc_continuations .add( @@ -120,22 +120,44 @@ impl Progress { .and_then(|a| a.expected_size) } async fn ready_for(watch: &mut watch::Receiver, size: u64) -> Result<(), Error> { - if let Some(e) = watch - .wait_for(|progress| progress.error.is_some() || progress.written >= size) + match &*watch + .wait_for(|progress| { + progress.error.is_some() + || progress.written >= size + || progress.expected_size.map_or(false, |e| e < size) + }) .await .map_err(|_| { Error::new( eyre!("failed to determine upload progress"), ErrorKind::Network, ) - })? - .error - .as_ref() - .map(|e| e.clone_output()) - { - Err(e) - } else { - Ok(()) + })? { + Progress { error: Some(e), .. } => Err(e.clone_output()), + Progress { + expected_size: Some(e), + .. + } if *e < size => Err(Error::new( + eyre!("file size is less than requested"), + ErrorKind::Network, + )), + _ => Ok(()), + } + } + async fn ready(watch: &mut watch::Receiver) -> Result<(), Error> { + match &*watch + .wait_for(|progress| { + progress.error.is_some() || Some(progress.written) == progress.expected_size + }) + .await + .map_err(|_| { + Error::new( + eyre!("failed to determine upload progress"), + ErrorKind::Network, + ) + })? { + Progress { error: Some(e), .. } => Err(e.clone_output()), + _ => Ok(()), } } fn complete(&mut self) -> bool { @@ -156,13 +178,25 @@ impl Progress { )); true } - Self { error, .. } if error.is_none() => { + Self { + error, + expected_size: Some(_), + .. + } if error.is_none() => { *error = Some(Error::new( eyre!("Connection closed or timed out before full file received"), ErrorKind::Network, )); true } + Self { + expected_size, + written, + .. + } if expected_size.is_none() => { + *expected_size = Some(*written); + true + } _ => false, } } @@ -204,6 +238,10 @@ impl ArchiveSource for UploadingFile { async fn size(&self) -> Option { Progress::expected_size(&mut self.progress.clone()).await } + async fn fetch_all(&self) -> Result { + Progress::ready(&mut self.progress.clone()).await?; + self.file.fetch_all().await + } async fn fetch(&self, position: u64, size: u64) -> Result { Progress::ready_for(&mut self.progress.clone(), position + size).await?; self.file.fetch(position, size).await diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index 75ab2c708..16bafd8f6 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -1,4 +1,4 @@ -use std::collections::VecDeque; +use std::collections::{BTreeSet, VecDeque}; use std::future::Future; use std::io::Cursor; use std::os::unix::prelude::MetadataExt; @@ -19,7 +19,7 @@ use tokio::io::{ duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf, }; use tokio::net::TcpStream; -use tokio::sync::Notify; +use tokio::sync::{Notify, OwnedMutexGuard}; use tokio::time::{Instant, Sleep}; use crate::prelude::*; @@ -804,7 +804,7 @@ pub struct TeeWriter { #[pin] writer2: W2, } -impl TeeWriter { +impl TeeWriter { pub fn new(writer1: W1, writer2: W2, capacity: usize) -> Self { Self { capacity, @@ -815,7 +815,6 @@ impl TeeWriter { } } } - impl TeeWriter { pub async fn into_inner(mut self) -> Result<(W1, W2), Error> { self.flush().await?; @@ -1007,3 +1006,114 @@ impl AsyncWrite for ParallelBlake3Writer { Poll::Pending } } + +#[pin_project::pin_project] +pub struct TrackingIO { + position: u64, + #[pin] + io: T, +} +impl TrackingIO { + pub fn new(start: u64, io: T) -> Self { + Self { + position: start, + io, + } + } + pub fn position(&self) -> u64 { + self.position + } + pub fn into_inner(self) -> T { + self.io + } +} +impl AsyncWrite for TrackingIO { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + let this = self.project(); + let written = futures::ready!(this.io.poll_write(cx, buf)?); + *this.position += written as u64; + Poll::Ready(Ok(written)) + } + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.project().io.poll_flush(cx) + } + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.project().io.poll_shutdown(cx) + } +} +impl AsyncRead for TrackingIO { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.project(); + let start = buf.filled().len(); + futures::ready!(this.io.poll_read(cx, buf)?); + *this.position += (buf.filled().len() - start) as u64; + Poll::Ready(Ok(())) + } +} +impl std::cmp::PartialEq for TrackingIO { + fn eq(&self, other: &Self) -> bool { + self.position.eq(&other.position) + } +} +impl std::cmp::Eq for TrackingIO {} +impl std::cmp::PartialOrd for TrackingIO { + fn partial_cmp(&self, other: &Self) -> Option { + self.position.partial_cmp(&other.position) + } +} +impl std::cmp::Ord for TrackingIO { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.position.cmp(&other.position) + } +} +impl std::borrow::Borrow for TrackingIO { + fn borrow(&self) -> &u64 { + &self.position + } +} + +pub struct MutexIO(OwnedMutexGuard); +impl AsyncRead for MutexIO { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut *self.get_mut().0).poll_read(cx, buf) + } +} +impl AsyncWrite for MutexIO { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut *self.get_mut().0).poll_write(cx, buf) + } + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + Pin::new(&mut *self.get_mut().0).poll_flush(cx) + } + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + Pin::new(&mut *self.get_mut().0).poll_shutdown(cx) + } +} diff --git a/core/startos/src/util/lshw.rs b/core/startos/src/util/lshw.rs index 2a658a8c8..eb5293d89 100644 --- a/core/startos/src/util/lshw.rs +++ b/core/startos/src/util/lshw.rs @@ -1,12 +1,13 @@ use models::{Error, ResultExt}; use serde::{Deserialize, Serialize}; use tokio::process::Command; +use ts_rs::TS; use crate::util::Invoke; const KNOWN_CLASSES: &[&str] = &["processor", "display"]; -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, TS)] #[serde(tag = "class")] #[serde(rename_all = "camelCase")] pub enum LshwDevice { @@ -28,12 +29,12 @@ impl LshwDevice { } } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, TS)] pub struct LshwProcessor { pub product: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, TS)] pub struct LshwDisplay { pub product: String, } diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index ec64c6f97..aa17b6d7a 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -15,7 +15,7 @@ use helpers::canonicalize; pub use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use lazy_static::lazy_static; -pub use models::Version; +pub use models::VersionString; use pin_project::pin_project; use sha2::Digest; use tokio::fs::File; diff --git a/core/startos/src/util/rpc.rs b/core/startos/src/util/rpc.rs index 7d2356877..e80aac6cb 100644 --- a/core/startos/src/util/rpc.rs +++ b/core/startos/src/util/rpc.rs @@ -1,3 +1,4 @@ +use std::path::Path; use clap::Parser; use rpc_toolkit::{from_fn_async, Context, ParentHandler}; @@ -9,9 +10,11 @@ use crate::context::CliContext; use crate::prelude::*; use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; -use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource}; +use crate::s9pk::merkle_archive::source::ArchiveSource; use crate::util::io::ParallelBlake3Writer; use crate::util::serde::Base16; +use crate::util::Apply; +use crate::CAP_10_MiB; pub fn util() -> ParentHandler { ParentHandler::new().subcommand("b3sum", from_fn_async(b3sum)) @@ -28,26 +31,29 @@ pub async fn b3sum( ctx: CliContext, B3sumParams { file, allow_mmap }: B3sumParams, ) -> Result, Error> { - let source = if let Ok(url) = file.parse::() { + async fn b3sum_source(source: S) -> Result, Error> { + let mut hasher = ParallelBlake3Writer::new(CAP_10_MiB); + source.copy_all_to(&mut hasher).await?; + hasher.finalize().await.map(|h| *h.as_bytes()).map(Base16) + } + async fn b3sum_file( + path: impl AsRef, + allow_mmap: bool, + ) -> Result, Error> { + let file = MultiCursorFile::from(File::open(path).await?); + if allow_mmap { + return file.blake3_mmap().await.map(|h| *h.as_bytes()).map(Base16); + } + b3sum_source(file).await + } + if let Ok(url) = file.parse::() { if url.scheme() == "file" { - let file = MultiCursorFile::from(File::open(url.path()).await?); - if allow_mmap { - return file.blake3_mmap().await.map(|h| *h.as_bytes()).map(Base16); - } - DynFileSource::new(file.section( - 0, - file.size().await.ok_or_else(|| { - Error::new(eyre!("failed to get file size"), ErrorKind::Filesystem) - })?, - )) + b3sum_file(url.path(), allow_mmap).await } else if url.scheme() == "http" || url.scheme() == "https" { - let file = HttpSource::new(ctx.client.clone(), url).await?; - DynFileSource::new(file.section( - 0, - file.size().await.ok_or_else(|| { - Error::new(eyre!("failed to get file size"), ErrorKind::Filesystem) - })?, - )) + HttpSource::new(ctx.client.clone(), url) + .await? + .apply(b3sum_source) + .await } else { return Err(Error::new( eyre!("unknown scheme: {}", url.scheme()), @@ -55,18 +61,6 @@ pub async fn b3sum( )); } } else { - let file = MultiCursorFile::from(File::open(file).await?); - if allow_mmap { - return file.blake3_mmap().await.map(|h| *h.as_bytes()).map(Base16); - } - DynFileSource::new(file.section( - 0, - file.size().await.ok_or_else(|| { - Error::new(eyre!("failed to get file size"), ErrorKind::Filesystem) - })?, - )) - }; - let mut hasher = ParallelBlake3Writer::new(crate::s9pk::merkle_archive::hash::BUFFER_CAPACITY); - source.copy(&mut hasher).await?; - hasher.finalize().await.map(|h| *h.as_bytes()).map(Base16) + b3sum_file(file, allow_mmap).await + } } diff --git a/core/startos/src/util/serde.rs b/core/startos/src/util/serde.rs index 9696f9c91..382ef2814 100644 --- a/core/startos/src/util/serde.rs +++ b/core/startos/src/util/serde.rs @@ -22,6 +22,7 @@ use ts_rs::TS; use super::IntoDoubleEndedIterator; use crate::prelude::*; +use crate::util::Apply; use crate::util::clap::FromStrParser; pub fn deserialize_from_str< @@ -999,6 +1000,11 @@ impl> std::fmt::Display for Base16 { #[derive(TS)] #[ts(type = "string", concrete(T = Vec))] pub struct Base32(pub T); +impl> std::fmt::Display for Base32 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + base32::encode(base32::Alphabet::RFC4648 { padding: true }, self.0.as_ref()).fmt(f) + } +} impl<'de, T: TryFrom>> Deserialize<'de> for Base32 { fn deserialize(deserializer: D) -> Result where @@ -1022,32 +1028,35 @@ impl> Serialize for Base32 { where S: Serializer, { - serializer.serialize_str(&base32::encode( - base32::Alphabet::RFC4648 { padding: true }, - self.0.as_ref(), - )) - } -} -impl> std::fmt::Display for Base32 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - base32::encode(base32::Alphabet::RFC4648 { padding: true }, self.0.as_ref()).fmt(f) + serialize_display(self, serializer) } } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, TS)] #[ts(type = "string", concrete(T = Vec))] pub struct Base64(pub T); +impl> std::fmt::Display for Base64 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&base64::encode(self.0.as_ref())) + } +} +impl>> FromStr for Base64 +{ + type Err = Error; + fn from_str(s: &str) -> Result { + base64::decode(&s) + .with_kind(ErrorKind::Deserialization)? + .apply(TryFrom::try_from) + .map(Self) + .map_err(|_| Error::new(eyre!("failed to create from buffer"), ErrorKind::Deserialization)) + } +} impl<'de, T: TryFrom>> Deserialize<'de> for Base64 { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - let s = String::deserialize(deserializer)?; - base64::decode(&s) - .map_err(serde::de::Error::custom)? - .try_into() - .map_err(|_| serde::de::Error::custom("invalid length")) - .map(Self) + deserialize_from_str(deserializer) } } impl> Serialize for Base64 { @@ -1055,7 +1064,7 @@ impl> Serialize for Base64 { where S: Serializer, { - serializer.serialize_str(&base64::encode(self.0.as_ref())) + serialize_display(self, serializer) } } impl Deref for Base64 { diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 1acce7896..7212b8801 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -28,7 +28,7 @@ enum Version { } impl Version { - fn from_util_version(version: crate::util::Version) -> Self { + fn from_util_version(version: crate::util::VersionString) -> Self { serde_json::to_value(version.clone()) .and_then(serde_json::from_value) .unwrap_or_else(|_e| { @@ -161,7 +161,7 @@ where T: VersionT, { fn deserialize>(deserializer: D) -> Result { - let v = crate::util::Version::deserialize(deserializer)?; + let v = crate::util::VersionString::deserialize(deserializer)?; let version = T::new(); if *v < version.semver() { Ok(Self(version, v.into_version())) @@ -186,7 +186,7 @@ where T: VersionT, { fn deserialize>(deserializer: D) -> Result { - let v = crate::util::Version::deserialize(deserializer)?; + let v = crate::util::VersionString::deserialize(deserializer)?; let version = T::new(); if *v == version.semver() { Ok(Wrapper(version)) diff --git a/core/startos/src/volume.rs b/core/startos/src/volume.rs index 32935bc7a..e801da79b 100644 --- a/core/startos/src/volume.rs +++ b/core/startos/src/volume.rs @@ -6,7 +6,7 @@ use models::{HostId, PackageId}; use crate::net::PACKAGE_CERT_PATH; use crate::prelude::*; -use crate::util::Version; +use crate::util::VersionString; pub const PKG_VOLUME_DIR: &str = "package-data/volumes"; pub const BACKUP_DIR: &str = "/media/startos/backups"; @@ -20,7 +20,7 @@ pub fn data_dir>(datadir: P, pkg_id: &PackageId, volume_id: &Volu .join(volume_id) } -pub fn asset_dir>(datadir: P, pkg_id: &PackageId, version: &Version) -> PathBuf { +pub fn asset_dir>(datadir: P, pkg_id: &PackageId, version: &VersionString) -> PathBuf { datadir .as_ref() .join(PKG_VOLUME_DIR) diff --git a/sdk/lib/osBindings/AcceptSigners.ts b/sdk/lib/osBindings/AcceptSigners.ts index 55d3997fe..1ef417623 100644 --- a/sdk/lib/osBindings/AcceptSigners.ts +++ b/sdk/lib/osBindings/AcceptSigners.ts @@ -1,7 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { SignerKey } from "./SignerKey" +import type { AnyVerifyingKey } from "./AnyVerifyingKey" export type AcceptSigners = - | { signer: SignerKey } + | { signer: AnyVerifyingKey } | { any: Array } | { all: Array } diff --git a/sdk/lib/osBindings/AddAdminParams.ts b/sdk/lib/osBindings/AddAdminParams.ts index a900f793b..9da08b54b 100644 --- a/sdk/lib/osBindings/AddAdminParams.ts +++ b/sdk/lib/osBindings/AddAdminParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Guid } from "./Guid" -export type AddAdminParams = { signer: string } +export type AddAdminParams = { signer: Guid } diff --git a/sdk/lib/osBindings/AddAssetParams.ts b/sdk/lib/osBindings/AddAssetParams.ts index 6a4d3fdf7..ce6128cf7 100644 --- a/sdk/lib/osBindings/AddAssetParams.ts +++ b/sdk/lib/osBindings/AddAssetParams.ts @@ -1,12 +1,13 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Signature } from "./Signature" -import type { SignerKey } from "./SignerKey" +import type { AnySignature } from "./AnySignature" +import type { Blake3Commitment } from "./Blake3Commitment" +import type { Version } from "./Version" export type AddAssetParams = { - url: string - signature: Signature - version: string + version: Version platform: string upload: boolean - __auth_signer: SignerKey + url: string + signature: AnySignature + commitment: Blake3Commitment } diff --git a/sdk/lib/osBindings/AddVersionParams.ts b/sdk/lib/osBindings/AddVersionParams.ts index 9fc281a6f..4ecbb7dcc 100644 --- a/sdk/lib/osBindings/AddVersionParams.ts +++ b/sdk/lib/osBindings/AddVersionParams.ts @@ -1,7 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Version } from "./Version" export type AddVersionParams = { - version: string + version: Version headline: string releaseNotes: string sourceVersion: string diff --git a/sdk/lib/osBindings/SignerKey.ts b/sdk/lib/osBindings/AnySignature.ts similarity index 55% rename from sdk/lib/osBindings/SignerKey.ts rename to sdk/lib/osBindings/AnySignature.ts index bd5c7c043..32b68b911 100644 --- a/sdk/lib/osBindings/SignerKey.ts +++ b/sdk/lib/osBindings/AnySignature.ts @@ -1,4 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Pem } from "./Pem" -export type SignerKey = { alg: "ed25519"; pubkey: Pem } +export type AnySignature = string diff --git a/sdk/lib/osBindings/AnySigningKey.ts b/sdk/lib/osBindings/AnySigningKey.ts new file mode 100644 index 000000000..4933251e8 --- /dev/null +++ b/sdk/lib/osBindings/AnySigningKey.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type AnySigningKey = string diff --git a/sdk/lib/osBindings/AnyVerifyingKey.ts b/sdk/lib/osBindings/AnyVerifyingKey.ts new file mode 100644 index 000000000..58a45aaa7 --- /dev/null +++ b/sdk/lib/osBindings/AnyVerifyingKey.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type AnyVerifyingKey = string diff --git a/sdk/lib/osBindings/Blake3Commitment.ts b/sdk/lib/osBindings/Blake3Commitment.ts new file mode 100644 index 000000000..690559122 --- /dev/null +++ b/sdk/lib/osBindings/Blake3Commitment.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Base64 } from "./Base64" + +export type Blake3Commitment = { hash: Base64; size: number } diff --git a/sdk/lib/osBindings/Category.ts b/sdk/lib/osBindings/Category.ts new file mode 100644 index 000000000..6e0815675 --- /dev/null +++ b/sdk/lib/osBindings/Category.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Description } from "./Description" + +export type Category = { name: string; description: Description } diff --git a/sdk/lib/osBindings/FullIndex.ts b/sdk/lib/osBindings/FullIndex.ts index 8da15b4c0..4d9914015 100644 --- a/sdk/lib/osBindings/FullIndex.ts +++ b/sdk/lib/osBindings/FullIndex.ts @@ -1,5 +1,13 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DataUrl } from "./DataUrl" +import type { Guid } from "./Guid" import type { OsIndex } from "./OsIndex" +import type { PackageIndex } from "./PackageIndex" import type { SignerInfo } from "./SignerInfo" -export type FullIndex = { os: OsIndex; signers: { [key: string]: SignerInfo } } +export type FullIndex = { + icon: DataUrl | null + package: PackageIndex + os: OsIndex + signers: { [key: Guid]: SignerInfo } +} diff --git a/sdk/lib/osBindings/GetOsAssetParams.ts b/sdk/lib/osBindings/GetOsAssetParams.ts index 100f711c7..9872d0b59 100644 --- a/sdk/lib/osBindings/GetOsAssetParams.ts +++ b/sdk/lib/osBindings/GetOsAssetParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Version } from "./Version" -export type GetOsAssetParams = { version: string; platform: string } +export type GetOsAssetParams = { version: Version; platform: string } diff --git a/sdk/lib/osBindings/GetPackageParams.ts b/sdk/lib/osBindings/GetPackageParams.ts new file mode 100644 index 000000000..6dfecc9b2 --- /dev/null +++ b/sdk/lib/osBindings/GetPackageParams.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PackageDetailLevel } from "./PackageDetailLevel" +import type { PackageId } from "./PackageId" + +export type GetPackageParams = { + id: PackageId | null + version: string | null + sourceVersion: string | null + otherVersions: PackageDetailLevel | null +} diff --git a/sdk/lib/osBindings/GetPackageResponse.ts b/sdk/lib/osBindings/GetPackageResponse.ts new file mode 100644 index 000000000..5bf24bfc0 --- /dev/null +++ b/sdk/lib/osBindings/GetPackageResponse.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PackageInfoShort } from "./PackageInfoShort" +import type { PackageVersionInfo } from "./PackageVersionInfo" +import type { Version } from "./Version" + +export type GetPackageResponse = { + best: { [key: Version]: PackageVersionInfo } + otherVersions?: { [key: Version]: PackageInfoShort } +} diff --git a/sdk/lib/osBindings/GetPackageResponseFull.ts b/sdk/lib/osBindings/GetPackageResponseFull.ts new file mode 100644 index 000000000..579924291 --- /dev/null +++ b/sdk/lib/osBindings/GetPackageResponseFull.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PackageVersionInfo } from "./PackageVersionInfo" +import type { Version } from "./Version" + +export type GetPackageResponseFull = { + best: { [key: Version]: PackageVersionInfo } + otherVersions: { [key: Version]: PackageVersionInfo } +} diff --git a/sdk/lib/osBindings/Pem.ts b/sdk/lib/osBindings/Guid.ts similarity index 80% rename from sdk/lib/osBindings/Pem.ts rename to sdk/lib/osBindings/Guid.ts index 1ec1cd375..28dc4d92d 100644 --- a/sdk/lib/osBindings/Pem.ts +++ b/sdk/lib/osBindings/Guid.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type Pem = string +export type Guid = string diff --git a/sdk/lib/osBindings/HardwareRequirements.ts b/sdk/lib/osBindings/HardwareRequirements.ts index bbe7c3ef5..4964bc66f 100644 --- a/sdk/lib/osBindings/HardwareRequirements.ts +++ b/sdk/lib/osBindings/HardwareRequirements.ts @@ -2,6 +2,6 @@ export type HardwareRequirements = { device: { [key: string]: string } - ram: bigint | null + ram: number | null arch: Array | null } diff --git a/sdk/lib/osBindings/ListVersionSignersParams.ts b/sdk/lib/osBindings/ListVersionSignersParams.ts index baf516bf2..d066fbeb4 100644 --- a/sdk/lib/osBindings/ListVersionSignersParams.ts +++ b/sdk/lib/osBindings/ListVersionSignersParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Version } from "./Version" -export type ListVersionSignersParams = { version: string } +export type ListVersionSignersParams = { version: Version } diff --git a/sdk/lib/osBindings/Manifest.ts b/sdk/lib/osBindings/Manifest.ts index 14f2224ef..7f06be25e 100644 --- a/sdk/lib/osBindings/Manifest.ts +++ b/sdk/lib/osBindings/Manifest.ts @@ -5,15 +5,15 @@ import type { Description } from "./Description" import type { HardwareRequirements } from "./HardwareRequirements" import type { ImageId } from "./ImageId" import type { PackageId } from "./PackageId" +import type { Version } from "./Version" import type { VolumeId } from "./VolumeId" export type Manifest = { id: PackageId title: string - version: string + version: Version releaseNotes: string license: string - replaces: Array wrapperRepo: string upstreamRepo: string supportSite: string @@ -27,6 +27,6 @@ export type Manifest = { dependencies: Dependencies hardwareRequirements: HardwareRequirements gitHash: string | null - osVersion: string + osVersion: Version hasConfig: boolean } diff --git a/sdk/lib/osBindings/Blake3Ed25519Signature.ts b/sdk/lib/osBindings/MerkleArchiveCommitment.ts similarity index 52% rename from sdk/lib/osBindings/Blake3Ed25519Signature.ts rename to sdk/lib/osBindings/MerkleArchiveCommitment.ts index 8d28cb9ec..5dcaa0aa3 100644 --- a/sdk/lib/osBindings/Blake3Ed25519Signature.ts +++ b/sdk/lib/osBindings/MerkleArchiveCommitment.ts @@ -1,10 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Base64 } from "./Base64" -import type { Pem } from "./Pem" -export type Blake3Ed25519Signature = { - hash: Base64 - size: bigint - pubkey: Pem - signature: Base64 +export type MerkleArchiveCommitment = { + rootSighash: Base64 + rootMaxsize: number } diff --git a/sdk/lib/osBindings/OsIndex.ts b/sdk/lib/osBindings/OsIndex.ts index 590e9f577..9fb795402 100644 --- a/sdk/lib/osBindings/OsIndex.ts +++ b/sdk/lib/osBindings/OsIndex.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { OsVersionInfo } from "./OsVersionInfo" +import type { Version } from "./Version" -export type OsIndex = { versions: { [key: string]: OsVersionInfo } } +export type OsIndex = { versions: { [key: Version]: OsVersionInfo } } diff --git a/sdk/lib/osBindings/OsVersionInfo.ts b/sdk/lib/osBindings/OsVersionInfo.ts index fc40ef842..7cd0fde9e 100644 --- a/sdk/lib/osBindings/OsVersionInfo.ts +++ b/sdk/lib/osBindings/OsVersionInfo.ts @@ -1,12 +1,14 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Blake3Commitment } from "./Blake3Commitment" +import type { Guid } from "./Guid" import type { RegistryAsset } from "./RegistryAsset" export type OsVersionInfo = { headline: string releaseNotes: string sourceVersion: string - signers: string[] - iso: { [key: string]: RegistryAsset } - squashfs: { [key: string]: RegistryAsset } - img: { [key: string]: RegistryAsset } + signers: Array + iso: { [key: string]: RegistryAsset } + squashfs: { [key: string]: RegistryAsset } + img: { [key: string]: RegistryAsset } } diff --git a/sdk/lib/osBindings/PackageDetailLevel.ts b/sdk/lib/osBindings/PackageDetailLevel.ts new file mode 100644 index 000000000..b5e1ae42b --- /dev/null +++ b/sdk/lib/osBindings/PackageDetailLevel.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PackageDetailLevel = "short" | "full" diff --git a/sdk/lib/osBindings/PackageIndex.ts b/sdk/lib/osBindings/PackageIndex.ts new file mode 100644 index 000000000..5e8c94945 --- /dev/null +++ b/sdk/lib/osBindings/PackageIndex.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Category } from "./Category" +import type { PackageId } from "./PackageId" +import type { PackageInfo } from "./PackageInfo" + +export type PackageIndex = { + categories: { [key: string]: Category } + packages: { [key: PackageId]: PackageInfo } +} diff --git a/sdk/lib/osBindings/PackageInfo.ts b/sdk/lib/osBindings/PackageInfo.ts new file mode 100644 index 000000000..af340424f --- /dev/null +++ b/sdk/lib/osBindings/PackageInfo.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Guid } from "./Guid" +import type { PackageVersionInfo } from "./PackageVersionInfo" +import type { Version } from "./Version" + +export type PackageInfo = { + signers: Array + versions: { [key: Version]: PackageVersionInfo } +} diff --git a/sdk/lib/osBindings/PackageInfoShort.ts b/sdk/lib/osBindings/PackageInfoShort.ts new file mode 100644 index 000000000..22c7fbea4 --- /dev/null +++ b/sdk/lib/osBindings/PackageInfoShort.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PackageInfoShort = { releaseNotes: string } diff --git a/sdk/lib/osBindings/PackageVersionInfo.ts b/sdk/lib/osBindings/PackageVersionInfo.ts new file mode 100644 index 000000000..da82540cd --- /dev/null +++ b/sdk/lib/osBindings/PackageVersionInfo.ts @@ -0,0 +1,25 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DataUrl } from "./DataUrl" +import type { Description } from "./Description" +import type { HardwareRequirements } from "./HardwareRequirements" +import type { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" +import type { RegistryAsset } from "./RegistryAsset" +import type { Version } from "./Version" + +export type PackageVersionInfo = { + title: string + icon: DataUrl + description: Description + releaseNotes: string + gitHash: string + license: string + wrapperRepo: string + upstreamRepo: string + supportSite: string + marketingSite: string + categories: string[] + osVersion: Version + hardwareRequirements: HardwareRequirements + sourceVersion: string | null + s9pk: RegistryAsset +} diff --git a/sdk/lib/osBindings/RegistryAsset.ts b/sdk/lib/osBindings/RegistryAsset.ts index cb878c66a..3eb13e8a4 100644 --- a/sdk/lib/osBindings/RegistryAsset.ts +++ b/sdk/lib/osBindings/RegistryAsset.ts @@ -1,4 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { SignatureInfo } from "./SignatureInfo" +import type { AnySignature } from "./AnySignature" +import type { AnyVerifyingKey } from "./AnyVerifyingKey" -export type RegistryAsset = { url: string; signatureInfo: SignatureInfo } +export type RegistryAsset = { + url: string + commitment: Commitment + signatures: { [key: AnyVerifyingKey]: AnySignature } +} diff --git a/sdk/lib/osBindings/RemoveVersionParams.ts b/sdk/lib/osBindings/RemoveVersionParams.ts index 2c974de56..d00a6ee9e 100644 --- a/sdk/lib/osBindings/RemoveVersionParams.ts +++ b/sdk/lib/osBindings/RemoveVersionParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Version } from "./Version" -export type RemoveVersionParams = { version: string } +export type RemoveVersionParams = { version: Version } diff --git a/sdk/lib/osBindings/Blake3Ed2551SignatureInfo.ts b/sdk/lib/osBindings/RequestCommitment.ts similarity index 51% rename from sdk/lib/osBindings/Blake3Ed2551SignatureInfo.ts rename to sdk/lib/osBindings/RequestCommitment.ts index efe18aca7..89df04e4a 100644 --- a/sdk/lib/osBindings/Blake3Ed2551SignatureInfo.ts +++ b/sdk/lib/osBindings/RequestCommitment.ts @@ -1,9 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Base64 } from "./Base64" -import type { Pem } from "./Pem" -export type Blake3Ed2551SignatureInfo = { - hash: Base64 - size: bigint - signatures: { [key: Pem]: Base64 } +export type RequestCommitment = { + timestamp: number + nonce: number + size: number + blake3: Base64 } diff --git a/sdk/lib/osBindings/ServerInfo.ts b/sdk/lib/osBindings/ServerInfo.ts index 935e3a99f..92b8b9b8f 100644 --- a/sdk/lib/osBindings/ServerInfo.ts +++ b/sdk/lib/osBindings/ServerInfo.ts @@ -2,6 +2,7 @@ import type { Governor } from "./Governor" import type { IpInfo } from "./IpInfo" import type { ServerStatus } from "./ServerStatus" +import type { Version } from "./Version" import type { WifiInfo } from "./WifiInfo" export type ServerInfo = { @@ -9,7 +10,7 @@ export type ServerInfo = { platform: string id: string hostname: string - version: string + version: Version lastBackup: string | null eosVersionCompat: string lanAddress: string diff --git a/sdk/lib/osBindings/SetMainStatus.ts b/sdk/lib/osBindings/SetMainStatus.ts index aa4aff0b2..6dcca73e9 100644 --- a/sdk/lib/osBindings/SetMainStatus.ts +++ b/sdk/lib/osBindings/SetMainStatus.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Status } from "./Status" +import type { SetMainStatusStatus } from "./SetMainStatusStatus" -export type SetMainStatus = { status: "running" | "stopped" | "starting" } +export type SetMainStatus = { status: SetMainStatusStatus } diff --git a/sdk/lib/osBindings/SetMainStatusStatus.ts b/sdk/lib/osBindings/SetMainStatusStatus.ts new file mode 100644 index 000000000..6db32e7bf --- /dev/null +++ b/sdk/lib/osBindings/SetMainStatusStatus.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetMainStatusStatus = "running" | "stopped" | "starting" diff --git a/sdk/lib/osBindings/SignAssetParams.ts b/sdk/lib/osBindings/SignAssetParams.ts index 3b3612c47..d55a061a7 100644 --- a/sdk/lib/osBindings/SignAssetParams.ts +++ b/sdk/lib/osBindings/SignAssetParams.ts @@ -1,8 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Signature } from "./Signature" +import type { AnySignature } from "./AnySignature" +import type { Version } from "./Version" export type SignAssetParams = { - version: string + version: Version platform: string - signature: Signature + signature: AnySignature } diff --git a/sdk/lib/osBindings/Signature.ts b/sdk/lib/osBindings/Signature.ts deleted file mode 100644 index a27ada0e7..000000000 --- a/sdk/lib/osBindings/Signature.ts +++ /dev/null @@ -1,4 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Blake3Ed25519Signature } from "./Blake3Ed25519Signature" - -export type Signature = { blake3Ed25519: Blake3Ed25519Signature } diff --git a/sdk/lib/osBindings/SignatureInfo.ts b/sdk/lib/osBindings/SignatureInfo.ts deleted file mode 100644 index 41a093742..000000000 --- a/sdk/lib/osBindings/SignatureInfo.ts +++ /dev/null @@ -1,7 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Blake3Ed2551SignatureInfo } from "./Blake3Ed2551SignatureInfo" - -export type SignatureInfo = { - context: string - "blake-3-ed-25-5i-9": Blake3Ed2551SignatureInfo | null -} diff --git a/sdk/lib/osBindings/SignerInfo.ts b/sdk/lib/osBindings/SignerInfo.ts index 06293cbab..7e7aa2588 100644 --- a/sdk/lib/osBindings/SignerInfo.ts +++ b/sdk/lib/osBindings/SignerInfo.ts @@ -1,9 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AnyVerifyingKey } from "./AnyVerifyingKey" import type { ContactInfo } from "./ContactInfo" -import type { SignerKey } from "./SignerKey" export type SignerInfo = { name: string contact: Array - keys: Array + keys: Array } diff --git a/sdk/lib/osBindings/Version.ts b/sdk/lib/osBindings/Version.ts new file mode 100644 index 000000000..b49b6e887 --- /dev/null +++ b/sdk/lib/osBindings/Version.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type Version = string diff --git a/sdk/lib/osBindings/VersionSignerParams.ts b/sdk/lib/osBindings/VersionSignerParams.ts index 70acdaa2e..102eecefd 100644 --- a/sdk/lib/osBindings/VersionSignerParams.ts +++ b/sdk/lib/osBindings/VersionSignerParams.ts @@ -1,3 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Guid } from "./Guid" +import type { Version } from "./Version" -export type VersionSignerParams = { version: string; signer: string } +export type VersionSignerParams = { version: Version; signer: Guid } diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 4fcb9a617..16fc56a74 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -11,14 +11,17 @@ export { Algorithm } from "./Algorithm" export { AllowedStatuses } from "./AllowedStatuses" export { AllPackageData } from "./AllPackageData" export { AlpnInfo } from "./AlpnInfo" +export { AnySignature } from "./AnySignature" +export { AnySigningKey } from "./AnySigningKey" +export { AnyVerifyingKey } from "./AnyVerifyingKey" export { BackupProgress } from "./BackupProgress" export { Base64 } from "./Base64" export { BindInfo } from "./BindInfo" export { BindOptions } from "./BindOptions" export { BindParams } from "./BindParams" -export { Blake3Ed25519Signature } from "./Blake3Ed25519Signature" -export { Blake3Ed2551SignatureInfo } from "./Blake3Ed2551SignatureInfo" +export { Blake3Commitment } from "./Blake3Commitment" export { Callback } from "./Callback" +export { Category } from "./Category" export { CheckDependenciesParam } from "./CheckDependenciesParam" export { CheckDependenciesResult } from "./CheckDependenciesResult" export { ChrootParams } from "./ChrootParams" @@ -48,6 +51,9 @@ export { FullProgress } from "./FullProgress" export { GetHostInfoParamsKind } from "./GetHostInfoParamsKind" export { GetHostInfoParams } from "./GetHostInfoParams" export { GetOsAssetParams } from "./GetOsAssetParams" +export { GetPackageParams } from "./GetPackageParams" +export { GetPackageResponseFull } from "./GetPackageResponseFull" +export { GetPackageResponse } from "./GetPackageResponse" export { GetPrimaryUrlParams } from "./GetPrimaryUrlParams" export { GetServiceInterfaceParams } from "./GetServiceInterfaceParams" export { GetServicePortForwardParams } from "./GetServicePortForwardParams" @@ -57,6 +63,7 @@ export { GetStoreParams } from "./GetStoreParams" export { GetSystemSmtpParams } from "./GetSystemSmtpParams" export { GetVersionParams } from "./GetVersionParams" export { Governor } from "./Governor" +export { Guid } from "./Guid" export { HardwareRequirements } from "./HardwareRequirements" export { HealthCheckId } from "./HealthCheckId" export { HealthCheckResult } from "./HealthCheckResult" @@ -75,24 +82,30 @@ export { ListVersionSignersParams } from "./ListVersionSignersParams" export { MainStatus } from "./MainStatus" export { Manifest } from "./Manifest" export { MaybeUtf8String } from "./MaybeUtf8String" +export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" export { MountParams } from "./MountParams" export { MountTarget } from "./MountTarget" export { NamedProgress } from "./NamedProgress" export { OsIndex } from "./OsIndex" export { OsVersionInfo } from "./OsVersionInfo" export { PackageDataEntry } from "./PackageDataEntry" +export { PackageDetailLevel } from "./PackageDetailLevel" export { PackageId } from "./PackageId" +export { PackageIndex } from "./PackageIndex" +export { PackageInfoShort } from "./PackageInfoShort" +export { PackageInfo } from "./PackageInfo" export { PackageState } from "./PackageState" +export { PackageVersionInfo } from "./PackageVersionInfo" export { ParamsMaybePackageId } from "./ParamsMaybePackageId" export { ParamsPackageId } from "./ParamsPackageId" export { PasswordType } from "./PasswordType" -export { Pem } from "./Pem" export { Progress } from "./Progress" export { Public } from "./Public" export { RegistryAsset } from "./RegistryAsset" export { RemoveActionParams } from "./RemoveActionParams" export { RemoveAddressParams } from "./RemoveAddressParams" export { RemoveVersionParams } from "./RemoveVersionParams" +export { RequestCommitment } from "./RequestCommitment" export { ReverseProxyBind } from "./ReverseProxyBind" export { ReverseProxyDestination } from "./ReverseProxyDestination" export { ReverseProxyHttp } from "./ReverseProxyHttp" @@ -111,16 +124,15 @@ export { Session } from "./Session" export { SetConfigured } from "./SetConfigured" export { SetDependenciesParams } from "./SetDependenciesParams" export { SetHealth } from "./SetHealth" +export { SetMainStatusStatus } from "./SetMainStatusStatus" export { SetMainStatus } from "./SetMainStatus" export { SetStoreParams } from "./SetStoreParams" export { SetSystemSmtpParams } from "./SetSystemSmtpParams" export { SignAssetParams } from "./SignAssetParams" -export { SignatureInfo } from "./SignatureInfo" -export { Signature } from "./Signature" export { SignerInfo } from "./SignerInfo" -export { SignerKey } from "./SignerKey" export { Status } from "./Status" export { UpdatingState } from "./UpdatingState" export { VersionSignerParams } from "./VersionSignerParams" +export { Version } from "./Version" export { VolumeId } from "./VolumeId" export { WifiInfo } from "./WifiInfo" diff --git a/web/projects/marketplace/src/pages/show/additional/additional.component.html b/web/projects/marketplace/src/pages/show/additional/additional.component.html index d362991e3..a75daa3db 100644 --- a/web/projects/marketplace/src/pages/show/additional/additional.component.html +++ b/web/projects/marketplace/src/pages/show/additional/additional.component.html @@ -1,14 +1,3 @@ - -
- Intended to replace -
    -
  • - {{ app }} -
    • -
-
- - Additional Info diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index 59ef6dd1e..2b0b96d23 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -51,7 +51,6 @@ export module Mock { short: 'A Bitcoin full node by Bitcoin Core.', long: 'Bitcoin is a decentralized consensus protocol and settlement network.', }, - replaces: ['banks', 'governments'], releaseNotes: 'Taproot, Schnorr, and more.', license: 'MIT', wrapperRepo: 'https://github.com/start9labs/bitcoind-wrapper', @@ -89,7 +88,6 @@ export module Mock { short: 'A bolt spec compliant client.', long: 'More info about LND. More info about LND. More info about LND.', }, - replaces: ['banks', 'governments'], releaseNotes: 'Dual funded channels!', license: 'MIT', wrapperRepo: 'https://github.com/start9labs/lnd-wrapper', @@ -158,7 +156,6 @@ export module Mock { optional: false, }, }, - replaces: [], hasConfig: false, images: ['main'], assets: [],