Feature/UI sideload (#2658)

* ui sideloading

* remove subtlecrypto import

* fix parser

* misc fixes

* allow docker pull during compat conversion

core/startos/src/registry/asset.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::sync::Arc;
 
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
@@ -7,10 +8,13 @@ use ts_rs::TS;
 use url::Url;
 
 use crate::prelude::*;
+use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
 use crate::registry::signer::commitment::{Commitment, Digestable};
 use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey};
 use crate::registry::signer::AcceptSigners;
 use crate::s9pk::merkle_archive::source::http::HttpSource;
+use crate::s9pk::merkle_archive::source::Section;
+use crate::s9pk::S9pk;
 
 #[derive(Debug, Deserialize, Serialize, TS)]
 #[serde(rename_all = "camelCase")]
@@ -52,3 +56,15 @@ impl<C: for<'a> Commitment<&'a HttpSource>> RegistryAsset<C> {
             .await
     }
 }
+impl RegistryAsset<MerkleArchiveCommitment> {
+    pub async fn deserialize_s9pk(
+        &self,
+        client: Client,
+    ) -> Result<S9pk<Section<Arc<HttpSource>>>, Error> {
+        S9pk::deserialize(
+            &Arc::new(HttpSource::new(client, self.url.clone()).await?),
+            Some(&self.commitment),
+        )
+        .await
+    }
+}

core/startos/src/registry/device_info.rs

@@ -4,7 +4,7 @@ use std::ops::Deref;
 
 use axum::extract::Request;
 use axum::response::Response;
-use emver::{Version, VersionRange};
+use exver::{Version, VersionRange};
 use http::HeaderValue;
 use imbl_value::InternedString;
 use rpc_toolkit::{Middleware, RpcRequest, RpcResponse};

core/startos/src/registry/mod.rs

@@ -1,5 +1,4 @@
 use std::collections::{BTreeMap, BTreeSet};
-use std::net::SocketAddr;
 
 use axum::Router;
 use futures::future::ready;

core/startos/src/registry/os/asset/add.rs

@@ -3,7 +3,6 @@ use std::panic::UnwindSafe;
 use std::path::PathBuf;
 
 use clap::Parser;
-use helpers::NonDetachingJoinHandle;
 use imbl_value::InternedString;
 use itertools::Itertools;
 use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler};
@@ -13,7 +12,7 @@ use url::Url;
 
 use crate::context::CliContext;
 use crate::prelude::*;
-use crate::progress::{FullProgressTracker, PhasedProgressBar};
+use crate::progress::{FullProgressTracker};
 use crate::registry::asset::RegistryAsset;
 use crate::registry::context::RegistryContext;
 use crate::registry::os::index::OsVersionInfo;
@@ -25,6 +24,7 @@ use crate::s9pk::merkle_archive::hash::VerifyingWriter;
 use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::util::io::open_file;
 use crate::util::serde::Base64;
 use crate::util::VersionString;
 
@@ -184,7 +184,7 @@ pub async fn cli_add_asset(
         }
     };
 
-    let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?);
+    let file = MultiCursorFile::from(open_file(&path).await?);
 
     let progress = FullProgressTracker::new();
     let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10));

core/startos/src/registry/os/asset/get.rs

@@ -20,6 +20,7 @@ use crate::registry::os::SIG_CONTEXT;
 use crate::registry::signer::commitment::blake3::Blake3Commitment;
 use crate::registry::signer::commitment::Commitment;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::util::io::open_file;
 use crate::util::VersionString;
 
 pub fn get_api<C: Context>() -> ParentHandler<C> {
@@ -158,9 +159,7 @@ async fn cli_get_os_asset(
         if let Some(mut reverify_phase) = reverify_phase {
             reverify_phase.start();
             res.commitment
-                .check(&MultiCursorFile::from(
-                    tokio::fs::File::open(download).await?,
-                ))
+                .check(&MultiCursorFile::from(open_file(download).await?))
                 .await?;
             reverify_phase.complete();
         }

core/startos/src/registry/os/asset/sign.rs

@@ -21,6 +21,7 @@ use crate::registry::signer::sign::ed25519::Ed25519;
 use crate::registry::signer::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::util::io::open_file;
 use crate::util::serde::Base64;
 use crate::util::VersionString;
 
@@ -166,7 +167,7 @@ pub async fn cli_sign_asset(
         }
     };
 
-    let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?);
+    let file = MultiCursorFile::from(open_file(&path).await?);
 
     let progress = FullProgressTracker::new();
     let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10));

core/startos/src/registry/os/index.rs

@@ -1,6 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet};
 
-use emver::VersionRange;
+use exver::VersionRange;
 use imbl_value::InternedString;
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;

core/startos/src/registry/os/version/mod.rs

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
 
 use chrono::Utc;
 use clap::Parser;
-use emver::VersionRange;
+use exver::VersionRange;
 use itertools::Itertools;
 use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
@@ -148,10 +148,11 @@ pub async fn get_version(
     if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, arch) {
         let created_at = Utc::now();
 
-        query!("INSERT INTO user_activity (created_at, server_id, arch) VALUES ($1, $2, $3)",
-        created_at,
-        server_id,
-        arch
+        query!(
+            "INSERT INTO user_activity (created_at, server_id, arch) VALUES ($1, $2, $3)",
+            created_at,
+            server_id,
+            arch
         )
         .execute(pool)
         .await?;

core/startos/src/registry/package/get.rs

@@ -1,7 +1,7 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use clap::{Parser, ValueEnum};
-use emver::{Version, VersionRange};
+use exver::{ExtendedVersion, VersionRange};
 use imbl_value::InternedString;
 use itertools::Itertools;
 use models::PackageId;
@@ -45,8 +45,7 @@ pub struct GetPackageParams {
     pub id: Option<PackageId>,
     #[ts(type = "string | null")]
     pub version: Option<VersionRange>,
-    #[ts(type = "string | null")]
-    pub source_version: Option<Version>,
+    pub source_version: Option<VersionString>,
     #[ts(skip)]
     #[arg(skip)]
     #[serde(rename = "__device_info")]
@@ -132,7 +131,7 @@ fn get_matching_models<'a>(
         device_info,
         ..
     }: &GetPackageParams,
-) -> Result<Vec<(PackageId, Version, &'a Model<PackageVersionInfo>)>, Error> {
+) -> Result<Vec<(PackageId, ExtendedVersion, &'a Model<PackageVersionInfo>)>, Error> {
     if let Some(id) = id {
         if let Some(pkg) = db.as_packages().as_idx(id) {
             vec![(id.clone(), pkg)]
@@ -166,7 +165,7 @@ fn get_matching_models<'a>(
                             .as_ref()
                             .map_or(Ok(true), |device_info| info.works_for_device(device_info))?
                     {
-                        Some((k.clone(), Version::from(v), info))
+                        Some((k.clone(), ExtendedVersion::from(v), info))
                     } else {
                         None
                     },

core/startos/src/registry/package/index.rs

@@ -1,6 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet};
 
-use emver::{Version, VersionRange};
+use exver::{Version, VersionRange};
 use imbl_value::InternedString;
 use models::{DataUrl, PackageId, VersionString};
 use serde::{Deserialize, Serialize};
@@ -70,7 +70,8 @@ pub struct PackageVersionInfo {
     pub support_site: Url,
     #[ts(type = "string")]
     pub marketing_site: Url,
-    pub os_version: VersionString,
+    #[ts(type = "string")]
+    pub os_version: Version,
     pub hardware_requirements: HardwareRequirements,
     #[ts(type = "string | null")]
     pub source_version: Option<VersionRange>,

core/startos/src/registry/signer/sign/mod.rs

@@ -4,7 +4,6 @@ use std::str::FromStr;
 use ::ed25519::pkcs8::BitStringRef;
 use clap::builder::ValueParserFactory;
 use der::referenced::OwnedToRef;
-use der::{Decode, Encode};
 use pkcs8::der::AnyRef;
 use pkcs8::{PrivateKeyInfo, SubjectPublicKeyInfo};
 use serde::{Deserialize, Serialize};