Refactor/project structure (#3085)

* refactor project structure

* environment-based default registry

* fix tests

* update build container

* use docker platform for iso build emulation

* simplify compat

* Fix docker platform spec in run-compat.sh

* handle riscv compat

* fix bug with dep error exists attr

* undo removal of sorting

* use qemu for iso stage

---------

Co-authored-by: Mariusz Kogen <k0gen@pm.me>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/src/registry/os/asset/add.rs

@@ -0,0 +1,353 @@
+use std::collections::{BTreeMap, HashMap};
+use std::panic::UnwindSafe;
+use std::path::PathBuf;
+
+use chrono::Utc;
+use clap::Parser;
+use exver::Version;
+use imbl_value::InternedString;
+use itertools::Itertools;
+use rpc_toolkit::{Context, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+use url::Url;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::progress::{FullProgressTracker, ProgressUnits};
+use crate::registry::asset::RegistryAsset;
+use crate::registry::context::RegistryContext;
+use crate::registry::os::SIG_CONTEXT;
+use crate::registry::os::index::OsVersionInfo;
+use crate::s9pk::merkle_archive::hash::VerifyingWriter;
+use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::s9pk::merkle_archive::source::http::HttpSource;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::sign::commitment::blake3::Blake3Commitment;
+use crate::sign::ed25519::Ed25519;
+use crate::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
+use crate::util::io::open_file;
+use crate::util::serde::Base64;
+
+pub fn add_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "iso",
+            from_fn_async(add_iso)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(add_img)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(add_squashfs)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+}
+
+pub fn remove_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "iso",
+            from_fn_async(remove_iso)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(remove_img)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(remove_squashfs)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+}
+
+#[derive(Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct AddAssetParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    #[ts(type = "string")]
+    pub platform: InternedString,
+    #[ts(type = "string")]
+    pub url: Url,
+    #[serde(rename = "__Auth_signer")]
+    #[ts(skip)]
+    pub signer: AnyVerifyingKey,
+    pub signature: AnySignature,
+    pub commitment: Blake3Commitment,
+}
+
+async fn add_asset(
+    ctx: RegistryContext,
+    AddAssetParams {
+        version,
+        platform,
+        url,
+        signer,
+        signature,
+        commitment,
+    }: AddAssetParams,
+    accessor: impl FnOnce(
+        &mut Model<OsVersionInfo>,
+    ) -> &mut Model<BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>>
+    + UnwindSafe
+    + Send,
+) -> Result<(), Error> {
+    signer
+        .scheme()
+        .verify_commitment(&signer, &commitment, SIG_CONTEXT, &signature)?;
+    ctx.db
+        .mutate(|db| {
+            let signer_guid = db.as_index().as_signers().get_signer(&signer)?;
+            if db
+                .as_index()
+                .as_os()
+                .as_versions()
+                .as_idx(&version)
+                .or_not_found(&version)?
+                .as_authorized()
+                .de()?
+                .contains(&signer_guid)
+                || db.as_admins().de()?.contains(&signer_guid)
+            {
+                accessor(
+                    db.as_index_mut()
+                        .as_os_mut()
+                        .as_versions_mut()
+                        .as_idx_mut(&version)
+                        .or_not_found(&version)?,
+                )
+                .upsert(&platform, || {
+                    Ok(RegistryAsset {
+                        published_at: Utc::now(),
+                        url,
+                        commitment: commitment.clone(),
+                        signatures: HashMap::new(),
+                    })
+                })?
+                .mutate(|s| {
+                    if s.commitment != commitment {
+                        Err(Error::new(
+                            eyre!("commitment does not match"),
+                            ErrorKind::InvalidSignature,
+                        ))
+                    } else {
+                        s.signatures.insert(signer, signature);
+                        Ok(())
+                    }
+                })?;
+                Ok(())
+            } else {
+                Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
+            }
+        })
+        .await
+        .result?;
+
+    Ok(())
+}
+
+pub async fn add_iso(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> {
+    add_asset(ctx, params, |m| m.as_iso_mut()).await
+}
+
+pub async fn add_img(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> {
+    add_asset(ctx, params, |m| m.as_img_mut()).await
+}
+
+pub async fn add_squashfs(ctx: RegistryContext, params: AddAssetParams) -> Result<(), Error> {
+    add_asset(ctx, params, |m| m.as_squashfs_mut()).await
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+pub struct CliAddAssetParams {
+    #[arg(short = 'p', long = "platform")]
+    pub platform: InternedString,
+    #[arg(short = 'v', long = "version")]
+    pub version: Version,
+    pub file: PathBuf,
+    pub url: Url,
+}
+
+pub async fn cli_add_asset(
+    HandlerArgs {
+        context: ctx,
+        parent_method,
+        method,
+        params:
+            CliAddAssetParams {
+                platform,
+                version,
+                file: path,
+                url,
+            },
+        ..
+    }: HandlerArgs<CliContext, CliAddAssetParams>,
+) -> Result<(), Error> {
+    let ext = match path.extension().and_then(|e| e.to_str()) {
+        Some("iso") => "iso",
+        Some("img") => "img",
+        Some("squashfs") => "squashfs",
+        _ => {
+            return Err(Error::new(
+                eyre!("Unknown extension"),
+                ErrorKind::InvalidRequest,
+            ));
+        }
+    };
+
+    let file = MultiCursorFile::from(open_file(&path).await?);
+
+    let progress = FullProgressTracker::new();
+    let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10));
+    let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100));
+    let mut index_phase = progress.add_phase(
+        InternedString::intern("Adding File to Registry Index"),
+        Some(1),
+    );
+
+    let progress_task =
+        progress.progress_bar_task(&format!("Adding {} to registry...", path.display()));
+
+    sign_phase.start();
+    let blake3 = file.blake3_mmap().await?;
+    let size = file
+        .size()
+        .await
+        .ok_or_else(|| Error::new(eyre!("failed to read file metadata"), ErrorKind::Filesystem))?;
+    let commitment = Blake3Commitment {
+        hash: Base64(*blake3.as_bytes()),
+        size,
+    };
+    let signature = AnySignature::Ed25519(Ed25519.sign_commitment(
+        ctx.developer_key()?,
+        &commitment,
+        SIG_CONTEXT,
+    )?);
+    sign_phase.complete();
+
+    verify_phase.start();
+    let src = HttpSource::new(ctx.client.clone(), url.clone()).await?;
+    if let Some(size) = src.size().await {
+        verify_phase.set_total(size);
+    }
+    verify_phase.set_units(Some(ProgressUnits::Bytes));
+    let mut writer = verify_phase.writer(VerifyingWriter::new(
+        tokio::io::sink(),
+        Some((blake3::Hash::from_bytes(*commitment.hash), commitment.size)),
+    ));
+    src.copy_all_to(&mut writer).await?;
+    let (verifier, mut verify_phase) = writer.into_inner();
+    verifier.verify().await?;
+    verify_phase.complete();
+
+    index_phase.start();
+    ctx.call_remote::<RegistryContext>(
+        &parent_method
+            .into_iter()
+            .chain(method)
+            .chain([ext])
+            .join("."),
+        imbl_value::json!({
+            "platform": platform,
+            "version": version,
+            "url": &url,
+            "signature": signature,
+            "commitment": commitment,
+        }),
+    )
+    .await?;
+    index_phase.complete();
+
+    progress.complete();
+
+    progress_task.await.with_kind(ErrorKind::Unknown)?;
+
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemoveAssetParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    #[ts(type = "string")]
+    pub platform: InternedString,
+    #[serde(rename = "__Auth_signer")]
+    #[ts(skip)]
+    pub signer: AnyVerifyingKey,
+}
+
+async fn remove_asset(
+    ctx: RegistryContext,
+    RemoveAssetParams {
+        version,
+        platform,
+        signer,
+    }: RemoveAssetParams,
+    accessor: impl FnOnce(
+        &mut Model<OsVersionInfo>,
+    ) -> &mut Model<BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>>
+    + UnwindSafe
+    + Send,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            let signer_guid = db.as_index().as_signers().get_signer(&signer)?;
+            if db
+                .as_index()
+                .as_os()
+                .as_versions()
+                .as_idx(&version)
+                .or_not_found(&version)?
+                .as_authorized()
+                .de()?
+                .contains(&signer_guid)
+                || db.as_admins().de()?.contains(&signer_guid)
+            {
+                accessor(
+                    db.as_index_mut()
+                        .as_os_mut()
+                        .as_versions_mut()
+                        .as_idx_mut(&version)
+                        .or_not_found(&version)?,
+                )
+                .remove(&platform)?;
+                Ok(())
+            } else {
+                Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
+            }
+        })
+        .await
+        .result?;
+
+    Ok(())
+}
+
+pub async fn remove_iso(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_iso_mut()).await
+}
+
+pub async fn remove_img(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_img_mut()).await
+}
+
+pub async fn remove_squashfs(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_squashfs_mut()).await
+}

core/src/registry/os/asset/get.rs

@@ -0,0 +1,213 @@
+use std::collections::BTreeMap;
+use std::panic::UnwindSafe;
+use std::path::{Path, PathBuf};
+
+use clap::Parser;
+use exver::Version;
+use imbl_value::{InternedString, json};
+use itertools::Itertools;
+use rpc_toolkit::{Context, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::progress::{FullProgressTracker, ProgressUnits};
+use crate::registry::asset::RegistryAsset;
+use crate::registry::context::RegistryContext;
+use crate::registry::os::SIG_CONTEXT;
+use crate::registry::os::index::OsVersionInfo;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::sign::commitment::Commitment;
+use crate::sign::commitment::blake3::Blake3Commitment;
+use crate::util::io::{AtomicFile, open_file};
+
+pub fn get_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "iso",
+            from_fn_async(get_iso)
+                .with_metadata("authenticated", Value::Bool(false))
+                .no_cli(),
+        )
+        .subcommand(
+            "iso",
+            from_fn_async(cli_get_os_asset)
+                .no_display()
+                .with_about("Download iso"),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(get_img)
+                .with_metadata("authenticated", Value::Bool(false))
+                .no_cli(),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(cli_get_os_asset)
+                .no_display()
+                .with_about("Download img"),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(get_squashfs)
+                .with_metadata("authenticated", Value::Bool(false))
+                .no_cli(),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(cli_get_os_asset)
+                .no_display()
+                .with_about("Download squashfs"),
+        )
+}
+
+#[derive(Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetOsAssetParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    #[ts(type = "string")]
+    pub platform: InternedString,
+}
+
+async fn get_os_asset(
+    ctx: RegistryContext,
+    GetOsAssetParams { version, platform }: GetOsAssetParams,
+    accessor: impl FnOnce(
+        &Model<OsVersionInfo>,
+    ) -> &Model<BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>>
+    + UnwindSafe
+    + Send,
+) -> Result<RegistryAsset<Blake3Commitment>, Error> {
+    accessor(
+        ctx.db
+            .peek()
+            .await
+            .as_index()
+            .as_os()
+            .as_versions()
+            .as_idx(&version)
+            .or_not_found(&version)?,
+    )
+    .as_idx(&platform)
+    .or_not_found(&platform)?
+    .de()
+}
+
+pub async fn get_iso(
+    ctx: RegistryContext,
+    params: GetOsAssetParams,
+) -> Result<RegistryAsset<Blake3Commitment>, Error> {
+    get_os_asset(ctx, params, |info| info.as_iso()).await
+}
+
+pub async fn get_img(
+    ctx: RegistryContext,
+    params: GetOsAssetParams,
+) -> Result<RegistryAsset<Blake3Commitment>, Error> {
+    get_os_asset(ctx, params, |info| info.as_img()).await
+}
+
+pub async fn get_squashfs(
+    ctx: RegistryContext,
+    params: GetOsAssetParams,
+) -> Result<RegistryAsset<Blake3Commitment>, Error> {
+    get_os_asset(ctx, params, |info| info.as_squashfs()).await
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+pub struct CliGetOsAssetParams {
+    pub version: Version,
+    pub platform: InternedString,
+    #[arg(
+        long = "download",
+        short = 'd',
+        help = "The path of the directory to download to"
+    )]
+    pub download: Option<PathBuf>,
+    #[arg(
+        long = "reverify",
+        short = 'r',
+        help = "verify the hash of the file a second time after download"
+    )]
+    pub reverify: bool,
+}
+
+async fn cli_get_os_asset(
+    HandlerArgs {
+        context: ctx,
+        parent_method,
+        method,
+        params:
+            CliGetOsAssetParams {
+                version,
+                platform,
+                download,
+                reverify,
+            },
+        ..
+    }: HandlerArgs<CliContext, CliGetOsAssetParams>,
+) -> Result<RegistryAsset<Blake3Commitment>, Error> {
+    let ext = method
+        .iter()
+        .last()
+        .or_else(|| parent_method.iter().last())
+        .unwrap_or(&"bin");
+
+    let res = from_value::<RegistryAsset<Blake3Commitment>>(
+        ctx.call_remote::<RegistryContext>(
+            &parent_method.iter().chain(&method).join("."),
+            json!({
+                "version": version,
+                "platform": platform,
+            }),
+        )
+        .await?,
+    )?;
+
+    res.validate(SIG_CONTEXT, res.all_signers())?;
+
+    if let Some(download) = download {
+        let download = download.join(format!("startos-{version}_{platform}.{ext}"));
+        let mut file = AtomicFile::new(&download, None::<&Path>).await?;
+
+        let progress = FullProgressTracker::new();
+        let mut download_phase =
+            progress.add_phase(InternedString::intern("Downloading File"), Some(100));
+        download_phase.set_total(res.commitment.size);
+        download_phase.set_units(Some(ProgressUnits::Bytes));
+        let reverify_phase = if reverify {
+            Some(progress.add_phase(InternedString::intern("Reverifying File"), Some(10)))
+        } else {
+            None
+        };
+
+        let progress_task = progress.progress_bar_task("Downloading...");
+
+        download_phase.start();
+        let mut download_writer = download_phase.writer(&mut *file);
+        res.download(ctx.client.clone(), &mut download_writer)
+            .await?;
+        let (_, mut download_phase) = download_writer.into_inner();
+        file.save().await?;
+        download_phase.complete();
+
+        if let Some(mut reverify_phase) = reverify_phase {
+            reverify_phase.start();
+            res.commitment
+                .check(&MultiCursorFile::from(open_file(download).await?))
+                .await?;
+            reverify_phase.complete();
+        }
+
+        progress.complete();
+
+        progress_task.await.with_kind(ErrorKind::Unknown)?;
+    }
+
+    Ok(res)
+}

core/src/registry/os/asset/mod.rs

@@ -0,0 +1,28 @@
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+
+pub mod add;
+pub mod get;
+pub mod sign;
+
+pub fn asset_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand("add", add::add_api::<C>())
+        .subcommand(
+            "add",
+            from_fn_async(add::cli_add_asset)
+                .no_display()
+                .with_about("Add asset to registry"),
+        )
+        .subcommand("remove", add::remove_api::<C>())
+        .subcommand("sign", sign::sign_api::<C>())
+        .subcommand(
+            "sign",
+            from_fn_async(sign::cli_sign_asset)
+                .no_display()
+                .with_about("Sign file and add to registry index"),
+        )
+        .subcommand(
+            "get",
+            get::get_api::<C>().with_about("Commands to download image, iso, or squashfs files"),
+        )
+}

core/src/registry/os/asset/sign.rs

@@ -0,0 +1,222 @@
+use std::collections::BTreeMap;
+use std::panic::UnwindSafe;
+use std::path::PathBuf;
+
+use clap::Parser;
+use exver::Version;
+use imbl_value::InternedString;
+use itertools::Itertools;
+use rpc_toolkit::{Context, HandlerArgs, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::progress::FullProgressTracker;
+use crate::registry::asset::RegistryAsset;
+use crate::registry::context::RegistryContext;
+use crate::registry::os::SIG_CONTEXT;
+use crate::registry::os::index::OsVersionInfo;
+use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::sign::commitment::blake3::Blake3Commitment;
+use crate::sign::ed25519::Ed25519;
+use crate::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
+use crate::util::io::open_file;
+use crate::util::serde::Base64;
+
+pub fn sign_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "iso",
+            from_fn_async(sign_iso)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(sign_img)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(sign_squashfs)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+}
+
+#[derive(Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct SignAssetParams {
+    #[ts(type = "string")]
+    version: Version,
+    #[ts(type = "string")]
+    platform: InternedString,
+    #[ts(skip)]
+    #[serde(rename = "__Auth_signer")]
+    signer: AnyVerifyingKey,
+    signature: AnySignature,
+}
+
+async fn sign_asset(
+    ctx: RegistryContext,
+    SignAssetParams {
+        version,
+        platform,
+        signer,
+        signature,
+    }: SignAssetParams,
+    accessor: impl FnOnce(
+        &mut Model<OsVersionInfo>,
+    ) -> &mut Model<BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>>
+    + UnwindSafe
+    + Send,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            let guid = db.as_index().as_signers().get_signer(&signer)?;
+            if !db
+                .as_index()
+                .as_os()
+                .as_versions()
+                .as_idx(&version)
+                .or_not_found(&version)?
+                .as_authorized()
+                .de()?
+                .contains(&guid)
+            {
+                return Err(Error::new(
+                    eyre!("signer {guid} is not authorized"),
+                    ErrorKind::Authorization,
+                ));
+            }
+
+            accessor(
+                db.as_index_mut()
+                    .as_os_mut()
+                    .as_versions_mut()
+                    .as_idx_mut(&version)
+                    .or_not_found(&version)?,
+            )
+            .as_idx_mut(&platform)
+            .or_not_found(&platform)?
+            .mutate(|s| {
+                signer.scheme().verify_commitment(
+                    &signer,
+                    &s.commitment,
+                    SIG_CONTEXT,
+                    &signature,
+                )?;
+                s.signatures.insert(signer, signature);
+                Ok(())
+            })?;
+
+            Ok(())
+        })
+        .await
+        .result
+}
+
+pub async fn sign_iso(ctx: RegistryContext, params: SignAssetParams) -> Result<(), Error> {
+    sign_asset(ctx, params, |m| m.as_iso_mut()).await
+}
+
+pub async fn sign_img(ctx: RegistryContext, params: SignAssetParams) -> Result<(), Error> {
+    sign_asset(ctx, params, |m| m.as_img_mut()).await
+}
+
+pub async fn sign_squashfs(ctx: RegistryContext, params: SignAssetParams) -> Result<(), Error> {
+    sign_asset(ctx, params, |m| m.as_squashfs_mut()).await
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+pub struct CliSignAssetParams {
+    #[arg(short = 'p', long = "platform")]
+    pub platform: InternedString,
+    #[arg(short = 'v', long = "version")]
+    pub version: Version,
+    pub file: PathBuf,
+}
+
+pub async fn cli_sign_asset(
+    HandlerArgs {
+        context: ctx,
+        parent_method,
+        method,
+        params:
+            CliSignAssetParams {
+                platform,
+                version,
+                file: path,
+            },
+        ..
+    }: HandlerArgs<CliContext, CliSignAssetParams>,
+) -> Result<(), Error> {
+    let ext = match path.extension().and_then(|e| e.to_str()) {
+        Some("iso") => "iso",
+        Some("img") => "img",
+        Some("squashfs") => "squashfs",
+        _ => {
+            return Err(Error::new(
+                eyre!("Unknown extension"),
+                ErrorKind::InvalidRequest,
+            ));
+        }
+    };
+
+    let file = MultiCursorFile::from(open_file(&path).await?);
+
+    let progress = FullProgressTracker::new();
+    let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10));
+    let mut index_phase = progress.add_phase(
+        InternedString::intern("Adding Signature to Registry Index"),
+        Some(1),
+    );
+
+    let progress_task =
+        progress.progress_bar_task(&format!("Adding {} to registry...", path.display()));
+
+    sign_phase.start();
+    let blake3 = file.blake3_mmap().await?;
+    let size = file
+        .size()
+        .await
+        .ok_or_else(|| Error::new(eyre!("failed to read file metadata"), ErrorKind::Filesystem))?;
+    let commitment = Blake3Commitment {
+        hash: Base64(*blake3.as_bytes()),
+        size,
+    };
+    let signature = AnySignature::Ed25519(Ed25519.sign_commitment(
+        ctx.developer_key()?,
+        &commitment,
+        SIG_CONTEXT,
+    )?);
+    sign_phase.complete();
+
+    index_phase.start();
+    ctx.call_remote::<RegistryContext>(
+        &parent_method
+            .into_iter()
+            .chain(method)
+            .chain([ext])
+            .join("."),
+        imbl_value::json!({
+            "platform": platform,
+            "version": version,
+            "signature": signature,
+        }),
+    )
+    .await?;
+    index_phase.complete();
+
+    progress.complete();
+
+    progress_task.await.with_kind(ErrorKind::Unknown)?;
+
+    Ok(())
+}

core/src/registry/os/index.rs

@@ -0,0 +1,54 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use exver::{Version, VersionRange};
+use imbl_value::InternedString;
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::prelude::*;
+use crate::registry::asset::RegistryAsset;
+use crate::registry::context::RegistryContext;
+use crate::rpc_continuations::Guid;
+use crate::sign::commitment::blake3::Blake3Commitment;
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+#[ts(export)]
+pub struct OsIndex {
+    pub versions: OsVersionInfoMap,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, TS)]
+pub struct OsVersionInfoMap(
+    #[ts(as = "BTreeMap::<String, OsVersionInfo>")] pub BTreeMap<Version, OsVersionInfo>,
+);
+impl Map for OsVersionInfoMap {
+    type Key = Version;
+    type Value = OsVersionInfo;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(InternedString::from_display(key))
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(InternedString::from_display(key))
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+#[ts(export)]
+pub struct OsVersionInfo {
+    pub headline: String,
+    pub release_notes: String,
+    #[ts(type = "string")]
+    pub source_version: VersionRange,
+    pub authorized: BTreeSet<Guid>,
+    pub iso: BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>, // platform (i.e. x86_64-nonfree) -> asset
+    pub squashfs: BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>, // platform (i.e. x86_64-nonfree) -> asset
+    pub img: BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>, // platform (i.e. raspberrypi) -> asset
+}
+
+pub async fn get_os_index(ctx: RegistryContext) -> Result<OsIndex, Error> {
+    ctx.db.peek().await.into_index().into_os().de()
+}

core/src/registry/os/mod.rs

@@ -0,0 +1,32 @@
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::util::serde::HandlerExtSerde;
+
+pub const SIG_CONTEXT: &str = "startos";
+
+pub mod asset;
+pub mod index;
+pub mod version;
+
+pub fn os_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "index",
+            from_fn_async(index::get_os_index)
+                .with_metadata("authenticated", Value::Bool(false))
+                .with_display_serializable()
+                .with_about("List index of OS versions")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "asset",
+            asset::asset_api::<C>().with_about("Commands to add, sign, or get registry assets"),
+        )
+        .subcommand(
+            "version",
+            version::version_api::<C>()
+                .with_about("Commands to add, remove, or list versions or version signers"),
+        )
+}

core/src/registry/os/version/mod.rs

@@ -0,0 +1,257 @@
+use std::collections::BTreeMap;
+
+use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};
+use clap::Parser;
+use exver::{Version, VersionRange};
+use imbl_value::InternedString;
+use itertools::Itertools;
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::registry::context::RegistryContext;
+use crate::registry::device_info::DeviceInfo;
+use crate::registry::os::index::OsVersionInfo;
+use crate::sign::AnyVerifyingKey;
+use crate::util::serde::{HandlerExtSerde, WithIoFormat, display_serializable};
+
+pub mod signer;
+
+pub fn version_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "add",
+            from_fn_async(add_version)
+                .with_metadata("admin", Value::Bool(true))
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_display()
+                .with_about("Add OS version")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_version)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Remove OS version")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "signer",
+            signer::signer_api::<C>().with_about("Add, remove, and list version signers"),
+        )
+        .subcommand(
+            "get",
+            from_fn_async(get_version)
+                .with_metadata("authenticated", Value::Bool(false))
+                .with_metadata("get_device_info", Value::Bool(true))
+                .with_display_serializable()
+                .with_custom_display_fn(|handle, result| {
+                    display_version_info(handle.params, result)
+                })
+                .with_about("Get OS versions and related version info")
+                .with_call_remote::<CliContext>(),
+        )
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct AddVersionParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    pub headline: String,
+    pub release_notes: String,
+    #[ts(type = "string")]
+    pub source_version: VersionRange,
+    #[arg(skip)]
+    #[ts(skip)]
+    #[serde(rename = "__Auth_signer")]
+    pub signer: Option<AnyVerifyingKey>,
+}
+
+pub async fn add_version(
+    ctx: RegistryContext,
+    AddVersionParams {
+        version,
+        headline,
+        release_notes,
+        source_version,
+        signer,
+    }: AddVersionParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            let signer = signer
+                .map(|s| db.as_index().as_signers().get_signer(&s))
+                .transpose()?;
+            db.as_index_mut()
+                .as_os_mut()
+                .as_versions_mut()
+                .upsert(&version, || Ok(OsVersionInfo::default()))?
+                .mutate(|i| {
+                    i.headline = headline;
+                    i.release_notes = release_notes;
+                    i.source_version = source_version;
+                    i.authorized.extend(signer);
+                    Ok(())
+                })
+        })
+        .await
+        .result
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemoveVersionParams {
+    #[ts(type = "string")]
+    pub version: Version,
+}
+
+pub async fn remove_version(
+    ctx: RegistryContext,
+    RemoveVersionParams { version }: RemoveVersionParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_index_mut()
+                .as_os_mut()
+                .as_versions_mut()
+                .remove(&version)?;
+            Ok(())
+        })
+        .await
+        .result
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct GetOsVersionParams {
+    #[ts(type = "string | null")]
+    #[arg(long = "src")]
+    pub source_version: Option<Version>,
+    #[ts(type = "string | null")]
+    #[arg(long)]
+    pub target_version: Option<VersionRange>,
+    #[arg(long = "id")]
+    server_id: Option<String>,
+    #[ts(type = "string | null")]
+    #[arg(long)]
+    platform: Option<InternedString>,
+    #[ts(skip)]
+    #[arg(skip)]
+    #[serde(rename = "__DeviceInfo_device_info")]
+    pub device_info: Option<DeviceInfo>,
+}
+
+struct PgDateTime(DateTime<Utc>);
+impl sqlx::Type<sqlx::Postgres> for PgDateTime {
+    fn type_info() -> <sqlx::Postgres as sqlx::Database>::TypeInfo {
+        sqlx::postgres::PgTypeInfo::with_oid(sqlx::postgres::types::Oid(1184))
+    }
+}
+impl sqlx::Encode<'_, sqlx::Postgres> for PgDateTime {
+    fn encode_by_ref(
+        &self,
+        buf: &mut <sqlx::Postgres as sqlx::Database>::ArgumentBuffer<'_>,
+    ) -> Result<sqlx::encode::IsNull, sqlx::error::BoxDynError> {
+        fn postgres_epoch_datetime() -> NaiveDateTime {
+            NaiveDate::from_ymd_opt(2000, 1, 1)
+                .expect("expected 2000-01-01 to be a valid NaiveDate")
+                .and_hms_opt(0, 0, 0)
+                .expect("expected 2000-01-01T00:00:00 to be a valid NaiveDateTime")
+        }
+        let micros = (self.0.naive_utc() - postgres_epoch_datetime())
+            .num_microseconds()
+            .ok_or_else(|| format!("NaiveDateTime out of range for Postgres: {:?}", self.0))?;
+        micros.encode(buf)
+    }
+    fn size_hint(&self) -> usize {
+        std::mem::size_of::<i64>()
+    }
+}
+
+pub async fn get_version(
+    ctx: RegistryContext,
+    GetOsVersionParams {
+        source_version: source,
+        target_version: target,
+        server_id,
+        platform,
+        device_info,
+    }: GetOsVersionParams,
+) -> Result<BTreeMap<Version, OsVersionInfo>, Error> {
+    let source = source.or_else(|| device_info.as_ref().map(|d| d.os.version.clone()));
+    let platform = platform.or_else(|| device_info.as_ref().map(|d| d.os.platform.clone()));
+    if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, &platform) {
+        let created_at = Utc::now();
+
+        sqlx::query("INSERT INTO user_activity (created_at, server_id, arch) VALUES ($1, $2, $3)")
+            .bind(PgDateTime(created_at))
+            .bind(server_id)
+            .bind(&**arch)
+            .execute(pool)
+            .await
+            .with_kind(ErrorKind::Database)?;
+    }
+    let target = target.unwrap_or(VersionRange::Any);
+    ctx.db
+        .peek()
+        .await
+        .into_index()
+        .into_os()
+        .into_versions()
+        .into_entries()?
+        .into_iter()
+        .map(|(v, i)| i.de().map(|i| (v, i)))
+        .filter_ok(|(version, info)| {
+            platform
+                .as_ref()
+                .map_or(true, |p| info.squashfs.contains_key(p))
+                && version.satisfies(&target)
+                && source
+                    .as_ref()
+                    .map_or(true, |s| s.satisfies(&info.source_version))
+        })
+        .collect()
+}
+
+pub fn display_version_info<T>(
+    params: WithIoFormat<T>,
+    info: BTreeMap<Version, OsVersionInfo>,
+) -> Result<(), Error> {
+    use prettytable::*;
+
+    if let Some(format) = params.format {
+        return display_serializable(format, info);
+    }
+
+    let mut table = Table::new();
+    table.add_row(row![bc =>
+        "VERSION",
+        "HEADLINE",
+        "RELEASE NOTES",
+        "ISO PLATFORMS",
+        "IMG PLATFORMS",
+        "SQUASHFS PLATFORMS",
+    ]);
+    for (version, info) in &info {
+        table.add_row(row![
+            &version.to_string(),
+            &info.headline,
+            &info.release_notes,
+            &info.iso.keys().into_iter().join(", "),
+            &info.img.keys().into_iter().join(", "),
+            &info.squashfs.keys().into_iter().join(", "),
+        ]);
+    }
+    table.print_tty(false)?;
+    Ok(())
+}

core/src/registry/os/version/signer.rs

@@ -0,0 +1,138 @@
+use std::collections::BTreeMap;
+
+use clap::Parser;
+use exver::Version;
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::registry::admin::display_signers;
+use crate::registry::context::RegistryContext;
+use crate::registry::signer::SignerInfo;
+use crate::rpc_continuations::Guid;
+use crate::util::serde::HandlerExtSerde;
+
+pub fn signer_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "add",
+            from_fn_async(add_version_signer)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Add version signer")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_version_signer)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Remove version signer")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "list",
+            from_fn_async(list_version_signers)
+                .with_metadata("authenticated", Value::Bool(false))
+                .with_display_serializable()
+                .with_custom_display_fn(|handle, result| display_signers(handle.params, result))
+                .with_about("List version signers and related signer info")
+                .with_call_remote::<CliContext>(),
+        )
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct VersionSignerParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    pub signer: Guid,
+}
+
+pub async fn add_version_signer(
+    ctx: RegistryContext,
+    VersionSignerParams { version, signer }: VersionSignerParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            ensure_code!(
+                db.as_index().as_signers().contains_key(&signer)?,
+                ErrorKind::InvalidRequest,
+                "unknown signer {signer}"
+            );
+
+            db.as_index_mut()
+                .as_os_mut()
+                .as_versions_mut()
+                .as_idx_mut(&version)
+                .or_not_found(&version)?
+                .as_authorized_mut()
+                .mutate(|s| Ok(s.insert(signer)))?;
+
+            Ok(())
+        })
+        .await
+        .result
+}
+
+pub async fn remove_version_signer(
+    ctx: RegistryContext,
+    VersionSignerParams { version, signer }: VersionSignerParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            if !db
+                .as_index_mut()
+                .as_os_mut()
+                .as_versions_mut()
+                .as_idx_mut(&version)
+                .or_not_found(&version)?
+                .as_authorized_mut()
+                .mutate(|s| Ok(s.remove(&signer)))?
+            {
+                return Err(Error::new(
+                    eyre!("signer {signer} is not authorized to sign for v{version}"),
+                    ErrorKind::NotFound,
+                ));
+            }
+
+            Ok(())
+        })
+        .await
+        .result
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ListVersionSignersParams {
+    #[ts(type = "string")]
+    pub version: Version,
+}
+
+pub async fn list_version_signers(
+    ctx: RegistryContext,
+    ListVersionSignersParams { version }: ListVersionSignersParams,
+) -> Result<BTreeMap<Guid, SignerInfo>, Error> {
+    let db = ctx.db.peek().await;
+    db.as_index()
+        .as_os()
+        .as_versions()
+        .as_idx(&version)
+        .or_not_found(&version)?
+        .as_authorized()
+        .de()?
+        .into_iter()
+        .filter_map(|guid| {
+            db.as_index()
+                .as_signers()
+                .as_idx(&guid)
+                .map(|s| s.de().map(|s| (guid, s)))
+        })
+        .collect()
+}