update marketplace url to reflect build version (#2914)

* update marketplace url to reflect build version

* adjust marketplace config

* use helper function to compare urls

* rework some registry stuff

* #2900, #2899, and other registry changes

* alpha.1

* trailing /

* add startosRegistry

* fix migration

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>

core/startos/src/registry/admin.rs

@@ -22,16 +22,30 @@ pub fn admin_api<C: Context>() -> ParentHandler<C> {
             "signer",
             signers_api::<C>().with_about("Commands to add or list signers"),
         )
-        .subcommand("add", from_fn_async(add_admin).no_cli())
+        .subcommand(
+            "add",
+            from_fn_async(add_admin)
+                .with_metadata("admin", Value::Bool(true))
+                .no_cli(),
+        )
         .subcommand(
             "add",
             from_fn_async(cli_add_admin)
                 .no_display()
                 .with_about("Add admin signer"),
         )
+        .subcommand(
+            "remove",
+            from_fn_async(remove_admin)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Remove an admin signer")
+                .with_call_remote::<CliContext>(),
+        )
         .subcommand(
             "list",
             from_fn_async(list_admins)
+                .with_metadata("admin", Value::Bool(true))
                 .with_display_serializable()
                 .with_custom_display_fn(|handle, result| Ok(display_signers(handle.params, result)))
                 .with_about("List admin signers")
@@ -285,6 +299,27 @@ pub async fn add_admin(
         .result
 }
 
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemoveAdminParams {
+    pub signer: Guid,
+}
+
+// TODO: don't allow removing self?
+pub async fn remove_admin(
+    ctx: RegistryContext,
+    RemoveAdminParams { signer }: RemoveAdminParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_admins_mut().mutate(|a| Ok(a.remove(&signer)))?;
+            Ok(())
+        })
+        .await
+        .result
+}
+
 #[derive(Debug, Deserialize, Serialize, Parser)]
 #[command(rename_all = "kebab-case")]
 #[serde(rename_all = "camelCase")]

core/startos/src/registry/os/asset/add.rs

@@ -51,6 +51,28 @@ pub fn add_api<C: Context>() -> ParentHandler<C> {
         )
 }
 
+pub fn remove_api<C: Context>() -> ParentHandler<C> {
+    ParentHandler::new()
+        .subcommand(
+            "iso",
+            from_fn_async(remove_iso)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "img",
+            from_fn_async(remove_img)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+        .subcommand(
+            "squashfs",
+            from_fn_async(remove_squashfs)
+                .with_metadata("get_signer", Value::Bool(true))
+                .no_cli(),
+        )
+}
+
 #[derive(Debug, Deserialize, Serialize, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export)]
@@ -99,6 +121,7 @@ async fn add_asset(
                 .as_authorized()
                 .de()?
                 .contains(&signer_guid)
+                || db.as_admins().de()?.contains(&signer_guid)
             {
                 accessor(
                     db.as_index_mut()
@@ -256,3 +279,74 @@ pub async fn cli_add_asset(
 
     Ok(())
 }
+
+#[derive(Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemoveAssetParams {
+    #[ts(type = "string")]
+    pub version: Version,
+    #[ts(type = "string")]
+    pub platform: InternedString,
+    #[serde(rename = "__auth_signer")]
+    #[ts(skip)]
+    pub signer: AnyVerifyingKey,
+}
+
+async fn remove_asset(
+    ctx: RegistryContext,
+    RemoveAssetParams {
+        version,
+        platform,
+        signer,
+    }: RemoveAssetParams,
+    accessor: impl FnOnce(
+            &mut Model<OsVersionInfo>,
+        ) -> &mut Model<BTreeMap<InternedString, RegistryAsset<Blake3Commitment>>>
+        + UnwindSafe
+        + Send,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            let signer_guid = db.as_index().as_signers().get_signer(&signer)?;
+            if db
+                .as_index()
+                .as_os()
+                .as_versions()
+                .as_idx(&version)
+                .or_not_found(&version)?
+                .as_authorized()
+                .de()?
+                .contains(&signer_guid)
+                || db.as_admins().de()?.contains(&signer_guid)
+            {
+                accessor(
+                    db.as_index_mut()
+                        .as_os_mut()
+                        .as_versions_mut()
+                        .as_idx_mut(&version)
+                        .or_not_found(&version)?,
+                )
+                .remove(&platform)?;
+                Ok(())
+            } else {
+                Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
+            }
+        })
+        .await
+        .result?;
+
+    Ok(())
+}
+
+pub async fn remove_iso(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_iso_mut()).await
+}
+
+pub async fn remove_img(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_img_mut()).await
+}
+
+pub async fn remove_squashfs(ctx: RegistryContext, params: RemoveAssetParams) -> Result<(), Error> {
+    remove_asset(ctx, params, |m| m.as_squashfs_mut()).await
+}

core/startos/src/registry/os/asset/mod.rs

@@ -13,6 +13,7 @@ pub fn asset_api<C: Context>() -> ParentHandler<C> {
                 .no_display()
                 .with_about("Add asset to registry"),
         )
+        .subcommand("remove", add::remove_api::<C>())
         .subcommand("sign", sign::sign_api::<C>())
         .subcommand(
             "sign",
@@ -20,6 +21,7 @@ pub fn asset_api<C: Context>() -> ParentHandler<C> {
                 .no_display()
                 .with_about("Sign file and add to registry index"),
         )
+        // TODO: remove signature api
         .subcommand(
             "get",
             get::get_api::<C>().with_about("Commands to download image, iso, or squashfs files"),

core/startos/src/registry/os/version/mod.rs

@@ -3,6 +3,7 @@ use std::collections::BTreeMap;
 use chrono::Utc;
 use clap::Parser;
 use exver::{Version, VersionRange};
+use imbl_value::InternedString;
 use itertools::Itertools;
 use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
@@ -12,6 +13,7 @@ use ts_rs::TS;
 use crate::context::CliContext;
 use crate::prelude::*;
 use crate::registry::context::RegistryContext;
+use crate::registry::device_info::DeviceInfo;
 use crate::registry::os::index::OsVersionInfo;
 use crate::registry::signer::sign::AnyVerifyingKey;
 use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
@@ -44,6 +46,7 @@ pub fn version_api<C: Context>() -> ParentHandler<C> {
         .subcommand(
             "get",
             from_fn_async(get_version)
+                .with_metadata("get_device_info", Value::Bool(true))
                 .with_display_serializable()
                 .with_custom_display_fn(|handle, result| {
                     Ok(display_version_info(handle.params, result))
@@ -133,35 +136,47 @@ pub async fn remove_version(
 pub struct GetOsVersionParams {
     #[ts(type = "string | null")]
     #[arg(long = "src")]
-    pub source: Option<Version>,
-    #[ts(type = "string | null")]
-    #[arg(long = "target")]
-    pub target: Option<VersionRange>,
+    pub source_version: Option<Version>,
     #[ts(type = "string | null")]
+    #[arg(long)]
+    pub target_version: Option<VersionRange>,
+    #[arg(long)]
+    pub include_prerelease: Option<bool>,
     #[arg(long = "id")]
     server_id: Option<String>,
     #[ts(type = "string | null")]
-    #[arg(long = "arch")]
-    arch: Option<String>,
+    #[arg(long)]
+    platform: Option<InternedString>,
+    #[ts(skip)]
+    #[arg(skip)]
+    #[serde(rename = "__device_info")]
+    pub device_info: Option<DeviceInfo>,
 }
 
 pub async fn get_version(
     ctx: RegistryContext,
     GetOsVersionParams {
-        source,
-        target,
+        source_version: source,
+        target_version: target,
+        include_prerelease,
         server_id,
-        arch,
+        platform,
+        device_info,
     }: GetOsVersionParams,
 ) -> Result<BTreeMap<Version, OsVersionInfo>, Error> {
-    if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, arch) {
+    let source = source.or_else(|| device_info.as_ref().map(|d| d.os.version.clone()));
+    let platform = platform.or_else(|| device_info.as_ref().map(|d| d.os.platform.clone()));
+    let include_prerelease = include_prerelease
+        .or_else(|| source.as_ref().map(|s| !s.prerelease().is_empty()))
+        .unwrap_or(cfg!(feature = "dev"));
+    if let (Some(pool), Some(server_id), Some(arch)) = (&ctx.pool, server_id, &platform) {
         let created_at = Utc::now();
 
         query!(
             "INSERT INTO user_activity (created_at, server_id, arch) VALUES ($1, $2, $3)",
             created_at,
             server_id,
-            arch
+            &**arch
         )
         .execute(pool)
         .await?;
@@ -177,7 +192,11 @@ pub async fn get_version(
         .into_iter()
         .map(|(v, i)| i.de().map(|i| (v, i)))
         .filter_ok(|(version, info)| {
-            version.satisfies(&target)
+            (version.prerelease().is_empty() || include_prerelease)
+                && platform
+                    .as_ref()
+                    .map_or(true, |p| info.squashfs.contains_key(p))
+                && version.satisfies(&target)
                 && source
                     .as_ref()
                     .map_or(true, |s| s.satisfies(&info.source_version))

core/startos/src/registry/package/add.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use clap::Parser;
 use imbl_value::InternedString;
 use itertools::Itertools;
+use models::{PackageId, VersionString};
 use rpc_toolkit::HandlerArgs;
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
@@ -158,3 +159,55 @@ pub async fn cli_add_package(
 
     Ok(())
 }
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemovePackageParams {
+    pub id: PackageId,
+    pub version: VersionString,
+    #[ts(skip)]
+    #[serde(rename = "__auth_signer")]
+    pub signer: AnyVerifyingKey,
+}
+
+pub async fn remove_package(
+    ctx: RegistryContext,
+    RemovePackageParams {
+        id,
+        version,
+        signer,
+    }: RemovePackageParams,
+) -> Result<(), Error> {
+    let peek = ctx.db.peek().await;
+    let signer_guid = peek.as_index().as_signers().get_signer(&signer)?;
+
+    ctx.db
+        .mutate(|db| {
+            if db.as_admins().de()?.contains(&signer_guid)
+                || db
+                    .as_index()
+                    .as_package()
+                    .as_packages()
+                    .as_idx(&id)
+                    .or_not_found(&id)?
+                    .as_authorized()
+                    .de()?
+                    .contains(&signer_guid)
+            {
+                if let Some(package) = db
+                    .as_index_mut()
+                    .as_package_mut()
+                    .as_packages_mut()
+                    .as_idx_mut(&id)
+                {
+                    package.as_versions_mut().remove(&version)?;
+                }
+                Ok(())
+            } else {
+                Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
+            }
+        })
+        .await
+        .result
+}

core/startos/src/registry/package/category.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 
 use clap::Parser;
 use imbl_value::InternedString;
+use models::PackageId;
 use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
@@ -10,7 +11,6 @@ use crate::context::CliContext;
 use crate::prelude::*;
 use crate::registry::context::RegistryContext;
 use crate::registry::package::index::Category;
-use crate::s9pk::manifest::Description;
 use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
 
 pub fn category_api<C: Context>() -> ParentHandler<C> {
@@ -31,6 +31,22 @@ pub fn category_api<C: Context>() -> ParentHandler<C> {
                 .with_about("Remove a category from the registry")
                 .with_call_remote::<CliContext>(),
         )
+        .subcommand(
+            "add-package",
+            from_fn_async(add_package)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Add a package to a category")
+                .with_call_remote::<CliContext>(),
+        )
+        .subcommand(
+            "remove-package",
+            from_fn_async(remove_package)
+                .with_metadata("admin", Value::Bool(true))
+                .no_display()
+                .with_about("Remove a package from a category")
+                .with_call_remote::<CliContext>(),
+        )
         .subcommand(
             "list",
             from_fn_async(list_categories)
@@ -50,33 +66,18 @@ pub struct AddCategoryParams {
     #[ts(type = "string")]
     pub id: InternedString,
     pub name: String,
-    #[arg(short, long, help = "Short description for the category")]
-    pub short: String,
-    #[arg(short, long, help = "Long description for the category")]
-    pub long: String,
 }
 
 pub async fn add_category(
     ctx: RegistryContext,
-    AddCategoryParams {
-        id,
-        name,
-        short,
-        long,
-    }: AddCategoryParams,
+    AddCategoryParams { id, name }: AddCategoryParams,
 ) -> Result<(), Error> {
     ctx.db
         .mutate(|db| {
             db.as_index_mut()
                 .as_package_mut()
                 .as_categories_mut()
-                .insert(
-                    &id,
-                    &Category {
-                        name,
-                        description: Description { short, long },
-                    },
-                )
+                .insert(&id, &Category { name })
         })
         .await
         .result?;
@@ -108,6 +109,64 @@ pub async fn remove_category(
     Ok(())
 }
 
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct AddPackageToCategoryParams {
+    #[ts(type = "string")]
+    pub id: InternedString,
+    pub package: PackageId,
+}
+
+pub async fn add_package(
+    ctx: RegistryContext,
+    AddPackageToCategoryParams { id, package }: AddPackageToCategoryParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_index_mut()
+                .as_package_mut()
+                .as_packages_mut()
+                .as_idx_mut(&package)
+                .or_not_found(&package)?
+                .as_categories_mut()
+                .mutate(|c| Ok(c.insert(id)))
+        })
+        .await
+        .result?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize, Parser, TS)]
+#[command(rename_all = "kebab-case")]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct RemovePackageFromCategoryParams {
+    #[ts(type = "string")]
+    pub id: InternedString,
+    pub package: PackageId,
+}
+
+pub async fn remove_package(
+    ctx: RegistryContext,
+    RemovePackageFromCategoryParams { id, package }: RemovePackageFromCategoryParams,
+) -> Result<(), Error> {
+    ctx.db
+        .mutate(|db| {
+            db.as_index_mut()
+                .as_package_mut()
+                .as_packages_mut()
+                .as_idx_mut(&package)
+                .or_not_found(&package)?
+                .as_categories_mut()
+                .mutate(|c| Ok(c.remove(&id)))
+        })
+        .await
+        .result?;
+    Ok(())
+}
+
 pub async fn list_categories(
     ctx: RegistryContext,
 ) -> Result<BTreeMap<InternedString, Category>, Error> {
@@ -134,16 +193,9 @@ pub fn display_categories<T>(
     table.add_row(row![bc =>
         "ID",
         "NAME",
-        "SHORT DESCRIPTION",
-        "LONG DESCRIPTION",
     ]);
     for (id, info) in categories {
-        table.add_row(row![
-            &*id,
-            &info.name,
-            &info.description.short,
-            &info.description.long,
-        ]);
+        table.add_row(row![&*id, &info.name]);
     }
     table.print_tty(false).unwrap();
 }

core/startos/src/registry/package/get.rs

@@ -45,7 +45,9 @@ pub struct PackageInfoShort {
 pub struct GetPackageParams {
     pub id: Option<PackageId>,
     #[ts(type = "string | null")]
-    pub version: Option<VersionRange>,
+    #[arg(long, short = 'v')]
+    pub target_version: Option<VersionRange>,
+    #[arg(long)]
     pub source_version: Option<VersionString>,
     #[ts(skip)]
     #[arg(skip)]
@@ -188,7 +190,7 @@ pub async fn get_package(ctx: RegistryContext, params: GetPackageParams) -> Resu
         let package_best = best.entry(id.clone()).or_default();
         let package_other = other.entry(id.clone()).or_default();
         if params
-            .version
+            .target_version
             .as_ref()
             .map_or(true, |v| version.satisfies(v))
             && package_best.keys().all(|k| !(**k > version))

core/startos/src/registry/package/index.rs

@@ -47,7 +47,6 @@ pub struct PackageInfo {
 #[ts(export)]
 pub struct Category {
     pub name: String,
-    pub description: Description,
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel, TS)]

core/startos/src/registry/package/mod.rs

@@ -31,6 +31,13 @@ pub fn package_api<C: Context>() -> ParentHandler<C> {
                 .no_display()
                 .with_about("Add package to registry index"),
         )
+        .subcommand(
+            "remove",
+            from_fn_async(add::remove_package)
+                .no_display()
+                .with_about("Add package to registry index")
+                .with_call_remote::<CliContext>(),
+        )
         .subcommand(
             "signer",
             signer::signer_api::<C>().with_about("Add, remove, and list package signers"),

core/startos/src/version/mod.rs

@@ -40,8 +40,9 @@ mod v0_3_6_alpha_17;
 mod v0_3_6_alpha_18;
 
 mod v0_4_0_alpha_0;
+mod v0_4_0_alpha_1;
 
-pub type Current = v0_4_0_alpha_0::Version; // VERSION_BUMP
+pub type Current = v0_4_0_alpha_1::Version; // VERSION_BUMP
 
 impl Current {
     #[instrument(skip(self, db))]
@@ -145,7 +146,8 @@ enum Version {
     V0_3_6_alpha_16(Wrapper<v0_3_6_alpha_16::Version>),
     V0_3_6_alpha_17(Wrapper<v0_3_6_alpha_17::Version>),
     V0_3_6_alpha_18(Wrapper<v0_3_6_alpha_18::Version>),
-    V0_4_0_alpha_0(Wrapper<v0_4_0_alpha_0::Version>), // VERSION_BUMP
+    V0_4_0_alpha_0(Wrapper<v0_4_0_alpha_0::Version>),
+    V0_4_0_alpha_1(Wrapper<v0_4_0_alpha_1::Version>), // VERSION_BUMP
     Other(exver::Version),
 }
 
@@ -187,8 +189,9 @@ impl Version {
             Self::V0_3_6_alpha_15(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_16(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_17(v) => DynVersion(Box::new(v.0)),
-            Self::V0_3_6_alpha_18(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
-            Self::V0_4_0_alpha_0(v) => DynVersion(Box::new(v.0)),  // VERSION_BUMP
+            Self::V0_3_6_alpha_18(v) => DynVersion(Box::new(v.0)),
+            Self::V0_4_0_alpha_0(v) => DynVersion(Box::new(v.0)),
+            Self::V0_4_0_alpha_1(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
             Self::Other(v) => {
                 return Err(Error::new(
                     eyre!("unknown version {v}"),
@@ -223,7 +226,8 @@ impl Version {
             Version::V0_3_6_alpha_16(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_17(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_18(Wrapper(x)) => x.semver(),
-            Version::V0_4_0_alpha_0(Wrapper(x)) => x.semver(), // VERSION_BUMP
+            Version::V0_4_0_alpha_0(Wrapper(x)) => x.semver(),
+            Version::V0_4_0_alpha_1(Wrapper(x)) => x.semver(), // VERSION_BUMP
             Version::Other(x) => x.clone(),
         }
     }

core/startos/src/version/v0_4_0_alpha_1.rs

@@ -0,0 +1,72 @@
+use exver::{PreReleaseSegment, VersionRange};
+use imbl_value::json;
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{v0_4_0_alpha_0, VersionT};
+use crate::prelude::*;
+
+lazy_static::lazy_static! {
+    static ref V0_4_0_alpha_1: exver::Version = exver::Version::new(
+        [0, 4, 0],
+        [PreReleaseSegment::String("alpha".into()), 1.into()]
+    );
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_4_0_alpha_0::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_4_0_alpha_1.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        let Some(ui) = db["public"]["ui"].as_object_mut() else {
+            return Err(Error::new(
+                eyre!("db.public.ui is not an object"),
+                ErrorKind::Database,
+            ));
+        };
+        ui.insert(
+            "registries".into(),
+            Value::Object(
+                ui.get("marketplace")
+                    .and_then(|m| m.get("knownHosts"))
+                    .and_then(|kh| kh.as_object())
+                    .into_iter()
+                    .flatten()
+                    .map(|(k, v)| (k.clone(), v["name"].clone()))
+                    .collect(),
+            ),
+        );
+        if let Some(highscore) = ui
+            .get_mut("gaming")
+            .and_then(|g| g.get_mut("snake"))
+            .and_then(|s| s.get_mut("highScore"))
+            .map(|hs| hs.take())
+            .filter(|s| s.is_number())
+        {
+            ui.insert("snakeHighScore".into(), highscore);
+        }
+        ui.insert(
+            "startosRegistry".into(),
+            json!("https://registry.start9.com/"),
+        );
+        ui.remove("marketplace");
+        ui.remove("gaming");
+        ui.remove("theme");
+
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}