registry admin script (#2426)

* registryadmin scripts

Add `start-sdk publish` command which can potentially replace
the Haskell implementation of `registry-publish upload`

* restructure modules

---------

Co-authored-by: Sam Sartor <me@samsartor.com>

backend/Cargo.lock

@@ -642,6 +642,19 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
+dependencies = [
+ "encode_unicode 0.3.6",
+ "lazy_static",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.4"
@@ -1292,6 +1305,12 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -2099,6 +2118,20 @@ dependencies = [
  "hashbrown 0.14.0",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.17.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730"
+dependencies = [
+ "console",
+ "instant",
+ "number_prefix",
+ "portable-atomic",
+ "tokio",
+ "unicode-width",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -2733,16 +2766,15 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.26.2"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
 dependencies = [
  "bitflags 1.3.2",
  "cfg-if 1.0.0",
  "libc",
  "memoffset 0.7.1",
  "pin-utils",
- "static_assertions",
 ]
 
 [[package]]
@@ -2902,6 +2934,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "object"
 version = "0.31.1"
@@ -3119,7 +3157,7 @@ dependencies = [
  "json-patch",
  "json-ptr",
  "lazy_static",
- "nix 0.26.2",
+ "nix 0.26.4",
  "patch-db-macro",
  "serde",
  "serde_cbor 0.11.1",
@@ -3312,6 +3350,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "portable-atomic"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -3331,7 +3375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
 dependencies = [
  "csv",
- "encode_unicode",
+ "encode_unicode 1.0.0",
  "is-terminal",
  "lazy_static",
  "term",
@@ -4541,6 +4585,7 @@ dependencies = [
  "ciborium",
  "clap 3.2.25",
  "color-eyre",
+ "console",
  "cookie",
  "cookie_store 0.19.1",
  "current_platform",
@@ -4565,6 +4610,7 @@ dependencies = [
  "imbl-value",
  "include_dir",
  "indexmap 1.9.3",
+ "indicatif",
  "ipnet",
  "iprange",
  "isocountry",

backend/Cargo.toml

@@ -158,6 +158,8 @@ url = { version = "2.2.2", features = ["serde"] }
 urlencoding = "2.1.2"
 uuid = { version = "1.1.2", features = ["v4"] }
 zeroize = "1.5.7"
+indicatif = { version = "0.17.6", features = ["tokio"] }
+console = "^0.15"
 
 [profile.test]
 opt-level = 3

backend/src/install/mod.rs

@@ -42,12 +42,12 @@ use crate::dependencies::{
 };
 use crate::install::cleanup::cleanup;
 use crate::install::progress::{InstallProgress, InstallProgressTracker};
-use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::prelude::*;
+use crate::registry::marketplace::with_query_params;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
-use crate::status::{DependencyConfigErrors, MainStatus, Status};
+use crate::status::{MainStatus, Status};
 use crate::util::docker::CONTAINER_TOOL;
 use crate::util::io::{copy_and_shutdown, response_to_reader};
 use crate::util::serde::{display_serializable, Port};

backend/src/lib.rs

@@ -34,7 +34,6 @@ pub mod inspect;
 pub mod install;
 pub mod logs;
 pub mod manager;
-pub mod marketplace;
 pub mod middleware;
 pub mod migration;
 pub mod net;
@@ -43,6 +42,7 @@ pub mod os_install;
 pub mod prelude;
 pub mod procedure;
 pub mod properties;
+pub mod registry;
 pub mod s9pk;
 pub mod setup;
 pub mod shutdown;
@@ -79,7 +79,7 @@ pub fn echo(#[arg] message: String) -> Result<String, RpcError> {
     disk::disk,
     notifications::notification,
     backup::backup,
-    marketplace::marketplace,
+    registry::marketplace::marketplace,
 ))]
 pub fn main_api() -> Result<(), RpcError> {
     Ok(())
@@ -124,7 +124,8 @@ pub fn package() -> Result<(), RpcError> {
     s9pk::pack,
     developer::verify,
     developer::init,
-    inspect::inspect
+    inspect::inspect,
+    registry::admin::publish,
 ))]
 pub fn portable_api() -> Result<(), RpcError> {
     Ok(())

backend/src/registry/admin.rs

@@ -0,0 +1,211 @@
+use std::path::PathBuf;
+use std::time::Duration;
+
+use color_eyre::eyre::eyre;
+use console::style;
+use futures::StreamExt;
+use indicatif::{ProgressBar, ProgressStyle};
+use reqwest::{header, Body, Client, Url};
+use rpc_toolkit::command;
+
+use crate::s9pk::reader::S9pkReader;
+use crate::util::display_none;
+use crate::{Error, ErrorKind};
+
+async fn registry_user_pass(location: &str) -> Result<(Url, String, String), Error> {
+    let mut url = Url::parse(location)?;
+    let user = url.username().to_string();
+    let pass = url.password().map(str::to_string);
+    if user.is_empty() || url.path() != "/" {
+        return Err(Error::new(
+            eyre!("{location:?} is not like \"https://user@registry.example.com/\""),
+            ErrorKind::ParseUrl,
+        ));
+    }
+    let _ = url.set_username("");
+    let _ = url.set_password(None);
+
+    let pass = match pass {
+        Some(p) => p,
+        None => {
+            let pass_prompt = format!("{} Password for {user}: ", style("?").yellow());
+            tokio::task::spawn_blocking(move || rpassword::prompt_password(pass_prompt))
+                .await
+                .unwrap()?
+        }
+    };
+    Ok((url, user.to_string(), pass.to_string()))
+}
+
+#[derive(serde::Serialize, Debug)]
+struct Package {
+    id: String,
+    version: String,
+    arches: Option<Vec<String>>,
+}
+
+async fn do_index(
+    httpc: &Client,
+    mut url: Url,
+    user: &str,
+    pass: &str,
+    pkg: &Package,
+) -> Result<(), Error> {
+    url.set_path("/admin/v0/index");
+    let mut req = httpc
+        .post(url)
+        .header(header::ACCEPT, "text/plain")
+        .basic_auth(user, Some(pass))
+        .json(pkg)
+        .build()?;
+    let res = httpc.execute(req).await?;
+    if !res.status().is_success() {
+        let info = res.text().await?;
+        return Err(Error::new(eyre!("{}", info), ErrorKind::Registry));
+    }
+    Ok(())
+}
+
+async fn do_upload(
+    httpc: &Client,
+    mut url: Url,
+    user: &str,
+    pass: &str,
+    body: Body,
+) -> Result<(), Error> {
+    url.set_path("/admin/v0/upload");
+    let mut req = httpc
+        .post(url)
+        .header(header::ACCEPT, "text/plain")
+        .basic_auth(user, Some(pass))
+        .body(body)
+        .build()?;
+    let res = httpc.execute(req).await?;
+    if !res.status().is_success() {
+        let info = res.text().await?;
+        return Err(Error::new(eyre!("{}", info), ErrorKind::Registry));
+    }
+    Ok(())
+}
+
+#[command(cli_only, display(display_none))]
+pub async fn publish(
+    #[arg] location: String,
+    #[arg] path: PathBuf,
+    #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
+    #[arg(rename = "no-upload", long = "no-upload")] no_upload: bool,
+    #[arg(rename = "no-index", long = "no-index")] no_index: bool,
+) -> Result<(), Error> {
+    // Prepare for progress bars.
+    let bytes_bar_style =
+        ProgressStyle::with_template("{percent}% {wide_bar} [{bytes}/{total_bytes}] [{eta}]")
+            .unwrap();
+    let plain_line_style =
+        ProgressStyle::with_template("{prefix:.bold.dim} {wide_msg}...").unwrap();
+    let spinner_line_style =
+        ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}...").unwrap();
+
+    // Read the file to get manifest information and check validity..
+    // Open file right away so it can not change out from under us.
+    let file = tokio::fs::File::open(&path).await?;
+
+    let manifest = if no_verify {
+        let pb = ProgressBar::new(1)
+            .with_style(spinner_line_style.clone())
+            .with_prefix("[1/3]")
+            .with_message("Querying s9pk");
+        pb.enable_steady_tick(Duration::from_millis(200));
+        let mut s9pk = S9pkReader::open(&path, false).await?;
+        let m = s9pk.manifest().await?.clone();
+        pb.set_style(plain_line_style.clone());
+        pb.abandon();
+        m
+    } else {
+        let pb = ProgressBar::new(1)
+            .with_style(spinner_line_style.clone())
+            .with_prefix("[1/3]")
+            .with_message("Verifying s9pk");
+        pb.enable_steady_tick(Duration::from_millis(200));
+        let mut s9pk = S9pkReader::open(&path, true).await?;
+        s9pk.validate().await?;
+        let m = s9pk.manifest().await?.clone();
+        pb.set_style(plain_line_style.clone());
+        pb.abandon();
+        m
+    };
+    let pkg = Package {
+        id: manifest.id.to_string(),
+        version: manifest.version.to_string(),
+        arches: manifest.hardware_requirements.arch.clone(),
+    };
+    println!("{} id = {}", style(">").green(), pkg.id);
+    println!("{} version = {}", style(">").green(), pkg.version);
+    if let Some(arches) = &pkg.arches {
+        println!("{} arches = {:?}", style(">").green(), arches);
+    } else {
+        println!(
+            "{} No architecture listed in hardware_requirements",
+            style(">").red()
+        );
+    }
+
+    // Process the url and get the user's password.
+    let (registry, user, pass) = registry_user_pass(&location).await?;
+
+    // Now prepare a stream of the file which will show a progress bar as it is consumed.
+    let file_size = file.metadata().await?.len();
+    let file_stream = tokio_util::io::ReaderStream::new(file);
+    ProgressBar::new(0)
+        .with_style(plain_line_style.clone())
+        .with_prefix("[2/3]")
+        .with_message("Uploading s9pk")
+        .abandon();
+    let pb = ProgressBar::new(file_size).with_style(bytes_bar_style.clone());
+    let stream_pb = pb.clone();
+    let file_stream = file_stream.inspect(move |bytes| {
+        if let Ok(bytes) = bytes {
+            stream_pb.inc(bytes.len() as u64);
+        }
+    });
+
+    let httpc = Client::builder().build().unwrap();
+    // And upload!
+    if no_upload {
+        println!("{} Skipping upload", style(">").yellow());
+    } else {
+        do_upload(
+            &httpc,
+            registry.clone(),
+            &user,
+            &pass,
+            Body::wrap_stream(file_stream),
+        )
+        .await?;
+    }
+    pb.finish_and_clear();
+
+    // Also index, so it will show up in the registry.
+    let pb = ProgressBar::new(0)
+        .with_style(spinner_line_style.clone())
+        .with_prefix("[3/3]")
+        .with_message("Indexing registry");
+    pb.enable_steady_tick(Duration::from_millis(200));
+    if no_index {
+        println!("{} Skipping index", style(">").yellow());
+    } else {
+        do_index(&httpc, registry.clone(), &user, &pass, &pkg).await?;
+    }
+    pb.set_style(plain_line_style.clone());
+    pb.abandon();
+
+    // All done
+    if !no_index {
+        println!(
+            "{} Package {} is now published to {}",
+            style(">").green(),
+            pkg.id,
+            registry
+        );
+    }
+    Ok(())
+}

backend/src/registry/mod.rs

@@ -0,0 +1,2 @@
+pub mod admin;
+pub mod marketplace;

backend/src/s9pk/manifest.rs

@@ -112,7 +112,7 @@ pub struct HardwareRequirements {
     #[serde(default)]
     device: BTreeMap<String, Regex>,
     ram: Option<u64>,
-    arch: Option<Vec<String>>,
+    pub arch: Option<Vec<String>>,
 }
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]

backend/src/update/mod.rs

@@ -17,9 +17,9 @@ use crate::db::model::UpdateProgress;
 use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::MountGuard;
-use crate::marketplace::with_query_params;
 use crate::notifications::NotificationLevel;
 use crate::prelude::*;
+use crate::registry::marketplace::with_query_params;
 use crate::sound::{
     CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4,
 };

libs/Cargo.lock

@@ -1897,16 +1897,15 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.26.2"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
 dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
  "memoffset 0.7.1",
  "pin-utils",
- "static_assertions",
 ]
 
 [[package]]
@@ -2181,7 +2180,7 @@ dependencies = [
  "json-patch",
  "json-ptr",
  "lazy_static",
- "nix 0.26.2",
+ "nix 0.26.4",
  "patch-db-macro",
  "serde",
  "serde_cbor 0.11.1",

libs/models/src/errors.rs

@@ -3,6 +3,7 @@ use std::fmt::Display;
 use color_eyre::eyre::eyre;
 use patch_db::Revision;
 use rpc_toolkit::hyper::http::uri::InvalidUri;
+use rpc_toolkit::reqwest;
 use rpc_toolkit::yajrc::RpcError;
 
 use crate::InvalidId;
@@ -272,7 +273,12 @@ impl From<ssh_key::Error> for Error {
 }
 impl From<reqwest::Error> for Error {
     fn from(e: reqwest::Error) -> Self {
-        Error::new(e, ErrorKind::Network)
+        let kind = match e {
+            _ if e.is_builder() => ErrorKind::ParseUrl,
+            _ if e.is_decode() => ErrorKind::Deserialization,
+            _ => ErrorKind::Network,
+        };
+        Error::new(e, kind)
     }
 }
 impl From<patch_db::value::Error> for Error {

system-images/compat/Cargo.lock

@@ -601,6 +601,19 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
+dependencies = [
+ "encode_unicode 0.3.6",
+ "lazy_static",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.1"
@@ -1157,6 +1170,12 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "encode_unicode"
 version = "1.0.0"
@@ -1904,6 +1923,20 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.17.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25"
+dependencies = [
+ "console",
+ "instant",
+ "number_prefix",
+ "portable-atomic",
+ "tokio",
+ "unicode-width",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -2581,6 +2614,12 @@ dependencies = [
  "syn 1.0.107",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "object"
 version = "0.30.2"
@@ -2984,6 +3023,12 @@ version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
+[[package]]
+name = "portable-atomic"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -3003,7 +3048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
 dependencies = [
  "csv",
- "encode_unicode",
+ "encode_unicode 1.0.0",
  "is-terminal",
  "lazy_static",
  "term",
@@ -4089,6 +4134,7 @@ dependencies = [
  "ciborium",
  "clap 3.2.23",
  "color-eyre",
+ "console",
  "cookie",
  "cookie_store 0.19.0",
  "current_platform",
@@ -4113,6 +4159,7 @@ dependencies = [
  "imbl-value",
  "include_dir",
  "indexmap",
+ "indicatif",
  "ipnet",
  "iprange",
  "isocountry",