From f5da5f4ef029befa47f8c12dfcd45641e5211aee Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:19:31 -0600 Subject: [PATCH] registry admin script (#2426) * registryadmin scripts Add `start-sdk publish` command which can potentially replace the Haskell implementation of `registry-publish upload` * restructure modules --------- Co-authored-by: Sam Sartor --- backend/Cargo.lock | 56 +++++- backend/Cargo.toml | 2 + backend/src/install/mod.rs | 4 +- backend/src/lib.rs | 7 +- backend/src/registry/admin.rs | 211 ++++++++++++++++++++++ backend/src/{ => registry}/marketplace.rs | 0 backend/src/registry/mod.rs | 2 + backend/src/s9pk/manifest.rs | 2 +- backend/src/update/mod.rs | 2 +- libs/Cargo.lock | 7 +- libs/models/src/errors.rs | 8 +- system-images/compat/Cargo.lock | 49 ++++- 12 files changed, 332 insertions(+), 18 deletions(-) create mode 100644 backend/src/registry/admin.rs rename backend/src/{ => registry}/marketplace.rs (100%) create mode 100644 backend/src/registry/mod.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9f0683e75..782823a4a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -642,6 +642,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode 0.3.6", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "const-oid" version = "0.9.4" @@ -1292,6 +1305,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -2099,6 +2118,20 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "indicatif" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-width", +] + [[package]] name = "instant" version = "0.1.12" @@ -2733,16 +2766,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", "memoffset 0.7.1", "pin-utils", - "static_assertions", ] [[package]] @@ -2902,6 +2934,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.31.1" @@ -3119,7 +3157,7 @@ dependencies = [ "json-patch", "json-ptr", "lazy_static", - "nix 0.26.2", + "nix 0.26.4", "patch-db-macro", "serde", "serde_cbor 0.11.1", @@ -3312,6 +3350,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "portable-atomic" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3331,7 +3375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" dependencies = [ "csv", - "encode_unicode", + "encode_unicode 1.0.0", "is-terminal", "lazy_static", "term", @@ -4541,6 +4585,7 @@ dependencies = [ "ciborium", "clap 3.2.25", "color-eyre", + "console", "cookie", "cookie_store 0.19.1", "current_platform", @@ -4565,6 +4610,7 @@ dependencies = [ "imbl-value", "include_dir", "indexmap 1.9.3", + "indicatif", "ipnet", "iprange", "isocountry", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d10c011a1..5501769a7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -158,6 +158,8 @@ url = { version = "2.2.2", features = ["serde"] } urlencoding = "2.1.2" uuid = { version = "1.1.2", features = ["v4"] } zeroize = "1.5.7" +indicatif = { version = "0.17.6", features = ["tokio"] } +console = "^0.15" [profile.test] opt-level = 3 diff --git a/backend/src/install/mod.rs b/backend/src/install/mod.rs index 3e3f07fb7..a85c7c823 100644 --- a/backend/src/install/mod.rs +++ b/backend/src/install/mod.rs @@ -42,12 +42,12 @@ use crate::dependencies::{ }; use crate::install::cleanup::cleanup; use crate::install::progress::{InstallProgress, InstallProgressTracker}; -use crate::marketplace::with_query_params; use crate::notifications::NotificationLevel; use crate::prelude::*; +use crate::registry::marketplace::with_query_params; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::s9pk::reader::S9pkReader; -use crate::status::{DependencyConfigErrors, MainStatus, Status}; +use crate::status::{MainStatus, Status}; use crate::util::docker::CONTAINER_TOOL; use crate::util::io::{copy_and_shutdown, response_to_reader}; use crate::util::serde::{display_serializable, Port}; diff --git a/backend/src/lib.rs b/backend/src/lib.rs index dda755dba..2b4818698 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -34,7 +34,6 @@ pub mod inspect; pub mod install; pub mod logs; pub mod manager; -pub mod marketplace; pub mod middleware; pub mod migration; pub mod net; @@ -43,6 +42,7 @@ pub mod os_install; pub mod prelude; pub mod procedure; pub mod properties; +pub mod registry; pub mod s9pk; pub mod setup; pub mod shutdown; @@ -79,7 +79,7 @@ pub fn echo(#[arg] message: String) -> Result { disk::disk, notifications::notification, backup::backup, - marketplace::marketplace, + registry::marketplace::marketplace, ))] pub fn main_api() -> Result<(), RpcError> { Ok(()) @@ -124,7 +124,8 @@ pub fn package() -> Result<(), RpcError> { s9pk::pack, developer::verify, developer::init, - inspect::inspect + inspect::inspect, + registry::admin::publish, ))] pub fn portable_api() -> Result<(), RpcError> { Ok(()) diff --git a/backend/src/registry/admin.rs b/backend/src/registry/admin.rs new file mode 100644 index 000000000..f6eecc1f2 --- /dev/null +++ b/backend/src/registry/admin.rs @@ -0,0 +1,211 @@ +use std::path::PathBuf; +use std::time::Duration; + +use color_eyre::eyre::eyre; +use console::style; +use futures::StreamExt; +use indicatif::{ProgressBar, ProgressStyle}; +use reqwest::{header, Body, Client, Url}; +use rpc_toolkit::command; + +use crate::s9pk::reader::S9pkReader; +use crate::util::display_none; +use crate::{Error, ErrorKind}; + +async fn registry_user_pass(location: &str) -> Result<(Url, String, String), Error> { + let mut url = Url::parse(location)?; + let user = url.username().to_string(); + let pass = url.password().map(str::to_string); + if user.is_empty() || url.path() != "/" { + return Err(Error::new( + eyre!("{location:?} is not like \"https://user@registry.example.com/\""), + ErrorKind::ParseUrl, + )); + } + let _ = url.set_username(""); + let _ = url.set_password(None); + + let pass = match pass { + Some(p) => p, + None => { + let pass_prompt = format!("{} Password for {user}: ", style("?").yellow()); + tokio::task::spawn_blocking(move || rpassword::prompt_password(pass_prompt)) + .await + .unwrap()? + } + }; + Ok((url, user.to_string(), pass.to_string())) +} + +#[derive(serde::Serialize, Debug)] +struct Package { + id: String, + version: String, + arches: Option>, +} + +async fn do_index( + httpc: &Client, + mut url: Url, + user: &str, + pass: &str, + pkg: &Package, +) -> Result<(), Error> { + url.set_path("/admin/v0/index"); + let mut req = httpc + .post(url) + .header(header::ACCEPT, "text/plain") + .basic_auth(user, Some(pass)) + .json(pkg) + .build()?; + let res = httpc.execute(req).await?; + if !res.status().is_success() { + let info = res.text().await?; + return Err(Error::new(eyre!("{}", info), ErrorKind::Registry)); + } + Ok(()) +} + +async fn do_upload( + httpc: &Client, + mut url: Url, + user: &str, + pass: &str, + body: Body, +) -> Result<(), Error> { + url.set_path("/admin/v0/upload"); + let mut req = httpc + .post(url) + .header(header::ACCEPT, "text/plain") + .basic_auth(user, Some(pass)) + .body(body) + .build()?; + let res = httpc.execute(req).await?; + if !res.status().is_success() { + let info = res.text().await?; + return Err(Error::new(eyre!("{}", info), ErrorKind::Registry)); + } + Ok(()) +} + +#[command(cli_only, display(display_none))] +pub async fn publish( + #[arg] location: String, + #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, + #[arg(rename = "no-upload", long = "no-upload")] no_upload: bool, + #[arg(rename = "no-index", long = "no-index")] no_index: bool, +) -> Result<(), Error> { + // Prepare for progress bars. + let bytes_bar_style = + ProgressStyle::with_template("{percent}% {wide_bar} [{bytes}/{total_bytes}] [{eta}]") + .unwrap(); + let plain_line_style = + ProgressStyle::with_template("{prefix:.bold.dim} {wide_msg}...").unwrap(); + let spinner_line_style = + ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}...").unwrap(); + + // Read the file to get manifest information and check validity.. + // Open file right away so it can not change out from under us. + let file = tokio::fs::File::open(&path).await?; + + let manifest = if no_verify { + let pb = ProgressBar::new(1) + .with_style(spinner_line_style.clone()) + .with_prefix("[1/3]") + .with_message("Querying s9pk"); + pb.enable_steady_tick(Duration::from_millis(200)); + let mut s9pk = S9pkReader::open(&path, false).await?; + let m = s9pk.manifest().await?.clone(); + pb.set_style(plain_line_style.clone()); + pb.abandon(); + m + } else { + let pb = ProgressBar::new(1) + .with_style(spinner_line_style.clone()) + .with_prefix("[1/3]") + .with_message("Verifying s9pk"); + pb.enable_steady_tick(Duration::from_millis(200)); + let mut s9pk = S9pkReader::open(&path, true).await?; + s9pk.validate().await?; + let m = s9pk.manifest().await?.clone(); + pb.set_style(plain_line_style.clone()); + pb.abandon(); + m + }; + let pkg = Package { + id: manifest.id.to_string(), + version: manifest.version.to_string(), + arches: manifest.hardware_requirements.arch.clone(), + }; + println!("{} id = {}", style(">").green(), pkg.id); + println!("{} version = {}", style(">").green(), pkg.version); + if let Some(arches) = &pkg.arches { + println!("{} arches = {:?}", style(">").green(), arches); + } else { + println!( + "{} No architecture listed in hardware_requirements", + style(">").red() + ); + } + + // Process the url and get the user's password. + let (registry, user, pass) = registry_user_pass(&location).await?; + + // Now prepare a stream of the file which will show a progress bar as it is consumed. + let file_size = file.metadata().await?.len(); + let file_stream = tokio_util::io::ReaderStream::new(file); + ProgressBar::new(0) + .with_style(plain_line_style.clone()) + .with_prefix("[2/3]") + .with_message("Uploading s9pk") + .abandon(); + let pb = ProgressBar::new(file_size).with_style(bytes_bar_style.clone()); + let stream_pb = pb.clone(); + let file_stream = file_stream.inspect(move |bytes| { + if let Ok(bytes) = bytes { + stream_pb.inc(bytes.len() as u64); + } + }); + + let httpc = Client::builder().build().unwrap(); + // And upload! + if no_upload { + println!("{} Skipping upload", style(">").yellow()); + } else { + do_upload( + &httpc, + registry.clone(), + &user, + &pass, + Body::wrap_stream(file_stream), + ) + .await?; + } + pb.finish_and_clear(); + + // Also index, so it will show up in the registry. + let pb = ProgressBar::new(0) + .with_style(spinner_line_style.clone()) + .with_prefix("[3/3]") + .with_message("Indexing registry"); + pb.enable_steady_tick(Duration::from_millis(200)); + if no_index { + println!("{} Skipping index", style(">").yellow()); + } else { + do_index(&httpc, registry.clone(), &user, &pass, &pkg).await?; + } + pb.set_style(plain_line_style.clone()); + pb.abandon(); + + // All done + if !no_index { + println!( + "{} Package {} is now published to {}", + style(">").green(), + pkg.id, + registry + ); + } + Ok(()) +} diff --git a/backend/src/marketplace.rs b/backend/src/registry/marketplace.rs similarity index 100% rename from backend/src/marketplace.rs rename to backend/src/registry/marketplace.rs diff --git a/backend/src/registry/mod.rs b/backend/src/registry/mod.rs new file mode 100644 index 000000000..27f541f1d --- /dev/null +++ b/backend/src/registry/mod.rs @@ -0,0 +1,2 @@ +pub mod admin; +pub mod marketplace; diff --git a/backend/src/s9pk/manifest.rs b/backend/src/s9pk/manifest.rs index 9e8e6f050..3eee540ed 100644 --- a/backend/src/s9pk/manifest.rs +++ b/backend/src/s9pk/manifest.rs @@ -112,7 +112,7 @@ pub struct HardwareRequirements { #[serde(default)] device: BTreeMap, ram: Option, - arch: Option>, + pub arch: Option>, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] diff --git a/backend/src/update/mod.rs b/backend/src/update/mod.rs index 92c73f01f..6e3d3e3b0 100644 --- a/backend/src/update/mod.rs +++ b/backend/src/update/mod.rs @@ -17,9 +17,9 @@ use crate::db::model::UpdateProgress; use crate::disk::mount::filesystem::bind::Bind; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::MountGuard; -use crate::marketplace::with_query_params; use crate::notifications::NotificationLevel; use crate::prelude::*; +use crate::registry::marketplace::with_query_params; use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, }; diff --git a/libs/Cargo.lock b/libs/Cargo.lock index 49db9a61e..af2585dbc 100644 --- a/libs/Cargo.lock +++ b/libs/Cargo.lock @@ -1897,16 +1897,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.7.1", "pin-utils", - "static_assertions", ] [[package]] @@ -2181,7 +2180,7 @@ dependencies = [ "json-patch", "json-ptr", "lazy_static", - "nix 0.26.2", + "nix 0.26.4", "patch-db-macro", "serde", "serde_cbor 0.11.1", diff --git a/libs/models/src/errors.rs b/libs/models/src/errors.rs index b0d786830..30a26b412 100644 --- a/libs/models/src/errors.rs +++ b/libs/models/src/errors.rs @@ -3,6 +3,7 @@ use std::fmt::Display; use color_eyre::eyre::eyre; use patch_db::Revision; use rpc_toolkit::hyper::http::uri::InvalidUri; +use rpc_toolkit::reqwest; use rpc_toolkit::yajrc::RpcError; use crate::InvalidId; @@ -272,7 +273,12 @@ impl From for Error { } impl From for Error { fn from(e: reqwest::Error) -> Self { - Error::new(e, ErrorKind::Network) + let kind = match e { + _ if e.is_builder() => ErrorKind::ParseUrl, + _ if e.is_decode() => ErrorKind::Deserialization, + _ => ErrorKind::Network, + }; + Error::new(e, kind) } } impl From for Error { diff --git a/system-images/compat/Cargo.lock b/system-images/compat/Cargo.lock index 3e05e0d4b..d603da538 100644 --- a/system-images/compat/Cargo.lock +++ b/system-images/compat/Cargo.lock @@ -601,6 +601,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode 0.3.6", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "const-oid" version = "0.9.1" @@ -1157,6 +1170,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1904,6 +1923,20 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-width", +] + [[package]] name = "instant" version = "0.1.12" @@ -2581,6 +2614,12 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.30.2" @@ -2984,6 +3023,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3003,7 +3048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" dependencies = [ "csv", - "encode_unicode", + "encode_unicode 1.0.0", "is-terminal", "lazy_static", "term", @@ -4089,6 +4134,7 @@ dependencies = [ "ciborium", "clap 3.2.23", "color-eyre", + "console", "cookie", "cookie_store 0.19.0", "current_platform", @@ -4113,6 +4159,7 @@ dependencies = [ "imbl-value", "include_dir", "indexmap", + "indicatif", "ipnet", "iprange", "isocountry",