minor bugfixes for alpha.14 (#3058)

* overwrite AllowedIPs in wg config
mute UnknownCA errors

* fix upgrade issues

* allow start9 user to access journal

* alpha.15

* sort actions lexicographically and show desc in marketplace details

* add registry package download cli command

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/Cargo.lock

@@ -7908,7 +7908,7 @@ dependencies = [
 
 [[package]]
 name = "start-os"
-version = "0.4.0-alpha.14"
+version = "0.4.0-alpha.15"
 dependencies = [
  "aes 0.7.5",
  "arti-client",

core/startos/Cargo.toml

@@ -15,7 +15,7 @@ license = "MIT"
 name = "start-os"
 readme = "README.md"
 repository = "https://github.com/Start9Labs/start-os"
-version = "0.4.0-alpha.14" # VERSION_BUMP
+version = "0.4.0-alpha.15" # VERSION_BUMP
 
 [lib]
 name = "startos"

core/startos/src/install/mod.rs

@@ -29,6 +29,7 @@ use crate::registry::context::{RegistryContext, RegistryUrlParams};
 use crate::registry::package::get::GetPackageResponse;
 use crate::rpc_continuations::{Guid, RpcContinuation};
 use crate::s9pk::manifest::PackageId;
+use crate::s9pk::v2::SIG_CONTEXT;
 use crate::upload::upload;
 use crate::util::Never;
 use crate::util::io::open_file;
@@ -154,6 +155,8 @@ pub async fn install(
         })?
         .s9pk;
 
+    asset.validate(SIG_CONTEXT, asset.all_signers())?;
+
     let progress_tracker = FullProgressTracker::new();
     let download_progress = progress_tracker.add_phase("Downloading".into(), Some(100));
     let download = ctx

core/startos/src/progress.rs

@@ -353,7 +353,7 @@ impl FullProgressTracker {
         }
     }
     pub fn progress_bar_task(&self, name: &str) -> NonDetachingJoinHandle<()> {
-        let mut stream = self.stream(None);
+        let mut stream = self.stream(Some(Duration::from_millis(200)));
         let mut bar = PhasedProgressBar::new(name);
         tokio::spawn(async move {
             while let Some(progress) = stream.next().await {

core/startos/src/registry/asset.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::path::Path;
 use std::sync::Arc;
 
 use chrono::{DateTime, Utc};
@@ -84,6 +85,26 @@ impl RegistryAsset<MerkleArchiveCommitment> {
         )
         .await
     }
+    pub async fn download_to(
+        &self,
+        path: impl AsRef<Path>,
+        client: Client,
+        progress: PhaseProgressTrackerHandle,
+    ) -> Result<
+        (
+            S9pk<Section<Arc<BufferedHttpSource>>>,
+            Arc<BufferedHttpSource>,
+        ),
+        Error,
+    > {
+        let source = Arc::new(
+            BufferedHttpSource::with_path(path, client, self.url.clone(), progress).await?,
+        );
+        Ok((
+            S9pk::deserialize(&source, Some(&self.commitment)).await?,
+            source,
+        ))
+    }
 }
 
 pub struct BufferedHttpSource {
@@ -91,6 +112,19 @@ pub struct BufferedHttpSource {
     file: UploadingFile,
 }
 impl BufferedHttpSource {
+    pub async fn with_path(
+        path: impl AsRef<Path>,
+        client: Client,
+        url: Url,
+        progress: PhaseProgressTrackerHandle,
+    ) -> Result<Self, Error> {
+        let (mut handle, file) = UploadingFile::with_path(path, progress).await?;
+        let response = client.get(url).send().await?;
+        Ok(Self {
+            _download: tokio::spawn(async move { handle.download(response).await }).into(),
+            file,
+        })
+    }
     pub async fn new(
         client: Client,
         url: Url,
@@ -103,6 +137,9 @@ impl BufferedHttpSource {
             file,
         })
     }
+    pub async fn wait_for_buffered(&self) -> Result<(), Error> {
+        self.file.wait_for_complete().await
+    }
 }
 impl ArchiveSource for BufferedHttpSource {
     type FetchReader = <UploadingFile as ArchiveSource>::FetchReader;

core/startos/src/registry/package/get.rs

@@ -1,19 +1,27 @@
 use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
 
 use clap::{Parser, ValueEnum};
 use exver::{ExtendedVersion, VersionRange};
-use imbl_value::InternedString;
+use helpers::to_tmp_path;
+use imbl_value::{InternedString, json};
 use itertools::Itertools;
 use models::PackageId;
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 
+use crate::context::CliContext;
 use crate::prelude::*;
+use crate::progress::{FullProgressTracker, ProgressUnits};
 use crate::registry::context::RegistryContext;
 use crate::registry::device_info::DeviceInfo;
 use crate::registry::package::index::{PackageIndex, PackageVersionInfo};
+use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::s9pk::v2::SIG_CONTEXT;
 use crate::util::VersionString;
+use crate::util::io::TrackingIO;
 use crate::util::serde::{WithIoFormat, display_serializable};
+use crate::util::tui::choose;
 
 #[derive(
     Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, TS, ValueEnum,
@@ -352,8 +360,7 @@ pub fn display_package_info(
     info: Value,
 ) -> Result<(), Error> {
     if let Some(format) = params.format {
-        display_serializable(format, info);
-        return Ok(());
+        return display_serializable(format, info);
     }
 
     if let Some(_) = params.rest.id {
@@ -387,3 +394,90 @@ pub fn display_package_info(
     }
     Ok(())
 }
+
+#[derive(Debug, Deserialize, Serialize, TS, Parser)]
+#[serde(rename_all = "camelCase")]
+pub struct CliDownloadParams {
+    pub id: PackageId,
+    #[arg(long, short = 'v')]
+    #[ts(type = "string | null")]
+    pub target_version: Option<VersionRange>,
+    #[arg(short, long)]
+    pub dest: Option<PathBuf>,
+}
+
+pub async fn cli_download(
+    ctx: CliContext,
+    CliDownloadParams {
+        ref id,
+        target_version,
+        dest,
+    }: CliDownloadParams,
+) -> Result<(), Error> {
+    let progress_tracker = FullProgressTracker::new();
+    let mut fetching_progress = progress_tracker.add_phase("Fetching".into(), Some(1));
+    let download_progress = progress_tracker.add_phase("Downloading".into(), Some(100));
+    let mut verify_progress = progress_tracker.add_phase("Verifying".into(), Some(10));
+
+    let progress = progress_tracker.progress_bar_task("Downloading S9PK...");
+
+    fetching_progress.start();
+    let mut res: GetPackageResponse = from_value(
+        ctx.call_remote::<RegistryContext>(
+            "package.get",
+            json!({
+                "id": &id,
+                "targetVersion": &target_version,
+            }),
+        )
+        .await?,
+    )?;
+    let PackageVersionInfo { s9pk, .. } = match res.best.len() {
+        0 => {
+            return Err(Error::new(
+                eyre!(
+                    "Could not find a version of {id} that satisfies {}",
+                    target_version.unwrap_or(VersionRange::Any)
+                ),
+                ErrorKind::NotFound,
+            ));
+        }
+        1 => res.best.pop_first().unwrap().1,
+        _ => {
+            let choices = res.best.keys().cloned().collect::<Vec<_>>();
+            let version = choose(
+                &format!("Multiple flavors of {id} available. Choose a version to download:"),
+                &choices,
+            )
+            .await?;
+            res.best.remove(version).unwrap()
+        }
+    };
+    s9pk.validate(SIG_CONTEXT, s9pk.all_signers())?;
+    fetching_progress.complete();
+
+    let dest = dest.unwrap_or_else(|| Path::new(".").join(id).with_extension("s9pk"));
+    let dest_tmp = to_tmp_path(&dest).with_kind(ErrorKind::Filesystem)?;
+    let (mut parsed, source) = s9pk
+        .download_to(&dest_tmp, ctx.client.clone(), download_progress)
+        .await?;
+    if let Some(size) = source.size().await {
+        verify_progress.set_total(size);
+    }
+    verify_progress.set_units(Some(ProgressUnits::Bytes));
+    let mut progress_sink = verify_progress.writer(tokio::io::sink());
+    parsed
+        .serialize(&mut TrackingIO::new(0, &mut progress_sink), true)
+        .await?;
+    progress_sink.into_inner().1.complete();
+
+    source.wait_for_buffered().await?;
+    tokio::fs::rename(dest_tmp, dest).await?;
+
+    progress_tracker.complete();
+    progress.await.unwrap();
+
+    println!("Download Complete");
+
+    Ok(())
+}

core/startos/src/registry/package/mod.rs

@@ -1,4 +1,4 @@
-use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async};
+use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async, from_fn_async_local};
 
 use crate::context::CliContext;
 use crate::prelude::*;
@@ -54,6 +54,12 @@ pub fn package_api<C: Context>() -> ParentHandler<C> {
                 .with_about("List installation candidate package(s)")
                 .with_call_remote::<CliContext>(),
         )
+        .subcommand(
+            "download",
+            from_fn_async_local(get::cli_download)
+                .no_display()
+                .with_about("Download an s9pk"),
+        )
         .subcommand(
             "category",
             category::category_api::<C>()

core/startos/src/upload.rs

@@ -1,4 +1,5 @@
 use std::io::SeekFrom;
+use std::path::Path;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::Poll;
@@ -56,6 +57,7 @@ struct Progress {
     tracker: PhaseProgressTrackerHandle,
     expected_size: Option<u64>,
     written: u64,
+    complete: bool,
     error: Option<Error>,
 }
 impl Progress {
@@ -111,9 +113,7 @@ impl Progress {
     }
     async fn ready(watch: &mut watch::Receiver<Self>) -> Result<(), Error> {
         match &*watch
-            .wait_for(|progress| {
-                progress.error.is_some() || Some(progress.written) == progress.expected_size
-            })
+            .wait_for(|progress| progress.error.is_some() || progress.complete)
             .await
             .map_err(|_| {
                 Error::new(
@@ -126,8 +126,9 @@ impl Progress {
         }
     }
     fn complete(&mut self) -> bool {
+        let mut changed = !self.complete;
         self.tracker.complete();
-        match self {
+        changed |= match self {
             Self {
                 expected_size: Some(size),
                 written,
@@ -165,18 +166,21 @@ impl Progress {
                 true
             }
             _ => false,
-        }
+        };
+        self.complete = true;
+        changed
     }
 }
 
 #[derive(Clone)]
 pub struct UploadingFile {
-    tmp_dir: Arc<TmpDir>,
+    tmp_dir: Option<Arc<TmpDir>>,
     file: MultiCursorFile,
     progress: watch::Receiver<Progress>,
 }
 impl UploadingFile {
-    pub async fn new(
+    pub async fn with_path(
+        path: impl AsRef<Path>,
         mut progress: PhaseProgressTrackerHandle,
     ) -> Result<(UploadHandle, Self), Error> {
         progress.set_units(Some(ProgressUnits::Bytes));
@@ -185,25 +189,35 @@ impl UploadingFile {
             expected_size: None,
             written: 0,
             error: None,
+            complete: false,
         });
-        let tmp_dir = Arc::new(TmpDir::new().await?);
-        let file = create_file(tmp_dir.join("upload.tmp")).await?;
+        let file = create_file(path).await?;
         let uploading = Self {
-            tmp_dir: tmp_dir.clone(),
+            tmp_dir: None,
             file: MultiCursorFile::open(&file).await?,
             progress: progress.1,
         };
         Ok((
             UploadHandle {
-                tmp_dir,
+                tmp_dir: None,
                 file,
                 progress: progress.0,
             },
             uploading,
         ))
     }
+    pub async fn new(progress: PhaseProgressTrackerHandle) -> Result<(UploadHandle, Self), Error> {
+        let tmp_dir = Arc::new(TmpDir::new().await?);
+        let (mut handle, mut file) = Self::with_path(tmp_dir.join("upload.tmp"), progress).await?;
+        handle.tmp_dir = Some(tmp_dir.clone());
+        file.tmp_dir = Some(tmp_dir);
+        Ok((handle, file))
+    }
+    pub async fn wait_for_complete(&self) -> Result<(), Error> {
+        Progress::ready(&mut self.progress.clone()).await
+    }
     pub async fn delete(self) -> Result<(), Error> {
-        if let Ok(tmp_dir) = Arc::try_unwrap(self.tmp_dir) {
+        if let Some(Ok(tmp_dir)) = self.tmp_dir.map(Arc::try_unwrap) {
             tmp_dir.delete().await?;
         }
         Ok(())
@@ -234,7 +248,7 @@ impl ArchiveSource for UploadingFile {
 
 #[pin_project::pin_project(project = UploadingFileReaderProjection)]
 pub struct UploadingFileReader {
-    tmp_dir: Arc<TmpDir>,
+    tmp_dir: Option<Arc<TmpDir>>,
     position: u64,
     to_seek: Option<SeekFrom>,
     #[pin]
@@ -330,7 +344,7 @@ impl AsyncSeek for UploadingFileReader {
 
 #[pin_project::pin_project(PinnedDrop)]
 pub struct UploadHandle {
-    tmp_dir: Arc<TmpDir>,
+    tmp_dir: Option<Arc<TmpDir>>,
     #[pin]
     file: File,
     progress: watch::Sender<Progress>,
@@ -377,6 +391,9 @@ impl UploadHandle {
                 break;
             }
         }
+        if let Err(e) = self.file.sync_all().await {
+            self.progress.send_if_modified(|p| p.handle_error(&e));
+        }
     }
 }
 #[pin_project::pinned_drop]

core/startos/src/version/mod.rs

@@ -54,8 +54,9 @@ mod v0_4_0_alpha_11;
 mod v0_4_0_alpha_12;
 mod v0_4_0_alpha_13;
 mod v0_4_0_alpha_14;
+mod v0_4_0_alpha_15;
 
-pub type Current = v0_4_0_alpha_14::Version; // VERSION_BUMP
+pub type Current = v0_4_0_alpha_15::Version; // VERSION_BUMP
 
 impl Current {
     #[instrument(skip(self, db))]
@@ -171,7 +172,8 @@ enum Version {
     V0_4_0_alpha_11(Wrapper<v0_4_0_alpha_11::Version>),
     V0_4_0_alpha_12(Wrapper<v0_4_0_alpha_12::Version>),
     V0_4_0_alpha_13(Wrapper<v0_4_0_alpha_13::Version>),
-    V0_4_0_alpha_14(Wrapper<v0_4_0_alpha_14::Version>), // VERSION_BUMP
+    V0_4_0_alpha_14(Wrapper<v0_4_0_alpha_14::Version>),
+    V0_4_0_alpha_15(Wrapper<v0_4_0_alpha_15::Version>), // VERSION_BUMP
     Other(exver::Version),
 }
 
@@ -228,7 +230,8 @@ impl Version {
             Self::V0_4_0_alpha_11(v) => DynVersion(Box::new(v.0)),
             Self::V0_4_0_alpha_12(v) => DynVersion(Box::new(v.0)),
             Self::V0_4_0_alpha_13(v) => DynVersion(Box::new(v.0)),
-            Self::V0_4_0_alpha_14(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
+            Self::V0_4_0_alpha_14(v) => DynVersion(Box::new(v.0)),
+            Self::V0_4_0_alpha_15(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP
             Self::Other(v) => {
                 return Err(Error::new(
                     eyre!("unknown version {v}"),
@@ -277,7 +280,8 @@ impl Version {
             Version::V0_4_0_alpha_11(Wrapper(x)) => x.semver(),
             Version::V0_4_0_alpha_12(Wrapper(x)) => x.semver(),
             Version::V0_4_0_alpha_13(Wrapper(x)) => x.semver(),
-            Version::V0_4_0_alpha_14(Wrapper(x)) => x.semver(), // VERSION_BUMP
+            Version::V0_4_0_alpha_14(Wrapper(x)) => x.semver(),
+            Version::V0_4_0_alpha_15(Wrapper(x)) => x.semver(), // VERSION_BUMP
             Version::Other(x) => x.clone(),
         }
     }

core/startos/src/version/v0_4_0_alpha_15.rs

@@ -0,0 +1,37 @@
+use exver::{PreReleaseSegment, VersionRange};
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{VersionT, v0_4_0_alpha_14};
+use crate::prelude::*;
+
+lazy_static::lazy_static! {
+    static ref V0_4_0_alpha_15: exver::Version = exver::Version::new(
+        [0, 4, 0],
+        [PreReleaseSegment::String("alpha".into()), 15.into()]
+    );
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_4_0_alpha_14::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_4_0_alpha_15.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    #[instrument(skip_all)]
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<Value, Error> {
+        Ok(Value::Null)
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}