Feature/fe new registry (#2647)

* bugfixes

* update fe types

* implement new registry types in marketplace and ui

* fix marketplace types to have default params

* add alt implementation toggle

* merge cleanup

* more cleanup and notes

* fix build

* cleanup sync with next/minor

* add exver JS parser

* parse ValidExVer to string

* update types to interface

* add VersionRange and comparative functions

* Parse ExtendedVersion from string

* add conjunction, disjunction, and inversion logic

* consider flavor in satisfiedBy fn

* consider prerelease for ordering

* add compare fn for sorting

* rename fns for consistency

* refactoring

* update compare fn to return null if flavors don't match

* begin simplifying dependencies

* under construction

* wip

* add dependency metadata to CurrentDependencyInfo

* ditch inheritance for recursive VersionRange constructor. Recursive 'satisfiedBy' fn wip

* preprocess manifest

* misc fixes

* use sdk version as osVersion in manifest

* chore: Change the type to just validate and not generate all solutions.

* add publishedAt

* fix pegjs exports

* integrate exver into sdk

* misc fixes

* complete satisfiedBy fn

* refactor - use greaterThanOrEqual and lessThanOrEqual fns

* fix tests

* update dependency details

* update types

* remove interim types

* rename alt implementation to flavor

* cleanup os update

* format exver.ts

* add s9pk parsing endpoints

* fix build

* update to exver

* exver and bug fixes

* update static endpoints + cleanup

* cleanup

* update static proxy verification

* make mocks more robust; fix dep icon fallback; cleanup

* refactor alert versions and update fixtures

* registry bugfixes

* misc fixes

* cleanup unused

* convert patchdb ui seed to camelCase

* update otherVersions type

* change otherVersions: null to 'none'

* refactor and complete feature

* improve static endpoints

* fix install params

* mask systemd-networkd-wait-online

* fix static file fetching

* include non-matching versions in otherVersions

* convert release notes to modal and clean up displayExver

* alert for no other versions

* Fix ack-instructions casing

* fix indeterminate loader on service install

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Shadowy Super Coder <musashidisciple@proton.me>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/startos/src/s9pk/v2/pack.rs

@@ -1,6 +1,4 @@
 use std::collections::BTreeSet;
-use std::ffi::OsStr;
-use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -10,25 +8,29 @@ use futures::{FutureExt, TryStreamExt};
 use imbl_value::InternedString;
 use models::{ImageId, PackageId, VersionString};
 use serde::{Deserialize, Serialize};
-use tokio::io::AsyncRead;
 use tokio::process::Command;
 use tokio::sync::OnceCell;
 use tokio_stream::wrappers::ReadDirStream;
+use tracing::{debug, warn};
 use ts_rs::TS;
 
 use crate::context::CliContext;
+use crate::dependencies::DependencyMetadata;
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
+use crate::s9pk::manifest::Manifest;
 use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::{
-    into_dyn_read, ArchiveSource, DynFileSource, FileSource, TmpSource,
+    into_dyn_read, ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource,
 };
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::v2::SIG_CONTEXT;
 use crate::s9pk::S9pk;
 use crate::util::io::{create_file, open_file, TmpDir};
-use crate::util::Invoke;
+use crate::util::serde::IoFormat;
+use crate::util::{new_guid, Invoke, PathOrUrl};
 
 #[cfg(not(feature = "docker"))]
 pub const CONTAINER_TOOL: &str = "podman";
@@ -83,7 +85,8 @@ pub enum PackSource {
     Squashfs(Arc<SqfsDir>),
 }
 impl FileSource for PackSource {
-    type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
+    type Reader = DynRead;
+    type SliceReader = DynRead;
     async fn size(&self) -> Result<u64, Error> {
         match self {
             Self::Buffered(a) => Ok(a.len() as u64),
@@ -102,11 +105,23 @@ impl FileSource for PackSource {
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         match self {
-            Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
-            Self::File(f) => Ok(into_dyn_read(open_file(f).await?)),
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::reader(a).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::reader(f).await?)),
             Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
         }
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        match self {
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::slice(a, position, size).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::slice(f, position, size).await?)),
+            Self::Squashfs(dir) => dir
+                .file()
+                .await?
+                .fetch(position, size)
+                .await
+                .map(into_dyn_read),
+        }
+    }
 }
 impl From<PackSource> for DynFileSource {
     fn from(value: PackSource) -> Self {
@@ -150,24 +165,71 @@ impl PackParams {
         if let Some(icon) = &self.icon {
             Ok(icon.clone())
         } else {
-            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
-                Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
-                Err(e) => Ok({
-                    let path = x.path();
-                    if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
-                    Ok(path)
-                    } else {
-                        Err(e)
-                    }
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("icon")),
+                    )
                 })
-            }}).await?
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("icon"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
         }
     }
-    fn license(&self) -> PathBuf {
-        self.license
-            .as_ref()
-            .cloned()
-            .unwrap_or_else(|| self.path().join("LICENSE.md"))
+    async fn license(&self) -> Result<PathBuf, Error> {
+        if let Some(license) = &self.license {
+            Ok(license.clone())
+        } else {
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("license")),
+                    )
+                })
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple licenses found in working directory, please specify which to use with `--license`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("license"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
+        }
     }
     fn instructions(&self) -> PathBuf {
         self.instructions
@@ -282,6 +344,15 @@ pub enum ImageSource {
     DockerTag(String),
 }
 impl ImageSource {
+    pub fn ingredients(&self) -> Vec<PathBuf> {
+        match self {
+            Self::Packed => Vec::new(),
+            Self::DockerBuild { dockerfile, .. } => {
+                vec![dockerfile.clone().unwrap_or_else(|| "Dockerfile".into())]
+            }
+            Self::DockerTag(_) => Vec::new(),
+        }
+    }
     #[instrument(skip_all)]
     pub fn load<'a, S: From<TmpSource<PackSource>> + FileSource + Clone>(
         &'a self,
@@ -320,7 +391,7 @@ impl ImageSource {
                         format!("--platform=linux/{arch}")
                     };
                     // docker buildx build ${path} -o type=image,name=start9/${id}
-                    let tag = format!("start9/{id}/{image_id}:{version}");
+                    let tag = format!("start9/{id}/{image_id}:{}", new_guid());
                     Command::new(CONTAINER_TOOL)
                         .arg("build")
                         .arg(workdir)
@@ -501,7 +572,7 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
         "LICENSE.md".into(),
         Entry::file(TmpSource::new(
             tmp_dir.clone(),
-            PackSource::File(params.license()),
+            PackSource::File(params.license().await?),
         )),
     );
     files.insert(
@@ -541,6 +612,54 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
 
     s9pk.load_images(tmp_dir.clone()).await?;
 
+    let mut to_insert = Vec::new();
+    for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
+        if let Some(s9pk) = dependency.s9pk.take() {
+            let s9pk = match s9pk {
+                PathOrUrl::Path(path) => {
+                    S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
+                        .await?
+                        .into_dyn()
+                }
+                PathOrUrl::Url(url) => {
+                    if url.scheme() == "http" || url.scheme() == "https" {
+                        S9pk::deserialize(
+                            &Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
+                            None,
+                        )
+                        .await?
+                        .into_dyn()
+                    } else {
+                        return Err(Error::new(
+                            eyre!("unknown scheme: {}", url.scheme()),
+                            ErrorKind::InvalidRequest,
+                        ));
+                    }
+                }
+            };
+            let dep_path = Path::new("dependencies").join(id);
+            to_insert.push((
+                dep_path.join("metadata.json"),
+                Entry::file(PackSource::Buffered(
+                    IoFormat::Json
+                        .to_vec(&DependencyMetadata {
+                            title: s9pk.as_manifest().title.clone(),
+                        })?
+                        .into(),
+                )),
+            ));
+            let icon = s9pk.icon().await?;
+            to_insert.push((
+                dep_path.join(&*icon.0),
+                Entry::file(PackSource::Buffered(
+                    icon.1.expect_file()?.to_vec(icon.1.hash()).await?.into(),
+                )),
+            ));
+        } else {
+            warn!("no s9pk specified for {id}, leaving metadata empty");
+        }
+    }
+
     s9pk.validate_and_filter(None)?;
 
     s9pk.serialize(
@@ -555,3 +674,58 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
 
     Ok(())
 }
+
+#[instrument(skip_all)]
+pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<PathBuf>, Error> {
+    let js_path = params.javascript().join("index.js");
+    let manifest: Manifest = match async {
+        serde_json::from_slice(
+            &Command::new("node")
+                .arg("-e")
+                .arg(format!(
+                    "console.log(JSON.stringify(require('{}').manifest))",
+                    js_path.display()
+                ))
+                .invoke(ErrorKind::Javascript)
+                .await?,
+        )
+        .with_kind(ErrorKind::Deserialization)
+    }
+    .await
+    {
+        Ok(m) => m,
+        Err(e) => {
+            warn!("failed to load manifest: {e}");
+            debug!("{e:?}");
+            return Ok(vec![
+                js_path,
+                params.icon().await?,
+                params.license().await?,
+                params.instructions(),
+            ]);
+        }
+    };
+    let mut ingredients = vec![
+        js_path,
+        params.icon().await?,
+        params.license().await?,
+        params.instructions(),
+    ];
+
+    for (_, dependency) in manifest.dependencies.0 {
+        if let Some(PathOrUrl::Path(p)) = dependency.s9pk {
+            ingredients.push(p);
+        }
+    }
+
+    let assets_dir = params.assets();
+    for assets in manifest.assets {
+        ingredients.push(assets_dir.join(assets));
+    }
+
+    for image in manifest.images.values() {
+        ingredients.extend(image.source.ingredients());
+    }
+
+    Ok(ingredients)
+}