Feature/fe new registry (#2647)

* bugfixes

* update fe types

* implement new registry types in marketplace and ui

* fix marketplace types to have default params

* add alt implementation toggle

* merge cleanup

* more cleanup and notes

* fix build

* cleanup sync with next/minor

* add exver JS parser

* parse ValidExVer to string

* update types to interface

* add VersionRange and comparative functions

* Parse ExtendedVersion from string

* add conjunction, disjunction, and inversion logic

* consider flavor in satisfiedBy fn

* consider prerelease for ordering

* add compare fn for sorting

* rename fns for consistency

* refactoring

* update compare fn to return null if flavors don't match

* begin simplifying dependencies

* under construction

* wip

* add dependency metadata to CurrentDependencyInfo

* ditch inheritance for recursive VersionRange constructor. Recursive 'satisfiedBy' fn wip

* preprocess manifest

* misc fixes

* use sdk version as osVersion in manifest

* chore: Change the type to just validate and not generate all solutions.

* add publishedAt

* fix pegjs exports

* integrate exver into sdk

* misc fixes

* complete satisfiedBy fn

* refactor - use greaterThanOrEqual and lessThanOrEqual fns

* fix tests

* update dependency details

* update types

* remove interim types

* rename alt implementation to flavor

* cleanup os update

* format exver.ts

* add s9pk parsing endpoints

* fix build

* update to exver

* exver and bug fixes

* update static endpoints + cleanup

* cleanup

* update static proxy verification

* make mocks more robust; fix dep icon fallback; cleanup

* refactor alert versions and update fixtures

* registry bugfixes

* misc fixes

* cleanup unused

* convert patchdb ui seed to camelCase

* update otherVersions type

* change otherVersions: null to 'none'

* refactor and complete feature

* improve static endpoints

* fix install params

* mask systemd-networkd-wait-online

* fix static file fetching

* include non-matching versions in otherVersions

* convert release notes to modal and clean up displayExver

* alert for no other versions

* Fix ack-instructions casing

* fix indeterminate loader on service install

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Shadowy Super Coder <musashidisciple@proton.me>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/startos/src/s9pk/merkle_archive/expected.rs

@@ -1,4 +1,3 @@
-
 use std::ffi::OsStr;
 use std::path::Path;
 
@@ -7,16 +6,16 @@ use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::merkle_archive::Entry;
 
-/// An object for tracking the files expected to be in an s9pk 
+/// An object for tracking the files expected to be in an s9pk
 pub struct Expected<'a, T> {
     keep: DirectoryContents<()>,
     dir: &'a DirectoryContents<T>,
 }
 impl<'a, T> Expected<'a, T> {
-    pub fn new(dir: &'a DirectoryContents<T>,) -> Self {
+    pub fn new(dir: &'a DirectoryContents<T>) -> Self {
         Self {
             keep: DirectoryContents::new(),
-            dir
+            dir,
         }
     }
 }
@@ -42,22 +41,23 @@ impl<'a, T: Clone> Expected<'a, T> {
         path: impl AsRef<Path>,
         mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
     ) -> Result<(), Error> {
-        let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
-            (
-                self.dir
-                    .get_path(parent)
-                    .and_then(|e| e.as_directory())
-                    .ok_or_else(|| {
-                        Error::new(
-                            eyre!("directory {} missing from archive", parent.display()),
-                            ErrorKind::ParseS9pk,
-                        )
-                    })?,
-                path.as_ref().strip_prefix(parent).unwrap(),
-            )
-        } else {
-            (self.dir, path.as_ref())
-        };
+        let (dir, stem) =
+            if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
+                (
+                    self.dir
+                        .get_path(parent)
+                        .and_then(|e| e.as_directory())
+                        .ok_or_else(|| {
+                            Error::new(
+                                eyre!("directory {} missing from archive", parent.display()),
+                                ErrorKind::ParseS9pk,
+                            )
+                        })?,
+                    path.as_ref().strip_prefix(parent).unwrap(),
+                )
+            } else {
+                (self.dir, path.as_ref())
+            };
         let name = dir
             .with_stem(&stem.as_os_str().to_string_lossy())
             .filter(|(_, e)| e.as_file().is_some())
@@ -69,7 +69,7 @@ impl<'a, T: Clone> Expected<'a, T> {
                     ),
                     ErrorKind::ParseS9pk,
                 )),
-                |acc, (name, _)| 
+                |acc, (name, _)|
                     if valid_extension(Path::new(&*name).extension()) {
                         match acc {
                             Ok(_) => Err(Error::new(
@@ -96,8 +96,10 @@ impl<'a, T: Clone> Expected<'a, T> {
 
 pub struct Filter(DirectoryContents<()>);
 impl Filter {
-    pub fn keep_checked<T: FileSource + Clone>(&self, dir: &mut DirectoryContents<T>) -> Result<(), Error> {
+    pub fn keep_checked<T: FileSource + Clone>(
+        &self,
+        dir: &mut DirectoryContents<T>,
+    ) -> Result<(), Error> {
         dir.filter(|path| self.0.get_path(path).is_some())
     }
 }
-

core/startos/src/s9pk/merkle_archive/mod.rs

@@ -233,6 +233,10 @@ impl<S> Entry<S> {
             _ => None,
         }
     }
+    pub fn expect_file(&self) -> Result<&FileContents<S>, Error> {
+        self.as_file()
+            .ok_or_else(|| Error::new(eyre!("not a file"), ErrorKind::ParseS9pk))
+    }
     pub fn as_directory(&self) -> Option<&DirectoryContents<S>> {
         match self.as_contents() {
             EntryContents::Directory(d) => Some(d),

core/startos/src/s9pk/merkle_archive/source/mod.rs

@@ -1,3 +1,5 @@
+use std::cmp::min;
+use std::io::SeekFrom;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -6,7 +8,7 @@ use blake3::Hash;
 use futures::future::BoxFuture;
 use futures::{Future, FutureExt};
 use tokio::fs::File;
-use tokio::io::{AsyncRead, AsyncWrite};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, Take};
 
 use crate::prelude::*;
 use crate::s9pk::merkle_archive::hash::VerifyingWriter;
@@ -17,8 +19,14 @@ pub mod multi_cursor_file;
 
 pub trait FileSource: Send + Sync + Sized + 'static {
     type Reader: AsyncRead + Unpin + Send;
+    type SliceReader: AsyncRead + Unpin + Send;
     fn size(&self) -> impl Future<Output = Result<u64, Error>> + Send;
     fn reader(&self) -> impl Future<Output = Result<Self::Reader, Error>> + Send;
+    fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> impl Future<Output = Result<Self::SliceReader, Error>> + Send;
     fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
         &self,
         w: &mut W,
@@ -65,12 +73,16 @@ pub trait FileSource: Send + Sync + Sized + 'static {
 
 impl<T: FileSource> FileSource for Arc<T> {
     type Reader = T::Reader;
+    type SliceReader = T::SliceReader;
     async fn size(&self) -> Result<u64, Error> {
         self.deref().size().await
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         self.deref().reader().await
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.deref().slice(position, size).await
+    }
     async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
         self.deref().copy(w).await
     }
@@ -95,12 +107,16 @@ impl DynFileSource {
 }
 impl FileSource for DynFileSource {
     type Reader = Box<dyn AsyncRead + Unpin + Send>;
+    type SliceReader = Box<dyn AsyncRead + Unpin + Send>;
     async fn size(&self) -> Result<u64, Error> {
         self.0.size().await
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         self.0.reader().await
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.0.slice(position, size).await
+    }
     async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
         &self,
         mut w: &mut W,
@@ -123,6 +139,11 @@ impl FileSource for DynFileSource {
 trait DynableFileSource: Send + Sync + 'static {
     async fn size(&self) -> Result<u64, Error>;
     async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
+    async fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
     async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error>;
     async fn copy_verify(
         &self,
@@ -139,6 +160,13 @@ impl<T: FileSource> DynableFileSource for T {
     async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
         Ok(Box::new(FileSource::reader(self).await?))
     }
+    async fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
+        Ok(Box::new(FileSource::slice(self, position, size).await?))
+    }
     async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error> {
         FileSource::copy(self, w).await
     }
@@ -156,22 +184,34 @@ impl<T: FileSource> DynableFileSource for T {
 
 impl FileSource for PathBuf {
     type Reader = File;
+    type SliceReader = Take<Self::Reader>;
     async fn size(&self) -> Result<u64, Error> {
         Ok(tokio::fs::metadata(self).await?.len())
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         Ok(open_file(self).await?)
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        let mut r = FileSource::reader(self).await?;
+        r.seek(SeekFrom::Start(position)).await?;
+        Ok(r.take(size))
+    }
 }
 
 impl FileSource for Arc<[u8]> {
     type Reader = std::io::Cursor<Self>;
+    type SliceReader = Take<Self::Reader>;
     async fn size(&self) -> Result<u64, Error> {
         Ok(self.len() as u64)
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         Ok(std::io::Cursor::new(self.clone()))
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        let mut r = FileSource::reader(self).await?;
+        r.seek(SeekFrom::Start(position)).await?;
+        Ok(r.take(size))
+    }
     async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
         use tokio::io::AsyncWriteExt;
 
@@ -272,12 +312,18 @@ pub struct Section<S> {
 }
 impl<S: ArchiveSource> FileSource for Section<S> {
     type Reader = S::FetchReader;
+    type SliceReader = S::FetchReader;
     async fn size(&self) -> Result<u64, Error> {
         Ok(self.size)
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         self.source.fetch(self.position, self.size).await
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.source
+            .fetch(self.position + position, min(size, self.size))
+            .await
+    }
     async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
         self.source.copy_to(self.position, self.size, w).await
     }
@@ -342,12 +388,16 @@ impl<S: FileSource> From<TmpSource<S>> for DynFileSource {
 
 impl<S: FileSource> FileSource for TmpSource<S> {
     type Reader = <S as FileSource>::Reader;
+    type SliceReader = <S as FileSource>::SliceReader;
     async fn size(&self) -> Result<u64, Error> {
         self.source.size().await
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         self.source.reader().await
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.source.slice(position, size).await
+    }
     async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
         &self,
         mut w: &mut W,

core/startos/src/s9pk/rpc.rs

@@ -15,14 +15,36 @@ use crate::s9pk::v2::pack::ImageConfig;
 use crate::s9pk::v2::SIG_CONTEXT;
 use crate::util::io::{create_file, open_file, TmpDir};
 use crate::util::serde::{apply_expr, HandlerExtSerde};
+use crate::util::Apply;
 
 pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
 
 pub fn s9pk() -> ParentHandler<CliContext> {
     ParentHandler::new()
         .subcommand("pack", from_fn_async(super::v2::pack::pack).no_display())
+        .subcommand(
+            "list-ingredients",
+            from_fn_async(super::v2::pack::list_ingredients).with_custom_display_fn(
+                |_, ingredients| {
+                    ingredients
+                        .into_iter()
+                        .map(Some)
+                        .apply(|i| itertools::intersperse(i, None))
+                        .for_each(|i| {
+                            if let Some(p) = i {
+                                print!("{}", p.display())
+                            } else {
+                                print!(" ")
+                            }
+                        });
+                    println!();
+                    Ok(())
+                },
+            ),
+        )
         .subcommand("edit", edit())
         .subcommand("inspect", inspect())
+        .subcommand("convert", from_fn_async(convert).no_display())
 }
 
 #[derive(Deserialize, Serialize, Parser)]
@@ -193,3 +215,17 @@ async fn inspect_manifest(
     .await?;
     Ok(s9pk.as_manifest().clone())
 }
+
+async fn convert(ctx: CliContext, S9pkPath { s9pk: s9pk_path }: S9pkPath) -> Result<(), Error> {
+    let mut s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+    s9pk.serialize(&mut create_file(&tmp_path).await?, true)
+        .await?;
+    tokio::fs::rename(tmp_path, s9pk_path).await?;
+    Ok(())
+}

core/startos/src/s9pk/v2/compat.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 use std::sync::Arc;
 
@@ -199,8 +199,9 @@ impl From<ManifestV1> for Manifest {
         let default_url = value.upstream_repo.clone();
         Self {
             id: value.id,
-            title: value.title,
+            title: value.title.into(),
             version: ExtendedVersion::from(value.version).into(),
+            satisfies: BTreeSet::new(),
             release_notes: value.release_notes,
             license: value.license.into(),
             wrapper_repo: value.wrapper_repo,
@@ -233,6 +234,7 @@ impl From<ManifestV1> for Manifest {
                             DepInfo {
                                 description: value.description,
                                 optional: !value.requirement.required(),
+                                s9pk: None,
                             },
                         )
                     })

core/startos/src/s9pk/v2/manifest.rs

@@ -31,8 +31,10 @@ fn current_version() -> Version {
 #[ts(export)]
 pub struct Manifest {
     pub id: PackageId,
-    pub title: String,
+    #[ts(type = "string")]
+    pub title: InternedString,
     pub version: VersionString,
+    pub satisfies: BTreeSet<VersionString>,
     pub release_notes: String,
     #[ts(type = "string")]
     pub license: InternedString, // type of license
@@ -81,6 +83,15 @@ impl Manifest {
         expected.check_file("LICENSE.md")?;
         expected.check_file("instructions.md")?;
         expected.check_file("javascript.squashfs")?;
+        for (dependency, _) in &self.dependencies.0 {
+            let dep_path = Path::new("dependencies").join(dependency);
+            let _ = expected.check_file(dep_path.join("metadata.json"));
+            let _ = expected.check_stem(dep_path.join("icon"), |ext| {
+                ext.and_then(|e| e.to_str())
+                    .and_then(mime)
+                    .map_or(false, |mime| mime.starts_with("image/"))
+            });
+        }
         for assets in &self.assets {
             expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?;
         }
@@ -148,7 +159,7 @@ impl Manifest {
 #[ts(export)]
 pub struct HardwareRequirements {
     #[serde(default)]
-    #[ts(type = "{ [key: string]: string }")] // TODO more specific key
+    #[ts(type = "{ device?: string, processor?: string }")]
     pub device: BTreeMap<String, Regex>,
     #[ts(type = "number | null")]
     pub ram: Option<u64>,

core/startos/src/s9pk/v2/mod.rs

@@ -6,10 +6,10 @@ use imbl_value::InternedString;
 use models::{mime, DataUrl, PackageId};
 use tokio::fs::File;
 
+use crate::dependencies::DependencyMetadata;
 use crate::prelude::*;
 use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment;
 use crate::s9pk::manifest::Manifest;
-use crate::s9pk::merkle_archive::file_contents::FileContents;
 use crate::s9pk::merkle_archive::sink::Sink;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::{
@@ -18,6 +18,7 @@ use crate::s9pk::merkle_archive::source::{
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::v2::pack::{ImageSource, PackSource};
 use crate::util::io::{open_file, TmpDir};
+use crate::util::serde::IoFormat;
 
 const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
 
@@ -33,6 +34,10 @@ pub mod pack;
     ├── icon.<ext>
     ├── LICENSE.md
     ├── instructions.md
+    ├── dependencies
+    │   └── <id>
+    │       ├── metadata.json
+    │       └── icon.<ext>
     ├── javascript.squashfs
     ├── assets
     │   └── <id>.squashfs (xN)
@@ -52,9 +57,10 @@ fn priority(s: &str) -> Option<usize> {
         a if Path::new(a).file_stem() == Some(OsStr::new("icon")) => Some(1),
         "LICENSE.md" => Some(2),
         "instructions.md" => Some(3),
-        "javascript.squashfs" => Some(4),
-        "assets" => Some(5),
-        "images" => Some(6),
+        "dependencies" => Some(4),
+        "javascript.squashfs" => Some(5),
+        "assets" => Some(6),
+        "images" => Some(7),
         _ => None,
     }
 }
@@ -101,22 +107,16 @@ impl<S: FileSource + Clone> S9pk<S> {
         filter.keep_checked(self.archive.contents_mut())
     }
 
-    pub async fn icon(&self) -> Result<(InternedString, FileContents<S>), Error> {
+    pub async fn icon(&self) -> Result<(InternedString, Entry<S>), Error> {
         let mut best_icon = None;
-        for (path, icon) in self
-            .archive
-            .contents()
-            .with_stem("icon")
-            .filter(|(p, _)| {
-                Path::new(&*p)
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .and_then(mime)
-                    .map_or(false, |e| e.starts_with("image/"))
-            })
-            .filter_map(|(k, v)| v.into_file().map(|f| (k, f)))
-        {
-            let size = icon.size().await?;
+        for (path, icon) in self.archive.contents().with_stem("icon").filter(|(p, v)| {
+            Path::new(&*p)
+                .extension()
+                .and_then(|e| e.to_str())
+                .and_then(mime)
+                .map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
+        }) {
+            let size = icon.expect_file()?.size().await?;
             best_icon = match best_icon {
                 Some((s, a)) if s >= size => Some((s, a)),
                 _ => Some((size, (path, icon))),
@@ -134,7 +134,75 @@ impl<S: FileSource + Clone> S9pk<S> {
             .and_then(|e| e.to_str())
             .and_then(mime)
             .unwrap_or("image/png");
-        DataUrl::from_reader(mime, contents.reader().await?, Some(contents.size().await?)).await
+        Ok(DataUrl::from_vec(
+            mime,
+            contents.expect_file()?.to_vec(contents.hash()).await?,
+        ))
+    }
+
+    pub async fn dependency_icon(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<(InternedString, Entry<S>)>, Error> {
+        let mut best_icon = None;
+        for (path, icon) in self
+            .archive
+            .contents()
+            .get_path(Path::new("dependencies").join(id))
+            .and_then(|p| p.as_directory())
+            .into_iter()
+            .flat_map(|d| {
+                d.with_stem("icon").filter(|(p, v)| {
+                    Path::new(&*p)
+                        .extension()
+                        .and_then(|e| e.to_str())
+                        .and_then(mime)
+                        .map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
+                })
+            })
+        {
+            let size = icon.expect_file()?.size().await?;
+            best_icon = match best_icon {
+                Some((s, a)) if s >= size => Some((s, a)),
+                _ => Some((size, (path, icon))),
+            };
+        }
+        Ok(best_icon.map(|(_, a)| a))
+    }
+
+    pub async fn dependency_icon_data_url(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<DataUrl<'static>>, Error> {
+        let Some((name, contents)) = self.dependency_icon(id).await? else {
+            return Ok(None);
+        };
+        let mime = Path::new(&*name)
+            .extension()
+            .and_then(|e| e.to_str())
+            .and_then(mime)
+            .unwrap_or("image/png");
+        Ok(Some(DataUrl::from_vec(
+            mime,
+            contents.expect_file()?.to_vec(contents.hash()).await?,
+        )))
+    }
+
+    pub async fn dependency_metadata(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<DependencyMetadata>, Error> {
+        if let Some(entry) = self
+            .archive
+            .contents()
+            .get_path(Path::new("dependencies").join(id).join("metadata.json"))
+        {
+            Ok(Some(IoFormat::Json.from_slice(
+                &entry.expect_file()?.to_vec(entry.hash()).await?,
+            )?))
+        } else {
+            Ok(None)
+        }
     }
 
     pub async fn serialize<W: Sink>(&mut self, w: &mut W, verify: bool) -> Result<(), Error> {

core/startos/src/s9pk/v2/pack.rs

@@ -1,6 +1,4 @@
 use std::collections::BTreeSet;
-use std::ffi::OsStr;
-use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -10,25 +8,29 @@ use futures::{FutureExt, TryStreamExt};
 use imbl_value::InternedString;
 use models::{ImageId, PackageId, VersionString};
 use serde::{Deserialize, Serialize};
-use tokio::io::AsyncRead;
 use tokio::process::Command;
 use tokio::sync::OnceCell;
 use tokio_stream::wrappers::ReadDirStream;
+use tracing::{debug, warn};
 use ts_rs::TS;
 
 use crate::context::CliContext;
+use crate::dependencies::DependencyMetadata;
 use crate::prelude::*;
 use crate::rpc_continuations::Guid;
+use crate::s9pk::manifest::Manifest;
 use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::{
-    into_dyn_read, ArchiveSource, DynFileSource, FileSource, TmpSource,
+    into_dyn_read, ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource,
 };
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::v2::SIG_CONTEXT;
 use crate::s9pk::S9pk;
 use crate::util::io::{create_file, open_file, TmpDir};
-use crate::util::Invoke;
+use crate::util::serde::IoFormat;
+use crate::util::{new_guid, Invoke, PathOrUrl};
 
 #[cfg(not(feature = "docker"))]
 pub const CONTAINER_TOOL: &str = "podman";
@@ -83,7 +85,8 @@ pub enum PackSource {
     Squashfs(Arc<SqfsDir>),
 }
 impl FileSource for PackSource {
-    type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
+    type Reader = DynRead;
+    type SliceReader = DynRead;
     async fn size(&self) -> Result<u64, Error> {
         match self {
             Self::Buffered(a) => Ok(a.len() as u64),
@@ -102,11 +105,23 @@ impl FileSource for PackSource {
     }
     async fn reader(&self) -> Result<Self::Reader, Error> {
         match self {
-            Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
-            Self::File(f) => Ok(into_dyn_read(open_file(f).await?)),
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::reader(a).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::reader(f).await?)),
             Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
         }
     }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        match self {
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::slice(a, position, size).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::slice(f, position, size).await?)),
+            Self::Squashfs(dir) => dir
+                .file()
+                .await?
+                .fetch(position, size)
+                .await
+                .map(into_dyn_read),
+        }
+    }
 }
 impl From<PackSource> for DynFileSource {
     fn from(value: PackSource) -> Self {
@@ -150,24 +165,71 @@ impl PackParams {
         if let Some(icon) = &self.icon {
             Ok(icon.clone())
         } else {
-            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
-                Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
-                Err(e) => Ok({
-                    let path = x.path();
-                    if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
-                    Ok(path)
-                    } else {
-                        Err(e)
-                    }
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("icon")),
+                    )
                 })
-            }}).await?
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("icon"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
         }
     }
-    fn license(&self) -> PathBuf {
-        self.license
-            .as_ref()
-            .cloned()
-            .unwrap_or_else(|| self.path().join("LICENSE.md"))
+    async fn license(&self) -> Result<PathBuf, Error> {
+        if let Some(license) = &self.license {
+            Ok(license.clone())
+        } else {
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("license")),
+                    )
+                })
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple licenses found in working directory, please specify which to use with `--license`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("license"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
+        }
     }
     fn instructions(&self) -> PathBuf {
         self.instructions
@@ -282,6 +344,15 @@ pub enum ImageSource {
     DockerTag(String),
 }
 impl ImageSource {
+    pub fn ingredients(&self) -> Vec<PathBuf> {
+        match self {
+            Self::Packed => Vec::new(),
+            Self::DockerBuild { dockerfile, .. } => {
+                vec![dockerfile.clone().unwrap_or_else(|| "Dockerfile".into())]
+            }
+            Self::DockerTag(_) => Vec::new(),
+        }
+    }
     #[instrument(skip_all)]
     pub fn load<'a, S: From<TmpSource<PackSource>> + FileSource + Clone>(
         &'a self,
@@ -320,7 +391,7 @@ impl ImageSource {
                         format!("--platform=linux/{arch}")
                     };
                     // docker buildx build ${path} -o type=image,name=start9/${id}
-                    let tag = format!("start9/{id}/{image_id}:{version}");
+                    let tag = format!("start9/{id}/{image_id}:{}", new_guid());
                     Command::new(CONTAINER_TOOL)
                         .arg("build")
                         .arg(workdir)
@@ -501,7 +572,7 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
         "LICENSE.md".into(),
         Entry::file(TmpSource::new(
             tmp_dir.clone(),
-            PackSource::File(params.license()),
+            PackSource::File(params.license().await?),
         )),
     );
     files.insert(
@@ -541,6 +612,54 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
 
     s9pk.load_images(tmp_dir.clone()).await?;
 
+    let mut to_insert = Vec::new();
+    for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
+        if let Some(s9pk) = dependency.s9pk.take() {
+            let s9pk = match s9pk {
+                PathOrUrl::Path(path) => {
+                    S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
+                        .await?
+                        .into_dyn()
+                }
+                PathOrUrl::Url(url) => {
+                    if url.scheme() == "http" || url.scheme() == "https" {
+                        S9pk::deserialize(
+                            &Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
+                            None,
+                        )
+                        .await?
+                        .into_dyn()
+                    } else {
+                        return Err(Error::new(
+                            eyre!("unknown scheme: {}", url.scheme()),
+                            ErrorKind::InvalidRequest,
+                        ));
+                    }
+                }
+            };
+            let dep_path = Path::new("dependencies").join(id);
+            to_insert.push((
+                dep_path.join("metadata.json"),
+                Entry::file(PackSource::Buffered(
+                    IoFormat::Json
+                        .to_vec(&DependencyMetadata {
+                            title: s9pk.as_manifest().title.clone(),
+                        })?
+                        .into(),
+                )),
+            ));
+            let icon = s9pk.icon().await?;
+            to_insert.push((
+                dep_path.join(&*icon.0),
+                Entry::file(PackSource::Buffered(
+                    icon.1.expect_file()?.to_vec(icon.1.hash()).await?.into(),
+                )),
+            ));
+        } else {
+            warn!("no s9pk specified for {id}, leaving metadata empty");
+        }
+    }
+
     s9pk.validate_and_filter(None)?;
 
     s9pk.serialize(
@@ -555,3 +674,58 @@ pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
 
     Ok(())
 }
+
+#[instrument(skip_all)]
+pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<PathBuf>, Error> {
+    let js_path = params.javascript().join("index.js");
+    let manifest: Manifest = match async {
+        serde_json::from_slice(
+            &Command::new("node")
+                .arg("-e")
+                .arg(format!(
+                    "console.log(JSON.stringify(require('{}').manifest))",
+                    js_path.display()
+                ))
+                .invoke(ErrorKind::Javascript)
+                .await?,
+        )
+        .with_kind(ErrorKind::Deserialization)
+    }
+    .await
+    {
+        Ok(m) => m,
+        Err(e) => {
+            warn!("failed to load manifest: {e}");
+            debug!("{e:?}");
+            return Ok(vec![
+                js_path,
+                params.icon().await?,
+                params.license().await?,
+                params.instructions(),
+            ]);
+        }
+    };
+    let mut ingredients = vec![
+        js_path,
+        params.icon().await?,
+        params.license().await?,
+        params.instructions(),
+    ];
+
+    for (_, dependency) in manifest.dependencies.0 {
+        if let Some(PathOrUrl::Path(p)) = dependency.s9pk {
+            ingredients.push(p);
+        }
+    }
+
+    let assets_dir = params.assets();
+    for assets in manifest.assets {
+        ingredients.push(assets_dir.join(assets));
+    }
+
+    for image in manifest.images.values() {
+        ingredients.extend(image.source.ingredients());
+    }
+
+    Ok(ingredients)
+}