Refactor/project structure (#3085)

* refactor project structure

* environment-based default registry

* fix tests

* update build container

* use docker platform for iso build emulation

* simplify compat

* Fix docker platform spec in run-compat.sh

* handle riscv compat

* fix bug with dep error exists attr

* undo removal of sorting

* use qemu for iso stage

---------

Co-authored-by: Mariusz Kogen <k0gen@pm.me>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

core/src/s9pk/git_hash.rs

@@ -0,0 +1,93 @@
+use std::ops::Deref;
+use std::path::Path;
+
+use tokio::process::Command;
+use ts_rs::TS;
+
+use crate::prelude::*;
+use crate::util::Invoke;
+
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, TS)]
+#[ts(type = "string")]
+pub struct GitHash(String);
+
+impl GitHash {
+    pub async fn from_path(path: impl AsRef<Path>) -> Result<GitHash, Error> {
+        let mut hash = String::from_utf8(
+            Command::new("git")
+                .arg("rev-parse")
+                .arg("HEAD")
+                .current_dir(&path)
+                .invoke(ErrorKind::Git)
+                .await?,
+        )?;
+        while hash.ends_with(|c: char| c.is_whitespace()) {
+            hash.pop();
+        }
+        if Command::new("git")
+            .arg("diff-index")
+            .arg("--quiet")
+            .arg("HEAD")
+            .arg("--")
+            .invoke(ErrorKind::Git)
+            .await
+            .is_err()
+        {
+            hash += "-modified";
+        }
+        Ok(GitHash(hash))
+    }
+    pub fn load_sync() -> Option<GitHash> {
+        let mut hash = String::from_utf8(
+            std::process::Command::new("git")
+                .arg("rev-parse")
+                .arg("HEAD")
+                .output()
+                .ok()?
+                .stdout,
+        )
+        .ok()?;
+        while hash.ends_with(|c: char| c.is_whitespace()) {
+            hash.pop();
+        }
+        if !std::process::Command::new("git")
+            .arg("diff-index")
+            .arg("--quiet")
+            .arg("HEAD")
+            .arg("--")
+            .output()
+            .ok()?
+            .status
+            .success()
+        {
+            hash += "-modified";
+        }
+
+        Some(GitHash(hash))
+    }
+}
+
+impl AsRef<str> for GitHash {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Deref for GitHash {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.as_ref()
+    }
+}
+
+// #[tokio::test]
+// async fn test_githash_for_current() {
+//     let answer: GitHash = GitHash::from_path(std::env::current_dir().unwrap())
+//         .await
+//         .unwrap();
+//     let answer_str: &str = answer.as_ref();
+//     assert!(
+//         !answer_str.is_empty(),
+//         "Should have a hash for this current working"
+//     );
+// }

core/src/s9pk/merkle_archive/directory_contents.rs

@@ -0,0 +1,328 @@
+use std::ffi::OsStr;
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use blake3::Hash;
+use futures::FutureExt;
+use futures::future::BoxFuture;
+use imbl::OrdMap;
+use imbl_value::InternedString;
+use itertools::Itertools;
+use tokio::io::AsyncRead;
+
+use crate::CAP_10_MiB;
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::sink::Sink;
+use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
+use crate::s9pk::merkle_archive::write_queue::WriteQueue;
+use crate::s9pk::merkle_archive::{Entry, EntryContents, varint};
+use crate::util::io::{ParallelBlake3Writer, TrackingIO};
+
+#[derive(Clone)]
+pub struct DirectoryContents<S> {
+    contents: OrdMap<InternedString, Entry<S>>,
+    /// used to optimize files to have earliest needed information up front
+    sort_by: Option<Arc<dyn Fn(&str, &str) -> std::cmp::Ordering + Send + Sync>>,
+}
+impl<S: Debug> Debug for DirectoryContents<S> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DirectoryContents")
+            .field("contents", &self.contents)
+            .finish_non_exhaustive()
+    }
+}
+impl<S> DirectoryContents<S> {
+    pub fn new() -> Self {
+        Self {
+            contents: OrdMap::new(),
+            sort_by: None,
+        }
+    }
+
+    pub fn sort_by(
+        &mut self,
+        sort_by: impl Fn(&str, &str) -> std::cmp::Ordering + Send + Sync + 'static,
+    ) {
+        self.sort_by = Some(Arc::new(sort_by))
+    }
+
+    #[instrument(skip_all)]
+    pub fn get_path(&self, path: impl AsRef<Path>) -> Option<&Entry<S>> {
+        let mut dir = Some(self);
+        let mut res = None;
+        for segment in path.as_ref().into_iter() {
+            let segment = segment.to_str()?;
+            if segment == "/" {
+                continue;
+            }
+            res = dir?.get(segment);
+            if let Some(EntryContents::Directory(d)) = res.as_ref().map(|e| e.as_contents()) {
+                dir = Some(d);
+            } else {
+                dir = None
+            }
+        }
+        res
+    }
+
+    pub fn file_paths(&self, prefix: impl AsRef<Path>) -> Vec<PathBuf> {
+        let prefix = prefix.as_ref();
+        let mut res = Vec::new();
+        for (name, entry) in &self.contents {
+            let path = prefix.join(name);
+            if let EntryContents::Directory(d) = entry.as_contents() {
+                res.push(path.join(""));
+                res.append(&mut d.file_paths(path));
+            } else {
+                res.push(path);
+            }
+        }
+        res
+    }
+
+    pub const fn header_size() -> u64 {
+        8 // position: u64 BE
+        + 8 // size: u64 BE
+    }
+
+    #[instrument(skip_all)]
+    pub async fn serialize_header<W: Sink>(&self, position: u64, w: &mut W) -> Result<u64, Error> {
+        use tokio::io::AsyncWriteExt;
+
+        let size = self.toc_size();
+
+        w.write_all(&position.to_be_bytes()).await?;
+        w.write_all(&size.to_be_bytes()).await?;
+
+        Ok(position)
+    }
+
+    pub fn toc_size(&self) -> u64 {
+        self.iter().fold(
+            varint::serialized_varint_size(self.len() as u64),
+            |acc, (name, entry)| {
+                acc + varint::serialized_varstring_size(&**name) + entry.header_size()
+            },
+        )
+    }
+}
+impl<S: Clone> DirectoryContents<S> {
+    pub fn with_stem(&self, stem: &str) -> impl Iterator<Item = (InternedString, Entry<S>)> {
+        let prefix = InternedString::intern(stem);
+        let (_, center, right) = self.split_lookup(&*stem);
+        center.map(|e| (prefix.clone(), e)).into_iter().chain(
+            right.into_iter().take_while(move |(k, _)| {
+                Path::new(&**k).file_stem() == Some(OsStr::new(&*prefix))
+            }),
+        )
+    }
+    pub fn insert_path(&mut self, path: impl AsRef<Path>, entry: Entry<S>) -> Result<(), Error> {
+        let path = path.as_ref();
+        let (parent, Some(file)) = (path.parent(), path.file_name().and_then(|f| f.to_str()))
+        else {
+            return Err(Error::new(
+                eyre!("cannot create file at root"),
+                ErrorKind::Pack,
+            ));
+        };
+        let mut dir = self;
+        for segment in parent.into_iter().flatten() {
+            let segment = segment
+                .to_str()
+                .ok_or_else(|| Error::new(eyre!("non-utf8 path segment"), ErrorKind::Utf8))?;
+            if segment == "/" {
+                continue;
+            }
+            if !dir.contains_key(segment) {
+                dir.insert(
+                    segment.into(),
+                    Entry::new(EntryContents::Directory(DirectoryContents::new())),
+                );
+            }
+            if let Some(EntryContents::Directory(d)) =
+                dir.get_mut(segment).map(|e| e.as_contents_mut())
+            {
+                dir = d;
+            } else {
+                return Err(Error::new(
+                    eyre!(
+                        "failed to insert entry at path {path:?}: ancestor exists and is not a directory"
+                    ),
+                    ErrorKind::Pack,
+                ));
+            }
+        }
+        dir.insert(file.into(), entry);
+        Ok(())
+    }
+}
+impl<S: ArchiveSource + Clone> DirectoryContents<Section<S>> {
+    #[instrument(skip_all)]
+    pub fn deserialize<'a>(
+        source: &'a S,
+        header: &'a mut (impl AsyncRead + Unpin + Send),
+        (sighash, max_size): (Hash, u64),
+    ) -> BoxFuture<'a, Result<Self, Error>> {
+        async move {
+            use tokio::io::AsyncReadExt;
+
+            let mut position = [0u8; 8];
+            header.read_exact(&mut position).await?;
+            let position = u64::from_be_bytes(position);
+
+            let mut size = [0u8; 8];
+            header.read_exact(&mut size).await?;
+            let size = u64::from_be_bytes(size);
+
+            ensure_code!(
+                size <= max_size,
+                ErrorKind::InvalidSignature,
+                "size is greater than signed"
+            );
+
+            let mut toc_reader = source.fetch(position, size).await?;
+
+            let len = varint::deserialize_varint(&mut toc_reader).await?;
+            let mut entries = OrdMap::new();
+            for _ in 0..len {
+                let name = varint::deserialize_varstring(&mut toc_reader).await?;
+                let entry = Entry::deserialize(source.clone(), &mut toc_reader).await?;
+                entries.insert(name.into(), entry);
+            }
+
+            let res = Self {
+                contents: entries,
+                sort_by: None,
+            };
+
+            if res.sighash().await? == sighash {
+                Ok(res)
+            } else {
+                Err(Error::new(
+                    eyre!("hash sum does not match"),
+                    ErrorKind::InvalidSignature,
+                ))
+            }
+        }
+        .boxed()
+    }
+}
+impl<S: FileSource + Clone> DirectoryContents<S> {
+    pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> {
+        for k in self.keys().cloned().collect::<Vec<_>>() {
+            let path = Path::new(&*k);
+            if let Some(v) = self.get_mut(&k) {
+                if !filter(path) {
+                    if v.hash.is_none() {
+                        return Err(Error::new(
+                            eyre!(
+                                "cannot filter out unhashed file {}, run `update_hashes` first",
+                                path.display()
+                            ),
+                            ErrorKind::InvalidRequest,
+                        ));
+                    }
+                    v.contents = EntryContents::Missing;
+                } else {
+                    let filter: Box<dyn Fn(&Path) -> bool> = Box::new(|p| filter(&path.join(p)));
+                    v.filter(filter)?;
+                }
+            }
+        }
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub fn update_hashes<'a>(&'a mut self, only_missing: bool) -> BoxFuture<'a, Result<(), Error>> {
+        async move {
+            for key in self.keys().cloned().collect::<Vec<_>>() {
+                if let Some(entry) = self.get_mut(&key) {
+                    entry.update_hash(only_missing).await?;
+                }
+            }
+            Ok(())
+        }
+        .boxed()
+    }
+
+    #[instrument(skip_all)]
+    pub fn sighash<'a>(&'a self) -> BoxFuture<'a, Result<Hash, Error>> {
+        async move {
+            let mut hasher = TrackingIO::new(0, ParallelBlake3Writer::new(CAP_10_MiB));
+            let mut sig_contents = OrdMap::new();
+            for (name, entry) in &**self {
+                sig_contents.insert(name.clone(), entry.to_missing().await?);
+            }
+            Self {
+                contents: sig_contents,
+                sort_by: None,
+            }
+            .serialize_toc(&mut WriteQueue::new(0), &mut hasher)
+            .await?;
+            let hash = hasher.into_inner().finalize().await?;
+            Ok(hash)
+        }
+        .boxed()
+    }
+
+    #[instrument(skip_all)]
+    pub async fn serialize_toc<'a, W: Sink>(
+        &'a self,
+        queue: &mut WriteQueue<'a, S>,
+        w: &mut W,
+    ) -> Result<(), Error> {
+        varint::serialize_varint(self.len() as u64, w).await?;
+        for (name, entry) in self.iter().sorted_by(|a, b| match (a, b, &self.sort_by) {
+            ((_, a), (_, b), _) if a.as_contents().is_dir() && !b.as_contents().is_dir() => {
+                std::cmp::Ordering::Less
+            }
+            ((_, a), (_, b), _) if !a.as_contents().is_dir() && b.as_contents().is_dir() => {
+                std::cmp::Ordering::Greater
+            }
+            ((_, a), (_, b), _)
+                if a.as_contents().is_missing() && !b.as_contents().is_missing() =>
+            {
+                std::cmp::Ordering::Greater
+            }
+            ((_, a), (_, b), _)
+                if !a.as_contents().is_missing() && b.as_contents().is_missing() =>
+            {
+                std::cmp::Ordering::Less
+            }
+            ((n_a, a), (n_b, b), _)
+                if a.as_contents().is_missing() && b.as_contents().is_missing() =>
+            {
+                n_a.cmp(n_b)
+            }
+            ((a, _), (b, _), Some(sort_by)) => sort_by(&***a, &***b),
+            _ => std::cmp::Ordering::Equal,
+        }) {
+            varint::serialize_varstring(&**name, w).await?;
+            entry.serialize_header(queue.add(entry).await?, w).await?;
+        }
+
+        Ok(())
+    }
+
+    pub fn into_dyn(self) -> DirectoryContents<DynFileSource> {
+        DirectoryContents {
+            contents: self
+                .contents
+                .into_iter()
+                .map(|(k, v)| (k, v.into_dyn()))
+                .collect(),
+            sort_by: self.sort_by,
+        }
+    }
+}
+impl<S> std::ops::Deref for DirectoryContents<S> {
+    type Target = OrdMap<InternedString, Entry<S>>;
+    fn deref(&self) -> &Self::Target {
+        &self.contents
+    }
+}
+impl<S> std::ops::DerefMut for DirectoryContents<S> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.contents
+    }
+}

core/src/s9pk/merkle_archive/expected.rs

@@ -0,0 +1,124 @@
+use std::ffi::OsStr;
+use std::path::Path;
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::Entry;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::FileSource;
+
+/// An object for tracking the files expected to be in an s9pk
+pub struct Expected<'a, T> {
+    keep: DirectoryContents<()>,
+    dir: &'a DirectoryContents<T>,
+}
+impl<'a, T> Expected<'a, T> {
+    pub fn new(dir: &'a DirectoryContents<T>) -> Self {
+        Self {
+            keep: DirectoryContents::new(),
+            dir,
+        }
+    }
+}
+impl<'a, T: Clone> Expected<'a, T> {
+    pub fn check_file(&mut self, path: impl AsRef<Path>) -> Result<(), Error> {
+        if self
+            .dir
+            .get_path(path.as_ref())
+            .and_then(|e| e.as_file())
+            .is_some()
+        {
+            self.keep.insert_path(path, Entry::file(()))?;
+            Ok(())
+        } else {
+            Err(Error::new(
+                eyre!("file {} missing from archive", path.as_ref().display()),
+                ErrorKind::ParseS9pk,
+            ))
+        }
+    }
+    pub fn check_dir(&mut self, path: impl AsRef<Path>) -> Result<(), Error> {
+        if let Some(dir) = self
+            .dir
+            .get_path(path.as_ref())
+            .and_then(|e| e.as_directory())
+        {
+            for entry in dir.file_paths(path.as_ref()) {
+                if !entry.to_string_lossy().ends_with("/") {
+                    self.keep.insert_path(entry, Entry::file(()))?;
+                }
+            }
+            Ok(())
+        } else {
+            Err(Error::new(
+                eyre!("directory {} missing from archive", path.as_ref().display()),
+                ErrorKind::ParseS9pk,
+            ))
+        }
+    }
+    pub fn check_stem(
+        &mut self,
+        path: impl AsRef<Path>,
+        mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
+    ) -> Result<(), Error> {
+        let (dir, stem) =
+            if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
+                (
+                    self.dir
+                        .get_path(parent)
+                        .and_then(|e| e.as_directory())
+                        .ok_or_else(|| {
+                            Error::new(
+                                eyre!("directory {} missing from archive", parent.display()),
+                                ErrorKind::ParseS9pk,
+                            )
+                        })?,
+                    path.as_ref().strip_prefix(parent).unwrap(),
+                )
+            } else {
+                (self.dir, path.as_ref())
+            };
+        let name = dir
+            .with_stem(&stem.as_os_str().to_string_lossy())
+            .filter(|(_, e)| e.as_file().is_some())
+            .try_fold(
+                Err(Error::new(
+                    eyre!(
+                        "file {} with valid extension missing from archive",
+                        path.as_ref().display()
+                    ),
+                    ErrorKind::ParseS9pk,
+                )),
+                |acc, (name, _)|
+                    if valid_extension(Path::new(&*name).extension()) {
+                        match acc {
+                            Ok(_) => Err(Error::new(
+                                eyre!(
+                                    "more than one file matching {} with valid extension in archive",
+                                    path.as_ref().display()
+                                ),
+                                ErrorKind::ParseS9pk,
+                            )),
+                            Err(_) => Ok(Ok(name))
+                        }
+                    } else {
+                        Ok(acc)
+                    }
+            )??;
+        self.keep
+            .insert_path(path.as_ref().with_file_name(name), Entry::file(()))?;
+        Ok(())
+    }
+    pub fn into_filter(self) -> Filter {
+        Filter(self.keep)
+    }
+}
+
+pub struct Filter(DirectoryContents<()>);
+impl Filter {
+    pub fn keep_checked<T: FileSource + Clone>(
+        &self,
+        dir: &mut DirectoryContents<T>,
+    ) -> Result<(), Error> {
+        dir.filter(|path| self.0.get_path(path).is_some())
+    }
+}

core/src/s9pk/merkle_archive/file_contents.rs

@@ -0,0 +1,70 @@
+use blake3::Hash;
+use tokio::io::AsyncRead;
+
+use crate::CAP_10_MiB;
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::sink::Sink;
+use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
+use crate::util::io::{ParallelBlake3Writer, TrackingIO};
+
+#[derive(Debug, Clone)]
+pub struct FileContents<S>(S);
+impl<S> FileContents<S> {
+    pub fn new(source: S) -> Self {
+        Self(source)
+    }
+    pub const fn header_size() -> u64 {
+        8 // position: u64 BE
+    }
+}
+impl<S: ArchiveSource> FileContents<Section<S>> {
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: S,
+        header: &mut (impl AsyncRead + Unpin + Send),
+        size: u64,
+    ) -> Result<Self, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut position = [0u8; 8];
+        header.read_exact(&mut position).await?;
+        let position = u64::from_be_bytes(position);
+
+        Ok(Self(source.section(position, size)))
+    }
+}
+impl<S: FileSource> FileContents<S> {
+    pub async fn hash(&self) -> Result<(Hash, u64), Error> {
+        let mut hasher = TrackingIO::new(0, ParallelBlake3Writer::new(CAP_10_MiB));
+        self.serialize_body(&mut hasher, None).await?;
+        let size = hasher.position();
+        let hash = hasher.into_inner().finalize().await?;
+        Ok((hash, size))
+    }
+    #[instrument(skip_all)]
+    pub async fn serialize_header<W: Sink>(&self, position: u64, w: &mut W) -> Result<u64, Error> {
+        use tokio::io::AsyncWriteExt;
+
+        w.write_all(&position.to_be_bytes()).await?;
+
+        Ok(position)
+    }
+    #[instrument(skip_all)]
+    pub async fn serialize_body<W: Sink>(
+        &self,
+        w: &mut W,
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error> {
+        self.0.copy_verify(w, verify).await?;
+        Ok(())
+    }
+    pub fn into_dyn(self) -> FileContents<DynFileSource> {
+        FileContents(DynFileSource::new(self.0))
+    }
+}
+impl<S> std::ops::Deref for FileContents<S> {
+    type Target = S;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

core/src/s9pk/merkle_archive/hash.rs

@@ -0,0 +1,92 @@
+use std::task::Poll;
+
+use blake3::Hash;
+use tokio::io::AsyncWrite;
+use tokio_util::either::Either;
+
+use crate::CAP_10_MiB;
+use crate::prelude::*;
+use crate::util::io::{ParallelBlake3Writer, TeeWriter};
+
+#[pin_project::pin_project]
+pub struct VerifyingWriter<W> {
+    verify: Option<(Hash, u64)>,
+    #[pin]
+    writer: Either<TeeWriter<W, ParallelBlake3Writer>, W>,
+}
+impl<W: AsyncWrite> VerifyingWriter<W> {
+    pub fn new(w: W, verify: Option<(Hash, u64)>) -> Self {
+        Self {
+            writer: if verify.is_some() {
+                Either::Left(TeeWriter::new(
+                    w,
+                    ParallelBlake3Writer::new(CAP_10_MiB),
+                    CAP_10_MiB,
+                ))
+            } else {
+                Either::Right(w)
+            },
+            verify,
+        }
+    }
+}
+impl<W: AsyncWrite + Unpin> VerifyingWriter<W> {
+    pub async fn verify(self) -> Result<W, Error> {
+        match self.writer {
+            Either::Left(writer) => {
+                let (writer, actual) = writer.into_inner().await?;
+                if let Some((expected, remaining)) = self.verify {
+                    ensure_code!(
+                        actual.finalize().await? == expected,
+                        ErrorKind::InvalidSignature,
+                        "hash sum mismatch"
+                    );
+                    ensure_code!(
+                        remaining == 0,
+                        ErrorKind::InvalidSignature,
+                        "file size mismatch"
+                    );
+                }
+                Ok(writer)
+            }
+            Either::Right(writer) => Ok(writer),
+        }
+    }
+}
+impl<W: AsyncWrite> AsyncWrite for VerifyingWriter<W> {
+    fn poll_write(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> Poll<Result<usize, std::io::Error>> {
+        let this = self.project();
+        if let Some((_, remaining)) = this.verify {
+            if *remaining < buf.len() as u64 {
+                return Poll::Ready(Err(std::io::Error::other(eyre!(
+                    "attempted to write more bytes than signed"
+                ))));
+            }
+        }
+        match this.writer.poll_write(cx, buf)? {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(n) => {
+                if let Some((_, remaining)) = this.verify {
+                    *remaining -= n as u64;
+                }
+                Poll::Ready(Ok(n))
+            }
+        }
+    }
+    fn poll_flush(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(), std::io::Error>> {
+        self.project().writer.poll_flush(cx)
+    }
+    fn poll_shutdown(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(), std::io::Error>> {
+        self.project().writer.poll_shutdown(cx)
+    }
+}

core/src/s9pk/merkle_archive/mod.rs

@@ -0,0 +1,451 @@
+use std::path::Path;
+
+use blake3::Hash;
+use ed25519_dalek::{Signature, SigningKey, VerifyingKey};
+use imbl_value::InternedString;
+use sha2::{Digest, Sha512};
+use tokio::io::AsyncRead;
+
+use crate::CAP_1_MiB;
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::file_contents::FileContents;
+use crate::s9pk::merkle_archive::sink::Sink;
+use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
+use crate::s9pk::merkle_archive::write_queue::WriteQueue;
+use crate::sign::SignatureScheme;
+use crate::sign::commitment::merkle_archive::MerkleArchiveCommitment;
+use crate::sign::ed25519::Ed25519;
+use crate::util::serde::Base64;
+
+pub mod directory_contents;
+pub mod expected;
+pub mod file_contents;
+pub mod hash;
+pub mod sink;
+pub mod source;
+#[cfg(test)]
+mod test;
+pub mod varint;
+pub mod write_queue;
+
+#[derive(Debug, Clone)]
+enum Signer {
+    Signed(VerifyingKey, Signature, u64, InternedString),
+    Signer(SigningKey, InternedString),
+}
+
+#[derive(Debug, Clone)]
+pub struct MerkleArchive<S> {
+    signer: Signer,
+    contents: DirectoryContents<S>,
+}
+impl<S> MerkleArchive<S> {
+    pub fn new(contents: DirectoryContents<S>, signer: SigningKey, context: &str) -> Self {
+        Self {
+            signer: Signer::Signer(signer, context.into()),
+            contents,
+        }
+    }
+    pub fn signer(&self) -> VerifyingKey {
+        match &self.signer {
+            Signer::Signed(k, _, _, _) => *k,
+            Signer::Signer(k, _) => k.verifying_key(),
+        }
+    }
+    pub const fn header_size() -> u64 {
+        32 // pubkey
+                 + 64 // signature
+                 + 32 // sighash
+                 + 8 // size
+                 + DirectoryContents::<Section<S>>::header_size()
+    }
+    pub fn contents(&self) -> &DirectoryContents<S> {
+        &self.contents
+    }
+    pub fn contents_mut(&mut self) -> &mut DirectoryContents<S> {
+        &mut self.contents
+    }
+    pub fn set_signer(&mut self, key: SigningKey, context: &str) {
+        self.signer = Signer::Signer(key, context.into());
+    }
+    pub fn sort_by(
+        &mut self,
+        sort_by: impl Fn(&str, &str) -> std::cmp::Ordering + Send + Sync + 'static,
+    ) {
+        self.contents.sort_by(sort_by)
+    }
+}
+impl<S: ArchiveSource + Clone> MerkleArchive<Section<S>> {
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: &S,
+        context: &str,
+        header: &mut (impl AsyncRead + Unpin + Send),
+        commitment: Option<&MerkleArchiveCommitment>,
+    ) -> Result<Self, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut pubkey = [0u8; 32];
+        header.read_exact(&mut pubkey).await?;
+        let pubkey = VerifyingKey::from_bytes(&pubkey)?;
+
+        let mut signature = [0u8; 64];
+        header.read_exact(&mut signature).await?;
+        let signature = Signature::from_bytes(&signature);
+
+        let mut sighash = [0u8; 32];
+        header.read_exact(&mut sighash).await?;
+        let sighash = Hash::from_bytes(sighash);
+
+        let mut max_size = [0u8; 8];
+        header.read_exact(&mut max_size).await?;
+        let max_size = u64::from_be_bytes(max_size);
+
+        pubkey.verify_prehashed_strict(
+            Sha512::new_with_prefix(sighash.as_bytes()).chain_update(&u64::to_be_bytes(max_size)),
+            Some(context.as_bytes()),
+            &signature,
+        )?;
+
+        if let Some(MerkleArchiveCommitment {
+            root_sighash,
+            root_maxsize,
+        }) = commitment
+        {
+            if sighash.as_bytes() != &**root_sighash {
+                return Err(Error::new(
+                    eyre!("merkle root mismatch"),
+                    ErrorKind::InvalidSignature,
+                ));
+            }
+            if max_size > *root_maxsize {
+                return Err(Error::new(
+                    eyre!("root directory max size too large"),
+                    ErrorKind::InvalidSignature,
+                ));
+            }
+        } else {
+            if max_size > CAP_1_MiB as u64 {
+                return Err(Error::new(
+                    eyre!(
+                        "root directory max size over 1MiB, cancelling download in case of DOS attack"
+                    ),
+                    ErrorKind::InvalidSignature,
+                ));
+            }
+        }
+
+        let contents = DirectoryContents::deserialize(source, header, (sighash, max_size)).await?;
+
+        Ok(Self {
+            signer: Signer::Signed(pubkey, signature, max_size, context.into()),
+            contents,
+        })
+    }
+}
+impl<S: FileSource + Clone> MerkleArchive<S> {
+    pub async fn update_hashes(&mut self, only_missing: bool) -> Result<(), Error> {
+        self.contents.update_hashes(only_missing).await
+    }
+    pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> {
+        self.contents.filter(filter)
+    }
+    pub async fn commitment(&self) -> Result<MerkleArchiveCommitment, Error> {
+        let root_maxsize = match self.signer {
+            Signer::Signed(_, _, s, _) => s,
+            _ => self.contents.toc_size(),
+        };
+        let root_sighash = self.contents.sighash().await?;
+        Ok(MerkleArchiveCommitment {
+            root_sighash: Base64(*root_sighash.as_bytes()),
+            root_maxsize,
+        })
+    }
+    pub async fn signature(&self) -> Result<Signature, Error> {
+        match &self.signer {
+            Signer::Signed(_, s, _, _) => Ok(*s),
+            Signer::Signer(k, context) => {
+                Ed25519.sign_commitment(k, &self.commitment().await?, context)
+            }
+        }
+    }
+    #[instrument(skip_all)]
+    pub async fn serialize<W: Sink>(&self, w: &mut W, verify: bool) -> Result<(), Error> {
+        use tokio::io::AsyncWriteExt;
+
+        let commitment = self.commitment().await?;
+
+        let (pubkey, signature) = match &self.signer {
+            Signer::Signed(pubkey, signature, _, _) => (*pubkey, *signature),
+            Signer::Signer(s, context) => {
+                (s.into(), Ed25519.sign_commitment(s, &commitment, context)?)
+            }
+        };
+
+        w.write_all(pubkey.as_bytes()).await?;
+        w.write_all(&signature.to_bytes()).await?;
+        w.write_all(&*commitment.root_sighash).await?;
+        w.write_all(&u64::to_be_bytes(commitment.root_maxsize))
+            .await?;
+        let mut next_pos = w.current_position().await?;
+        next_pos += DirectoryContents::<S>::header_size();
+        self.contents.serialize_header(next_pos, w).await?;
+        next_pos += self.contents.toc_size();
+        let mut queue = WriteQueue::new(next_pos);
+        self.contents.serialize_toc(&mut queue, w).await?;
+        queue.serialize(w, verify).await?;
+        Ok(())
+    }
+    pub fn into_dyn(self) -> MerkleArchive<DynFileSource> {
+        MerkleArchive {
+            signer: self.signer,
+            contents: self.contents.into_dyn(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Entry<S> {
+    hash: Option<(Hash, u64)>,
+    contents: EntryContents<S>,
+}
+impl<S> Entry<S> {
+    pub fn new(contents: EntryContents<S>) -> Self {
+        Self {
+            hash: None,
+            contents,
+        }
+    }
+    pub fn file(source: S) -> Self {
+        Self::new(EntryContents::File(FileContents::new(source)))
+    }
+    pub fn directory(directory: DirectoryContents<S>) -> Self {
+        Self::new(EntryContents::Directory(directory))
+    }
+    pub fn hash(&self) -> Option<(Hash, u64)> {
+        self.hash
+    }
+    pub fn as_contents(&self) -> &EntryContents<S> {
+        &self.contents
+    }
+    pub fn as_file(&self) -> Option<&FileContents<S>> {
+        match self.as_contents() {
+            EntryContents::File(f) => Some(f),
+            _ => None,
+        }
+    }
+    pub fn expect_file(&self) -> Result<&FileContents<S>, Error> {
+        self.as_file()
+            .ok_or_else(|| Error::new(eyre!("not a file"), ErrorKind::ParseS9pk))
+    }
+    pub fn as_directory(&self) -> Option<&DirectoryContents<S>> {
+        match self.as_contents() {
+            EntryContents::Directory(d) => Some(d),
+            _ => None,
+        }
+    }
+    pub fn as_contents_mut(&mut self) -> &mut EntryContents<S> {
+        self.hash = None;
+        &mut self.contents
+    }
+    pub fn into_contents(self) -> EntryContents<S> {
+        self.contents
+    }
+    pub fn into_file(self) -> Option<FileContents<S>> {
+        match self.into_contents() {
+            EntryContents::File(f) => Some(f),
+            _ => None,
+        }
+    }
+    pub fn into_directory(self) -> Option<DirectoryContents<S>> {
+        match self.into_contents() {
+            EntryContents::Directory(d) => Some(d),
+            _ => None,
+        }
+    }
+    pub fn header_size(&self) -> u64 {
+        32 // hash
+        + 8 // size: u64 BE
+        + self.contents.header_size()
+    }
+}
+impl<S: ArchiveSource + Clone> Entry<Section<S>> {
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: S,
+        header: &mut (impl AsyncRead + Unpin + Send),
+    ) -> Result<Self, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut hash = [0u8; 32];
+        header.read_exact(&mut hash).await?;
+        let hash = Hash::from_bytes(hash);
+
+        let mut size = [0u8; 8];
+        header.read_exact(&mut size).await?;
+        let size = u64::from_be_bytes(size);
+
+        let contents = EntryContents::deserialize(source, header, (hash, size)).await?;
+
+        Ok(Self {
+            hash: Some((hash, size)),
+            contents,
+        })
+    }
+}
+impl<S: FileSource + Clone> Entry<S> {
+    pub fn filter(&mut self, filter: impl Fn(&Path) -> bool) -> Result<(), Error> {
+        if let EntryContents::Directory(d) = &mut self.contents {
+            d.filter(filter)?;
+        }
+        Ok(())
+    }
+    pub async fn update_hash(&mut self, only_missing: bool) -> Result<(), Error> {
+        if let EntryContents::Directory(d) = &mut self.contents {
+            d.update_hashes(only_missing).await?;
+        }
+        self.hash = Some(self.contents.hash().await?);
+        Ok(())
+    }
+    pub async fn to_missing(&self) -> Result<Self, Error> {
+        let hash = if let Some(hash) = self.hash {
+            hash
+        } else {
+            self.contents.hash().await?
+        };
+        Ok(Self {
+            hash: Some(hash),
+            contents: EntryContents::Missing,
+        })
+    }
+    #[instrument(skip_all)]
+    pub async fn serialize_header<W: Sink>(
+        &self,
+        position: u64,
+        w: &mut W,
+    ) -> Result<Option<u64>, Error> {
+        use tokio::io::AsyncWriteExt;
+
+        let (hash, size) = if let Some(hash) = self.hash {
+            hash
+        } else {
+            self.contents.hash().await?
+        };
+        w.write_all(hash.as_bytes()).await?;
+        w.write_all(&u64::to_be_bytes(size)).await?;
+        self.contents.serialize_header(position, w).await
+    }
+    pub fn into_dyn(self) -> Entry<DynFileSource> {
+        Entry {
+            hash: self.hash,
+            contents: self.contents.into_dyn(),
+        }
+    }
+}
+impl<S: FileSource> Entry<S> {
+    pub async fn read_file_to_vec(&self) -> Result<Vec<u8>, Error> {
+        match self.as_contents() {
+            EntryContents::File(f) => Ok(f.to_vec(self.hash).await?),
+            EntryContents::Directory(_) => Err(Error::new(
+                eyre!("expected file, found directory"),
+                ErrorKind::ParseS9pk,
+            )),
+            EntryContents::Missing => {
+                Err(Error::new(eyre!("entry is missing"), ErrorKind::ParseS9pk))
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum EntryContents<S> {
+    Missing,
+    File(FileContents<S>),
+    Directory(DirectoryContents<S>),
+}
+impl<S> EntryContents<S> {
+    fn type_id(&self) -> u8 {
+        match self {
+            Self::Missing => 0,
+            Self::File(_) => 1,
+            Self::Directory(_) => 2,
+        }
+    }
+    pub fn header_size(&self) -> u64 {
+        1 // type
+        + match self {
+            Self::Missing => 0,
+            Self::File(_) => FileContents::<S>::header_size(),
+            Self::Directory(_) => DirectoryContents::<S>::header_size(),
+        }
+    }
+    pub fn is_dir(&self) -> bool {
+        matches!(self, &EntryContents::Directory(_))
+    }
+    pub fn is_missing(&self) -> bool {
+        matches!(self, &EntryContents::Missing)
+    }
+}
+impl<S: ArchiveSource + Clone> EntryContents<Section<S>> {
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: S,
+        header: &mut (impl AsyncRead + Unpin + Send),
+        (hash, size): (Hash, u64),
+    ) -> Result<Self, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut type_id = [0u8];
+        header.read_exact(&mut type_id).await?;
+        match type_id[0] {
+            0 => Ok(Self::Missing),
+            1 => Ok(Self::File(
+                FileContents::deserialize(source, header, size).await?,
+            )),
+            2 => Ok(Self::Directory(
+                DirectoryContents::deserialize(&source, header, (hash, size)).await?,
+            )),
+            id => Err(Error::new(
+                eyre!("Unknown type id {id} found in MerkleArchive"),
+                ErrorKind::ParseS9pk,
+            )),
+        }
+    }
+}
+impl<S: FileSource + Clone> EntryContents<S> {
+    pub async fn hash(&self) -> Result<(Hash, u64), Error> {
+        match self {
+            Self::Missing => Err(Error::new(
+                eyre!("Cannot compute hash of missing file"),
+                ErrorKind::Pack,
+            )),
+            Self::File(f) => f.hash().await,
+            Self::Directory(d) => Ok((d.sighash().await?, d.toc_size())),
+        }
+    }
+    pub fn into_dyn(self) -> EntryContents<DynFileSource> {
+        match self {
+            Self::Missing => EntryContents::Missing,
+            Self::File(f) => EntryContents::File(f.into_dyn()),
+            Self::Directory(d) => EntryContents::Directory(d.into_dyn()),
+        }
+    }
+}
+impl<S: FileSource> EntryContents<S> {
+    #[instrument(skip_all)]
+    pub async fn serialize_header<W: Sink>(
+        &self,
+        position: u64,
+        w: &mut W,
+    ) -> Result<Option<u64>, Error> {
+        use tokio::io::AsyncWriteExt;
+
+        w.write_all(&[self.type_id()]).await?;
+        Ok(match self {
+            Self::Missing => None,
+            Self::File(f) => Some(f.serialize_header(position, w).await?),
+            Self::Directory(d) => Some(d.serialize_header(position, w).await?),
+        })
+    }
+}

core/src/s9pk/merkle_archive/sink.rs

@@ -0,0 +1,22 @@
+use tokio::io::{AsyncSeek, AsyncWrite};
+
+use crate::prelude::*;
+use crate::util::io::TrackingIO;
+
+pub trait Sink: AsyncWrite + Unpin + Send {
+    fn current_position(&mut self) -> impl Future<Output = Result<u64, Error>> + Send + '_;
+}
+
+impl<S: AsyncWrite + AsyncSeek + Unpin + Send> Sink for S {
+    async fn current_position(&mut self) -> Result<u64, Error> {
+        use tokio::io::AsyncSeekExt;
+
+        Ok(self.stream_position().await?)
+    }
+}
+
+impl<W: AsyncWrite + Unpin + Send> Sink for TrackingIO<W> {
+    async fn current_position(&mut self) -> Result<u64, Error> {
+        Ok(self.position())
+    }
+}

core/src/s9pk/merkle_archive/source/http.rs

@@ -0,0 +1,191 @@
+use std::collections::BTreeSet;
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use std::task::Poll;
+
+use bytes::Bytes;
+use futures::{Stream, TryStreamExt};
+use reqwest::header::{ACCEPT_RANGES, CONTENT_LENGTH, RANGE};
+use reqwest::{Client, Url};
+use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf, Take};
+use tokio_util::io::StreamReader;
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::source::ArchiveSource;
+use crate::util::Apply;
+use crate::util::io::TrackingIO;
+
+pub struct HttpSource {
+    url: Url,
+    client: Client,
+    size: Option<u64>,
+    range_support: Result<(), Arc<Mutex<BTreeSet<TrackingIO<HttpBodyReader>>>>>,
+}
+impl HttpSource {
+    pub async fn new(client: Client, url: Url) -> Result<Self, Error> {
+        let head = client
+            .head(url.clone())
+            .send()
+            .await
+            .with_kind(ErrorKind::Network)?
+            .error_for_status()
+            .with_kind(ErrorKind::Network)?;
+        let range_support = head
+            .headers()
+            .get(ACCEPT_RANGES)
+            .and_then(|s| s.to_str().ok())
+            == Some("bytes")
+            && false;
+        let size = head
+            .headers()
+            .get(CONTENT_LENGTH)
+            .and_then(|s| s.to_str().ok())
+            .and_then(|s| s.parse().ok());
+        Ok(Self {
+            url,
+            client,
+            size,
+            range_support: if range_support {
+                Ok(())
+            } else {
+                Err(Arc::new(Mutex::new(BTreeSet::new())))
+            },
+        })
+    }
+}
+impl ArchiveSource for HttpSource {
+    type FetchReader = HttpReader;
+    type FetchAllReader = StreamReader<BoxStream<'static, Result<Bytes, std::io::Error>>, Bytes>;
+    async fn size(&self) -> Option<u64> {
+        self.size
+    }
+    async fn fetch_all(&self) -> Result<Self::FetchAllReader, Error> {
+        Ok(StreamReader::new(
+            self.client
+                .get(self.url.clone())
+                .send()
+                .await
+                .with_kind(ErrorKind::Network)?
+                .error_for_status()
+                .with_kind(ErrorKind::Network)?
+                .bytes_stream()
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+                .apply(boxed),
+        ))
+    }
+    async fn fetch(&self, position: u64, size: u64) -> Result<Self::FetchReader, Error> {
+        match &self.range_support {
+            Ok(_) => Ok(HttpReader::Range(
+                StreamReader::new(if size > 0 {
+                    self.client
+                        .get(self.url.clone())
+                        .header(RANGE, format!("bytes={}-{}", position, position + size - 1))
+                        .send()
+                        .await
+                        .with_kind(ErrorKind::Network)?
+                        .error_for_status()
+                        .with_kind(ErrorKind::Network)?
+                        .bytes_stream()
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+                        .apply(boxed)
+                } else {
+                    futures::stream::empty().apply(boxed)
+                })
+                .take(size),
+            )),
+            Err(pool) => {
+                fn get_reader_for(
+                    pool: &Arc<Mutex<BTreeSet<TrackingIO<HttpBodyReader>>>>,
+                    position: u64,
+                ) -> Option<TrackingIO<HttpBodyReader>> {
+                    let mut lock = pool.lock().unwrap();
+                    let pos = lock.range(..position).last()?.position();
+                    lock.take(&pos)
+                }
+                let reader = get_reader_for(pool, position);
+                let mut reader = if let Some(reader) = reader {
+                    reader
+                } else {
+                    TrackingIO::new(
+                        0,
+                        StreamReader::new(
+                            self.client
+                                .get(self.url.clone())
+                                .send()
+                                .await
+                                .with_kind(ErrorKind::Network)?
+                                .error_for_status()
+                                .with_kind(ErrorKind::Network)?
+                                .bytes_stream()
+                                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
+                                .apply(boxed),
+                        ),
+                    )
+                };
+                if reader.position() < position {
+                    let to_skip = position - reader.position();
+                    tokio::io::copy(&mut (&mut reader).take(to_skip), &mut tokio::io::sink())
+                        .await?;
+                }
+                Ok(HttpReader::Rangeless {
+                    pool: pool.clone(),
+                    reader: Some(reader.take(size)),
+                })
+            }
+        }
+    }
+}
+
+type BoxStream<'a, T> = Pin<Box<dyn Stream<Item = T> + Send + Sync + 'a>>;
+fn boxed<'a, T>(stream: impl Stream<Item = T> + Send + Sync + 'a) -> BoxStream<'a, T> {
+    Box::pin(stream)
+}
+type HttpBodyReader = StreamReader<BoxStream<'static, Result<Bytes, std::io::Error>>, Bytes>;
+
+#[pin_project::pin_project(project = HttpReaderProj, PinnedDrop)]
+pub enum HttpReader {
+    Range(#[pin] Take<HttpBodyReader>),
+    Rangeless {
+        pool: Arc<Mutex<BTreeSet<TrackingIO<HttpBodyReader>>>>,
+        #[pin]
+        reader: Option<Take<TrackingIO<HttpBodyReader>>>,
+    },
+}
+impl AsyncRead for HttpReader {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        match self.project() {
+            HttpReaderProj::Range(r) => r.poll_read(cx, buf),
+            HttpReaderProj::Rangeless { mut reader, .. } => {
+                let mut finished = false;
+                if let Some(reader) = reader.as_mut().as_pin_mut() {
+                    let start = buf.filled().len();
+                    futures::ready!(reader.poll_read(cx, buf)?);
+                    finished = start == buf.filled().len();
+                }
+                if finished {
+                    reader.take();
+                }
+                Poll::Ready(Ok(()))
+            }
+        }
+    }
+}
+#[pin_project::pinned_drop]
+impl PinnedDrop for HttpReader {
+    fn drop(self: Pin<&mut Self>) {
+        match self.project() {
+            HttpReaderProj::Range(_) => (),
+            HttpReaderProj::Rangeless { pool, mut reader } => {
+                if let Some(reader) = reader.take() {
+                    pool.lock().unwrap().insert(reader.into_inner());
+                }
+            }
+        }
+    }
+}
+
+// type RangelessReader = StreamReader<BoxStream<'static, Bytes>, Bytes>;

core/src/s9pk/merkle_archive/source/mod.rs

@@ -0,0 +1,429 @@
+use std::cmp::min;
+use std::io::SeekFrom;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use blake3::Hash;
+use futures::future::BoxFuture;
+use futures::{Future, FutureExt};
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, Take};
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::hash::VerifyingWriter;
+use crate::util::io::{TmpDir, open_file};
+
+pub mod http;
+pub mod multi_cursor_file;
+
+pub trait FileSource: Send + Sync + Sized + 'static {
+    type Reader: AsyncRead + Unpin + Send;
+    type SliceReader: AsyncRead + Unpin + Send;
+    fn size(&self) -> impl Future<Output = Result<u64, Error>> + Send;
+    fn reader(&self) -> impl Future<Output = Result<Self::Reader, Error>> + Send;
+    fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> impl Future<Output = Result<Self::SliceReader, Error>> + Send;
+    fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+    ) -> impl Future<Output = Result<(), Error>> + Send {
+        async move {
+            tokio::io::copy(&mut self.reader().await?, w).await?;
+            Ok(())
+        }
+    }
+    fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+        verify: Option<(Hash, u64)>,
+    ) -> impl Future<Output = Result<(), Error>> + Send {
+        async move {
+            let mut w = VerifyingWriter::new(w, verify);
+            tokio::io::copy(&mut self.reader().await?, &mut w).await?;
+            w.verify().await?;
+            Ok(())
+        }
+    }
+    fn to_vec(
+        &self,
+        verify: Option<(Hash, u64)>,
+    ) -> impl Future<Output = Result<Vec<u8>, Error>> + Send {
+        fn to_vec(
+            src: &impl FileSource,
+            verify: Option<(Hash, u64)>,
+        ) -> BoxFuture<'_, Result<Vec<u8>, Error>> {
+            async move {
+                let mut vec = Vec::with_capacity(if let Some((_, size)) = &verify {
+                    *size
+                } else {
+                    src.size().await?
+                } as usize);
+                src.copy_verify(&mut vec, verify).await?;
+                Ok(vec)
+            }
+            .boxed()
+        }
+        to_vec(self, verify)
+    }
+}
+
+impl<T: FileSource> FileSource for Arc<T> {
+    type Reader = T::Reader;
+    type SliceReader = T::SliceReader;
+    async fn size(&self) -> Result<u64, Error> {
+        self.deref().size().await
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        self.deref().reader().await
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.deref().slice(position, size).await
+    }
+    async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
+        self.deref().copy(w).await
+    }
+    async fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error> {
+        self.deref().copy_verify(w, verify).await
+    }
+    async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
+        self.deref().to_vec(verify).await
+    }
+}
+
+#[derive(Clone)]
+pub struct DynFileSource(Arc<dyn DynableFileSource>);
+impl DynFileSource {
+    pub fn new<T: FileSource>(source: T) -> Self {
+        Self(Arc::new(source))
+    }
+}
+impl FileSource for DynFileSource {
+    type Reader = Box<dyn AsyncRead + Unpin + Send>;
+    type SliceReader = Box<dyn AsyncRead + Unpin + Send>;
+    async fn size(&self) -> Result<u64, Error> {
+        self.0.size().await
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        self.0.reader().await
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.0.slice(position, size).await
+    }
+    async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        mut w: &mut W,
+    ) -> Result<(), Error> {
+        self.0.copy(&mut w).await
+    }
+    async fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        mut w: &mut W,
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error> {
+        self.0.copy_verify(&mut w, verify).await
+    }
+    async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
+        self.0.to_vec(verify).await
+    }
+}
+
+#[async_trait::async_trait]
+trait DynableFileSource: Send + Sync + 'static {
+    async fn size(&self) -> Result<u64, Error>;
+    async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
+    async fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error>;
+    async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error>;
+    async fn copy_verify(
+        &self,
+        w: &mut (dyn AsyncWrite + Unpin + Send),
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error>;
+    async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error>;
+}
+#[async_trait::async_trait]
+impl<T: FileSource> DynableFileSource for T {
+    async fn size(&self) -> Result<u64, Error> {
+        FileSource::size(self).await
+    }
+    async fn reader(&self) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
+        Ok(Box::new(FileSource::reader(self).await?))
+    }
+    async fn slice(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> Result<Box<dyn AsyncRead + Unpin + Send>, Error> {
+        Ok(Box::new(FileSource::slice(self, position, size).await?))
+    }
+    async fn copy(&self, w: &mut (dyn AsyncWrite + Unpin + Send)) -> Result<(), Error> {
+        FileSource::copy(self, w).await
+    }
+    async fn copy_verify(
+        &self,
+        w: &mut (dyn AsyncWrite + Unpin + Send),
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error> {
+        FileSource::copy_verify(self, w, verify).await
+    }
+    async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
+        FileSource::to_vec(self, verify).await
+    }
+}
+
+impl FileSource for PathBuf {
+    type Reader = File;
+    type SliceReader = Take<Self::Reader>;
+    async fn size(&self) -> Result<u64, Error> {
+        Ok(tokio::fs::metadata(self).await?.len())
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        Ok(open_file(self).await?)
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        let mut r = FileSource::reader(self).await?;
+        r.seek(SeekFrom::Start(position)).await?;
+        Ok(r.take(size))
+    }
+}
+
+impl FileSource for Arc<[u8]> {
+    type Reader = std::io::Cursor<Self>;
+    type SliceReader = Take<Self::Reader>;
+    async fn size(&self) -> Result<u64, Error> {
+        Ok(self.len() as u64)
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        Ok(std::io::Cursor::new(self.clone()))
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        let mut r = FileSource::reader(self).await?;
+        r.seek(SeekFrom::Start(position)).await?;
+        Ok(r.take(size))
+    }
+    async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
+        use tokio::io::AsyncWriteExt;
+
+        w.write_all(&*self).await?;
+        Ok(())
+    }
+}
+
+pub trait ArchiveSource: Send + Sync + Sized + 'static {
+    type FetchReader: AsyncRead + Unpin + Send;
+    type FetchAllReader: AsyncRead + Unpin + Send;
+    fn size(&self) -> impl Future<Output = Option<u64>> + Send {
+        async { None }
+    }
+    fn fetch_all(&self) -> impl Future<Output = Result<Self::FetchAllReader, Error>> + Send;
+    fn fetch(
+        &self,
+        position: u64,
+        size: u64,
+    ) -> impl Future<Output = Result<Self::FetchReader, Error>> + Send;
+    fn copy_all_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+    ) -> impl Future<Output = Result<(), Error>> + Send {
+        async move {
+            tokio::io::copy(&mut self.fetch_all().await?, w).await?;
+            Ok(())
+        }
+    }
+    fn copy_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        position: u64,
+        size: u64,
+        w: &mut W,
+    ) -> impl Future<Output = Result<(), Error>> + Send {
+        async move {
+            tokio::io::copy(&mut self.fetch(position, size).await?, w).await?;
+            Ok(())
+        }
+    }
+    fn section(self, position: u64, size: u64) -> Section<Self> {
+        Section {
+            source: self,
+            position,
+            size,
+        }
+    }
+}
+
+impl<T: ArchiveSource> ArchiveSource for Arc<T> {
+    type FetchReader = T::FetchReader;
+    type FetchAllReader = T::FetchAllReader;
+    async fn size(&self) -> Option<u64> {
+        self.deref().size().await
+    }
+    async fn fetch_all(&self) -> Result<Self::FetchAllReader, Error> {
+        self.deref().fetch_all().await
+    }
+    async fn fetch(&self, position: u64, size: u64) -> Result<Self::FetchReader, Error> {
+        self.deref().fetch(position, size).await
+    }
+    async fn copy_all_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+    ) -> Result<(), Error> {
+        self.deref().copy_all_to(w).await
+    }
+    async fn copy_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        position: u64,
+        size: u64,
+        w: &mut W,
+    ) -> Result<(), Error> {
+        self.deref().copy_to(position, size, w).await
+    }
+}
+
+impl ArchiveSource for Arc<[u8]> {
+    type FetchReader = tokio::io::Take<std::io::Cursor<Self>>;
+    type FetchAllReader = std::io::Cursor<Self>;
+    async fn fetch_all(&self) -> Result<Self::FetchAllReader, Error> {
+        Ok(std::io::Cursor::new(self.clone()))
+    }
+    async fn fetch(&self, position: u64, size: u64) -> Result<Self::FetchReader, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut cur = std::io::Cursor::new(self.clone());
+        cur.set_position(position);
+        Ok(cur.take(size))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Section<S> {
+    source: S,
+    position: u64,
+    size: u64,
+}
+impl<S> Section<S> {
+    pub fn source(&self) -> &S {
+        &self.source
+    }
+    pub fn null(source: S) -> Self {
+        Self {
+            source,
+            position: 0,
+            size: 0,
+        }
+    }
+}
+impl<S: ArchiveSource> FileSource for Section<S> {
+    type Reader = S::FetchReader;
+    type SliceReader = S::FetchReader;
+    async fn size(&self) -> Result<u64, Error> {
+        Ok(self.size)
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        self.source.fetch(self.position, self.size).await
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.source
+            .fetch(self.position + position, min(size, self.size))
+            .await
+    }
+    async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
+        self.source.copy_to(self.position, self.size, w).await
+    }
+}
+
+pub type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
+pub fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
+    Box::new(r)
+}
+
+#[derive(Clone)]
+pub struct TmpSource<S> {
+    tmp_dir: Arc<TmpDir>,
+    source: S,
+}
+impl<S> TmpSource<S> {
+    pub fn new(tmp_dir: Arc<TmpDir>, source: S) -> Self {
+        Self { tmp_dir, source }
+    }
+    pub async fn gc(self) -> Result<(), Error> {
+        self.tmp_dir.gc().await
+    }
+}
+impl<S> std::ops::Deref for TmpSource<S> {
+    type Target = S;
+    fn deref(&self) -> &Self::Target {
+        &self.source
+    }
+}
+impl<S: ArchiveSource> ArchiveSource for TmpSource<S> {
+    type FetchReader = <S as ArchiveSource>::FetchReader;
+    type FetchAllReader = <S as ArchiveSource>::FetchAllReader;
+    async fn size(&self) -> Option<u64> {
+        self.source.size().await
+    }
+    async fn fetch_all(&self) -> Result<Self::FetchAllReader, Error> {
+        self.source.fetch_all().await
+    }
+    async fn fetch(&self, position: u64, size: u64) -> Result<Self::FetchReader, Error> {
+        self.source.fetch(position, size).await
+    }
+    async fn copy_all_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        w: &mut W,
+    ) -> Result<(), Error> {
+        self.source.copy_all_to(w).await
+    }
+    async fn copy_to<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        position: u64,
+        size: u64,
+        w: &mut W,
+    ) -> Result<(), Error> {
+        self.source.copy_to(position, size, w).await
+    }
+}
+impl<S: FileSource> From<TmpSource<S>> for DynFileSource {
+    fn from(value: TmpSource<S>) -> Self {
+        DynFileSource::new(value)
+    }
+}
+
+impl<S: FileSource> FileSource for TmpSource<S> {
+    type Reader = <S as FileSource>::Reader;
+    type SliceReader = <S as FileSource>::SliceReader;
+    async fn size(&self) -> Result<u64, Error> {
+        self.source.size().await
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        self.source.reader().await
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        self.source.slice(position, size).await
+    }
+    async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        mut w: &mut W,
+    ) -> Result<(), Error> {
+        self.source.copy(&mut w).await
+    }
+    async fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
+        &self,
+        mut w: &mut W,
+        verify: Option<(Hash, u64)>,
+    ) -> Result<(), Error> {
+        self.source.copy_verify(&mut w, verify).await
+    }
+    async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
+        self.source.to_vec(verify).await
+    }
+}

core/src/s9pk/merkle_archive/source/multi_cursor_file.rs

@@ -0,0 +1,146 @@
+use std::io::SeekFrom;
+use std::os::fd::{AsRawFd, RawFd};
+use std::path::{Path, PathBuf};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::Poll;
+
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, ReadBuf, Take};
+use tokio::sync::{Mutex, OwnedMutexGuard};
+
+use crate::disk::mount::filesystem::loop_dev::LoopDev;
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::source::{ArchiveSource, Section};
+use crate::util::io::open_file;
+
+fn path_from_fd(fd: RawFd) -> Result<PathBuf, Error> {
+    #[cfg(target_os = "linux")]
+    let path = Path::new("/proc/self/fd").join(fd.to_string());
+    #[cfg(target_os = "macos")] // here be dragons
+    let path = unsafe {
+        let mut buf = [0u8; libc::PATH_MAX as usize];
+        if libc::fcntl(fd, libc::F_GETPATH, buf.as_mut_ptr().cast::<libc::c_char>()) == -1 {
+            return Err(std::io::Error::last_os_error().into());
+        }
+        Path::new(
+            &*std::ffi::CStr::from_bytes_until_nul(&buf)
+                .with_kind(ErrorKind::Utf8)?
+                .to_string_lossy(),
+        )
+        .to_owned()
+    };
+    Ok(path)
+}
+
+#[derive(Clone)]
+pub struct MultiCursorFile {
+    fd: RawFd,
+    file: Arc<Mutex<File>>,
+}
+impl MultiCursorFile {
+    pub fn path(&self) -> Result<PathBuf, Error> {
+        path_from_fd(self.fd)
+    }
+    pub async fn open(fd: &impl AsRawFd) -> Result<Self, Error> {
+        let f = open_file(path_from_fd(fd.as_raw_fd())?).await?;
+        Ok(Self::from(f))
+    }
+    pub async fn cursor(&self) -> Result<FileCursor, Error> {
+        Ok(FileCursor(
+            if let Ok(file) = self.file.clone().try_lock_owned() {
+                file
+            } else {
+                Arc::new(Mutex::new(open_file(self.path()?).await?))
+                    .try_lock_owned()
+                    .expect("freshly created")
+            },
+        ))
+    }
+    pub async fn blake3_mmap(&self) -> Result<blake3::Hash, Error> {
+        let path = self.path()?;
+        tokio::task::spawn_blocking(move || {
+            let mut hasher = blake3::Hasher::new();
+            hasher.update_mmap_rayon(path)?;
+            Ok(hasher.finalize())
+        })
+        .await
+        .with_kind(ErrorKind::Unknown)?
+    }
+}
+impl From<File> for MultiCursorFile {
+    fn from(value: File) -> Self {
+        Self {
+            fd: value.as_raw_fd(),
+            file: Arc::new(Mutex::new(value)),
+        }
+    }
+}
+
+#[pin_project::pin_project]
+pub struct FileCursor(#[pin] OwnedMutexGuard<File>);
+impl AsyncRead for FileCursor {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let this = self.project();
+        Pin::new(&mut (&mut **this.0.get_mut())).poll_read(cx, buf)
+    }
+}
+impl AsyncSeek for FileCursor {
+    fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> {
+        let this = self.project();
+        Pin::new(&mut (&mut **this.0.get_mut())).start_seek(position)
+    }
+    fn poll_complete(
+        self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<std::io::Result<u64>> {
+        let this = self.project();
+        Pin::new(&mut (&mut **this.0.get_mut())).poll_complete(cx)
+    }
+}
+impl std::ops::Deref for FileCursor {
+    type Target = File;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl std::ops::DerefMut for FileCursor {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut *self.0
+    }
+}
+
+impl ArchiveSource for MultiCursorFile {
+    type FetchReader = Take<FileCursor>;
+    type FetchAllReader = FileCursor;
+    async fn size(&self) -> Option<u64> {
+        tokio::fs::metadata(self.path().ok()?)
+            .await
+            .ok()
+            .map(|m| m.len())
+    }
+    async fn fetch_all(&self) -> Result<Self::FetchAllReader, Error> {
+        use tokio::io::AsyncSeekExt;
+
+        let mut file = self.cursor().await?;
+        file.0.seek(SeekFrom::Start(0)).await?;
+        Ok(file)
+    }
+    async fn fetch(&self, position: u64, size: u64) -> Result<Self::FetchReader, Error> {
+        use tokio::io::AsyncSeekExt;
+
+        let mut file = self.cursor().await?;
+        file.0.seek(SeekFrom::Start(position)).await?;
+        Ok(file.take(size))
+    }
+}
+
+impl From<&Section<MultiCursorFile>> for LoopDev<PathBuf> {
+    fn from(value: &Section<MultiCursorFile>) -> Self {
+        LoopDev::new(value.source.path().unwrap(), value.position, value.size)
+    }
+}

core/src/s9pk/merkle_archive/test.rs

@@ -0,0 +1,142 @@
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use ed25519_dalek::SigningKey;
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::file_contents::FileContents;
+use crate::s9pk::merkle_archive::source::FileSource;
+use crate::s9pk::merkle_archive::{Entry, EntryContents, MerkleArchive};
+use crate::util::io::TrackingIO;
+
+/// Creates a MerkleArchive (a1) with the provided files at the provided paths. NOTE: later files can overwrite previous files/directories at the same path
+/// Tests:
+///     - a1.update_hashes(): returns Ok(_)
+///     - a1.serialize(verify: true): returns Ok(s1)
+///     - MerkleArchive::deserialize(s1): returns Ok(a2)
+///     - a2: contains all expected files with expected content
+///     - a2.serialize(verify: true): returns Ok(s2)
+///     - s1 == s2
+#[instrument]
+fn test(files: Vec<(PathBuf, String)>) -> Result<(), Error> {
+    let mut root = DirectoryContents::<Arc<[u8]>>::new();
+    let mut check_set = BTreeMap::<PathBuf, String>::new();
+    for (path, content) in files {
+        if let Err(e) = root.insert_path(
+            &path,
+            Entry::new(EntryContents::File(FileContents::new(
+                content.clone().into_bytes().into(),
+            ))),
+        ) {
+            eprintln!("failed to insert file at {path:?}: {e}");
+        } else {
+            let path = path.strip_prefix("/").unwrap_or(&path);
+            let mut remaining = check_set.split_off(path);
+            while {
+                if let Some((p, s)) = remaining.pop_first() {
+                    if !p.starts_with(path) {
+                        remaining.insert(p, s);
+                        false
+                    } else {
+                        true
+                    }
+                } else {
+                    false
+                }
+            } {}
+            check_set.append(&mut remaining);
+            check_set.insert(path.to_owned(), content);
+        }
+    }
+    let key = SigningKey::generate(&mut ssh_key::rand_core::OsRng::default());
+    let mut a1 = MerkleArchive::new(root, key, "test");
+    tokio::runtime::Builder::new_current_thread()
+        .enable_io()
+        .build()
+        .unwrap()
+        .block_on(async move {
+            a1.update_hashes(true).await?;
+            let mut s1 = Vec::new();
+            a1.serialize(&mut TrackingIO::new(0, &mut s1), true).await?;
+            let s1: Arc<[u8]> = s1.into();
+            let a2 = MerkleArchive::deserialize(
+                &s1,
+                "test",
+                &mut Cursor::new(s1.clone()),
+                Some(&a1.commitment().await?),
+            )
+            .await?;
+
+            for (path, content) in check_set {
+                match a2
+                    .contents
+                    .get_path(&path)
+                    .map(|e| (e.as_contents(), e.hash()))
+                {
+                    Some((EntryContents::File(f), hash)) => {
+                        ensure_code!(
+                            &f.to_vec(hash).await? == content.as_bytes(),
+                            ErrorKind::ParseS9pk,
+                            "File at {path:?} does not match input"
+                        )
+                    }
+                    _ => {
+                        return Err(Error::new(
+                            eyre!("expected file at {path:?}"),
+                            ErrorKind::ParseS9pk,
+                        ));
+                    }
+                }
+            }
+
+            let mut s2 = Vec::new();
+            a2.serialize(&mut TrackingIO::new(0, &mut s2), true).await?;
+            let s2: Arc<[u8]> = s2.into();
+            ensure_code!(s1 == s2, ErrorKind::Pack, "s1 does not match s2");
+
+            Ok(())
+        })
+}
+
+proptest::proptest! {
+    #[test]
+    fn property_test(files: Vec<(PathBuf, String)>) {
+        let files: Vec<(PathBuf, String)> = files.into_iter().filter(|(p, _)| p.file_name().is_some() && p.iter().all(|s| s.to_str().is_some())).collect();
+        if let Err(e) = test(files.clone()) {
+            panic!("{e}\nInput: {files:#?}\n{e:?}");
+        }
+    }
+}
+
+#[test]
+fn test_example_1() {
+    if let Err(e) = test(vec![(Path::new("foo").into(), "bar".into())]) {
+        panic!("{e}\n{e:?}");
+    }
+}
+
+#[test]
+fn test_example_2() {
+    if let Err(e) = test(vec![
+        (Path::new("a/a.txt").into(), "a.txt".into()),
+        (Path::new("a/b/a.txt").into(), "a.txt".into()),
+        (Path::new("a/b/b/a.txt").into(), "a.txt".into()),
+        (Path::new("a/b/c.txt").into(), "c.txt".into()),
+        (Path::new("a/c.txt").into(), "c.txt".into()),
+    ]) {
+        panic!("{e}\n{e:?}");
+    }
+}
+
+#[test]
+fn test_example_3() {
+    if let Err(e) = test(vec![
+        (Path::new("b/a").into(), "𑦪".into()),
+        (Path::new("a/c/a").into(), "·".into()),
+    ]) {
+        panic!("{e}\n{e:?}");
+    }
+}

core/src/s9pk/merkle_archive/varint.rs

@@ -0,0 +1,157 @@
+use integer_encoding::VarInt;
+use tokio::io::{AsyncRead, AsyncWrite};
+
+use crate::prelude::*;
+
+/// Most-significant bit, == 0x80
+pub const MSB: u8 = 0b1000_0000;
+
+const MAX_STR_LEN: u64 = 1024 * 1024; // 1 MiB
+
+pub fn serialized_varint_size(n: u64) -> u64 {
+    VarInt::required_space(n) as u64
+}
+
+pub async fn serialize_varint<W: AsyncWrite + Unpin + Send>(
+    n: u64,
+    w: &mut W,
+) -> Result<(), Error> {
+    use tokio::io::AsyncWriteExt;
+
+    let mut buf = [0 as u8; 10];
+    let b = n.encode_var(&mut buf);
+    w.write_all(&buf[0..b]).await?;
+
+    Ok(())
+}
+
+pub fn serialized_varstring_size(s: &str) -> u64 {
+    serialized_varint_size(s.len() as u64) + s.len() as u64
+}
+
+pub async fn serialize_varstring<W: AsyncWrite + Unpin + Send>(
+    s: &str,
+    w: &mut W,
+) -> Result<(), Error> {
+    use tokio::io::AsyncWriteExt;
+    serialize_varint(s.len() as u64, w).await?;
+    w.write_all(s.as_bytes()).await?;
+    Ok(())
+}
+
+const MAX_SIZE: usize = (std::mem::size_of::<u64>() * 8 + 7) / 7;
+
+#[derive(Default)]
+struct VarIntProcessor {
+    buf: [u8; MAX_SIZE],
+    i: usize,
+}
+
+impl VarIntProcessor {
+    fn new() -> VarIntProcessor {
+        Self::default()
+    }
+    fn push(&mut self, b: u8) -> Result<(), Error> {
+        if self.i >= MAX_SIZE {
+            return Err(Error::new(
+                eyre!("Unterminated varint"),
+                ErrorKind::ParseS9pk,
+            ));
+        }
+        self.buf[self.i] = b;
+        self.i += 1;
+        Ok(())
+    }
+    fn finished(&self) -> bool {
+        self.i > 0 && (self.buf[self.i - 1] & MSB == 0)
+    }
+    fn decode(&self) -> Option<u64> {
+        Some(u64::decode_var(&self.buf[0..self.i])?.0)
+    }
+}
+
+pub async fn deserialize_varint<R: AsyncRead + Unpin>(r: &mut R) -> Result<u64, Error> {
+    use tokio::io::AsyncReadExt;
+
+    let mut buf = [0 as u8; 1];
+    let mut p = VarIntProcessor::new();
+
+    while !p.finished() {
+        r.read_exact(&mut buf).await?;
+
+        p.push(buf[0])?;
+    }
+
+    p.decode()
+        .ok_or_else(|| Error::new(eyre!("Reached EOF"), ErrorKind::ParseS9pk))
+}
+
+pub async fn deserialize_varstring<R: AsyncRead + Unpin>(r: &mut R) -> Result<String, Error> {
+    use tokio::io::AsyncReadExt;
+
+    let len = std::cmp::min(deserialize_varint(r).await?, MAX_STR_LEN);
+    let mut res = String::with_capacity(len as usize);
+    r.take(len).read_to_string(&mut res).await?;
+    Ok(res)
+}
+
+#[cfg(test)]
+mod test {
+    use std::io::Cursor;
+
+    use crate::prelude::*;
+
+    fn test_int(n: u64) -> Result<(), Error> {
+        let n1 = n;
+        tokio::runtime::Builder::new_current_thread()
+            .enable_io()
+            .build()
+            .unwrap()
+            .block_on(async move {
+                let mut v = Vec::new();
+                super::serialize_varint(n1, &mut v).await?;
+                let n2 = super::deserialize_varint(&mut Cursor::new(v)).await?;
+
+                ensure_code!(n1 == n2, ErrorKind::Deserialization, "n1 does not match n2");
+
+                Ok(())
+            })
+    }
+
+    fn test_string(s: &str) -> Result<(), Error> {
+        let s1 = s;
+        tokio::runtime::Builder::new_current_thread()
+            .enable_io()
+            .build()
+            .unwrap()
+            .block_on(async move {
+                let mut v: Vec<u8> = Vec::new();
+                super::serialize_varstring(&s1, &mut v).await?;
+                let s2 = super::deserialize_varstring(&mut Cursor::new(v)).await?;
+
+                ensure_code!(
+                    s1 == &s2,
+                    ErrorKind::Deserialization,
+                    "s1 does not match s2"
+                );
+
+                Ok(())
+            })
+    }
+
+    proptest::proptest! {
+        #[test]
+        fn proptest_int(n: u64) {
+            if let Err(e) = test_int(n) {
+                panic!("{e}\nInput: {n}\n{e:?}");
+            }
+        }
+
+        #[test]
+        fn proptest_string(s: String) {
+            if let Err(e) = test_string(&s) {
+                panic!("{e}\nInput: {s:?}\n{e:?}");
+            }
+        }
+    }
+}

core/src/s9pk/merkle_archive/write_queue.rs

@@ -0,0 +1,48 @@
+use std::collections::VecDeque;
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::sink::Sink;
+use crate::s9pk::merkle_archive::source::FileSource;
+use crate::s9pk::merkle_archive::{Entry, EntryContents};
+
+pub struct WriteQueue<'a, S> {
+    next_available_position: u64,
+    queue: VecDeque<&'a Entry<S>>,
+}
+
+impl<'a, S> WriteQueue<'a, S> {
+    pub fn new(next_available_position: u64) -> Self {
+        Self {
+            next_available_position,
+            queue: VecDeque::new(),
+        }
+    }
+}
+impl<'a, S: FileSource> WriteQueue<'a, S> {
+    pub async fn add(&mut self, entry: &'a Entry<S>) -> Result<u64, Error> {
+        let res = self.next_available_position;
+        let size = match entry.as_contents() {
+            EntryContents::Missing => return Ok(0),
+            EntryContents::File(f) => f.size().await?,
+            EntryContents::Directory(d) => d.toc_size(),
+        };
+        self.next_available_position += size;
+        self.queue.push_back(entry);
+        Ok(res)
+    }
+}
+impl<'a, S: FileSource + Clone> WriteQueue<'a, S> {
+    pub async fn serialize<W: Sink>(&mut self, w: &mut W, verify: bool) -> Result<(), Error> {
+        loop {
+            let Some(next) = self.queue.pop_front() else {
+                break;
+            };
+            match next.as_contents() {
+                EntryContents::Missing => (),
+                EntryContents::File(f) => f.serialize_body(w, next.hash.filter(|_| verify)).await?,
+                EntryContents::Directory(d) => d.serialize_toc(self, w).await?,
+            }
+        }
+        Ok(())
+    }
+}

core/src/s9pk/mod.rs

@@ -0,0 +1,60 @@
+pub mod git_hash;
+pub mod merkle_archive;
+pub mod rpc;
+pub mod v1;
+pub mod v2;
+
+use std::sync::Arc;
+
+use tokio::io::{AsyncReadExt, AsyncSeek};
+pub use v2::{S9pk, manifest};
+
+use crate::prelude::*;
+use crate::progress::FullProgressTracker;
+use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource};
+use crate::s9pk::v1::reader::S9pkReader;
+use crate::s9pk::v2::compat::MAGIC_AND_VERSION;
+use crate::util::io::TmpDir;
+
+pub async fn load<S, K>(
+    source: S,
+    key: K,
+    progress: Option<&FullProgressTracker>,
+) -> Result<S9pk<DynFileSource>, Error>
+where
+    S: ArchiveSource,
+    S::FetchAllReader: AsyncSeek + Sync,
+    K: FnOnce() -> Result<ed25519_dalek::SigningKey, Error>,
+{
+    // TODO: return s9pk
+    const MAGIC_LEN: usize = MAGIC_AND_VERSION.len();
+    let mut magic = [0_u8; MAGIC_LEN];
+    source.fetch(0, 3).await?.read_exact(&mut magic).await?;
+    if magic == v2::compat::MAGIC_AND_VERSION {
+        let phase = if let Some(progress) = progress {
+            let mut phase = progress.add_phase(
+                "Converting Package to V2".into(),
+                Some(source.size().await.unwrap_or(60)),
+            );
+            phase.start();
+            Some(phase)
+        } else {
+            None
+        };
+        tracing::info!("Converting package to v2 s9pk");
+        let tmp_dir = TmpDir::new().await?;
+        let s9pk = S9pk::from_v1(
+            S9pkReader::from_reader(source.fetch_all().await?, true).await?,
+            Arc::new(tmp_dir),
+            key()?,
+        )
+        .await?;
+        tracing::info!("Converted s9pk successfully");
+        if let Some(mut phase) = phase {
+            phase.complete();
+        }
+        Ok(s9pk.into_dyn())
+    } else {
+        Ok(S9pk::deserialize(&Arc::new(source), None).await?.into_dyn())
+    }
+}

core/src/s9pk/rpc.rs

@@ -0,0 +1,254 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use clap::Parser;
+use rpc_toolkit::{Empty, HandlerExt, ParentHandler, from_fn_async};
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+use crate::ImageId;
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::s9pk::manifest::Manifest;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::s9pk::v2::SIG_CONTEXT;
+use crate::s9pk::v2::pack::ImageConfig;
+use crate::util::Apply;
+use crate::util::io::{TmpDir, create_file, open_file};
+use crate::util::serde::{HandlerExtSerde, apply_expr};
+
+pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
+
+pub fn s9pk() -> ParentHandler<CliContext> {
+    ParentHandler::new()
+        .subcommand(
+            "pack",
+            from_fn_async(super::v2::pack::pack)
+                .no_display()
+                .with_about("Package s9pk input files into valid s9pk"),
+        )
+        .subcommand(
+            "list-ingredients",
+            from_fn_async(super::v2::pack::list_ingredients)
+                .with_custom_display_fn(|_, ingredients| {
+                    ingredients
+                        .into_iter()
+                        .map(Some)
+                        .apply(|i| itertools::intersperse(i, None))
+                        .for_each(|i| {
+                            if let Some(p) = i {
+                                print!("{}", p.display())
+                            } else {
+                                print!(" ")
+                            }
+                        });
+                    println!();
+                    Ok(())
+                })
+                .with_about("List paths of package ingredients"),
+        )
+        .subcommand(
+            "edit",
+            edit().with_about("Commands to add an image to an s9pk or edit the manifest"),
+        )
+        .subcommand(
+            "inspect",
+            inspect().with_about("Commands to display file paths, file contents, or manifest"),
+        )
+        .subcommand(
+            "convert",
+            from_fn_async(convert)
+                .no_display()
+                .with_about("Convert s9pk from v1 to v2"),
+        )
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+struct S9pkPath {
+    s9pk: PathBuf,
+}
+
+fn edit() -> ParentHandler<CliContext, S9pkPath> {
+    let only_parent = |a, _| a;
+    ParentHandler::new()
+        .subcommand(
+            "add-image",
+            from_fn_async(add_image)
+                .with_inherited(only_parent)
+                .no_display()
+                .with_about("Add image to s9pk"),
+        )
+        .subcommand(
+            "manifest",
+            from_fn_async(edit_manifest)
+                .with_inherited(only_parent)
+                .with_display_serializable()
+                .with_about("Edit s9pk manifest"),
+        )
+}
+
+fn inspect() -> ParentHandler<CliContext, S9pkPath> {
+    let only_parent = |a, _| a;
+    ParentHandler::new()
+        .subcommand(
+            "file-tree",
+            from_fn_async(file_tree)
+                .with_inherited(only_parent)
+                .with_display_serializable()
+                .with_about("Display list of paths"),
+        )
+        .subcommand(
+            "cat",
+            from_fn_async(cat)
+                .with_inherited(only_parent)
+                .no_display()
+                .with_about("Display file contents"),
+        )
+        .subcommand(
+            "manifest",
+            from_fn_async(inspect_manifest)
+                .with_inherited(only_parent)
+                .with_display_serializable()
+                .with_about("Display s9pk manifest"),
+        )
+}
+
+#[derive(Deserialize, Serialize, Parser, TS)]
+struct AddImageParams {
+    id: ImageId,
+    #[command(flatten)]
+    config: ImageConfig,
+}
+async fn add_image(
+    ctx: CliContext,
+    AddImageParams { id, config }: AddImageParams,
+    S9pkPath { s9pk: s9pk_path }: S9pkPath,
+) -> Result<(), Error> {
+    let mut s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    s9pk.as_manifest_mut().images.insert(id, config);
+    let tmp_dir = Arc::new(TmpDir::new().await?);
+    s9pk.load_images(tmp_dir.clone()).await?;
+    s9pk.validate_and_filter(None)?;
+    let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+    let mut tmp_file = create_file(&tmp_path).await?;
+    s9pk.serialize(&mut tmp_file, true).await?;
+    drop(s9pk);
+    tmp_file.sync_all().await?;
+    tokio::fs::rename(&tmp_path, &s9pk_path).await?;
+
+    tmp_dir.gc().await?;
+
+    Ok(())
+}
+
+#[derive(Deserialize, Serialize, Parser, TS)]
+struct EditManifestParams {
+    expression: String,
+}
+async fn edit_manifest(
+    ctx: CliContext,
+    EditManifestParams { expression }: EditManifestParams,
+    S9pkPath { s9pk: s9pk_path }: S9pkPath,
+) -> Result<Manifest, Error> {
+    let mut s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    let old = serde_json::to_value(s9pk.as_manifest()).with_kind(ErrorKind::Serialization)?;
+    *s9pk.as_manifest_mut() = serde_json::from_value(apply_expr(old.into(), &expression)?.into())
+        .with_kind(ErrorKind::Serialization)?;
+    let manifest = s9pk.as_manifest().clone();
+    let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+    let mut tmp_file = create_file(&tmp_path).await?;
+    s9pk.as_archive_mut()
+        .set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
+    s9pk.serialize(&mut tmp_file, true).await?;
+    tmp_file.sync_all().await?;
+    tokio::fs::rename(&tmp_path, &s9pk_path).await?;
+
+    Ok(manifest)
+}
+
+async fn file_tree(
+    ctx: CliContext,
+    _: Empty,
+    S9pkPath { s9pk: s9pk_path }: S9pkPath,
+) -> Result<Vec<PathBuf>, Error> {
+    let s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    Ok(s9pk.as_archive().contents().file_paths(""))
+}
+
+#[derive(Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[command(rename_all = "kebab-case")]
+struct CatParams {
+    file_path: PathBuf,
+}
+async fn cat(
+    ctx: CliContext,
+    CatParams { file_path }: CatParams,
+    S9pkPath { s9pk: s9pk_path }: S9pkPath,
+) -> Result<(), Error> {
+    use crate::s9pk::merkle_archive::source::FileSource;
+
+    let s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    tokio::io::copy(
+        &mut s9pk
+            .as_archive()
+            .contents()
+            .get_path(&file_path)
+            .or_not_found(&file_path.display())?
+            .as_file()
+            .or_not_found(&file_path.display())?
+            .reader()
+            .await?,
+        &mut tokio::io::stdout(),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn inspect_manifest(
+    ctx: CliContext,
+    _: Empty,
+    S9pkPath { s9pk: s9pk_path }: S9pkPath,
+) -> Result<Manifest, Error> {
+    let s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    Ok(s9pk.as_manifest().clone())
+}
+
+async fn convert(ctx: CliContext, S9pkPath { s9pk: s9pk_path }: S9pkPath) -> Result<(), Error> {
+    let mut s9pk = super::load(
+        MultiCursorFile::from(open_file(&s9pk_path).await?),
+        || ctx.developer_key().cloned(),
+        None,
+    )
+    .await?;
+    let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+    s9pk.serialize(&mut create_file(&tmp_path).await?, true)
+        .await?;
+    tokio::fs::rename(tmp_path, s9pk_path).await?;
+    Ok(())
+}

core/src/s9pk/v1/builder.rs

@@ -0,0 +1,145 @@
+use sha2::{Digest, Sha512};
+use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom};
+use tracing::instrument;
+use typed_builder::TypedBuilder;
+
+use super::SIG_CONTEXT;
+use super::header::{FileSection, Header};
+use super::manifest::Manifest;
+use crate::util::HashWriter;
+use crate::util::io::to_cbor_async_writer;
+use crate::{Error, ResultExt};
+
+#[derive(TypedBuilder)]
+pub struct S9pkPacker<
+    'a,
+    W: AsyncWriteExt + AsyncSeekExt,
+    RLicense: AsyncReadExt + Unpin,
+    RInstructions: AsyncReadExt + Unpin,
+    RIcon: AsyncReadExt + Unpin,
+    RDockerImages: AsyncReadExt + Unpin,
+    RAssets: AsyncReadExt + Unpin,
+    RScripts: AsyncReadExt + Unpin,
+> {
+    writer: W,
+    manifest: &'a Manifest,
+    license: RLicense,
+    instructions: RInstructions,
+    icon: RIcon,
+    docker_images: RDockerImages,
+    assets: RAssets,
+    scripts: Option<RScripts>,
+}
+impl<
+    'a,
+    W: AsyncWriteExt + AsyncSeekExt + Unpin,
+    RLicense: AsyncReadExt + Unpin,
+    RInstructions: AsyncReadExt + Unpin,
+    RIcon: AsyncReadExt + Unpin,
+    RDockerImages: AsyncReadExt + Unpin,
+    RAssets: AsyncReadExt + Unpin,
+    RScripts: AsyncReadExt + Unpin,
+> S9pkPacker<'a, W, RLicense, RInstructions, RIcon, RDockerImages, RAssets, RScripts>
+{
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub async fn pack(mut self, key: &ed25519_dalek::SigningKey) -> Result<(), Error> {
+        let header_pos = self.writer.stream_position().await?;
+        if header_pos != 0 {
+            tracing::warn!("Appending to non-empty file.");
+        }
+        let mut header = Header::placeholder();
+        header.serialize(&mut self.writer).await.with_ctx(|_| {
+            (
+                crate::ErrorKind::Serialization,
+                "Writing Placeholder Header",
+            )
+        })?;
+        let mut position = self.writer.stream_position().await?;
+
+        let mut writer = HashWriter::new(Sha512::new(), &mut self.writer);
+        // manifest
+        to_cbor_async_writer(&mut writer, self.manifest).await?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.manifest = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // license
+        tokio::io::copy(&mut self.license, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying License"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.license = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // instructions
+        tokio::io::copy(&mut self.instructions, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Instructions"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.instructions = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // icon
+        tokio::io::copy(&mut self.icon, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Icon"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.icon = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // docker_images
+        tokio::io::copy(&mut self.docker_images, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Docker Images"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.docker_images = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // assets
+        tokio::io::copy(&mut self.assets, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Assets"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.assets = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // scripts
+        if let Some(mut scripts) = self.scripts {
+            tokio::io::copy(&mut scripts, &mut writer)
+                .await
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Scripts"))?;
+            let new_pos = writer.inner_mut().stream_position().await?;
+            header.table_of_contents.scripts = Some(FileSection {
+                position,
+                length: new_pos - position,
+            });
+            position = new_pos;
+        }
+
+        // header
+        let (hash, _) = writer.finish();
+        self.writer.seek(SeekFrom::Start(header_pos)).await?;
+        header.pubkey = key.into();
+        header.signature = key.sign_prehashed(hash, Some(SIG_CONTEXT))?;
+        header
+            .serialize(&mut self.writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Serialization, "Writing Header"))?;
+        self.writer.seek(SeekFrom::Start(position)).await?;
+
+        Ok(())
+    }
+}

core/src/s9pk/v1/docker.rs

@@ -0,0 +1,159 @@
+use std::collections::BTreeSet;
+use std::io::SeekFrom;
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::eyre;
+use futures::{FutureExt, TryStreamExt};
+use serde::{Deserialize, Serialize};
+use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
+use tokio_tar::{Archive, Entry};
+
+use crate::util::io::{from_cbor_async_reader, from_json_async_reader};
+use crate::{Error, ErrorKind};
+
+#[derive(Default, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DockerMultiArch {
+    pub default: String,
+    pub available: BTreeSet<String>,
+}
+
+#[pin_project::pin_project(project = DockerReaderProject)]
+#[derive(Debug)]
+pub enum DockerReader<R: AsyncRead + Unpin> {
+    SingleArch(#[pin] R),
+    MultiArch(#[pin] Entry<Archive<R>>),
+}
+impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> DockerReader<R> {
+    pub async fn list_arches(rdr: &mut R) -> Result<BTreeSet<String>, Error> {
+        if let Some(multiarch) = tokio_tar::Archive::new(&mut *rdr)
+            .entries()?
+            .try_filter_map(|e| {
+                async move {
+                    Ok(if &*e.path()? == Path::new("multiarch.cbor") {
+                        Some(e)
+                    } else {
+                        None
+                    })
+                }
+                .boxed()
+            })
+            .try_next()
+            .await?
+        {
+            let multiarch: DockerMultiArch = from_cbor_async_reader(multiarch).await?;
+            return Ok(multiarch.available);
+        }
+
+        let Some(manifest) = tokio_tar::Archive::new(&mut *rdr)
+            .entries()?
+            .try_filter_map(|e| {
+                async move {
+                    Ok(if &*e.path()? == Path::new("manifest.json") {
+                        Some(e)
+                    } else {
+                        None
+                    })
+                }
+                .boxed()
+            })
+            .try_next()
+            .await?
+        else {
+            return Err(Error::new(
+                eyre!("Single arch legacy s9pk is malformed"),
+                ErrorKind::ParseS9pk,
+            ));
+        };
+
+        #[derive(Deserialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct Manifest {
+            config: PathBuf,
+        }
+        let Manifest { config } = from_json_async_reader(manifest).await?;
+        rdr.seek(SeekFrom::Start(0)).await?;
+        let Some(config) = tokio_tar::Archive::new(rdr)
+            .entries()?
+            .try_filter_map(|e| {
+                let config = config.clone();
+                async move { Ok(if &*e.path()? == config { Some(e) } else { None }) }.boxed()
+            })
+            .try_next()
+            .await?
+        else {
+            return Err(Error::new(
+                eyre!("Single arch legacy s9pk is malformed"),
+                ErrorKind::ParseS9pk,
+            ));
+        };
+
+        #[derive(Deserialize)]
+        struct Config {
+            architecture: String,
+        }
+        let Config { architecture } = from_json_async_reader(config).await?;
+
+        let mut arches = BTreeSet::new();
+        arches.insert(architecture);
+
+        Ok(arches)
+    }
+    pub async fn new(mut rdr: R, arch: &str) -> Result<Self, Error> {
+        rdr.seek(SeekFrom::Start(0)).await?;
+        if tokio_tar::Archive::new(&mut rdr)
+            .entries()?
+            .try_filter_map(|e| {
+                async move {
+                    Ok(if &*e.path()? == Path::new("multiarch.cbor") {
+                        Some(e)
+                    } else {
+                        None
+                    })
+                }
+                .boxed()
+            })
+            .try_next()
+            .await?
+            .is_some()
+        {
+            rdr.seek(SeekFrom::Start(0)).await?;
+            if let Some(image) = tokio_tar::Archive::new(rdr)
+                .entries()?
+                .try_filter_map(|e| {
+                    async move {
+                        Ok(if &*e.path()? == Path::new(&format!("{}.tar", arch)) {
+                            Some(e)
+                        } else {
+                            None
+                        })
+                    }
+                    .boxed()
+                })
+                .try_next()
+                .await?
+            {
+                Ok(Self::MultiArch(image))
+            } else {
+                Err(Error::new(
+                    eyre!("Docker image section does not contain tarball for architecture"),
+                    ErrorKind::ParseS9pk,
+                ))
+            }
+        } else {
+            Ok(Self::SingleArch(rdr))
+        }
+    }
+}
+impl<R: AsyncRead + Unpin + Send + Sync> AsyncRead for DockerReader<R> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        match self.project() {
+            DockerReaderProject::SingleArch(r) => r.poll_read(cx, buf),
+            DockerReaderProject::MultiArch(r) => r.poll_read(cx, buf),
+        }
+    }
+}

core/src/s9pk/v1/header.rs

@@ -0,0 +1,187 @@
+use std::collections::BTreeMap;
+
+use color_eyre::eyre::eyre;
+use ed25519_dalek::{Signature, VerifyingKey};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
+
+use crate::Error;
+
+pub const MAGIC: [u8; 2] = [59, 59];
+pub const VERSION: u8 = 1;
+
+#[derive(Debug)]
+pub struct Header {
+    pub pubkey: VerifyingKey,
+    pub signature: Signature,
+    pub table_of_contents: TableOfContents,
+}
+impl Header {
+    pub fn placeholder() -> Self {
+        Header {
+            pubkey: VerifyingKey::default(),
+            signature: Signature::from_bytes(&[0; 64]),
+            table_of_contents: Default::default(),
+        }
+    }
+    // MUST BE SAME SIZE REGARDLESS OF DATA
+    pub async fn serialize<W: AsyncWriteExt + Unpin>(&self, mut writer: W) -> std::io::Result<()> {
+        writer.write_all(&MAGIC).await?;
+        writer.write_all(&[VERSION]).await?;
+        writer.write_all(self.pubkey.as_bytes()).await?;
+        writer.write_all(&self.signature.to_bytes()).await?;
+        self.table_of_contents.serialize(writer).await?;
+        Ok(())
+    }
+    pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> Result<Self, Error> {
+        let mut magic = [0; 2];
+        reader.read_exact(&mut magic).await?;
+        if magic != MAGIC {
+            return Err(Error::new(
+                eyre!("Incorrect Magic: {:?}", magic),
+                crate::ErrorKind::ParseS9pk,
+            ));
+        }
+        let mut version = [0];
+        reader.read_exact(&mut version).await?;
+        if version[0] != VERSION {
+            return Err(Error::new(
+                eyre!("Unknown Version: {}", version[0]),
+                crate::ErrorKind::ParseS9pk,
+            ));
+        }
+        let mut pubkey_bytes = [0; 32];
+        reader.read_exact(&mut pubkey_bytes).await?;
+        let pubkey = VerifyingKey::from_bytes(&pubkey_bytes)
+            .map_err(|e| Error::new(e, crate::ErrorKind::ParseS9pk))?;
+        let mut sig_bytes = [0; 64];
+        reader.read_exact(&mut sig_bytes).await?;
+        let signature = Signature::from_bytes(&sig_bytes);
+        let table_of_contents = TableOfContents::deserialize(reader).await?;
+
+        Ok(Header {
+            pubkey,
+            signature,
+            table_of_contents,
+        })
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct TableOfContents {
+    pub manifest: FileSection,
+    pub license: FileSection,
+    pub instructions: FileSection,
+    pub icon: FileSection,
+    pub docker_images: FileSection,
+    pub assets: FileSection,
+    pub scripts: Option<FileSection>,
+}
+impl TableOfContents {
+    pub async fn serialize<W: AsyncWriteExt + Unpin>(&self, mut writer: W) -> std::io::Result<()> {
+        let len: u32 = ((1 + "manifest".len() + 16)
+            + (1 + "license".len() + 16)
+            + (1 + "instructions".len() + 16)
+            + (1 + "icon".len() + 16)
+            + (1 + "docker_images".len() + 16)
+            + (1 + "assets".len() + 16)
+            + (1 + "scripts".len() + 16)) as u32;
+        writer.write_all(&u32::to_be_bytes(len)).await?;
+        self.manifest
+            .serialize_entry("manifest", &mut writer)
+            .await?;
+        self.license.serialize_entry("license", &mut writer).await?;
+        self.instructions
+            .serialize_entry("instructions", &mut writer)
+            .await?;
+        self.icon.serialize_entry("icon", &mut writer).await?;
+        self.docker_images
+            .serialize_entry("docker_images", &mut writer)
+            .await?;
+        self.assets.serialize_entry("assets", &mut writer).await?;
+        self.scripts
+            .unwrap_or_default()
+            .serialize_entry("scripts", &mut writer)
+            .await?;
+        Ok(())
+    }
+    pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> std::io::Result<Self> {
+        let mut toc_len = [0; 4];
+        reader.read_exact(&mut toc_len).await?;
+        let toc_len = u32::from_be_bytes(toc_len);
+        let mut reader = reader.take(toc_len as u64);
+        let mut table = BTreeMap::new();
+        while let Some((label, section)) = FileSection::deserialize_entry(&mut reader).await? {
+            table.insert(label, section);
+        }
+        fn from_table(
+            table: &BTreeMap<Vec<u8>, FileSection>,
+            label: &str,
+        ) -> std::io::Result<FileSection> {
+            table.get(label.as_bytes()).copied().ok_or_else(|| {
+                std::io::Error::new(
+                    std::io::ErrorKind::UnexpectedEof,
+                    format!("Missing Required Label: {}", label),
+                )
+            })
+        }
+        #[allow(dead_code)]
+        fn as_opt(fs: FileSection) -> Option<FileSection> {
+            if fs.position | fs.length == 0 {
+                // 0/0 is not a valid file section
+                None
+            } else {
+                Some(fs)
+            }
+        }
+        Ok(TableOfContents {
+            manifest: from_table(&table, "manifest")?,
+            license: from_table(&table, "license")?,
+            instructions: from_table(&table, "instructions")?,
+            icon: from_table(&table, "icon")?,
+            docker_images: from_table(&table, "docker_images")?,
+            assets: from_table(&table, "assets")?,
+            scripts: table.get("scripts".as_bytes()).cloned(),
+        })
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct FileSection {
+    pub position: u64,
+    pub length: u64,
+}
+impl FileSection {
+    pub async fn serialize_entry<W: AsyncWriteExt + Unpin>(
+        self,
+        label: &str,
+        mut writer: W,
+    ) -> std::io::Result<()> {
+        writer.write_all(&[label.len() as u8]).await?;
+        writer.write_all(label.as_bytes()).await?;
+        writer.write_all(&u64::to_be_bytes(self.position)).await?;
+        writer.write_all(&u64::to_be_bytes(self.length)).await?;
+        Ok(())
+    }
+    pub async fn deserialize_entry<R: AsyncRead + Unpin>(
+        mut reader: R,
+    ) -> std::io::Result<Option<(Vec<u8>, Self)>> {
+        let mut label_len = [0];
+        let read = reader.read(&mut label_len).await?;
+        if read == 0 {
+            return Ok(None);
+        }
+        let mut label = vec![0; label_len[0] as usize];
+        reader.read_exact(&mut label).await?;
+        let mut pos = [0; 8];
+        reader.read_exact(&mut pos).await?;
+        let mut len = [0; 8];
+        reader.read_exact(&mut len).await?;
+        Ok(Some((
+            label,
+            FileSection {
+                position: u64::from_be_bytes(pos),
+                length: u64::from_be_bytes(len),
+            },
+        )))
+    }
+}

core/src/s9pk/v1/manifest.rs

@@ -0,0 +1,265 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+use exver::{Version, VersionRange};
+use imbl_value::InternedString;
+use indexmap::IndexMap;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+pub use crate::PackageId;
+use crate::prelude::*;
+use crate::s9pk::git_hash::GitHash;
+use crate::s9pk::manifest::{Alerts, Description};
+use crate::util::serde::{Duration, IoFormat, Regex};
+use crate::{ActionId, HealthCheckId, ImageId, VolumeId};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Manifest {
+    pub eos_version: Version,
+    pub id: PackageId,
+    #[serde(default)]
+    pub git_hash: Option<GitHash>,
+    pub title: String,
+    pub version: String,
+    pub description: Description,
+    #[serde(default)]
+    pub assets: Assets,
+    #[serde(default)]
+    pub build: Option<Vec<String>>,
+    pub release_notes: String,
+    pub license: String, // type of license
+    pub wrapper_repo: Url,
+    pub upstream_repo: Url,
+    pub support_site: Option<Url>,
+    pub marketing_site: Option<Url>,
+    pub donation_url: Option<Url>,
+    #[serde(default)]
+    pub alerts: Alerts,
+    pub main: PackageProcedure,
+    pub health_checks: HealthChecks,
+    pub config: Option<ConfigActions>,
+    pub properties: Option<PackageProcedure>,
+    pub volumes: BTreeMap<VolumeId, Value>,
+    // #[serde(default)]
+    // pub interfaces: Interfaces,
+    // #[serde(default)]
+    pub backup: BackupActions,
+    #[serde(default)]
+    pub migrations: Migrations,
+    #[serde(default)]
+    pub actions: BTreeMap<ActionId, Action>,
+    // #[serde(default)]
+    // pub permissions: Permissions,
+    #[serde(default)]
+    pub dependencies: BTreeMap<PackageId, DepInfo>,
+
+    #[serde(default)]
+    pub replaces: Vec<String>,
+
+    #[serde(default)]
+    pub hardware_requirements: HardwareRequirements,
+}
+
+impl Manifest {
+    pub fn package_procedures(&self) -> impl Iterator<Item = &PackageProcedure> {
+        use std::iter::once;
+        let main = once(&self.main);
+        let cfg_get = self.config.as_ref().map(|a| &a.get).into_iter();
+        let cfg_set = self.config.as_ref().map(|a| &a.set).into_iter();
+        let props = self.properties.iter();
+        let backups = vec![&self.backup.create, &self.backup.restore].into_iter();
+        let migrations = self
+            .migrations
+            .to
+            .values()
+            .chain(self.migrations.from.values());
+        let actions = self.actions.values().map(|a| &a.implementation);
+        main.chain(cfg_get)
+            .chain(cfg_set)
+            .chain(props)
+            .chain(backups)
+            .chain(migrations)
+            .chain(actions)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[serde(tag = "type")]
+#[model = "Model<Self>"]
+pub enum PackageProcedure {
+    Docker(DockerProcedure),
+    Script(Value),
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DockerProcedure {
+    pub image: ImageId,
+    #[serde(default)]
+    pub system: bool,
+    pub entrypoint: String,
+    #[serde(default)]
+    pub args: Vec<String>,
+    #[serde(default)]
+    pub inject: bool,
+    #[serde(default)]
+    pub mounts: BTreeMap<VolumeId, PathBuf>,
+    #[serde(default)]
+    pub io_format: Option<IoFormat>,
+    #[serde(default)]
+    pub sigterm_timeout: Option<Duration>,
+    #[serde(default)]
+    pub shm_size_mb: Option<usize>, // TODO: use postfix sizing? like 1k vs 1m vs 1g
+    #[serde(default)]
+    pub gpu_acceleration: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct HealthChecks(pub BTreeMap<HealthCheckId, HealthCheck>);
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HealthCheck {
+    pub name: String,
+    pub success_message: Option<String>,
+    #[serde(flatten)]
+    implementation: PackageProcedure,
+    pub timeout: Option<Duration>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConfigActions {
+    pub get: PackageProcedure,
+    pub set: PackageProcedure,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct BackupActions {
+    pub create: PackageProcedure,
+    pub restore: PackageProcedure,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Migrations {
+    pub from: IndexMap<VersionRange, PackageProcedure>,
+    pub to: IndexMap<VersionRange, PackageProcedure>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Action {
+    pub name: String,
+    pub description: String,
+    #[serde(default)]
+    pub warning: Option<String>,
+    pub implementation: PackageProcedure,
+    // pub allowed_statuses: Vec<DockerStatus>,
+    // #[serde(default)]
+    // pub input_spec: ConfigSpec,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DepInfo {
+    pub version: VersionRange,
+    pub requirement: DependencyRequirement,
+    pub description: Option<String>,
+    #[serde(default)]
+    pub config: Option<DependencyConfig>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DependencyConfig {
+    check: PackageProcedure,
+    auto_configure: PackageProcedure,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(tag = "type")]
+pub enum DependencyRequirement {
+    OptIn { how: String },
+    OptOut { how: String },
+    Required,
+}
+impl DependencyRequirement {
+    pub fn required(&self) -> bool {
+        matches!(self, &DependencyRequirement::Required)
+    }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HardwareRequirements {
+    #[serde(default)]
+    pub device: BTreeMap<InternedString, Regex>,
+    pub ram: Option<u64>,
+    pub arch: Option<BTreeSet<InternedString>>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Assets {
+    #[serde(default)]
+    pub license: Option<PathBuf>,
+    #[serde(default)]
+    pub instructions: Option<PathBuf>,
+    #[serde(default)]
+    pub icon: Option<PathBuf>,
+    #[serde(default)]
+    pub docker_images: Option<PathBuf>,
+    #[serde(default)]
+    pub assets: Option<PathBuf>,
+    #[serde(default)]
+    pub scripts: Option<PathBuf>,
+}
+impl Assets {
+    pub fn license_path(&self) -> &Path {
+        self.license
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("LICENSE.md"))
+    }
+    pub fn instructions_path(&self) -> &Path {
+        self.instructions
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("INSTRUCTIONS.md"))
+    }
+    pub fn icon_path(&self) -> &Path {
+        self.icon
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("icon.png"))
+    }
+    pub fn icon_type(&self) -> &str {
+        self.icon
+            .as_ref()
+            .and_then(|icon| icon.extension())
+            .and_then(|ext| ext.to_str())
+            .unwrap_or("png")
+    }
+    pub fn docker_images_path(&self) -> &Path {
+        self.docker_images
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("docker-images"))
+    }
+    pub fn assets_path(&self) -> &Path {
+        self.assets
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("assets"))
+    }
+    pub fn scripts_path(&self) -> &Path {
+        self.scripts
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("scripts"))
+    }
+}

core/src/s9pk/v1/mod.rs

@@ -0,0 +1,20 @@
+use std::path::PathBuf;
+
+use clap::Parser;
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+
+pub mod builder;
+pub mod docker;
+pub mod header;
+pub mod manifest;
+pub mod reader;
+
+pub const SIG_CONTEXT: &[u8] = b"s9pk";
+
+#[derive(Deserialize, Serialize, Parser, TS)]
+#[serde(rename_all = "camelCase")]
+#[command(rename_all = "kebab-case")]
+pub struct VerifyParams {
+    pub path: PathBuf,
+}

core/src/s9pk/v1/reader.rs

@@ -0,0 +1,275 @@
+use std::collections::BTreeSet;
+use std::io::SeekFrom;
+use std::ops::Range;
+use std::path::Path;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::task::{Context, Poll};
+
+use color_eyre::eyre::eyre;
+use digest::Output;
+use ed25519_dalek::VerifyingKey;
+use sha2::{Digest, Sha512};
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, BufReader, ReadBuf};
+use tracing::instrument;
+
+use super::SIG_CONTEXT;
+use super::header::{FileSection, Header, TableOfContents};
+use crate::prelude::*;
+use crate::s9pk::v1::docker::DockerReader;
+use crate::util::VersionString;
+use crate::util::io::open_file;
+use crate::{ImageId, PackageId};
+
+#[pin_project::pin_project]
+#[derive(Debug)]
+pub struct ReadHandle<'a, R = File> {
+    pos: &'a mut u64,
+    range: Range<u64>,
+    #[pin]
+    rdr: &'a mut R,
+}
+impl<'a, R: AsyncRead + Unpin> ReadHandle<'a, R> {
+    pub async fn to_vec(mut self) -> std::io::Result<Vec<u8>> {
+        let mut buf = vec![0; (self.range.end - self.range.start) as usize];
+        self.read_exact(&mut buf).await?;
+        Ok(buf)
+    }
+}
+impl<'a, R: AsyncRead + Unpin> AsyncRead for ReadHandle<'a, R> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let this = self.project();
+        let start = buf.filled().len();
+        let mut take_buf = buf.take(this.range.end.saturating_sub(**this.pos) as usize);
+        let res = AsyncRead::poll_read(this.rdr, cx, &mut take_buf);
+        let n = take_buf.filled().len();
+        unsafe { buf.assume_init(start + n) };
+        buf.advance(n);
+        **this.pos += n as u64;
+        res
+    }
+}
+impl<'a, R: AsyncSeek + Unpin> AsyncSeek for ReadHandle<'a, R> {
+    fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> {
+        let this = self.project();
+        AsyncSeek::start_seek(
+            this.rdr,
+            match position {
+                SeekFrom::Current(n) => SeekFrom::Current(n),
+                SeekFrom::End(n) => SeekFrom::Start((this.range.end as i64 + n) as u64),
+                SeekFrom::Start(n) => SeekFrom::Start(this.range.start + n),
+            },
+        )
+    }
+    fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<u64>> {
+        let this = self.project();
+        match AsyncSeek::poll_complete(this.rdr, cx) {
+            Poll::Ready(Ok(n)) => {
+                let res = n.saturating_sub(this.range.start);
+                **this.pos = this.range.start + res;
+                Poll::Ready(Ok(res))
+            }
+            a => a,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ImageTag {
+    pub package_id: PackageId,
+    pub image_id: ImageId,
+    pub version: VersionString,
+}
+impl ImageTag {
+    #[instrument(skip_all)]
+    pub fn validate(&self, id: &PackageId, version: &VersionString) -> Result<(), Error> {
+        if id != &self.package_id {
+            return Err(Error::new(
+                eyre!(
+                    "Contains image for incorrect package: id {}",
+                    self.package_id,
+                ),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if version != &self.version {
+            return Err(Error::new(
+                eyre!(
+                    "Contains image with incorrect version: expected {} received {}",
+                    version,
+                    self.version,
+                ),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        Ok(())
+    }
+}
+impl FromStr for ImageTag {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let rest = s.strip_prefix("start9/").ok_or_else(|| {
+            Error::new(
+                eyre!("Invalid image tag prefix: expected start9/"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        let (package, rest) = rest.split_once("/").ok_or_else(|| {
+            Error::new(
+                eyre!("Image tag missing image id"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        let (image, version) = rest.split_once(":").ok_or_else(|| {
+            Error::new(
+                eyre!("Image tag missing version"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        Ok(ImageTag {
+            package_id: package.parse()?,
+            image_id: image.parse()?,
+            version: version.parse()?,
+        })
+    }
+}
+
+pub struct S9pkReader<R: AsyncRead + AsyncSeek + Unpin + Send + Sync = BufReader<File>> {
+    hash: Option<Output<Sha512>>,
+    hash_string: Option<String>,
+    developer_key: VerifyingKey,
+    toc: TableOfContents,
+    pos: u64,
+    rdr: R,
+}
+impl S9pkReader {
+    pub async fn open<P: AsRef<Path>>(path: P, check_sig: bool) -> Result<Self, Error> {
+        let p = path.as_ref();
+        let rdr = open_file(p).await?;
+
+        Self::from_reader(BufReader::new(rdr), check_sig).await
+    }
+}
+impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
+    #[instrument(skip_all)]
+    pub async fn from_reader(mut rdr: R, check_sig: bool) -> Result<Self, Error> {
+        let header = Header::deserialize(&mut rdr).await?;
+
+        let (hash, hash_string) = if check_sig {
+            let mut hasher = Sha512::new();
+            let mut buf = [0; 1024];
+            let mut read;
+            while {
+                read = rdr.read(&mut buf).await?;
+                read != 0
+            } {
+                hasher.update(&buf[0..read]);
+            }
+            let hash = hasher.clone().finalize();
+            header
+                .pubkey
+                .verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?;
+            (
+                Some(hash),
+                Some(base32::encode(
+                    base32::Alphabet::Rfc4648 { padding: false },
+                    hash.as_slice(),
+                )),
+            )
+        } else {
+            (None, None)
+        };
+
+        let pos = rdr.stream_position().await?;
+
+        Ok(S9pkReader {
+            hash_string,
+            hash,
+            developer_key: header.pubkey,
+            toc: header.table_of_contents,
+            pos,
+            rdr,
+        })
+    }
+
+    pub fn hash(&self) -> Option<&Output<Sha512>> {
+        self.hash.as_ref()
+    }
+
+    pub fn hash_str(&self) -> Option<&str> {
+        self.hash_string.as_ref().map(|s| s.as_str())
+    }
+
+    pub fn developer_key(&self) -> &VerifyingKey {
+        &self.developer_key
+    }
+
+    pub async fn reset(&mut self) -> Result<(), Error> {
+        self.rdr.seek(SeekFrom::Start(0)).await?;
+        Ok(())
+    }
+
+    async fn read_handle<'a>(
+        &'a mut self,
+        section: FileSection,
+    ) -> Result<ReadHandle<'a, R>, Error> {
+        if self.pos != section.position {
+            self.rdr.seek(SeekFrom::Start(section.position)).await?;
+            self.pos = section.position;
+        }
+        Ok(ReadHandle {
+            range: self.pos..(self.pos + section.length),
+            pos: &mut self.pos,
+            rdr: &mut self.rdr,
+        })
+    }
+
+    pub async fn manifest_raw(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.manifest).await
+    }
+
+    pub async fn manifest(&mut self) -> Result<Value, Error> {
+        let slice = self.manifest_raw().await?.to_vec().await?;
+        serde_cbor::de::from_reader(slice.as_slice())
+            .with_ctx(|_| (crate::ErrorKind::ParseS9pk, "Deserializing Manifest (CBOR)"))
+    }
+
+    pub async fn license(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.license).await
+    }
+
+    pub async fn instructions(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.instructions).await
+    }
+
+    pub async fn icon(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.icon).await
+    }
+
+    pub async fn docker_arches(&mut self) -> Result<BTreeSet<String>, Error> {
+        DockerReader::list_arches(&mut self.read_handle(self.toc.docker_images).await?).await
+    }
+
+    pub async fn docker_images(
+        &mut self,
+        arch: &str,
+    ) -> Result<DockerReader<ReadHandle<'_, R>>, Error> {
+        DockerReader::new(self.read_handle(self.toc.docker_images).await?, arch).await
+    }
+
+    pub async fn assets(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.assets).await
+    }
+
+    pub async fn scripts(&mut self) -> Result<Option<ReadHandle<'_, R>>, Error> {
+        Ok(match self.toc.scripts {
+            None => None,
+            Some(a) => Some(self.read_handle(a).await?),
+        })
+    }
+}

core/src/s9pk/v2/compat.rs

@@ -0,0 +1,256 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use exver::{ExtendedVersion, VersionRange};
+use tokio::io::{AsyncRead, AsyncSeek, AsyncWriteExt};
+use tokio::process::Command;
+
+use crate::dependencies::{DepInfo, Dependencies};
+use crate::prelude::*;
+use crate::s9pk::manifest::{DeviceFilter, Manifest};
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::TmpSource;
+use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
+use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure};
+use crate::s9pk::v1::reader::S9pkReader;
+use crate::s9pk::v2::pack::{CONTAINER_TOOL, ImageSource, PackSource};
+use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
+use crate::util::Invoke;
+use crate::util::io::{TmpDir, create_file};
+use crate::{ImageId, VolumeId};
+
+pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01];
+
+impl S9pk<TmpSource<PackSource>> {
+    #[instrument(skip_all)]
+    pub async fn from_v1<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
+        mut reader: S9pkReader<R>,
+        tmp_dir: Arc<TmpDir>,
+        signer: ed25519_dalek::SigningKey,
+    ) -> Result<Self, Error> {
+        Command::new(*CONTAINER_TOOL)
+            .arg("run")
+            .arg("--rm")
+            .arg("--privileged")
+            .arg("tonistiigi/binfmt")
+            .arg("--install")
+            .arg("all")
+            .invoke(ErrorKind::Docker)
+            .await?;
+
+        let mut archive = DirectoryContents::<TmpSource<PackSource>>::new();
+
+        // manifest.json
+        let manifest_raw = reader.manifest().await?;
+        let manifest = from_value::<ManifestV1>(manifest_raw.clone())?;
+        let mut new_manifest = Manifest::try_from(manifest.clone())?;
+
+        let images: BTreeSet<(ImageId, bool)> = manifest
+            .package_procedures()
+            .filter_map(|p| {
+                if let PackageProcedure::Docker(p) = p {
+                    Some((p.image.clone(), p.system))
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        // LICENSE.md
+        let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into();
+        archive.insert_path(
+            "LICENSE.md",
+            Entry::file(TmpSource::new(
+                tmp_dir.clone(),
+                PackSource::Buffered(license.into()),
+            )),
+        )?;
+
+        // icon.*
+        let icon: Arc<[u8]> = reader.icon().await?.to_vec().await?.into();
+        archive.insert_path(
+            format!("icon.{}", manifest.assets.icon_type()),
+            Entry::file(TmpSource::new(
+                tmp_dir.clone(),
+                PackSource::Buffered(icon.into()),
+            )),
+        )?;
+
+        // images
+        for arch in reader.docker_arches().await? {
+            Command::new(*CONTAINER_TOOL)
+                .arg("load")
+                .input(Some(&mut reader.docker_images(&arch).await?))
+                .invoke(ErrorKind::Docker)
+                .await?;
+            for (image, system) in &images {
+                let mut image_config = new_manifest.images.remove(image).unwrap_or_default();
+                image_config.arch.insert(arch.as_str().into());
+                new_manifest.images.insert(image.clone(), image_config);
+                let image_name = if *system {
+                    format!("start9/{}:latest", image)
+                } else {
+                    format!("start9/{}/{}:{}", manifest.id, image, manifest.version)
+                };
+                ImageSource::DockerTag(image_name.clone())
+                    .load(
+                        tmp_dir.clone(),
+                        &new_manifest.id,
+                        &new_manifest.version,
+                        image,
+                        &arch,
+                        &mut archive,
+                    )
+                    .await?;
+                Command::new(*CONTAINER_TOOL)
+                    .arg("rmi")
+                    .arg("-f")
+                    .arg(&image_name)
+                    .invoke(ErrorKind::Docker)
+                    .await?;
+            }
+        }
+
+        // assets
+        let asset_dir = tmp_dir.join("assets");
+        tokio::fs::create_dir_all(&asset_dir).await?;
+        tokio_tar::Archive::new(reader.assets().await?)
+            .unpack(&asset_dir)
+            .await?;
+        let sqfs_path = asset_dir.with_extension("squashfs");
+        Command::new("mksquashfs")
+            .arg(&asset_dir)
+            .arg(&sqfs_path)
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        archive.insert_path(
+            "assets.squashfs",
+            Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(sqfs_path))),
+        )?;
+
+        // javascript
+        let js_dir = tmp_dir.join("javascript");
+        let sqfs_path = js_dir.with_extension("squashfs");
+        tokio::fs::create_dir_all(&js_dir).await?;
+        if let Some(mut scripts) = reader.scripts().await? {
+            let mut js_file = create_file(js_dir.join("embassy.js")).await?;
+            tokio::io::copy(&mut scripts, &mut js_file).await?;
+            js_file.sync_all().await?;
+        }
+        {
+            let mut js_file = create_file(js_dir.join("embassyManifest.json")).await?;
+            js_file
+                .write_all(&serde_json::to_vec(&manifest_raw).with_kind(ErrorKind::Serialization)?)
+                .await?;
+            js_file.sync_all().await?;
+        }
+        Command::new("mksquashfs")
+            .arg(&js_dir)
+            .arg(&sqfs_path)
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        archive.insert_path(
+            Path::new("javascript.squashfs"),
+            Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(sqfs_path))),
+        )?;
+
+        archive.insert_path(
+            "manifest.json",
+            Entry::file(TmpSource::new(
+                tmp_dir.clone(),
+                PackSource::Buffered(
+                    serde_json::to_vec::<Manifest>(&new_manifest)
+                        .with_kind(ErrorKind::Serialization)?
+                        .into(),
+                ),
+            )),
+        )?;
+
+        let mut res = S9pk::new(MerkleArchive::new(archive, signer, SIG_CONTEXT), None).await?;
+        res.as_archive_mut().update_hashes(true).await?;
+        Ok(res)
+    }
+}
+
+impl TryFrom<ManifestV1> for Manifest {
+    type Error = Error;
+    fn try_from(value: ManifestV1) -> Result<Self, Self::Error> {
+        let default_url = value.upstream_repo.clone();
+        let mut version = ExtendedVersion::from(
+            exver::emver::Version::from_str(&value.version)
+                .with_kind(ErrorKind::Deserialization)?,
+        );
+        if &*value.id == "bitcoind" && value.title.to_ascii_lowercase().contains("knots") {
+            version = version.with_flavor("knots");
+        } else if &*value.id == "lnd" || &*value.id == "ride-the-lightning" || &*value.id == "datum"
+        {
+            version = version.map_upstream(|v| v.with_prerelease(["beta".into()]));
+        } else if &*value.id == "lightning-terminal" || &*value.id == "robosats" {
+            version = version.map_upstream(|v| v.with_prerelease(["alpha".into()]));
+        }
+        Ok(Self {
+            id: value.id,
+            title: format!("{} (Legacy)", value.title).into(),
+            version: version.into(),
+            satisfies: BTreeSet::new(),
+            release_notes: value.release_notes,
+            can_migrate_from: VersionRange::any(),
+            can_migrate_to: VersionRange::none(),
+            license: value.license.into(),
+            wrapper_repo: value.wrapper_repo,
+            upstream_repo: value.upstream_repo,
+            support_site: value.support_site.unwrap_or_else(|| default_url.clone()),
+            marketing_site: value.marketing_site.unwrap_or_else(|| default_url.clone()),
+            donation_url: value.donation_url,
+            docs_url: None,
+            description: value.description,
+            images: BTreeMap::new(),
+            volumes: value
+                .volumes
+                .iter()
+                .filter(|(_, v)| v.get("type").and_then(|v| v.as_str()) == Some("data"))
+                .map(|(id, _)| id.clone())
+                .chain([VolumeId::from_str("embassy").unwrap()])
+                .collect(),
+            alerts: value.alerts,
+            dependencies: Dependencies(
+                value
+                    .dependencies
+                    .into_iter()
+                    .map(|(id, value)| {
+                        (
+                            id,
+                            DepInfo {
+                                description: value.description,
+                                optional: !value.requirement.required(),
+                                metadata: None,
+                            },
+                        )
+                    })
+                    .collect(),
+            ),
+            hardware_requirements: super::manifest::HardwareRequirements {
+                arch: value.hardware_requirements.arch,
+                ram: value.hardware_requirements.ram,
+                device: value
+                    .hardware_requirements
+                    .device
+                    .into_iter()
+                    .map(|(class, product)| DeviceFilter {
+                        pattern_description: format!(
+                            "a {class} device matching the expression {}",
+                            product.as_ref()
+                        ),
+                        class,
+                        pattern: product,
+                    })
+                    .collect(),
+            },
+            git_hash: value.git_hash,
+            os_version: value.eos_version,
+            sdk_version: None,
+        })
+    }
+}

core/src/s9pk/v2/manifest.rs

@@ -0,0 +1,224 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use exver::{Version, VersionRange};
+use imbl_value::InternedString;
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+use url::Url;
+
+pub use crate::PackageId;
+use crate::dependencies::Dependencies;
+use crate::prelude::*;
+use crate::s9pk::git_hash::GitHash;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::expected::{Expected, Filter};
+use crate::s9pk::v2::pack::ImageConfig;
+use crate::util::serde::Regex;
+use crate::util::{VersionString, mime};
+use crate::version::{Current, VersionT};
+use crate::{ImageId, VolumeId};
+
+fn current_version() -> Version {
+    Current::default().semver()
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]
+#[serde(rename_all = "camelCase")]
+#[model = "Model<Self>"]
+#[ts(export)]
+pub struct Manifest {
+    pub id: PackageId,
+    #[ts(type = "string")]
+    pub title: InternedString,
+    pub version: VersionString,
+    pub satisfies: BTreeSet<VersionString>,
+    pub release_notes: String,
+    #[ts(type = "string")]
+    pub can_migrate_to: VersionRange,
+    #[ts(type = "string")]
+    pub can_migrate_from: VersionRange,
+    #[ts(type = "string")]
+    pub license: InternedString, // type of license
+    #[ts(type = "string")]
+    pub wrapper_repo: Url,
+    #[ts(type = "string")]
+    pub upstream_repo: Url,
+    #[ts(type = "string")]
+    pub support_site: Url,
+    #[ts(type = "string")]
+    pub marketing_site: Url,
+    #[ts(type = "string | null")]
+    pub donation_url: Option<Url>,
+    #[ts(type = "string | null")]
+    pub docs_url: Option<Url>,
+    pub description: Description,
+    pub images: BTreeMap<ImageId, ImageConfig>,
+    pub volumes: BTreeSet<VolumeId>,
+    #[serde(default)]
+    pub alerts: Alerts,
+    #[serde(default)]
+    pub dependencies: Dependencies,
+    #[serde(default)]
+    pub hardware_requirements: HardwareRequirements,
+    pub git_hash: Option<GitHash>,
+    #[serde(default = "current_version")]
+    #[ts(type = "string")]
+    pub os_version: Version,
+    #[ts(type = "string | null")]
+    pub sdk_version: Option<Version>,
+}
+impl Manifest {
+    pub fn validate_for<'a, T: Clone>(
+        &self,
+        arch: Option<&str>,
+        archive: &'a DirectoryContents<T>,
+    ) -> Result<Filter, Error> {
+        let mut expected = Expected::new(archive);
+        expected.check_file("manifest.json")?;
+        expected.check_stem("icon", |ext| {
+            ext.and_then(|e| e.to_str())
+                .and_then(mime)
+                .map_or(false, |mime| mime.starts_with("image/"))
+        })?;
+        expected.check_file("LICENSE.md")?;
+        expected.check_file("javascript.squashfs")?;
+        for (dependency, _) in &self.dependencies.0 {
+            let dep_path = Path::new("dependencies").join(dependency);
+            let _ = expected.check_file(dep_path.join("metadata.json"));
+            let _ = expected.check_stem(dep_path.join("icon"), |ext| {
+                ext.and_then(|e| e.to_str())
+                    .and_then(mime)
+                    .map_or(false, |mime| mime.starts_with("image/"))
+            });
+        }
+        if let Err(e) = expected.check_file(Path::new("assets.squashfs")) {
+            // backwards compatibility for alpha s9pks - remove eventually
+            if expected.check_dir("assets").is_err() {
+                tracing::warn!("{e}");
+                tracing::debug!("{e:?}");
+                // return Err(e);
+            }
+        }
+        for (image_id, config) in &self.images {
+            let mut check_arch = |arch: &str| {
+                let mut arch = arch;
+                if let Err(e) = expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("squashfs"),
+                ) {
+                    if let Some(emulate_as) = &config.emulate_missing_as {
+                        expected.check_file(
+                            Path::new("images")
+                                .join(arch)
+                                .join(image_id)
+                                .with_extension("squashfs"),
+                        )?;
+                        arch = &**emulate_as;
+                    } else {
+                        return Err(e);
+                    }
+                }
+                expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("json"),
+                )?;
+                expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("env"),
+                )?;
+                Ok(())
+            };
+            if let Some(arch) = arch {
+                check_arch(arch)?;
+            } else if let Some(arches) = &self.hardware_requirements.arch {
+                for arch in arches {
+                    check_arch(arch)?;
+                }
+            } else if let Some(arch) = config.emulate_missing_as.as_deref() {
+                if !config.arch.contains(arch) {
+                    return Err(Error::new(
+                        eyre!("`emulateMissingAs` must match an included `arch`"),
+                        ErrorKind::ParseS9pk,
+                    ));
+                }
+                for arch in &config.arch {
+                    check_arch(&arch)?;
+                }
+            } else {
+                return Err(Error::new(
+                    eyre!(
+                        "`emulateMissingAs` required for all images if no `arch` specified in `hardwareRequirements`"
+                    ),
+                    ErrorKind::ParseS9pk,
+                ));
+            }
+        }
+        Ok(expected.into_filter())
+    }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct HardwareRequirements {
+    #[serde(default)]
+    pub device: Vec<DeviceFilter>,
+    #[ts(type = "number | null")]
+    pub ram: Option<u64>,
+    #[ts(type = "string[] | null")]
+    pub arch: Option<BTreeSet<InternedString>>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct DeviceFilter {
+    #[ts(type = "\"processor\" | \"display\"")]
+    pub class: InternedString,
+    #[ts(type = "string")]
+    pub pattern: Regex,
+    pub pattern_description: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, TS)]
+#[ts(export)]
+pub struct Description {
+    pub short: String,
+    pub long: String,
+}
+impl Description {
+    pub fn validate(&self) -> Result<(), Error> {
+        if self.short.chars().skip(160).next().is_some() {
+            return Err(Error::new(
+                eyre!("Short description must be 160 characters or less."),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if self.long.chars().skip(5000).next().is_some() {
+            return Err(Error::new(
+                eyre!("Long description must be 5000 characters or less."),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct Alerts {
+    pub install: Option<String>,
+    pub uninstall: Option<String>,
+    pub restore: Option<String>,
+    pub start: Option<String>,
+    pub stop: Option<String>,
+}

core/src/s9pk/v2/mod.rs

@@ -0,0 +1,346 @@
+use std::ffi::OsStr;
+use std::path::Path;
+use std::sync::Arc;
+
+use imbl_value::InternedString;
+use tokio::fs::File;
+
+use crate::PackageId;
+use crate::dependencies::DependencyMetadata;
+use crate::prelude::*;
+use crate::s9pk::manifest::Manifest;
+use crate::s9pk::merkle_archive::sink::Sink;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::s9pk::merkle_archive::source::{
+    ArchiveSource, DynFileSource, FileSource, Section, TmpSource,
+};
+use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
+use crate::s9pk::v2::pack::{ImageSource, PackSource};
+use crate::sign::commitment::merkle_archive::MerkleArchiveCommitment;
+use crate::util::io::{TmpDir, open_file};
+use crate::util::serde::IoFormat;
+use crate::util::{DataUrl, mime};
+
+const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
+
+pub const SIG_CONTEXT: &str = "s9pk";
+
+pub mod compat;
+pub mod manifest;
+pub mod pack;
+pub mod recipe;
+
+/**
+    /
+    ├── manifest.json
+    ├── icon.<ext>
+    ├── LICENSE.md
+    ├── dependencies
+    │   └── <id>
+    │       ├── metadata.json
+    │       └── icon.<ext>
+    ├── javascript.squashfs
+    ├── assets
+    │   └── <id>.squashfs (xN)
+    └── images
+        └── <arch>
+            ├── <id>.json (xN)
+            ├── <id>.env (xN)
+            └── <id>.squashfs (xN)
+*/
+
+// this sorts the s9pk to optimize such that the parts that are used first appear earlier in the s9pk
+// this is useful for manipulating an s9pk while partially downloaded on a source that does not support
+// random access
+fn priority(s: &str) -> Option<usize> {
+    match s {
+        "manifest.json" => Some(0),
+        a if Path::new(a).file_stem() == Some(OsStr::new("icon")) => Some(1),
+        "LICENSE.md" => Some(2),
+        "dependencies" => Some(3),
+        "javascript.squashfs" => Some(4),
+        "assets.squashfs" => Some(5),
+        "images" => Some(6),
+        _ => None,
+    }
+}
+
+#[derive(Clone)]
+pub struct S9pk<S = Section<MultiCursorFile>> {
+    pub manifest: Manifest,
+    manifest_dirty: bool,
+    archive: MerkleArchive<S>,
+    size: Option<u64>,
+}
+impl<S> S9pk<S> {
+    pub fn as_manifest(&self) -> &Manifest {
+        &self.manifest
+    }
+    pub fn as_manifest_mut(&mut self) -> &mut Manifest {
+        self.manifest_dirty = true;
+        &mut self.manifest
+    }
+    pub fn as_archive(&self) -> &MerkleArchive<S> {
+        &self.archive
+    }
+    pub fn as_archive_mut(&mut self) -> &mut MerkleArchive<S> {
+        &mut self.archive
+    }
+    pub fn size(&self) -> Option<u64> {
+        self.size
+    }
+}
+
+impl<S: FileSource + Clone> S9pk<S> {
+    pub async fn new(archive: MerkleArchive<S>, size: Option<u64>) -> Result<Self, Error> {
+        let manifest = extract_manifest(&archive).await?;
+        Ok(Self {
+            manifest,
+            manifest_dirty: false,
+            archive,
+            size,
+        })
+    }
+
+    pub fn new_with_manifest(
+        archive: MerkleArchive<S>,
+        size: Option<u64>,
+        manifest: Manifest,
+    ) -> Self {
+        Self {
+            manifest,
+            manifest_dirty: true,
+            archive,
+            size,
+        }
+    }
+
+    pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> {
+        let filter = self.manifest.validate_for(arch, self.archive.contents())?;
+        filter.keep_checked(self.archive.contents_mut())
+    }
+
+    pub async fn icon(&self) -> Result<(InternedString, Entry<S>), Error> {
+        let mut best_icon = None;
+        for (path, icon) in self.archive.contents().with_stem("icon").filter(|(p, v)| {
+            Path::new(&*p)
+                .extension()
+                .and_then(|e| e.to_str())
+                .and_then(mime)
+                .map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
+        }) {
+            let size = icon.expect_file()?.size().await?;
+            best_icon = match best_icon {
+                Some((s, a)) if s >= size => Some((s, a)),
+                _ => Some((size, (path, icon))),
+            };
+        }
+        best_icon
+            .map(|(_, a)| a)
+            .ok_or_else(|| Error::new(eyre!("no icon found in archive"), ErrorKind::ParseS9pk))
+    }
+
+    pub async fn icon_data_url(&self) -> Result<DataUrl<'static>, Error> {
+        let (name, contents) = self.icon().await?;
+        let mime = Path::new(&*name)
+            .extension()
+            .and_then(|e| e.to_str())
+            .and_then(mime)
+            .unwrap_or("image/png");
+        Ok(DataUrl::from_vec(
+            mime,
+            contents.expect_file()?.to_vec(contents.hash()).await?,
+        ))
+    }
+
+    pub async fn dependency_icon(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<(InternedString, Entry<S>)>, Error> {
+        let mut best_icon = None;
+        for (path, icon) in self
+            .archive
+            .contents()
+            .get_path(Path::new("dependencies").join(id))
+            .and_then(|p| p.as_directory())
+            .into_iter()
+            .flat_map(|d| {
+                d.with_stem("icon").filter(|(p, v)| {
+                    Path::new(&*p)
+                        .extension()
+                        .and_then(|e| e.to_str())
+                        .and_then(mime)
+                        .map_or(false, |e| e.starts_with("image/") && v.as_file().is_some())
+                })
+            })
+        {
+            let size = icon.expect_file()?.size().await?;
+            best_icon = match best_icon {
+                Some((s, a)) if s >= size => Some((s, a)),
+                _ => Some((size, (path, icon))),
+            };
+        }
+        Ok(best_icon.map(|(_, a)| a))
+    }
+
+    pub async fn dependency_icon_data_url(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<DataUrl<'static>>, Error> {
+        let Some((name, contents)) = self.dependency_icon(id).await? else {
+            return Ok(None);
+        };
+        let mime = Path::new(&*name)
+            .extension()
+            .and_then(|e| e.to_str())
+            .and_then(mime)
+            .unwrap_or("image/png");
+        Ok(Some(DataUrl::from_vec(
+            mime,
+            contents.expect_file()?.to_vec(contents.hash()).await?,
+        )))
+    }
+
+    pub async fn dependency_metadata(
+        &self,
+        id: &PackageId,
+    ) -> Result<Option<DependencyMetadata>, Error> {
+        if let Some(entry) = self
+            .archive
+            .contents()
+            .get_path(Path::new("dependencies").join(id).join("metadata.json"))
+        {
+            Ok(Some(IoFormat::Json.from_slice(
+                &entry.expect_file()?.to_vec(entry.hash()).await?,
+            )?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub async fn serialize<W: Sink>(&mut self, w: &mut W, verify: bool) -> Result<(), Error> {
+        use tokio::io::AsyncWriteExt;
+
+        w.write_all(MAGIC_AND_VERSION).await?;
+        if !self.manifest_dirty {
+            self.archive.serialize(w, verify).await?;
+        } else {
+            let mut dyn_s9pk = self.clone().into_dyn();
+            dyn_s9pk.as_archive_mut().contents_mut().insert_path(
+                "manifest.json",
+                Entry::file(DynFileSource::new(Arc::<[u8]>::from(
+                    serde_json::to_vec(&self.manifest).with_kind(ErrorKind::Serialization)?,
+                ))),
+            )?;
+            dyn_s9pk.archive.serialize(w, verify).await?;
+        }
+
+        Ok(())
+    }
+
+    pub fn into_dyn(self) -> S9pk<DynFileSource> {
+        S9pk {
+            manifest: self.manifest,
+            manifest_dirty: self.manifest_dirty,
+            archive: self.archive.into_dyn(),
+            size: self.size,
+        }
+    }
+}
+
+impl<S: From<TmpSource<PackSource>> + FileSource + Clone> S9pk<S> {
+    pub async fn load_images(&mut self, tmp_dir: Arc<TmpDir>) -> Result<(), Error> {
+        let id = &self.manifest.id;
+        let version = &self.manifest.version;
+        for (image_id, image_config) in &mut self.manifest.images {
+            self.manifest_dirty = true;
+            for arch in &image_config.arch {
+                image_config
+                    .source
+                    .load(
+                        tmp_dir.clone(),
+                        id,
+                        version,
+                        image_id,
+                        arch,
+                        self.archive.contents_mut(),
+                    )
+                    .await?;
+            }
+            image_config.source = ImageSource::Packed;
+        }
+
+        Ok(())
+    }
+}
+
+impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
+    #[instrument(skip_all)]
+    pub async fn archive(
+        source: &S,
+        commitment: Option<&MerkleArchiveCommitment>,
+    ) -> Result<MerkleArchive<Section<S>>, Error> {
+        use tokio::io::AsyncReadExt;
+
+        let mut header = source
+            .fetch(
+                0,
+                MAGIC_AND_VERSION.len() as u64 + MerkleArchive::<Section<S>>::header_size(),
+            )
+            .await?;
+
+        let mut magic_version = [0u8; MAGIC_AND_VERSION.len()];
+        header.read_exact(&mut magic_version).await?;
+        ensure_code!(
+            &magic_version == MAGIC_AND_VERSION,
+            ErrorKind::ParseS9pk,
+            "Invalid Magic or Unexpected Version"
+        );
+        MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await
+    }
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: &S,
+        commitment: Option<&MerkleArchiveCommitment>,
+    ) -> Result<Self, Error> {
+        let mut archive = Self::archive(source, commitment).await?;
+
+        archive.sort_by(|a, b| match (priority(a), priority(b)) {
+            (Some(a), Some(b)) => a.cmp(&b),
+            (Some(_), None) => std::cmp::Ordering::Less,
+            (None, Some(_)) => std::cmp::Ordering::Greater,
+            (None, None) => std::cmp::Ordering::Equal,
+        });
+
+        Self::new(archive, source.size().await).await
+    }
+}
+impl S9pk {
+    pub async fn from_file(file: File) -> Result<Self, Error> {
+        Self::deserialize(&MultiCursorFile::from(file), None).await
+    }
+    pub async fn open(path: impl AsRef<Path>, id: Option<&PackageId>) -> Result<Self, Error> {
+        let res = Self::from_file(open_file(path).await?).await?;
+        if let Some(id) = id {
+            ensure_code!(
+                &res.as_manifest().id == id,
+                ErrorKind::ValidateS9pk,
+                "manifest.id does not match expected"
+            );
+        }
+        Ok(res)
+    }
+}
+
+async fn extract_manifest<S: FileSource>(archive: &MerkleArchive<S>) -> Result<Manifest, Error> {
+    let manifest = serde_json::from_slice(
+        &archive
+            .contents()
+            .get_path("manifest.json")
+            .or_not_found("manifest.json")?
+            .read_file_to_vec()
+            .await?,
+    )
+    .with_kind(ErrorKind::Deserialization)?;
+    Ok(manifest)
+}

core/src/s9pk/v2/pack.rs

@@ -0,0 +1,864 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+use std::process::Stdio;
+use std::sync::{Arc, LazyLock, OnceLock};
+
+use clap::Parser;
+use futures::future::{BoxFuture, ready};
+use futures::{FutureExt, TryStreamExt};
+use imbl_value::InternedString;
+use serde::{Deserialize, Serialize};
+use tokio::process::Command;
+use tokio::sync::OnceCell;
+use tokio_stream::wrappers::ReadDirStream;
+use tracing::{debug, warn};
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::dependencies::{DependencyMetadata, MetadataSrc};
+use crate::prelude::*;
+use crate::rpc_continuations::Guid;
+use crate::s9pk::S9pk;
+use crate::s9pk::git_hash::GitHash;
+use crate::s9pk::manifest::Manifest;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::http::HttpSource;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::s9pk::merkle_archive::source::{
+    ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource, into_dyn_read,
+};
+use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
+use crate::s9pk::v2::SIG_CONTEXT;
+use crate::util::io::{TmpDir, create_file, open_file};
+use crate::util::serde::IoFormat;
+use crate::util::{DataUrl, Invoke, PathOrUrl, VersionString, new_guid};
+use crate::{ImageId, PackageId};
+
+pub static PREFER_DOCKER: OnceLock<bool> = OnceLock::new();
+
+pub static CONTAINER_TOOL: LazyLock<&'static str> = LazyLock::new(|| {
+    if *PREFER_DOCKER.get_or_init(|| false) {
+        if std::process::Command::new("which")
+            .arg("docker")
+            .stdout(Stdio::null())
+            .status()
+            .map_or(false, |o| o.success())
+        {
+            "docker"
+        } else {
+            "podman"
+        }
+    } else {
+        "podman"
+    }
+});
+pub static CONTAINER_DATADIR: LazyLock<&'static str> = LazyLock::new(|| {
+    if *CONTAINER_TOOL == "docker" {
+        "/var/lib/docker"
+    } else {
+        "/var/lib/containers"
+    }
+});
+
+pub struct SqfsDir {
+    path: PathBuf,
+    tmpdir: Arc<TmpDir>,
+    sqfs: OnceCell<MultiCursorFile>,
+}
+impl SqfsDir {
+    pub fn new(path: PathBuf, tmpdir: Arc<TmpDir>) -> Self {
+        Self {
+            path,
+            tmpdir,
+            sqfs: OnceCell::new(),
+        }
+    }
+    async fn file(&self) -> Result<&MultiCursorFile, Error> {
+        self.sqfs
+            .get_or_try_init(|| async move {
+                let guid = Guid::new();
+                let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs");
+                if self.path.extension().and_then(|s| s.to_str()) == Some("tar") {
+                    tar2sqfs(&self.path)?
+                        .input(Some(&mut open_file(&self.path).await?))
+                        .invoke(ErrorKind::Filesystem)
+                        .await?;
+                } else {
+                    Command::new("mksquashfs")
+                        .arg(&self.path)
+                        .arg(&path)
+                        .arg("-quiet")
+                        .invoke(ErrorKind::Filesystem)
+                        .await?;
+                }
+
+                Ok(MultiCursorFile::from(
+                    open_file(&path)
+                        .await
+                        .with_ctx(|_| (ErrorKind::Filesystem, path.display()))?,
+                ))
+            })
+            .await
+    }
+}
+
+#[derive(Clone)]
+pub enum PackSource {
+    Buffered(Arc<[u8]>),
+    File(PathBuf),
+    Squashfs(Arc<SqfsDir>),
+}
+impl FileSource for PackSource {
+    type Reader = DynRead;
+    type SliceReader = DynRead;
+    async fn size(&self) -> Result<u64, Error> {
+        match self {
+            Self::Buffered(a) => Ok(a.len() as u64),
+            Self::File(f) => Ok(tokio::fs::metadata(f)
+                .await
+                .with_ctx(|_| (ErrorKind::Filesystem, f.display()))?
+                .len()),
+            Self::Squashfs(dir) => dir
+                .file()
+                .await
+                .with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))?
+                .size()
+                .await
+                .or_not_found("file metadata"),
+        }
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        match self {
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::reader(a).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::reader(f).await?)),
+            Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
+        }
+    }
+    async fn slice(&self, position: u64, size: u64) -> Result<Self::SliceReader, Error> {
+        match self {
+            Self::Buffered(a) => Ok(into_dyn_read(FileSource::slice(a, position, size).await?)),
+            Self::File(f) => Ok(into_dyn_read(FileSource::slice(f, position, size).await?)),
+            Self::Squashfs(dir) => dir
+                .file()
+                .await?
+                .fetch(position, size)
+                .await
+                .map(into_dyn_read),
+        }
+    }
+}
+impl From<PackSource> for DynFileSource {
+    fn from(value: PackSource) -> Self {
+        DynFileSource::new(value)
+    }
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct PackParams {
+    pub path: Option<PathBuf>,
+    #[arg(short, long)]
+    pub output: Option<PathBuf>,
+    #[arg(long)]
+    pub javascript: Option<PathBuf>,
+    #[arg(long)]
+    pub icon: Option<PathBuf>,
+    #[arg(long)]
+    pub license: Option<PathBuf>,
+    #[arg(long, conflicts_with = "no-assets")]
+    pub assets: Option<PathBuf>,
+    #[arg(long, conflicts_with = "assets")]
+    pub no_assets: bool,
+    #[arg(long, help = "Architecture Mask")]
+    pub arch: Vec<InternedString>,
+}
+impl PackParams {
+    fn path(&self) -> &Path {
+        self.path.as_deref().unwrap_or(Path::new("."))
+    }
+    fn output(&self, id: &PackageId) -> PathBuf {
+        self.output
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join(id).with_extension("s9pk"))
+    }
+    fn javascript(&self) -> PathBuf {
+        self.javascript
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("javascript"))
+    }
+    async fn icon(&self) -> Result<PathBuf, Error> {
+        if let Some(icon) = &self.icon {
+            Ok(icon.clone())
+        } else {
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("icon")),
+                    )
+                })
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("icon"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
+        }
+    }
+    async fn license(&self) -> Result<PathBuf, Error> {
+        if let Some(license) = &self.license {
+            Ok(license.clone())
+        } else {
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?)
+                .try_filter(|x| {
+                    ready(
+                        x.path()
+                            .file_stem()
+                            .map_or(false, |s| s.eq_ignore_ascii_case("license")),
+                    )
+                })
+                .map_err(Error::from)
+                .try_fold(
+                    Err(Error::new(eyre!("license not found"), ErrorKind::NotFound)),
+                    |acc, x| async move {
+                        match acc {
+                            Ok(_) => Err(Error::new(eyre!("multiple licenses found in working directory, please specify which to use with `--license`"), ErrorKind::InvalidRequest)),
+                            Err(e) => Ok({
+                                let path = x.path();
+                                if path
+                                    .file_stem()
+                                    .map_or(false, |s| s.eq_ignore_ascii_case("license"))
+                                {
+                                    Ok(path)
+                                } else {
+                                    Err(e)
+                                }
+                            }),
+                        }
+                    },
+                )
+                .await?
+        }
+    }
+    fn assets(&self) -> PathBuf {
+        self.assets
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("assets"))
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ImageConfig {
+    pub source: ImageSource,
+    #[ts(type = "string[]")]
+    pub arch: BTreeSet<InternedString>,
+    #[ts(type = "string | null")]
+    pub emulate_missing_as: Option<InternedString>,
+}
+impl Default for ImageConfig {
+    fn default() -> Self {
+        Self {
+            source: ImageSource::Packed,
+            arch: BTreeSet::new(),
+            emulate_missing_as: None,
+        }
+    }
+}
+
+#[derive(Parser)]
+struct CliImageConfig {
+    #[arg(long, conflicts_with("docker-tag"))]
+    docker_build: bool,
+    #[arg(long, requires("docker-build"))]
+    dockerfile: Option<PathBuf>,
+    #[arg(long, requires("docker-build"))]
+    workdir: Option<PathBuf>,
+    #[arg(long, conflicts_with_all(["dockerfile", "workdir"]))]
+    docker_tag: Option<String>,
+    #[arg(long)]
+    arch: Vec<InternedString>,
+    #[arg(long)]
+    emulate_missing_as: Option<InternedString>,
+}
+impl TryFrom<CliImageConfig> for ImageConfig {
+    type Error = clap::Error;
+    fn try_from(value: CliImageConfig) -> Result<Self, Self::Error> {
+        let res = Self {
+            source: if value.docker_build {
+                ImageSource::DockerBuild {
+                    dockerfile: value.dockerfile,
+                    workdir: value.workdir,
+                    build_args: None,
+                }
+            } else if let Some(tag) = value.docker_tag {
+                ImageSource::DockerTag(tag)
+            } else {
+                ImageSource::Packed
+            },
+            arch: value.arch.into_iter().collect(),
+            emulate_missing_as: value.emulate_missing_as,
+        };
+        res.emulate_missing_as
+            .as_ref()
+            .map(|a| {
+                if !res.arch.contains(a) {
+                    Err(clap::Error::raw(
+                        clap::error::ErrorKind::InvalidValue,
+                        "`emulate-missing-as` must match one of the provided `arch`es",
+                    ))
+                } else {
+                    Ok(())
+                }
+            })
+            .transpose()?;
+        Ok(res)
+    }
+}
+impl clap::Args for ImageConfig {
+    fn augment_args(cmd: clap::Command) -> clap::Command {
+        CliImageConfig::augment_args(cmd)
+    }
+    fn augment_args_for_update(cmd: clap::Command) -> clap::Command {
+        CliImageConfig::augment_args_for_update(cmd)
+    }
+}
+impl clap::FromArgMatches for ImageConfig {
+    fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
+        Self::try_from(CliImageConfig::from_arg_matches(matches)?)
+    }
+    fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
+        *self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[serde(untagged)]
+#[ts(export)]
+pub enum BuildArg {
+    String(String),
+    EnvVar { env: String },
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum ImageSource {
+    Packed,
+    #[serde(rename_all = "camelCase")]
+    DockerBuild {
+        #[ts(optional)]
+        workdir: Option<PathBuf>,
+        #[ts(optional)]
+        dockerfile: Option<PathBuf>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        #[ts(optional)]
+        build_args: Option<BTreeMap<String, BuildArg>>,
+    },
+    DockerTag(String),
+    // Recipe(DirRecipe),
+}
+impl ImageSource {
+    pub fn ingredients(&self) -> Vec<PathBuf> {
+        match self {
+            Self::Packed => Vec::new(),
+            Self::DockerBuild {
+                dockerfile,
+                workdir,
+                ..
+            } => {
+                vec![
+                    workdir
+                        .as_deref()
+                        .unwrap_or(Path::new("."))
+                        .join(dockerfile.as_deref().unwrap_or(Path::new("Dockerfile"))),
+                ]
+            }
+            Self::DockerTag(_) => Vec::new(),
+        }
+    }
+    #[instrument(skip_all)]
+    pub fn load<'a, S: From<TmpSource<PackSource>> + FileSource + Clone>(
+        &'a self,
+        tmp_dir: Arc<TmpDir>,
+        id: &'a PackageId,
+        version: &'a VersionString,
+        image_id: &'a ImageId,
+        arch: &'a str,
+        into: &'a mut DirectoryContents<S>,
+    ) -> BoxFuture<'a, Result<(), Error>> {
+        #[derive(Deserialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct DockerImageConfig {
+            env: Vec<String>,
+            #[serde(default)]
+            working_dir: PathBuf,
+            #[serde(default)]
+            user: String,
+            entrypoint: Option<Vec<String>>,
+            cmd: Option<Vec<String>>,
+        }
+        async move {
+            match self {
+                ImageSource::Packed => Ok(()),
+                ImageSource::DockerBuild {
+                    workdir,
+                    dockerfile,
+                    build_args,
+                } => {
+                    let workdir = workdir.as_deref().unwrap_or(Path::new("."));
+                    let dockerfile = dockerfile
+                        .clone()
+                        .unwrap_or_else(|| workdir.join("Dockerfile"));
+                    let docker_platform = if arch == "x86_64" {
+                        "--platform=linux/amd64".to_owned()
+                    } else if arch == "aarch64" {
+                        "--platform=linux/arm64".to_owned()
+                    } else {
+                        format!("--platform=linux/{arch}")
+                    };
+                    // docker buildx build ${path} -o type=image,name=start9/${id}
+                    let tag = format!("start9/{id}/{image_id}:{}", new_guid());
+                    let mut command = Command::new(*CONTAINER_TOOL);
+                    if *CONTAINER_TOOL == "docker" {
+                        command.arg("buildx");
+                    }
+                    command
+                        .arg("build")
+                        .arg(workdir)
+                        .arg("-f")
+                        .arg(dockerfile)
+                        .arg("-t")
+                        .arg(&tag)
+                        .arg(&docker_platform)
+                        .arg("--build-arg")
+                        .arg(format!("ARCH={}", arch));
+
+                    // add build arguments
+                    if let Some(build_args) = build_args {
+                        for (key, value) in build_args {
+                            let build_arg_value = match value {
+                                BuildArg::String(val) => val.to_string(),
+                                BuildArg::EnvVar { env } => {
+                                    match std::env::var(&env) {
+                                        Ok(val) => val,
+                                        Err(_) => continue, // skip if env var not set or invalid
+                                    }
+                                }
+                            };
+
+                            command
+                                .arg("--build-arg")
+                                .arg(format!("{}={}", key, build_arg_value));
+                        }
+                    }
+
+                    command
+                        .arg("-o")
+                        .arg("type=docker,dest=-")
+                        .capture(false)
+                        .pipe(Command::new(*CONTAINER_TOOL).arg("load"))
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    ImageSource::DockerTag(tag.clone())
+                        .load(tmp_dir, id, version, image_id, arch, into)
+                        .await?;
+                    Command::new(*CONTAINER_TOOL)
+                        .arg("rmi")
+                        .arg("-f")
+                        .arg(&tag)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    Ok(())
+                }
+                ImageSource::DockerTag(tag) => {
+                    let docker_platform = if arch == "x86_64" {
+                        "--platform=linux/amd64".to_owned()
+                    } else if arch == "aarch64" {
+                        "--platform=linux/arm64".to_owned()
+                    } else {
+                        format!("--platform=linux/{arch}")
+                    };
+                    let container = String::from_utf8(
+                        Command::new(*CONTAINER_TOOL)
+                            .arg("create")
+                            .arg(&docker_platform)
+                            .arg(&tag)
+                            .invoke(ErrorKind::Docker)
+                            .await?,
+                    )?;
+                    let container = container.trim();
+                    let config = serde_json::from_slice::<DockerImageConfig>(
+                        &Command::new(*CONTAINER_TOOL)
+                            .arg("container")
+                            .arg("inspect")
+                            .arg("--format")
+                            .arg("{{json .Config}}")
+                            .arg(container)
+                            .invoke(ErrorKind::Docker)
+                            .await?,
+                    )
+                    .with_kind(ErrorKind::Deserialization)?;
+                    let base_path = Path::new("images").join(arch).join(image_id);
+                    into.insert_path(
+                        base_path.with_extension("json"),
+                        Entry::file(
+                            TmpSource::new(
+                                tmp_dir.clone(),
+                                PackSource::Buffered(
+                                    serde_json::to_vec(&ImageMetadata {
+                                        workdir: if config.working_dir == Path::new("") {
+                                            "/".into()
+                                        } else {
+                                            config.working_dir
+                                        },
+                                        user: if config.user.is_empty() {
+                                            "root".into()
+                                        } else {
+                                            config.user.into()
+                                        },
+                                        entrypoint: config.entrypoint,
+                                        cmd: config.cmd,
+                                    })
+                                    .with_kind(ErrorKind::Serialization)?
+                                    .into(),
+                                ),
+                            )
+                            .into(),
+                        ),
+                    )?;
+                    into.insert_path(
+                        base_path.with_extension("env"),
+                        Entry::file(
+                            TmpSource::new(
+                                tmp_dir.clone(),
+                                PackSource::Buffered(config.env.join("\n").into_bytes().into()),
+                            )
+                            .into(),
+                        ),
+                    )?;
+                    let dest = tmp_dir
+                        .join(Guid::new().as_ref())
+                        .with_extension("squashfs");
+
+                    Command::new(*CONTAINER_TOOL)
+                        .arg("export")
+                        .arg(container)
+                        .pipe(&mut tar2sqfs(&dest)?)
+                        .capture(false)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    Command::new(*CONTAINER_TOOL)
+                        .arg("rm")
+                        .arg(container)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    into.insert_path(
+                        base_path.with_extension("squashfs"),
+                        Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(dest)).into()),
+                    )?;
+
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+fn tar2sqfs(dest: impl AsRef<Path>) -> Result<Command, Error> {
+    let dest = dest.as_ref();
+
+    Ok({
+        #[cfg(target_os = "linux")]
+        {
+            let mut command = Command::new("tar2sqfs");
+            command.arg("-q").arg(&dest);
+            command
+        }
+        #[cfg(target_os = "macos")]
+        {
+            let directory = dest
+                .parent()
+                .unwrap_or_else(|| Path::new("/"))
+                .to_path_buf();
+            let mut command = Command::new(*CONTAINER_TOOL);
+            command
+                .arg("run")
+                .arg("-i")
+                .arg("--rm")
+                .arg("--mount")
+                .arg(format!("type=bind,src={},dst=/data", directory.display()))
+                .arg("ghcr.io/start9labs/sdk/utils:latest")
+                .arg("tar2sqfs")
+                .arg("-q")
+                .arg(Path::new("/data").join(&dest.file_name().unwrap_or_default()));
+            command
+        }
+    })
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ImageMetadata {
+    pub workdir: PathBuf,
+    #[ts(type = "string")]
+    pub user: InternedString,
+    pub entrypoint: Option<Vec<String>>,
+    pub cmd: Option<Vec<String>>,
+}
+
+#[instrument(skip_all)]
+pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
+    let tmp_dir = Arc::new(TmpDir::new().await?);
+    let mut files = DirectoryContents::<TmpSource<PackSource>>::new();
+    let js_dir = params.javascript();
+    let manifest: Arc<[u8]> = Command::new("node")
+        .arg("-e")
+        .arg(format!(
+            "console.log(JSON.stringify(require('{}/index.js').manifest))",
+            js_dir.display()
+        ))
+        .invoke(ErrorKind::Javascript)
+        .await?
+        .into();
+    files.insert(
+        "manifest.json".into(),
+        Entry::file(TmpSource::new(
+            tmp_dir.clone(),
+            PackSource::Buffered(manifest.clone()),
+        )),
+    );
+    let icon = params.icon().await?;
+    let icon_ext = icon
+        .extension()
+        .or_not_found("icon file extension")?
+        .to_string_lossy();
+    files.insert(
+        InternedString::from_display(&lazy_format!("icon.{}", icon_ext)),
+        Entry::file(TmpSource::new(tmp_dir.clone(), PackSource::File(icon))),
+    );
+    files.insert(
+        "LICENSE.md".into(),
+        Entry::file(TmpSource::new(
+            tmp_dir.clone(),
+            PackSource::File(params.license().await?),
+        )),
+    );
+    files.insert(
+        "javascript.squashfs".into(),
+        Entry::file(TmpSource::new(
+            tmp_dir.clone(),
+            PackSource::Squashfs(Arc::new(SqfsDir::new(js_dir, tmp_dir.clone()))),
+        )),
+    );
+
+    let mut s9pk = S9pk::new(
+        MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT),
+        None,
+    )
+    .await?;
+
+    let manifest = s9pk.as_manifest_mut();
+    manifest.git_hash = Some(GitHash::from_path(params.path()).await?);
+    if !params.arch.is_empty() {
+        let arches = match manifest.hardware_requirements.arch.take() {
+            Some(a) => params
+                .arch
+                .iter()
+                .filter(|x| a.contains(*x))
+                .cloned()
+                .collect(),
+            None => params.arch.iter().cloned().collect(),
+        };
+        manifest
+            .images
+            .values_mut()
+            .for_each(|c| c.arch = c.arch.intersection(&arches).cloned().collect());
+        manifest.hardware_requirements.arch = Some(arches);
+    }
+
+    if !params.no_assets {
+        let assets_dir = params.assets();
+        s9pk.as_archive_mut().contents_mut().insert_path(
+            "assets.squashfs",
+            Entry::file(TmpSource::new(
+                tmp_dir.clone(),
+                PackSource::Squashfs(Arc::new(SqfsDir::new(assets_dir, tmp_dir.clone()))),
+            )),
+        )?;
+    }
+
+    s9pk.load_images(tmp_dir.clone()).await?;
+
+    let mut to_insert = Vec::new();
+    for (id, dependency) in &mut s9pk.as_manifest_mut().dependencies.0 {
+        if let Some((title, icon)) = match dependency.metadata.take() {
+            Some(MetadataSrc::Metadata(metadata)) => {
+                let icon = match metadata.icon {
+                    PathOrUrl::Path(path) => DataUrl::from_path(path).await?,
+                    PathOrUrl::Url(url) => {
+                        if url.scheme() == "http" || url.scheme() == "https" {
+                            DataUrl::from_response(ctx.client.get(url).send().await?).await?
+                        } else if url.scheme() == "data" {
+                            url.as_str().parse()?
+                        } else {
+                            return Err(Error::new(
+                                eyre!("unknown scheme: {}", url.scheme()),
+                                ErrorKind::InvalidRequest,
+                            ));
+                        }
+                    }
+                };
+                Some((metadata.title, icon))
+            }
+            Some(MetadataSrc::S9pk(Some(s9pk))) => {
+                let s9pk = match s9pk {
+                    PathOrUrl::Path(path) => {
+                        S9pk::deserialize(&MultiCursorFile::from(open_file(path).await?), None)
+                            .await?
+                            .into_dyn()
+                    }
+                    PathOrUrl::Url(url) => {
+                        if url.scheme() == "http" || url.scheme() == "https" {
+                            S9pk::deserialize(
+                                &Arc::new(HttpSource::new(ctx.client.clone(), url).await?),
+                                None,
+                            )
+                            .await?
+                            .into_dyn()
+                        } else {
+                            return Err(Error::new(
+                                eyre!("unknown scheme: {}", url.scheme()),
+                                ErrorKind::InvalidRequest,
+                            ));
+                        }
+                    }
+                };
+                Some((
+                    s9pk.as_manifest().title.clone(),
+                    s9pk.icon_data_url().await?,
+                ))
+            }
+            Some(MetadataSrc::S9pk(None)) | None => {
+                warn!("no metadata specified for {id}, leaving metadata empty");
+                None
+            }
+        } {
+            let dep_path = Path::new("dependencies").join(id);
+            to_insert.push((
+                dep_path.join("metadata.json"),
+                Entry::file(TmpSource::new(
+                    tmp_dir.clone(),
+                    PackSource::Buffered(
+                        IoFormat::Json.to_vec(&DependencyMetadata { title })?.into(),
+                    ),
+                )),
+            ));
+            to_insert.push((
+                dep_path
+                    .join("icon")
+                    .with_extension(icon.canonical_ext().unwrap_or("ico")),
+                Entry::file(TmpSource::new(
+                    tmp_dir.clone(),
+                    PackSource::Buffered(icon.data.into_owned().into()),
+                )),
+            ));
+        }
+    }
+    for (path, source) in to_insert {
+        s9pk.as_archive_mut()
+            .contents_mut()
+            .insert_path(path, source)?;
+    }
+
+    s9pk.validate_and_filter(None)?;
+
+    s9pk.serialize(
+        &mut create_file(params.output(&s9pk.as_manifest().id)).await?,
+        false,
+    )
+    .await?;
+
+    drop(s9pk);
+
+    tmp_dir.gc().await?;
+
+    Ok(())
+}
+
+#[instrument(skip_all)]
+pub async fn list_ingredients(_: CliContext, params: PackParams) -> Result<Vec<PathBuf>, Error> {
+    let js_path = params.javascript().join("index.js");
+    let manifest: Manifest = match async {
+        serde_json::from_slice(
+            &Command::new("node")
+                .arg("-e")
+                .arg(format!(
+                    "console.log(JSON.stringify(require('{}').manifest))",
+                    js_path.display()
+                ))
+                .invoke(ErrorKind::Javascript)
+                .await?,
+        )
+        .with_kind(ErrorKind::Deserialization)
+    }
+    .await
+    {
+        Ok(m) => m,
+        Err(e) => {
+            warn!("failed to load manifest: {e}");
+            debug!("{e:?}");
+            return Ok(vec![js_path, params.icon().await?, params.license().await?]);
+        }
+    };
+    let mut ingredients = vec![js_path, params.icon().await?, params.license().await?];
+
+    for (_, dependency) in manifest.dependencies.0 {
+        match dependency.metadata {
+            Some(MetadataSrc::Metadata(crate::dependencies::Metadata {
+                icon: PathOrUrl::Path(icon),
+                ..
+            })) => {
+                ingredients.push(icon);
+            }
+            Some(MetadataSrc::S9pk(Some(PathOrUrl::Path(s9pk)))) => {
+                ingredients.push(s9pk);
+            }
+            _ => (),
+        }
+    }
+
+    if !params.no_assets {
+        let assets_dir = params.assets();
+        ingredients.push(assets_dir);
+    }
+
+    for image in manifest.images.values() {
+        ingredients.extend(image.source.ingredients());
+    }
+
+    Ok(ingredients)
+}

core/src/s9pk/v2/recipe.rs

@@ -0,0 +1,21 @@
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+use serde::{Deserialize, Serialize};
+use ts_rs::TS;
+use url::Url;
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+pub struct DirRecipe(BTreeMap<PathBuf, Recipe>);
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+pub enum Recipe {
+    Make(PathBuf),
+    Wget {
+        #[ts(type = "string")]
+        url: Url,
+        checksum: String,
+    },
+    Recipe(DirRecipe),
+}

core/src/s9pk/v2/specv2.md

@@ -0,0 +1,89 @@
+## Magic
+
+`0x3b3b`
+
+## Version
+
+`0x02` (varint)
+
+## Merkle Archive
+
+### Header
+
+- ed25519 pubkey (32B)
+- ed25519 signature of TOC sighash (64B)
+- TOC sighash: (32B)
+- TOC position: (8B: u64 BE)
+- TOC size: (8B: u64 BE)
+
+### TOC
+
+- number of entries (varint)
+- FOREACH section
+  - name (varstring)
+  - hash (32B: BLAKE-3 of file contents / TOC sighash)
+  - TYPE (1B)
+    - TYPE=MISSING (`0x00`)
+    - TYPE=FILE (`0x01`)
+      - position (8B: u64 BE)
+      - size (8B: u64 BE)
+    - TYPE=TOC (`0x02`)
+      - position (8B: u64 BE)
+      - size (8B: u64 BE)
+
+#### SigHash
+Hash of TOC with all contents MISSING
+
+### FILE
+
+`<File contents>`
+
+# Example
+
+`foo/bar/baz.txt`
+
+ROOT TOC:
+  - 1 section
+    - name: foo
+      hash: sighash('a)
+      type: TOC
+      position: 'a
+      length: _
+
+'a:
+  - 1 section
+    - name: bar
+      hash: sighash('b)
+      type: TOC
+      position: 'b
+      size: _
+
+'b:
+  - 2 sections
+    - name: baz.txt
+      hash: hash('c)
+      type: FILE
+      position: 'c
+      length: _
+    - name: qux
+      hash: `<unverifiable>`
+      type: MISSING
+
+'c: `<CONTENTS OF baz.txt>`
+
+"foo/"
+hash: _
+size: 15b
+
+"bar.txt"
+hash: _
+size: 5b
+
+`<CONTENTS OF foo/>` (
+  "baz.txt"
+  hash: _
+  size: 2b
+)
+`<CONTENTS OF bar.txt>` ("hello")
+`<CONTENTS OF baz.txt>` ("hi")
+