rename frontend to web and update contributing guide (#2509)

* rename frontend to web and update contributing guide

* rename this time

* fix build

* restructure rust code

* update documentation

* update descriptions

* Update CONTRIBUTING.md

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>

core/startos/src/s9pk/builder.rs

@@ -0,0 +1,145 @@
+use sha2::{Digest, Sha512};
+use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom};
+use tracing::instrument;
+use typed_builder::TypedBuilder;
+
+use super::header::{FileSection, Header};
+use super::manifest::Manifest;
+use super::SIG_CONTEXT;
+use crate::util::io::to_cbor_async_writer;
+use crate::util::HashWriter;
+use crate::{Error, ResultExt};
+
+#[derive(TypedBuilder)]
+pub struct S9pkPacker<
+    'a,
+    W: AsyncWriteExt + AsyncSeekExt,
+    RLicense: AsyncReadExt + Unpin,
+    RInstructions: AsyncReadExt + Unpin,
+    RIcon: AsyncReadExt + Unpin,
+    RDockerImages: AsyncReadExt + Unpin,
+    RAssets: AsyncReadExt + Unpin,
+    RScripts: AsyncReadExt + Unpin,
+> {
+    writer: W,
+    manifest: &'a Manifest,
+    license: RLicense,
+    instructions: RInstructions,
+    icon: RIcon,
+    docker_images: RDockerImages,
+    assets: RAssets,
+    scripts: Option<RScripts>,
+}
+impl<
+        'a,
+        W: AsyncWriteExt + AsyncSeekExt + Unpin,
+        RLicense: AsyncReadExt + Unpin,
+        RInstructions: AsyncReadExt + Unpin,
+        RIcon: AsyncReadExt + Unpin,
+        RDockerImages: AsyncReadExt + Unpin,
+        RAssets: AsyncReadExt + Unpin,
+        RScripts: AsyncReadExt + Unpin,
+    > S9pkPacker<'a, W, RLicense, RInstructions, RIcon, RDockerImages, RAssets, RScripts>
+{
+    /// BLOCKING
+    #[instrument(skip_all)]
+    pub async fn pack(mut self, key: &ed25519_dalek::SigningKey) -> Result<(), Error> {
+        let header_pos = self.writer.stream_position().await?;
+        if header_pos != 0 {
+            tracing::warn!("Appending to non-empty file.");
+        }
+        let mut header = Header::placeholder();
+        header.serialize(&mut self.writer).await.with_ctx(|_| {
+            (
+                crate::ErrorKind::Serialization,
+                "Writing Placeholder Header",
+            )
+        })?;
+        let mut position = self.writer.stream_position().await?;
+
+        let mut writer = HashWriter::new(Sha512::new(), &mut self.writer);
+        // manifest
+        to_cbor_async_writer(&mut writer, self.manifest).await?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.manifest = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // license
+        tokio::io::copy(&mut self.license, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying License"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.license = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // instructions
+        tokio::io::copy(&mut self.instructions, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Instructions"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.instructions = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // icon
+        tokio::io::copy(&mut self.icon, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Icon"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.icon = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // docker_images
+        tokio::io::copy(&mut self.docker_images, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Docker Images"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.docker_images = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // assets
+        tokio::io::copy(&mut self.assets, &mut writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Assets"))?;
+        let new_pos = writer.inner_mut().stream_position().await?;
+        header.table_of_contents.assets = FileSection {
+            position,
+            length: new_pos - position,
+        };
+        position = new_pos;
+        // scripts
+        if let Some(mut scripts) = self.scripts {
+            tokio::io::copy(&mut scripts, &mut writer)
+                .await
+                .with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Scripts"))?;
+            let new_pos = writer.inner_mut().stream_position().await?;
+            header.table_of_contents.scripts = Some(FileSection {
+                position,
+                length: new_pos - position,
+            });
+            position = new_pos;
+        }
+
+        // header
+        let (hash, _) = writer.finish();
+        self.writer.seek(SeekFrom::Start(header_pos)).await?;
+        header.pubkey = key.into();
+        header.signature = key.sign_prehashed(hash, Some(SIG_CONTEXT))?;
+        header
+            .serialize(&mut self.writer)
+            .await
+            .with_ctx(|_| (crate::ErrorKind::Serialization, "Writing Header"))?;
+        self.writer.seek(SeekFrom::Start(position)).await?;
+
+        Ok(())
+    }
+}

core/startos/src/s9pk/docker.rs

@@ -0,0 +1,95 @@
+use std::borrow::Cow;
+use std::collections::BTreeSet;
+use std::io::SeekFrom;
+use std::path::Path;
+
+use color_eyre::eyre::eyre;
+use futures::{FutureExt, TryStreamExt};
+use serde::{Deserialize, Serialize};
+use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt};
+use tokio_tar::{Archive, Entry};
+
+use crate::util::io::from_cbor_async_reader;
+use crate::{Error, ErrorKind, ARCH};
+
+#[derive(Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DockerMultiArch {
+    pub default: String,
+    pub available: BTreeSet<String>,
+}
+
+#[pin_project::pin_project(project = DockerReaderProject)]
+#[derive(Debug)]
+pub enum DockerReader<R: AsyncRead + Unpin> {
+    SingleArch(#[pin] R),
+    MultiArch(#[pin] Entry<Archive<R>>),
+}
+impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> DockerReader<R> {
+    pub async fn new(mut rdr: R) -> Result<Self, Error> {
+        let arch = if let Some(multiarch) = tokio_tar::Archive::new(&mut rdr)
+            .entries()?
+            .try_filter_map(|e| {
+                async move {
+                    Ok(if &*e.path()? == Path::new("multiarch.cbor") {
+                        Some(e)
+                    } else {
+                        None
+                    })
+                }
+                .boxed()
+            })
+            .try_next()
+            .await?
+        {
+            let multiarch: DockerMultiArch = from_cbor_async_reader(multiarch).await?;
+            Some(if multiarch.available.contains(&**ARCH) {
+                Cow::Borrowed(&**ARCH)
+            } else {
+                Cow::Owned(multiarch.default)
+            })
+        } else {
+            None
+        };
+        rdr.seek(SeekFrom::Start(0)).await?;
+        if let Some(arch) = arch {
+            if let Some(image) = tokio_tar::Archive::new(rdr)
+                .entries()?
+                .try_filter_map(|e| {
+                    let arch = arch.clone();
+                    async move {
+                        Ok(if &*e.path()? == Path::new(&format!("{}.tar", arch)) {
+                            Some(e)
+                        } else {
+                            None
+                        })
+                    }
+                    .boxed()
+                })
+                .try_next()
+                .await?
+            {
+                Ok(Self::MultiArch(image))
+            } else {
+                Err(Error::new(
+                    eyre!("Docker image section does not contain tarball for architecture"),
+                    ErrorKind::ParseS9pk,
+                ))
+            }
+        } else {
+            Ok(Self::SingleArch(rdr))
+        }
+    }
+}
+impl<R: AsyncRead + Unpin + Send + Sync> AsyncRead for DockerReader<R> {
+    fn poll_read(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        match self.project() {
+            DockerReaderProject::SingleArch(r) => r.poll_read(cx, buf),
+            DockerReaderProject::MultiArch(r) => r.poll_read(cx, buf),
+        }
+    }
+}

core/startos/src/s9pk/git_hash.rs

@@ -0,0 +1,41 @@
+use std::path::Path;
+
+use crate::Error;
+
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+pub struct GitHash(String);
+
+impl GitHash {
+    pub async fn from_path(path: impl AsRef<Path>) -> Result<GitHash, Error> {
+        let hash = tokio::process::Command::new("git")
+            .args(["describe", "--always", "--abbrev=40", "--dirty=-modified"])
+            .current_dir(path)
+            .output()
+            .await?;
+        if !hash.status.success() {
+            return Err(Error::new(
+                color_eyre::eyre::eyre!("Could not get hash: {}", String::from_utf8(hash.stderr)?),
+                crate::ErrorKind::Filesystem,
+            ));
+        }
+        Ok(GitHash(String::from_utf8(hash.stdout)?))
+    }
+}
+
+impl AsRef<str> for GitHash {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+// #[tokio::test]
+// async fn test_githash_for_current() {
+//     let answer: GitHash = GitHash::from_path(std::env::current_dir().unwrap())
+//         .await
+//         .unwrap();
+//     let answer_str: &str = answer.as_ref();
+//     assert!(
+//         !answer_str.is_empty(),
+//         "Should have a hash for this current working"
+//     );
+// }

core/startos/src/s9pk/header.rs

@@ -0,0 +1,187 @@
+use std::collections::BTreeMap;
+
+use color_eyre::eyre::eyre;
+use ed25519_dalek::{Signature, VerifyingKey};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
+
+use crate::Error;
+
+pub const MAGIC: [u8; 2] = [59, 59];
+pub const VERSION: u8 = 1;
+
+#[derive(Debug)]
+pub struct Header {
+    pub pubkey: VerifyingKey,
+    pub signature: Signature,
+    pub table_of_contents: TableOfContents,
+}
+impl Header {
+    pub fn placeholder() -> Self {
+        Header {
+            pubkey: VerifyingKey::default(),
+            signature: Signature::from_bytes(&[0; 64]),
+            table_of_contents: Default::default(),
+        }
+    }
+    // MUST BE SAME SIZE REGARDLESS OF DATA
+    pub async fn serialize<W: AsyncWriteExt + Unpin>(&self, mut writer: W) -> std::io::Result<()> {
+        writer.write_all(&MAGIC).await?;
+        writer.write_all(&[VERSION]).await?;
+        writer.write_all(self.pubkey.as_bytes()).await?;
+        writer.write_all(&self.signature.to_bytes()).await?;
+        self.table_of_contents.serialize(writer).await?;
+        Ok(())
+    }
+    pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> Result<Self, Error> {
+        let mut magic = [0; 2];
+        reader.read_exact(&mut magic).await?;
+        if magic != MAGIC {
+            return Err(Error::new(
+                eyre!("Incorrect Magic: {:?}", magic),
+                crate::ErrorKind::ParseS9pk,
+            ));
+        }
+        let mut version = [0];
+        reader.read_exact(&mut version).await?;
+        if version[0] != VERSION {
+            return Err(Error::new(
+                eyre!("Unknown Version: {}", version[0]),
+                crate::ErrorKind::ParseS9pk,
+            ));
+        }
+        let mut pubkey_bytes = [0; 32];
+        reader.read_exact(&mut pubkey_bytes).await?;
+        let pubkey = VerifyingKey::from_bytes(&pubkey_bytes)
+            .map_err(|e| Error::new(e, crate::ErrorKind::ParseS9pk))?;
+        let mut sig_bytes = [0; 64];
+        reader.read_exact(&mut sig_bytes).await?;
+        let signature = Signature::from_bytes(&sig_bytes);
+        let table_of_contents = TableOfContents::deserialize(reader).await?;
+
+        Ok(Header {
+            pubkey,
+            signature,
+            table_of_contents,
+        })
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct TableOfContents {
+    pub manifest: FileSection,
+    pub license: FileSection,
+    pub instructions: FileSection,
+    pub icon: FileSection,
+    pub docker_images: FileSection,
+    pub assets: FileSection,
+    pub scripts: Option<FileSection>,
+}
+impl TableOfContents {
+    pub async fn serialize<W: AsyncWriteExt + Unpin>(&self, mut writer: W) -> std::io::Result<()> {
+        let len: u32 = ((1 + "manifest".len() + 16)
+            + (1 + "license".len() + 16)
+            + (1 + "instructions".len() + 16)
+            + (1 + "icon".len() + 16)
+            + (1 + "docker_images".len() + 16)
+            + (1 + "assets".len() + 16)
+            + (1 + "scripts".len() + 16)) as u32;
+        writer.write_all(&u32::to_be_bytes(len)).await?;
+        self.manifest
+            .serialize_entry("manifest", &mut writer)
+            .await?;
+        self.license.serialize_entry("license", &mut writer).await?;
+        self.instructions
+            .serialize_entry("instructions", &mut writer)
+            .await?;
+        self.icon.serialize_entry("icon", &mut writer).await?;
+        self.docker_images
+            .serialize_entry("docker_images", &mut writer)
+            .await?;
+        self.assets.serialize_entry("assets", &mut writer).await?;
+        self.scripts
+            .unwrap_or_default()
+            .serialize_entry("scripts", &mut writer)
+            .await?;
+        Ok(())
+    }
+    pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> std::io::Result<Self> {
+        let mut toc_len = [0; 4];
+        reader.read_exact(&mut toc_len).await?;
+        let toc_len = u32::from_be_bytes(toc_len);
+        let mut reader = reader.take(toc_len as u64);
+        let mut table = BTreeMap::new();
+        while let Some((label, section)) = FileSection::deserialize_entry(&mut reader).await? {
+            table.insert(label, section);
+        }
+        fn from_table(
+            table: &BTreeMap<Vec<u8>, FileSection>,
+            label: &str,
+        ) -> std::io::Result<FileSection> {
+            table.get(label.as_bytes()).copied().ok_or_else(|| {
+                std::io::Error::new(
+                    std::io::ErrorKind::UnexpectedEof,
+                    format!("Missing Required Label: {}", label),
+                )
+            })
+        }
+        #[allow(dead_code)]
+        fn as_opt(fs: FileSection) -> Option<FileSection> {
+            if fs.position | fs.length == 0 {
+                // 0/0 is not a valid file section
+                None
+            } else {
+                Some(fs)
+            }
+        }
+        Ok(TableOfContents {
+            manifest: from_table(&table, "manifest")?,
+            license: from_table(&table, "license")?,
+            instructions: from_table(&table, "instructions")?,
+            icon: from_table(&table, "icon")?,
+            docker_images: from_table(&table, "docker_images")?,
+            assets: from_table(&table, "assets")?,
+            scripts: table.get("scripts".as_bytes()).cloned(),
+        })
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct FileSection {
+    pub position: u64,
+    pub length: u64,
+}
+impl FileSection {
+    pub async fn serialize_entry<W: AsyncWriteExt + Unpin>(
+        self,
+        label: &str,
+        mut writer: W,
+    ) -> std::io::Result<()> {
+        writer.write_all(&[label.len() as u8]).await?;
+        writer.write_all(label.as_bytes()).await?;
+        writer.write_all(&u64::to_be_bytes(self.position)).await?;
+        writer.write_all(&u64::to_be_bytes(self.length)).await?;
+        Ok(())
+    }
+    pub async fn deserialize_entry<R: AsyncRead + Unpin>(
+        mut reader: R,
+    ) -> std::io::Result<Option<(Vec<u8>, Self)>> {
+        let mut label_len = [0];
+        let read = reader.read(&mut label_len).await?;
+        if read == 0 {
+            return Ok(None);
+        }
+        let mut label = vec![0; label_len[0] as usize];
+        reader.read_exact(&mut label).await?;
+        let mut pos = [0; 8];
+        reader.read_exact(&mut pos).await?;
+        let mut len = [0; 8];
+        reader.read_exact(&mut len).await?;
+        Ok(Some((
+            label,
+            FileSection {
+                position: u64::from_be_bytes(pos),
+                length: u64::from_be_bytes(len),
+            },
+        )))
+    }
+}

core/startos/src/s9pk/manifest.rs

@@ -0,0 +1,211 @@
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use color_eyre::eyre::eyre;
+pub use models::PackageId;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+use super::git_hash::GitHash;
+use crate::action::Actions;
+use crate::backup::BackupActions;
+use crate::config::action::ConfigActions;
+use crate::dependencies::Dependencies;
+use crate::migration::Migrations;
+use crate::net::interface::Interfaces;
+use crate::prelude::*;
+use crate::procedure::docker::DockerContainers;
+use crate::procedure::PackageProcedure;
+use crate::status::health_check::HealthChecks;
+use crate::util::serde::Regex;
+use crate::util::Version;
+use crate::version::{Current, VersionT};
+use crate::volume::Volumes;
+use crate::Error;
+
+fn current_version() -> Version {
+    Current::new().semver().into()
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct Manifest {
+    #[serde(default = "current_version")]
+    pub eos_version: Version,
+    pub id: PackageId,
+    #[serde(default)]
+    pub git_hash: Option<GitHash>,
+    pub title: String,
+    pub version: Version,
+    pub description: Description,
+    #[serde(default)]
+    pub assets: Assets,
+    #[serde(default)]
+    pub build: Option<Vec<String>>,
+    pub release_notes: String,
+    pub license: String, // type of license
+    pub wrapper_repo: Url,
+    pub upstream_repo: Url,
+    pub support_site: Option<Url>,
+    pub marketing_site: Option<Url>,
+    pub donation_url: Option<Url>,
+    #[serde(default)]
+    pub alerts: Alerts,
+    pub main: PackageProcedure,
+    pub health_checks: HealthChecks,
+    pub config: Option<ConfigActions>,
+    pub properties: Option<PackageProcedure>,
+    pub volumes: Volumes,
+    // #[serde(default)]
+    pub interfaces: Interfaces,
+    // #[serde(default)]
+    pub backup: BackupActions,
+    #[serde(default)]
+    pub migrations: Migrations,
+    #[serde(default)]
+    pub actions: Actions,
+    // #[serde(default)]
+    // pub permissions: Permissions,
+    #[serde(default)]
+    pub dependencies: Dependencies,
+    pub containers: Option<DockerContainers>,
+
+    #[serde(default)]
+    pub replaces: Vec<String>,
+
+    #[serde(default)]
+    pub hardware_requirements: HardwareRequirements,
+}
+
+impl Manifest {
+    pub fn package_procedures(&self) -> impl Iterator<Item = &PackageProcedure> {
+        use std::iter::once;
+        let main = once(&self.main);
+        let cfg_get = self.config.as_ref().map(|a| &a.get).into_iter();
+        let cfg_set = self.config.as_ref().map(|a| &a.set).into_iter();
+        let props = self.properties.iter();
+        let backups = vec![&self.backup.create, &self.backup.restore].into_iter();
+        let migrations = self
+            .migrations
+            .to
+            .values()
+            .chain(self.migrations.from.values());
+        let actions = self.actions.0.values().map(|a| &a.implementation);
+        main.chain(cfg_get)
+            .chain(cfg_set)
+            .chain(props)
+            .chain(backups)
+            .chain(migrations)
+            .chain(actions)
+    }
+
+    pub fn with_git_hash(mut self, git_hash: GitHash) -> Self {
+        self.git_hash = Some(git_hash);
+        self
+    }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HardwareRequirements {
+    #[serde(default)]
+    device: BTreeMap<String, Regex>,
+    ram: Option<u64>,
+    pub arch: Option<Vec<String>>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Assets {
+    #[serde(default)]
+    pub license: Option<PathBuf>,
+    #[serde(default)]
+    pub instructions: Option<PathBuf>,
+    #[serde(default)]
+    pub icon: Option<PathBuf>,
+    #[serde(default)]
+    pub docker_images: Option<PathBuf>,
+    #[serde(default)]
+    pub assets: Option<PathBuf>,
+    #[serde(default)]
+    pub scripts: Option<PathBuf>,
+}
+impl Assets {
+    pub fn license_path(&self) -> &Path {
+        self.license
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("LICENSE.md"))
+    }
+    pub fn instructions_path(&self) -> &Path {
+        self.instructions
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("INSTRUCTIONS.md"))
+    }
+    pub fn icon_path(&self) -> &Path {
+        self.icon
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("icon.png"))
+    }
+    pub fn icon_type(&self) -> &str {
+        self.icon
+            .as_ref()
+            .and_then(|icon| icon.extension())
+            .and_then(|ext| ext.to_str())
+            .unwrap_or("png")
+    }
+    pub fn docker_images_path(&self) -> &Path {
+        self.docker_images
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("docker-images"))
+    }
+    pub fn assets_path(&self) -> &Path {
+        self.assets
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("assets"))
+    }
+    pub fn scripts_path(&self) -> &Path {
+        self.scripts
+            .as_ref()
+            .map(|a| a.as_path())
+            .unwrap_or(Path::new("scripts"))
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Description {
+    pub short: String,
+    pub long: String,
+}
+impl Description {
+    pub fn validate(&self) -> Result<(), Error> {
+        if self.short.chars().skip(160).next().is_some() {
+            return Err(Error::new(
+                eyre!("Short description must be 160 characters or less."),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if self.long.chars().skip(5000).next().is_some() {
+            return Err(Error::new(
+                eyre!("Long description must be 5000 characters or less."),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Alerts {
+    pub install: Option<String>,
+    pub uninstall: Option<String>,
+    pub restore: Option<String>,
+    pub start: Option<String>,
+    pub stop: Option<String>,
+}

core/startos/src/s9pk/mod.rs

@@ -0,0 +1,246 @@
+use std::ffi::OsStr;
+use std::path::PathBuf;
+
+use color_eyre::eyre::eyre;
+use futures::TryStreamExt;
+use imbl::OrdMap;
+use rpc_toolkit::command;
+use serde_json::Value;
+use tokio::io::AsyncRead;
+use tracing::instrument;
+
+use crate::context::SdkContext;
+use crate::s9pk::builder::S9pkPacker;
+use crate::s9pk::docker::DockerMultiArch;
+use crate::s9pk::git_hash::GitHash;
+use crate::s9pk::manifest::Manifest;
+use crate::s9pk::reader::S9pkReader;
+use crate::util::display_none;
+use crate::util::io::BufferedWriteReader;
+use crate::util::serde::IoFormat;
+use crate::volume::Volume;
+use crate::{Error, ErrorKind, ResultExt};
+
+pub mod builder;
+pub mod docker;
+pub mod git_hash;
+pub mod header;
+pub mod manifest;
+pub mod reader;
+
+pub const SIG_CONTEXT: &[u8] = b"s9pk";
+
+#[command(cli_only, display(display_none))]
+#[instrument(skip_all)]
+pub async fn pack(#[context] ctx: SdkContext, #[arg] path: Option<PathBuf>) -> Result<(), Error> {
+    use tokio::fs::File;
+
+    let path = if let Some(path) = path {
+        path
+    } else {
+        std::env::current_dir()?
+    };
+    let manifest_value: Value = if path.join("manifest.toml").exists() {
+        IoFormat::Toml
+            .from_async_reader(File::open(path.join("manifest.toml")).await?)
+            .await?
+    } else if path.join("manifest.yaml").exists() {
+        IoFormat::Yaml
+            .from_async_reader(File::open(path.join("manifest.yaml")).await?)
+            .await?
+    } else if path.join("manifest.json").exists() {
+        IoFormat::Json
+            .from_async_reader(File::open(path.join("manifest.json")).await?)
+            .await?
+    } else {
+        return Err(Error::new(
+            eyre!("manifest not found"),
+            crate::ErrorKind::Pack,
+        ));
+    };
+
+    let manifest: Manifest = serde_json::from_value::<Manifest>(manifest_value.clone())
+        .with_kind(crate::ErrorKind::Deserialization)?
+        .with_git_hash(GitHash::from_path(&path).await?);
+    let extra_keys =
+        enumerate_extra_keys(&serde_json::to_value(&manifest).unwrap(), &manifest_value);
+    for k in extra_keys {
+        tracing::warn!("Unrecognized Manifest Key: {}", k);
+    }
+
+    let outfile_path = path.join(format!("{}.s9pk", manifest.id));
+    let mut outfile = File::create(outfile_path).await?;
+    S9pkPacker::builder()
+        .manifest(&manifest)
+        .writer(&mut outfile)
+        .license(
+            File::open(path.join(manifest.assets.license_path()))
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        manifest.assets.license_path().display().to_string(),
+                    )
+                })?,
+        )
+        .icon(
+            File::open(path.join(manifest.assets.icon_path()))
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        manifest.assets.icon_path().display().to_string(),
+                    )
+                })?,
+        )
+        .instructions(
+            File::open(path.join(manifest.assets.instructions_path()))
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        manifest.assets.instructions_path().display().to_string(),
+                    )
+                })?,
+        )
+        .docker_images({
+            let docker_images_path = path.join(manifest.assets.docker_images_path());
+            let res: Box<dyn AsyncRead + Unpin + Send + Sync> = if tokio::fs::metadata(&docker_images_path).await?.is_dir() {
+                let tars: Vec<_> = tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(&docker_images_path).await?).try_collect().await?;
+                let mut arch_info = DockerMultiArch::default();
+                for tar in &tars {
+                    if tar.path().extension() == Some(OsStr::new("tar")) {
+                        arch_info.available.insert(tar.path().file_stem().unwrap_or_default().to_str().unwrap_or_default().to_owned());
+                    }
+                }
+                if arch_info.available.contains("aarch64") {
+                    arch_info.default = "aarch64".to_owned();
+                } else {
+                    arch_info.default = arch_info.available.iter().next().cloned().unwrap_or_default();
+                }
+                let arch_info_cbor = IoFormat::Cbor.to_vec(&arch_info)?;
+                Box::new(BufferedWriteReader::new(|w| async move {
+                    let mut docker_images = tokio_tar::Builder::new(w);
+                    let mut multiarch_header = tokio_tar::Header::new_gnu();
+                    multiarch_header.set_path("multiarch.cbor")?;
+                    multiarch_header.set_size(arch_info_cbor.len() as u64);
+                    multiarch_header.set_cksum();
+                    docker_images.append(&multiarch_header, std::io::Cursor::new(arch_info_cbor)).await?;
+                    for tar in tars
+                    {
+                        docker_images
+                            .append_path_with_name(
+                                tar.path(),
+                                tar.file_name(),
+                            )
+                            .await?;
+                    }
+                    Ok::<_, std::io::Error>(())
+                }, 1024 * 1024))
+            } else {
+                Box::new(File::open(docker_images_path)
+                    .await
+                    .with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            manifest.assets.docker_images_path().display().to_string(),
+                        )
+                    })?)
+            };
+            res
+        })
+        .assets({
+            let asset_volumes = manifest
+            .volumes
+            .iter()
+            .filter(|(_, v)| matches!(v, &&Volume::Assets {})).map(|(id, _)| id.clone()).collect::<Vec<_>>();
+            let assets_path = manifest.assets.assets_path().to_owned();
+            let path = path.clone();
+
+            BufferedWriteReader::new(|w| async move {
+                let mut assets = tokio_tar::Builder::new(w);
+                for asset_volume in asset_volumes
+                {
+                    assets
+                        .append_dir_all(
+                            &asset_volume,
+                            path.join(&assets_path).join(&asset_volume),
+                        )
+                        .await?;
+                }
+                Ok::<_, std::io::Error>(())
+            }, 1024 * 1024)
+        })
+        .scripts({
+            let script_path = path.join(manifest.assets.scripts_path()).join("embassy.js");
+            let needs_script = manifest.package_procedures().any(|a| a.is_script());
+            let has_script = script_path.exists();
+            match (needs_script, has_script) {
+                (true, true) => Some(File::open(script_path).await?),
+                (true, false) => {
+                    return Err(Error::new(eyre!("Script is declared in manifest, but no such script exists at ./scripts/embassy.js"), ErrorKind::Pack).into())
+                }
+                (false, true) => {
+                    tracing::warn!("Manifest does not declare any actions that use scripts, but a script exists at ./scripts/embassy.js");
+                    None
+                }
+                (false, false) => None
+            }
+        })
+        .build()
+        .pack(&ctx.developer_key()?)
+        .await?;
+    outfile.sync_all().await?;
+
+    Ok(())
+}
+
+#[command(rename = "s9pk", cli_only, display(display_none))]
+pub async fn verify(#[arg] path: PathBuf) -> Result<(), Error> {
+    let mut s9pk = S9pkReader::open(path, true).await?;
+    s9pk.validate().await?;
+
+    Ok(())
+}
+
+fn enumerate_extra_keys(reference: &Value, candidate: &Value) -> Vec<String> {
+    match (reference, candidate) {
+        (Value::Object(m_r), Value::Object(m_c)) => {
+            let om_r: OrdMap<String, Value> = m_r.clone().into_iter().collect();
+            let om_c: OrdMap<String, Value> = m_c.clone().into_iter().collect();
+            let common = om_r.clone().intersection(om_c.clone());
+            let top_extra = common.clone().symmetric_difference(om_c.clone());
+            let mut all_extra = top_extra
+                .keys()
+                .map(|s| format!(".{}", s))
+                .collect::<Vec<String>>();
+            for (k, v) in common {
+                all_extra.extend(
+                    enumerate_extra_keys(&v, om_c.get(&k).unwrap())
+                        .into_iter()
+                        .map(|s| format!(".{}{}", k, s)),
+                )
+            }
+            all_extra
+        }
+        (_, Value::Object(m1)) => m1.clone().keys().map(|s| format!(".{}", s)).collect(),
+        _ => Vec::new(),
+    }
+}
+
+#[test]
+fn test_enumerate_extra_keys() {
+    use serde_json::json;
+    let extras = enumerate_extra_keys(
+        &json!({
+            "test": 1,
+            "test2": null,
+        }),
+        &json!({
+            "test": 1,
+            "test2": { "test3": null },
+            "test4": null
+        }),
+    );
+    println!("{:?}", extras)
+}

core/startos/src/s9pk/reader.rs

@@ -0,0 +1,406 @@
+use std::collections::BTreeSet;
+use std::io::SeekFrom;
+use std::ops::Range;
+use std::path::Path;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::task::{Context, Poll};
+
+use color_eyre::eyre::eyre;
+use digest::Output;
+use ed25519_dalek::VerifyingKey;
+use futures::TryStreamExt;
+use models::ImageId;
+use sha2::{Digest, Sha512};
+use tokio::fs::File;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf};
+use tracing::instrument;
+
+use super::header::{FileSection, Header, TableOfContents};
+use super::manifest::{Manifest, PackageId};
+use super::SIG_CONTEXT;
+use crate::install::progress::InstallProgressTracker;
+use crate::s9pk::docker::DockerReader;
+use crate::util::Version;
+use crate::{Error, ResultExt};
+
+const MAX_REPLACES: usize = 10;
+const MAX_TITLE_LEN: usize = 30;
+
+#[pin_project::pin_project]
+#[derive(Debug)]
+pub struct ReadHandle<'a, R = File> {
+    pos: &'a mut u64,
+    range: Range<u64>,
+    #[pin]
+    rdr: &'a mut R,
+}
+impl<'a, R: AsyncRead + Unpin> ReadHandle<'a, R> {
+    pub async fn to_vec(mut self) -> std::io::Result<Vec<u8>> {
+        let mut buf = vec![0; (self.range.end - self.range.start) as usize];
+        self.read_exact(&mut buf).await?;
+        Ok(buf)
+    }
+}
+impl<'a, R: AsyncRead + Unpin> AsyncRead for ReadHandle<'a, R> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let this = self.project();
+        let start = buf.filled().len();
+        let mut take_buf = buf.take(this.range.end.saturating_sub(**this.pos) as usize);
+        let res = AsyncRead::poll_read(this.rdr, cx, &mut take_buf);
+        let n = take_buf.filled().len();
+        unsafe { buf.assume_init(start + n) };
+        buf.advance(n);
+        **this.pos += n as u64;
+        res
+    }
+}
+impl<'a, R: AsyncSeek + Unpin> AsyncSeek for ReadHandle<'a, R> {
+    fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> {
+        let this = self.project();
+        AsyncSeek::start_seek(
+            this.rdr,
+            match position {
+                SeekFrom::Current(n) => SeekFrom::Current(n),
+                SeekFrom::End(n) => SeekFrom::Start((this.range.end as i64 + n) as u64),
+                SeekFrom::Start(n) => SeekFrom::Start(this.range.start + n),
+            },
+        )
+    }
+    fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<u64>> {
+        let this = self.project();
+        match AsyncSeek::poll_complete(this.rdr, cx) {
+            Poll::Ready(Ok(n)) => {
+                let res = n.saturating_sub(this.range.start);
+                **this.pos = this.range.start + res;
+                Poll::Ready(Ok(res))
+            }
+            a => a,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ImageTag {
+    pub package_id: PackageId,
+    pub image_id: ImageId,
+    pub version: Version,
+}
+impl ImageTag {
+    #[instrument(skip_all)]
+    pub fn validate(&self, id: &PackageId, version: &Version) -> Result<(), Error> {
+        if id != &self.package_id {
+            return Err(Error::new(
+                eyre!(
+                    "Contains image for incorrect package: id {}",
+                    self.package_id,
+                ),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if version != &self.version {
+            return Err(Error::new(
+                eyre!(
+                    "Contains image with incorrect version: expected {} received {}",
+                    version,
+                    self.version,
+                ),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        Ok(())
+    }
+}
+impl FromStr for ImageTag {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let rest = s.strip_prefix("start9/").ok_or_else(|| {
+            Error::new(
+                eyre!("Invalid image tag prefix: expected start9/"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        let (package, rest) = rest.split_once("/").ok_or_else(|| {
+            Error::new(
+                eyre!("Image tag missing image id"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        let (image, version) = rest.split_once(":").ok_or_else(|| {
+            Error::new(
+                eyre!("Image tag missing version"),
+                crate::ErrorKind::ValidateS9pk,
+            )
+        })?;
+        Ok(ImageTag {
+            package_id: package.parse()?,
+            image_id: image.parse()?,
+            version: version.parse()?,
+        })
+    }
+}
+
+pub struct S9pkReader<R: AsyncRead + AsyncSeek + Unpin + Send + Sync = File> {
+    hash: Option<Output<Sha512>>,
+    hash_string: Option<String>,
+    developer_key: VerifyingKey,
+    toc: TableOfContents,
+    pos: u64,
+    rdr: R,
+}
+impl S9pkReader {
+    pub async fn open<P: AsRef<Path>>(path: P, check_sig: bool) -> Result<Self, Error> {
+        let p = path.as_ref();
+        let rdr = File::open(p)
+            .await
+            .with_ctx(|_| (crate::error::ErrorKind::Filesystem, p.display().to_string()))?;
+
+        Self::from_reader(rdr, check_sig).await
+    }
+}
+impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<InstallProgressTracker<R>> {
+    pub fn validated(&mut self) {
+        self.rdr.validated()
+    }
+}
+impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
+    #[instrument(skip_all)]
+    pub async fn validate(&mut self) -> Result<(), Error> {
+        if self.toc.icon.length > 102_400 {
+            // 100 KiB
+            return Err(Error::new(
+                eyre!("icon must be less than 100KiB"),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        let image_tags = self.image_tags().await?;
+        let man = self.manifest().await?;
+        let containers = &man.containers;
+        let validated_image_ids = image_tags
+            .into_iter()
+            .map(|i| i.validate(&man.id, &man.version).map(|_| i.image_id))
+            .collect::<Result<BTreeSet<ImageId>, _>>()?;
+        man.description.validate()?;
+        man.actions.0.iter().try_for_each(|(_, action)| {
+            action.validate(
+                containers,
+                &man.eos_version,
+                &man.volumes,
+                &validated_image_ids,
+            )
+        })?;
+        man.backup.validate(
+            containers,
+            &man.eos_version,
+            &man.volumes,
+            &validated_image_ids,
+        )?;
+        if let Some(cfg) = &man.config {
+            cfg.validate(
+                containers,
+                &man.eos_version,
+                &man.volumes,
+                &validated_image_ids,
+            )?;
+        }
+        man.health_checks
+            .validate(&man.eos_version, &man.volumes, &validated_image_ids)?;
+        man.interfaces.validate()?;
+        man.main
+            .validate(&man.eos_version, &man.volumes, &validated_image_ids, false)
+            .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Main"))?;
+        man.migrations.validate(
+            containers,
+            &man.eos_version,
+            &man.volumes,
+            &validated_image_ids,
+        )?;
+
+        #[cfg(feature = "js-engine")]
+        if man.containers.is_some()
+            || matches!(man.main, crate::procedure::PackageProcedure::Script(_))
+        {
+            return Err(Error::new(
+                eyre!("Right now we don't support the containers and the long running main"),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+
+        if man.replaces.len() >= MAX_REPLACES {
+            return Err(Error::new(
+                eyre!("Cannot have more than {MAX_REPLACES} replaces"),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if let Some(too_big) = man.replaces.iter().find(|x| x.len() >= MAX_REPLACES) {
+            return Err(Error::new(
+                eyre!("We have found a replaces of ({too_big}) that exceeds the max length of {MAX_TITLE_LEN} "),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if man.title.len() >= MAX_TITLE_LEN {
+            return Err(Error::new(
+                eyre!("Cannot have more than a length of {MAX_TITLE_LEN} for title"),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+
+        if man.containers.is_some()
+            && matches!(man.main, crate::procedure::PackageProcedure::Docker(_))
+        {
+            return Err(Error::new(
+                eyre!("Cannot have a main docker and a main in containers"),
+                crate::ErrorKind::ValidateS9pk,
+            ));
+        }
+        if let Some(props) = &man.properties {
+            props
+                .validate(&man.eos_version, &man.volumes, &validated_image_ids, true)
+                .with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Properties"))?;
+        }
+        man.volumes.validate(&man.interfaces)?;
+
+        Ok(())
+    }
+    #[instrument(skip_all)]
+    pub async fn image_tags(&mut self) -> Result<Vec<ImageTag>, Error> {
+        let mut tar = tokio_tar::Archive::new(self.docker_images().await?);
+        let mut entries = tar.entries()?;
+        while let Some(mut entry) = entries.try_next().await? {
+            if &*entry.path()? != Path::new("manifest.json") {
+                continue;
+            }
+            let mut buf = Vec::with_capacity(entry.header().size()? as usize);
+            entry.read_to_end(&mut buf).await?;
+            #[derive(serde::Deserialize)]
+            struct ManEntry {
+                #[serde(rename = "RepoTags")]
+                tags: Vec<String>,
+            }
+            let man_entries = serde_json::from_slice::<Vec<ManEntry>>(&buf)
+                .with_ctx(|_| (crate::ErrorKind::Deserialization, "manifest.json"))?;
+            return man_entries
+                .iter()
+                .flat_map(|e| &e.tags)
+                .map(|t| t.parse())
+                .collect();
+        }
+        Err(Error::new(
+            eyre!("image.tar missing manifest.json"),
+            crate::ErrorKind::ParseS9pk,
+        ))
+    }
+    #[instrument(skip_all)]
+    pub async fn from_reader(mut rdr: R, check_sig: bool) -> Result<Self, Error> {
+        let header = Header::deserialize(&mut rdr).await?;
+
+        let (hash, hash_string) = if check_sig {
+            let mut hasher = Sha512::new();
+            let mut buf = [0; 1024];
+            let mut read;
+            while {
+                read = rdr.read(&mut buf).await?;
+                read != 0
+            } {
+                hasher.update(&buf[0..read]);
+            }
+            let hash = hasher.clone().finalize();
+            header
+                .pubkey
+                .verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?;
+            (
+                Some(hash),
+                Some(base32::encode(
+                    base32::Alphabet::RFC4648 { padding: false },
+                    hash.as_slice(),
+                )),
+            )
+        } else {
+            (None, None)
+        };
+
+        let pos = rdr.stream_position().await?;
+
+        Ok(S9pkReader {
+            hash_string,
+            hash,
+            developer_key: header.pubkey,
+            toc: header.table_of_contents,
+            pos,
+            rdr,
+        })
+    }
+
+    pub fn hash(&self) -> Option<&Output<Sha512>> {
+        self.hash.as_ref()
+    }
+
+    pub fn hash_str(&self) -> Option<&str> {
+        self.hash_string.as_ref().map(|s| s.as_str())
+    }
+
+    pub fn developer_key(&self) -> &VerifyingKey {
+        &self.developer_key
+    }
+
+    pub async fn reset(&mut self) -> Result<(), Error> {
+        self.rdr.seek(SeekFrom::Start(0)).await?;
+        Ok(())
+    }
+
+    async fn read_handle<'a>(
+        &'a mut self,
+        section: FileSection,
+    ) -> Result<ReadHandle<'a, R>, Error> {
+        if self.pos != section.position {
+            self.rdr.seek(SeekFrom::Start(section.position)).await?;
+            self.pos = section.position;
+        }
+        Ok(ReadHandle {
+            range: self.pos..(self.pos + section.length),
+            pos: &mut self.pos,
+            rdr: &mut self.rdr,
+        })
+    }
+
+    pub async fn manifest_raw(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.manifest).await
+    }
+
+    pub async fn manifest(&mut self) -> Result<Manifest, Error> {
+        let slice = self.manifest_raw().await?.to_vec().await?;
+        serde_cbor::de::from_reader(slice.as_slice())
+            .with_ctx(|_| (crate::ErrorKind::ParseS9pk, "Deserializing Manifest (CBOR)"))
+    }
+
+    pub async fn license(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.license).await
+    }
+
+    pub async fn instructions(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.instructions).await
+    }
+
+    pub async fn icon(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.icon).await
+    }
+
+    pub async fn docker_images(&mut self) -> Result<DockerReader<ReadHandle<'_, R>>, Error> {
+        DockerReader::new(self.read_handle(self.toc.docker_images).await?).await
+    }
+
+    pub async fn assets(&mut self) -> Result<ReadHandle<'_, R>, Error> {
+        self.read_handle(self.toc.assets).await
+    }
+
+    pub async fn scripts(&mut self) -> Result<Option<ReadHandle<'_, R>>, Error> {
+        Ok(match self.toc.scripts {
+            None => None,
+            Some(a) => Some(self.read_handle(a).await?),
+        })
+    }
+}

core/startos/src/s9pk/specv2.md

@@ -0,0 +1,28 @@
+## Header
+
+### Magic
+
+2B: `0x3b3b`
+
+### Version
+
+varint: `0x02`
+
+### Pubkey
+
+32B: ed25519 pubkey
+
+### TOC
+
+- number of sections (varint)
+- FOREACH section
+  - sig (32B: ed25519 signature of BLAKE-3 of rest of section)
+  - name (varstring)
+  - TYPE (varint)
+    - TYPE=FILE (`0x01`)
+      - mime (varstring)
+      - pos (32B: u64 BE)
+      - len (32B: u64 BE)
+      - hash (32B: BLAKE-3 of file contents)
+    - TYPE=TOC (`0x02`)
+      - recursively defined