Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 20:14:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/new registry (#2612)

Browse Source 
* wip

* overhaul boot process

* wip: new registry

* wip

* wip

* wip

* wip

* wip

* wip

* os registry complete

* ui fixes

* fixes

* fixes

* more fixes

* fix merkle archive
This commit is contained in:
Aiden McClelland
2024-05-06 10:20:44 -06:00
committed by GitHub
parent 8a38666105
commit 9b14d714ca
167 changed files with 6297 additions and 3190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
core/startos/src/s9pk/merkle_archive/directory_contents.rs
View File

@@ -3,6 +3,7 @@ use std::fmt::Debug;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use blake3::Hash;
use futures::future::BoxFuture;
use futures::FutureExt;
use imbl::OrdMap;
@@ -11,11 +12,11 @@ use itertools::Itertools;
use tokio::io::AsyncRead;
use crate::prelude::*;
use crate::s9pk::merkle_archive::hash::{Hash, HashWriter};
use crate::s9pk::merkle_archive::sink::{Sink, TrackingWriter};
use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
use crate::s9pk::merkle_archive::write_queue::WriteQueue;
use crate::s9pk::merkle_archive::{varint, Entry, EntryContents};
use crate::util::io::ParallelBlake3Writer;
#[derive(Clone)]
pub struct DirectoryContents<S> {
@@ -155,7 +156,7 @@ impl<S: ArchiveSource> DirectoryContents<Section<S>> {
pub fn deserialize<'a>(
source: &'a S,
header: &'a mut (impl AsyncRead + Unpin + Send),
sighash: Hash,
(sighash, max_size): (Hash, u64),
) -> BoxFuture<'a, Result<Self, Error>> {
async move {
use tokio::io::AsyncReadExt;
@@ -168,15 +169,20 @@ impl<S: ArchiveSource> DirectoryContents<Section<S>> {
header.read_exact(&mut size).await?;
let size = u64::from_be_bytes(size);
ensure_code!(
size <= max_size,
ErrorKind::InvalidSignature,
"size is greater than signed"
);
let mut toc_reader = source.fetch(position, size).await?;
let len = varint::deserialize_varint(&mut toc_reader).await?;
let mut entries = OrdMap::new();
for _ in 0..len {
entries.insert(
varint::deserialize_varstring(&mut toc_reader).await?.into(),
Entry::deserialize(source, &mut toc_reader).await?,
);
let name = varint::deserialize_varstring(&mut toc_reader).await?;
let entry = Entry::deserialize(source, &mut toc_reader).await?;
entries.insert(name.into(), entry);
}
let res = Self {
@@ -233,7 +239,8 @@ impl<S: FileSource> DirectoryContents<S> {
#[instrument(skip_all)]
pub fn sighash<'a>(&'a self) -> BoxFuture<'a, Result<Hash, Error>> {
async move {
let mut hasher = TrackingWriter::new(0, HashWriter::new());
let mut hasher =
TrackingWriter::new(0, ParallelBlake3Writer::new(super::hash::BUFFER_CAPACITY));
let mut sig_contents = OrdMap::new();
for (name, entry) in &**self {
sig_contents.insert(name.clone(), entry.to_missing().await?);
@@ -244,7 +251,8 @@ impl<S: FileSource> DirectoryContents<S> {
}
.serialize_toc(&mut WriteQueue::new(0), &mut hasher)
.await?;
Ok(hasher.into_inner().finalize())
let hash = hasher.into_inner().finalize().await?;
Ok(hash)
}
.boxed()
}
@@ -267,7 +275,9 @@ impl<S: FileSource> DirectoryContents<S> {
_ => std::cmp::Ordering::Equal,
}) {
varint::serialize_varstring(&**name, w).await?;
entry.serialize_header(queue.add(entry).await?, w).await?;
if let Some(pos) = entry.serialize_header(queue.add(entry).await?, w).await? {
eprintln!("DEBUG ====> {name} @ {pos}");
}
}
Ok(())

35
core/startos/src/s9pk/merkle_archive/file_contents.rs
View File

@@ -1,9 +1,10 @@
use blake3::Hash;
use tokio::io::AsyncRead;
use crate::prelude::*;
use crate::s9pk::merkle_archive::hash::{Hash, HashWriter};
use crate::s9pk::merkle_archive::sink::{Sink, TrackingWriter};
use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
use crate::util::io::ParallelBlake3Writer;
#[derive(Debug, Clone)]
pub struct FileContents<S>(S);
@@ -13,7 +14,6 @@ impl<S> FileContents<S> {
}
pub const fn header_size() -> u64 {
8 // position: u64 BE
+ 8 // size: u64 BE
}
}
impl<S: ArchiveSource> FileContents<Section<S>> {
@@ -21,6 +21,7 @@ impl<S: ArchiveSource> FileContents<Section<S>> {
pub async fn deserialize(
source: &S,
header: &mut (impl AsyncRead + Unpin + Send),
size: u64,
) -> Result<Self, Error> {
use tokio::io::AsyncReadExt;
@@ -28,27 +29,23 @@ impl<S: ArchiveSource> FileContents<Section<S>> {
header.read_exact(&mut position).await?;
let position = u64::from_be_bytes(position);
let mut size = [0u8; 8];
header.read_exact(&mut size).await?;
let size = u64::from_be_bytes(size);
Ok(Self(source.section(position, size)))
}
}
impl<S: FileSource> FileContents<S> {
pub async fn hash(&self) -> Result<Hash, Error> {
let mut hasher = TrackingWriter::new(0, HashWriter::new());
pub async fn hash(&self) -> Result<(Hash, u64), Error> {
let mut hasher =
TrackingWriter::new(0, ParallelBlake3Writer::new(super::hash::BUFFER_CAPACITY));
self.serialize_body(&mut hasher, None).await?;
Ok(hasher.into_inner().finalize())
let size = hasher.position();
let hash = hasher.into_inner().finalize().await?;
Ok((hash, size))
}
#[instrument(skip_all)]
pub async fn serialize_header<W: Sink>(&self, position: u64, w: &mut W) -> Result<u64, Error> {
use tokio::io::AsyncWriteExt;
let size = self.0.size().await?;
w.write_all(&position.to_be_bytes()).await?;
w.write_all(&size.to_be_bytes()).await?;
Ok(position)
}
@@ -56,21 +53,9 @@ impl<S: FileSource> FileContents<S> {
pub async fn serialize_body<W: Sink>(
&self,
w: &mut W,
verify: Option<Hash>,
verify: Option<(Hash, u64)>,
) -> Result<(), Error> {
let start = if verify.is_some() {
Some(w.current_position().await?)
} else {
None
};
self.0.copy_verify(w, verify).await?;
if let Some(start) = start {
ensure_code!(
w.current_position().await? - start == self.0.size().await?,
ErrorKind::Pack,
"FileSource::copy wrote a number of bytes that does not match FileSource::size"
);
}
Ok(())
}
pub fn into_dyn(self) -> FileContents<DynFileSource> {

118
core/startos/src/s9pk/merkle_archive/hash.rs
View File

@@ -1,68 +1,57 @@
pub use blake3::Hash;
use blake3::Hasher;
use std::task::Poll;
use blake3::Hash;
use tokio::io::AsyncWrite;
use tokio_util::either::Either;
use crate::prelude::*;
use crate::util::io::{ParallelBlake3Writer, TeeWriter};
#[pin_project::pin_project]
pub struct HashWriter {
hasher: Hasher,
}
impl HashWriter {
pub fn new() -> Self {
Self {
hasher: Hasher::new(),
}
}
pub fn finalize(self) -> Hash {
self.hasher.finalize()
}
}
impl AsyncWrite for HashWriter {
fn poll_write(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<Result<usize, std::io::Error>> {
self.project().hasher.update(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
std::task::Poll::Ready(Ok(()))
}
}
pub const BUFFER_CAPACITY: usize = 10 * 1024 * 1024; // 10MiB
#[pin_project::pin_project]
pub struct VerifyingWriter<W> {
verify: Option<(Hasher, Hash)>,
verify: Option<(Hash, u64)>,
#[pin]
writer: W,
writer: Either<TeeWriter<W, ParallelBlake3Writer>, W>,
}
impl<W: AsyncWrite> VerifyingWriter<W> {
pub fn new(w: W, verify: Option<Hash>) -> Self {
pub fn new(w: W, verify: Option<(Hash, u64)>) -> Self {
Self {
verify: verify.map(|v| (Hasher::new(), v)),
writer: w,
writer: if verify.is_some() {
Either::Left(TeeWriter::new(
w,
ParallelBlake3Writer::new(BUFFER_CAPACITY),
BUFFER_CAPACITY,
))
} else {
Either::Right(w)
},
verify,
}
}
pub fn verify(self) -> Result<W, Error> {
if let Some((actual, expected)) = self.verify {
ensure_code!(
actual.finalize() == expected,
ErrorKind::InvalidSignature,
"hash sum does not match"
);
}
impl<W: AsyncWrite + Unpin> VerifyingWriter<W> {
pub async fn verify(self) -> Result<W, Error> {
match self.writer {
Either::Left(writer) => {
let (writer, actual) = writer.into_inner().await?;
if let Some((expected, remaining)) = self.verify {
ensure_code!(
actual.finalize().await? == expected,
ErrorKind::InvalidSignature,
"hash sum mismatch"
);
ensure_code!(
remaining == 0,
ErrorKind::InvalidSignature,
"file size mismatch"
);
}
Ok(writer)
}
Either::Right(writer) => Ok(writer),
}
Ok(self.writer)
}
}
impl<W: AsyncWrite> AsyncWrite for VerifyingWriter<W> {
@@ -70,28 +59,35 @@ impl<W: AsyncWrite> AsyncWrite for VerifyingWriter<W> {
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<Result<usize, std::io::Error>> {
) -> Poll<Result<usize, std::io::Error>> {
let this = self.project();
match this.writer.poll_write(cx, buf) {
std::task::Poll::Ready(Ok(written)) => {
if let Some((h, _)) = this.verify {
h.update(&buf[..written]);
}
std::task::Poll::Ready(Ok(written))
if let Some((_, remaining)) = this.verify {
if *remaining < buf.len() as u64 {
return Poll::Ready(Err(std::io::Error::other(eyre!(
"attempted to write more bytes than signed"
))));
}
}
match this.writer.poll_write(cx, buf)? {
Poll::Pending => Poll::Pending,
Poll::Ready(n) => {
if let Some((_, remaining)) = this.verify {
*remaining -= n as u64;
}
Poll::Ready(Ok(n))
}
a => a,
}
}
fn poll_flush(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
) -> Poll<Result<(), std::io::Error>> {
self.project().writer.poll_flush(cx)
}
fn poll_shutdown(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
) -> Poll<Result<(), std::io::Error>> {
self.project().writer.poll_shutdown(cx)
}
}

81
core/startos/src/s9pk/merkle_archive/mod.rs
View File

@@ -1,12 +1,14 @@
use std::path::Path;
use blake3::Hash;
use ed25519_dalek::{Signature, SigningKey, VerifyingKey};
use imbl_value::InternedString;
use sha2::{Digest, Sha512};
use tokio::io::AsyncRead;
use crate::prelude::*;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::file_contents::FileContents;
use crate::s9pk::merkle_archive::hash::Hash;
use crate::s9pk::merkle_archive::sink::Sink;
use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
use crate::s9pk::merkle_archive::write_queue::WriteQueue;
@@ -23,8 +25,8 @@ pub mod write_queue;
#[derive(Debug, Clone)]
enum Signer {
Signed(VerifyingKey, Signature),
Signer(SigningKey),
Signed(VerifyingKey, Signature, u64, InternedString),
Signer(SigningKey, InternedString),
}
#[derive(Debug, Clone)]
@@ -33,22 +35,23 @@ pub struct MerkleArchive<S> {
contents: DirectoryContents<S>,
}
impl<S> MerkleArchive<S> {
pub fn new(contents: DirectoryContents<S>, signer: SigningKey) -> Self {
pub fn new(contents: DirectoryContents<S>, signer: SigningKey, context: &str) -> Self {
Self {
signer: Signer::Signer(signer),
signer: Signer::Signer(signer, context.into()),
contents,
}
}
pub fn signer(&self) -> VerifyingKey {
match &self.signer {
Signer::Signed(k, _) => *k,
Signer::Signer(k) => k.verifying_key(),
Signer::Signed(k, _, _, _) => *k,
Signer::Signer(k, _) => k.verifying_key(),
}
}
pub const fn header_size() -> u64 {
32 // pubkey
+ 64 // signature
+ 32 // sighash
+ 8 // size
+ DirectoryContents::<Section<S>>::header_size()
}
pub fn contents(&self) -> &DirectoryContents<S> {
@@ -57,8 +60,8 @@ impl<S> MerkleArchive<S> {
pub fn contents_mut(&mut self) -> &mut DirectoryContents<S> {
&mut self.contents
}
pub fn set_signer(&mut self, key: SigningKey) {
self.signer = Signer::Signer(key);
pub fn set_signer(&mut self, key: SigningKey, context: &str) {
self.signer = Signer::Signer(key, context.into());
}
pub fn sort_by(
&mut self,
@@ -71,6 +74,7 @@ impl<S: ArchiveSource> MerkleArchive<Section<S>> {
#[instrument(skip_all)]
pub async fn deserialize(
source: &S,
context: &str,
header: &mut (impl AsyncRead + Unpin + Send),
) -> Result<Self, Error> {
use tokio::io::AsyncReadExt;
@@ -87,12 +91,20 @@ impl<S: ArchiveSource> MerkleArchive<Section<S>> {
header.read_exact(&mut sighash).await?;
let sighash = Hash::from_bytes(sighash);
let contents = DirectoryContents::deserialize(source, header, sighash).await?;
let mut max_size = [0u8; 8];
header.read_exact(&mut max_size).await?;
let max_size = u64::from_be_bytes(max_size);
pubkey.verify_strict(contents.sighash().await?.as_bytes(), &signature)?;
pubkey.verify_prehashed_strict(
Sha512::new_with_prefix(sighash.as_bytes()).chain_update(&u64::to_be_bytes(max_size)),
Some(context.as_bytes()),
&signature,
)?;
let contents = DirectoryContents::deserialize(source, header, (sighash, max_size)).await?;
Ok(Self {
signer: Signer::Signed(pubkey, signature),
signer: Signer::Signed(pubkey, signature, max_size, context.into()),
contents,
})
}
@@ -109,15 +121,26 @@ impl<S: FileSource> MerkleArchive<S> {
use tokio::io::AsyncWriteExt;
let sighash = self.contents.sighash().await?;
let size = self.contents.toc_size();
let (pubkey, signature) = match &self.signer {
Signer::Signed(pubkey, signature) => (*pubkey, *signature),
Signer::Signer(s) => (s.into(), ed25519_dalek::Signer::sign(s, sighash.as_bytes())),
let (pubkey, signature, max_size) = match &self.signer {
Signer::Signed(pubkey, signature, max_size, _) => (*pubkey, *signature, *max_size),
Signer::Signer(s, context) => (
s.into(),
ed25519_dalek::SigningKey::sign_prehashed(
s,
Sha512::new_with_prefix(sighash.as_bytes())
.chain_update(&u64::to_be_bytes(size)),
Some(context.as_bytes()),
)?,
size,
),
};
w.write_all(pubkey.as_bytes()).await?;
w.write_all(&signature.to_bytes()).await?;
w.write_all(sighash.as_bytes()).await?;
w.write_all(&u64::to_be_bytes(max_size)).await?;
let mut next_pos = w.current_position().await?;
next_pos += DirectoryContents::<S>::header_size();
self.contents.serialize_header(next_pos, w).await?;
@@ -137,7 +160,7 @@ impl<S: FileSource> MerkleArchive<S> {
#[derive(Debug, Clone)]
pub struct Entry<S> {
hash: Option<Hash>,
hash: Option<(Hash, u64)>,
contents: EntryContents<S>,
}
impl<S> Entry<S> {
@@ -150,7 +173,7 @@ impl<S> Entry<S> {
pub fn file(source: S) -> Self {
Self::new(EntryContents::File(FileContents::new(source)))
}
pub fn hash(&self) -> Option<Hash> {
pub fn hash(&self) -> Option<(Hash, u64)> {
self.hash
}
pub fn as_contents(&self) -> &EntryContents<S> {
@@ -189,6 +212,7 @@ impl<S> Entry<S> {
}
pub fn header_size(&self) -> u64 {
32 // hash
+ 8 // size: u64 BE
+ self.contents.header_size()
}
}
@@ -205,10 +229,14 @@ impl<S: ArchiveSource> Entry<Section<S>> {
header.read_exact(&mut hash).await?;
let hash = Hash::from_bytes(hash);
let contents = EntryContents::deserialize(source, header, hash).await?;
let mut size = [0u8; 8];
header.read_exact(&mut size).await?;
let size = u64::from_be_bytes(size);
let contents = EntryContents::deserialize(source, header, (hash, size)).await?;
Ok(Self {
hash: Some(hash),
hash: Some((hash, size)),
contents,
})
}
@@ -258,12 +286,13 @@ impl<S: FileSource> Entry<S> {
) -> Result<Option<u64>, Error> {
use tokio::io::AsyncWriteExt;
let hash = if let Some(hash) = self.hash {
let (hash, size) = if let Some(hash) = self.hash {
hash
} else {
self.contents.hash().await?
};
w.write_all(hash.as_bytes()).await?;
w.write_all(&u64::to_be_bytes(size)).await?;
self.contents.serialize_header(position, w).await
}
pub fn into_dyn(self) -> Entry<DynFileSource> {
@@ -305,7 +334,7 @@ impl<S: ArchiveSource> EntryContents<Section<S>> {
pub async fn deserialize(
source: &S,
header: &mut (impl AsyncRead + Unpin + Send),
hash: Hash,
(hash, size): (Hash, u64),
) -> Result<Self, Error> {
use tokio::io::AsyncReadExt;
@@ -313,9 +342,11 @@ impl<S: ArchiveSource> EntryContents<Section<S>> {
header.read_exact(&mut type_id).await?;
match type_id[0] {
0 => Ok(Self::Missing),
1 => Ok(Self::File(FileContents::deserialize(source, header).await?)),
1 => Ok(Self::File(
FileContents::deserialize(source, header, size).await?,
)),
2 => Ok(Self::Directory(
DirectoryContents::deserialize(source, header, hash).await?,
DirectoryContents::deserialize(source, header, (hash, size)).await?,
)),
id => Err(Error::new(
eyre!("Unknown type id {id} found in MerkleArchive"),
@@ -325,14 +356,14 @@ impl<S: ArchiveSource> EntryContents<Section<S>> {
}
}
impl<S: FileSource> EntryContents<S> {
pub async fn hash(&self) -> Result<Hash, Error> {
pub async fn hash(&self) -> Result<(Hash, u64), Error> {
match self {
Self::Missing => Err(Error::new(
eyre!("Cannot compute hash of missing file"),
ErrorKind::Pack,
)),
Self::File(f) => f.hash().await,
Self::Directory(d) => d.sighash().await,
Self::Directory(d) => Ok((d.sighash().await?, d.toc_size())),
}
}
#[instrument(skip_all)]

3
core/startos/src/s9pk/merkle_archive/sink.rs
View File

@@ -36,6 +36,9 @@ impl<W> TrackingWriter<W> {
writer: w,
}
}
pub fn position(&self) -> u64 {
self.position
}
pub fn into_inner(self) -> W {
self.writer
}

8
core/startos/src/s9pk/merkle_archive/source/http.rs
View File

@@ -3,7 +3,7 @@ use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use reqwest::header::{ACCEPT_RANGES, CONTENT_LENGTH, RANGE};
use reqwest::{Client, Url};
use tokio::io::AsyncRead;
use tokio::io::{AsyncRead, AsyncReadExt, Take};
use tokio_util::io::StreamReader;
use crate::prelude::*;
@@ -50,9 +50,8 @@ impl HttpSource {
})
}
}
#[async_trait::async_trait]
impl ArchiveSource for HttpSource {
type Reader = HttpReader;
type Reader = Take<HttpReader>;
async fn size(&self) -> Option<u64> {
self.size
}
@@ -72,7 +71,8 @@ impl ArchiveSource for HttpSource {
.boxed()
} else {
futures::stream::empty().boxed()
}))),
}))
.take(size)),
_ => todo!(),
}
}

98
core/startos/src/s9pk/merkle_archive/source/mod.rs
View File

@@ -2,6 +2,8 @@ use std::path::PathBuf;
use std::sync::Arc;
use blake3::Hash;
use futures::future::BoxFuture;
use futures::{Future, FutureExt};
use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncWrite};
@@ -11,29 +13,51 @@ use crate::s9pk::merkle_archive::hash::VerifyingWriter;
pub mod http;
pub mod multi_cursor_file;
#[async_trait::async_trait]
pub trait FileSource: Clone + Send + Sync + Sized + 'static {
type Reader: AsyncRead + Unpin + Send;
async fn size(&self) -> Result<u64, Error>;
async fn reader(&self) -> Result<Self::Reader, Error>;
async fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(&self, w: &mut W) -> Result<(), Error> {
tokio::io::copy(&mut self.reader().await?, w).await?;
Ok(())
}
async fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
fn size(&self) -> impl Future<Output = Result<u64, Error>> + Send;
fn reader(&self) -> impl Future<Output = Result<Self::Reader, Error>> + Send;
fn copy<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
w: &mut W,
verify: Option<Hash>,
) -> Result<(), Error> {
let mut w = VerifyingWriter::new(w, verify);
tokio::io::copy(&mut self.reader().await?, &mut w).await?;
w.verify()?;
Ok(())
) -> impl Future<Output = Result<(), Error>> + Send {
async move {
tokio::io::copy(&mut self.reader().await?, w).await?;
Ok(())
}
}
async fn to_vec(&self, verify: Option<Hash>) -> Result<Vec<u8>, Error> {
let mut vec = Vec::with_capacity(self.size().await? as usize);
self.copy_verify(&mut vec, verify).await?;
Ok(vec)
fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
w: &mut W,
verify: Option<(Hash, u64)>,
) -> impl Future<Output = Result<(), Error>> + Send {
async move {
let mut w = VerifyingWriter::new(w, verify);
tokio::io::copy(&mut self.reader().await?, &mut w).await?;
w.verify().await?;
Ok(())
}
}
fn to_vec(
&self,
verify: Option<(Hash, u64)>,
) -> impl Future<Output = Result<Vec<u8>, Error>> + Send {
fn to_vec(
src: &impl FileSource,
verify: Option<(Hash, u64)>,
) -> BoxFuture<Result<Vec<u8>, Error>> {
async move {
let mut vec = Vec::with_capacity(if let Some((_, size)) = &verify {
*size
} else {
src.size().await?
} as usize);
src.copy_verify(&mut vec, verify).await?;
Ok(vec)
}
.boxed()
}
to_vec(self, verify)
}
}
@@ -44,7 +68,6 @@ impl DynFileSource {
Self(Arc::new(source))
}
}
#[async_trait::async_trait]
impl FileSource for DynFileSource {
type Reader = Box<dyn AsyncRead + Unpin + Send>;
async fn size(&self) -> Result<u64, Error> {
@@ -62,11 +85,11 @@ impl FileSource for DynFileSource {
async fn copy_verify<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
mut w: &mut W,
verify: Option<Hash>,
verify: Option<(Hash, u64)>,
) -> Result<(), Error> {
self.0.copy_verify(&mut w, verify).await
}
async fn to_vec(&self, verify: Option<Hash>) -> Result<Vec<u8>, Error> {
async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
self.0.to_vec(verify).await
}
}
@@ -79,9 +102,9 @@ trait DynableFileSource: Send + Sync + 'static {
async fn copy_verify(
&self,
w: &mut (dyn AsyncWrite + Unpin + Send),
verify: Option<Hash>,
verify: Option<(Hash, u64)>,
) -> Result<(), Error>;
async fn to_vec(&self, verify: Option<Hash>) -> Result<Vec<u8>, Error>;
async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error>;
}
#[async_trait::async_trait]
impl<T: FileSource> DynableFileSource for T {
@@ -97,16 +120,15 @@ impl<T: FileSource> DynableFileSource for T {
async fn copy_verify(
&self,
w: &mut (dyn AsyncWrite + Unpin + Send),
verify: Option<Hash>,
verify: Option<(Hash, u64)>,
) -> Result<(), Error> {
FileSource::copy_verify(self, w, verify).await
}
async fn to_vec(&self, verify: Option<Hash>) -> Result<Vec<u8>, Error> {
async fn to_vec(&self, verify: Option<(Hash, u64)>) -> Result<Vec<u8>, Error> {
FileSource::to_vec(self, verify).await
}
}
#[async_trait::async_trait]
impl FileSource for PathBuf {
type Reader = File;
async fn size(&self) -> Result<u64, Error> {
@@ -117,7 +139,6 @@ impl FileSource for PathBuf {
}
}
#[async_trait::async_trait]
impl FileSource for Arc<[u8]> {
type Reader = std::io::Cursor<Self>;
async fn size(&self) -> Result<u64, Error> {
@@ -134,21 +155,26 @@ impl FileSource for Arc<[u8]> {
}
}
#[async_trait::async_trait]
pub trait ArchiveSource: Clone + Send + Sync + Sized + 'static {
type Reader: AsyncRead + Unpin + Send;
async fn size(&self) -> Option<u64> {
None
fn size(&self) -> impl Future<Output = Option<u64>> + Send {
async { None }
}
async fn fetch(&self, position: u64, size: u64) -> Result<Self::Reader, Error>;
async fn copy_to<W: AsyncWrite + Unpin + Send + ?Sized>(
fn fetch(
&self,
position: u64,
size: u64,
) -> impl Future<Output = Result<Self::Reader, Error>> + Send;
fn copy_to<W: AsyncWrite + Unpin + Send + ?Sized>(
&self,
position: u64,
size: u64,
w: &mut W,
) -> Result<(), Error> {
tokio::io::copy(&mut self.fetch(position, size).await?, w).await?;
Ok(())
) -> impl Future<Output = Result<(), Error>> + Send {
async move {
tokio::io::copy(&mut self.fetch(position, size).await?, w).await?;
Ok(())
}
}
fn section(&self, position: u64, size: u64) -> Section<Self> {
Section {
@@ -159,7 +185,6 @@ pub trait ArchiveSource: Clone + Send + Sync + Sized + 'static {
}
}
#[async_trait::async_trait]
impl ArchiveSource for Arc<[u8]> {
type Reader = tokio::io::Take<std::io::Cursor<Self>>;
async fn fetch(&self, position: u64, size: u64) -> Result<Self::Reader, Error> {
@@ -177,7 +202,6 @@ pub struct Section<S> {
position: u64,
size: u64,
}
#[async_trait::async_trait]
impl<S: ArchiveSource> FileSource for Section<S> {
type Reader = S::Reader;
async fn size(&self) -> Result<u64, Error> {

11
core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs
View File

@@ -31,6 +31,16 @@ impl MultiCursorFile {
file: Arc::new(Mutex::new(File::open(path_from_fd(fd)).await?)),
})
}
pub async fn blake3_mmap(&self) -> Result<blake3::Hash, Error> {
let path = self.path();
tokio::task::spawn_blocking(move || {
let mut hasher = blake3::Hasher::new();
hasher.update_mmap_rayon(path)?;
Ok(hasher.finalize())
})
.await
.with_kind(ErrorKind::Unknown)?
}
}
impl From<File> for MultiCursorFile {
fn from(value: File) -> Self {
@@ -67,7 +77,6 @@ impl AsyncRead for FileSectionReader {
}
}
#[async_trait::async_trait]
impl ArchiveSource for MultiCursorFile {
type Reader = FileSectionReader;
async fn size(&self) -> Option<u64> {

4
core/startos/src/s9pk/merkle_archive/test.rs
View File

@@ -52,7 +52,7 @@ fn test(files: Vec<(PathBuf, String)>) -> Result<(), Error> {
}
}
let key = SigningKey::generate(&mut rand::thread_rng());
let mut a1 = MerkleArchive::new(root, key);
let mut a1 = MerkleArchive::new(root, key, "test");
tokio::runtime::Builder::new_current_thread()
.enable_io()
.build()
@@ -63,7 +63,7 @@ fn test(files: Vec<(PathBuf, String)>) -> Result<(), Error> {
a1.serialize(&mut TrackingWriter::new(0, &mut s1), true)
.await?;
let s1: Arc<[u8]> = s1.into();
let a2 = MerkleArchive::deserialize(&s1, &mut Cursor::new(s1.clone())).await?;
let a2 = MerkleArchive::deserialize(&s1, "test", &mut Cursor::new(s1.clone())).await?;
for (path, content) in check_set {
match a2

15
core/startos/src/s9pk/rpc.rs
View File

@@ -17,6 +17,7 @@ use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::source::DynFileSource;
use crate::s9pk::merkle_archive::Entry;
use crate::s9pk::v2::compat::CONTAINER_TOOL;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TmpDir;
use crate::util::serde::{apply_expr, HandlerExtSerde};
@@ -24,7 +25,7 @@ use crate::util::Invoke;
pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
pub fn s9pk() -> ParentHandler {
pub fn s9pk() -> ParentHandler<CliContext> {
ParentHandler::new()
.subcommand("edit", edit())
.subcommand("inspect", inspect())
@@ -35,9 +36,9 @@ struct S9pkPath {
s9pk: PathBuf,
}
fn edit() -> ParentHandler<S9pkPath> {
fn edit() -> ParentHandler<CliContext, S9pkPath> {
let only_parent = |a, _| a;
ParentHandler::<S9pkPath>::new()
ParentHandler::new()
.subcommand(
"add-image",
from_fn_async(add_image)
@@ -52,9 +53,9 @@ fn edit() -> ParentHandler<S9pkPath> {
)
}
fn inspect() -> ParentHandler<S9pkPath> {
fn inspect() -> ParentHandler<CliContext, S9pkPath> {
let only_parent = |a, _| a;
ParentHandler::<S9pkPath>::new()
ParentHandler::new()
.subcommand(
"file-tree",
from_fn_async(file_tree)
@@ -158,7 +159,7 @@ async fn add_image(
.invoke(ErrorKind::Docker)
.await?;
let archive = s9pk.as_archive_mut();
archive.set_signer(ctx.developer_key()?.clone());
archive.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
@@ -213,7 +214,7 @@ async fn edit_manifest(
let tmp_path = s9pk_path.with_extension("s9pk.tmp");
let mut tmp_file = File::create(&tmp_path).await?;
s9pk.as_archive_mut()
.set_signer(ctx.developer_key()?.clone());
.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
s9pk.serialize(&mut tmp_file, true).await?;
tmp_file.sync_all().await?;
tokio::fs::rename(&tmp_path, &s9pk_path).await?;

1
core/startos/src/s9pk/v1/docker.rs
View File

@@ -55,7 +55,6 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> DockerReader<R> {
if let Some(image) = tokio_tar::Archive::new(rdr)
.entries()?
.try_filter_map(|e| {
let arch = arch.clone();
async move {
Ok(if &*e.path()? == Path::new(&format!("{}.tar", arch)) {
Some(e)

5
core/startos/src/s9pk/v2/compat.rs
View File

@@ -18,7 +18,7 @@ use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::rpc::SKIP_ENV;
use crate::s9pk::v1::manifest::Manifest as ManifestV1;
use crate::s9pk::v1::reader::S9pkReader;
use crate::s9pk::v2::S9pk;
use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
use crate::util::io::TmpDir;
use crate::util::Invoke;
@@ -40,7 +40,6 @@ enum CompatSource {
Buffered(Arc<[u8]>),
File(PathBuf),
}
#[async_trait::async_trait]
impl FileSource for CompatSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
async fn size(&self) -> Result<u64, Error> {
@@ -315,7 +314,7 @@ impl S9pk<Section<MultiCursorFile>> {
)),
)?;
let mut s9pk = S9pk::new(MerkleArchive::new(archive, signer), None).await?;
let mut s9pk = S9pk::new(MerkleArchive::new(archive, signer, SIG_CONTEXT), None).await?;
let mut dest_file = File::create(destination.as_ref()).await?;
s9pk.serialize(&mut dest_file, false).await?;
dest_file.sync_all().await?;

4
core/startos/src/s9pk/v2/mod.rs
View File

@@ -17,6 +17,8 @@ use crate::ARCH;
const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
pub const SIG_CONTEXT: &str = "s9pk";
pub mod compat;
pub mod manifest;
@@ -191,7 +193,7 @@ impl<S: ArchiveSource> S9pk<Section<S>> {
"Invalid Magic or Unexpected Version"
);
let mut archive = MerkleArchive::deserialize(source, &mut header).await?;
let mut archive = MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header).await?;
if apply_filter {
archive.filter(filter)?;