feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

core/startos/src/util/actor/concurrent.rs

@@ -8,6 +8,7 @@ use helpers::NonDetachingJoinHandle;
 use tokio::sync::{mpsc, oneshot};
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner};
 use crate::util::actor::{Actor, ConflictFn, Handler, PendingMessageStrategy, Request};
 
@@ -18,6 +19,7 @@ struct ConcurrentRunner<A> {
     waiting: Vec<Request<A>>,
     recv: mpsc::UnboundedReceiver<Request<A>>,
     handlers: Vec<(
+        Guid,
         Arc<ConflictFn<A>>,
         oneshot::Sender<Box<dyn Any + Send>>,
         BoxFuture<'static, Box<dyn Any + Send>>,
@@ -41,16 +43,21 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
             }
         });
         if this.shutdown.is_some() {
-            while let std::task::Poll::Ready(Some((msg, reply))) = this.recv.poll_recv(cx) {
-                if this.handlers.iter().any(|(f, _, _)| f(&*msg)) {
-                    this.waiting.push((msg, reply));
+            while let std::task::Poll::Ready(Some((id, msg, reply))) = this.recv.poll_recv(cx) {
+                if this
+                    .handlers
+                    .iter()
+                    .any(|(hid, f, _, _)| &id != hid && f(&*msg))
+                {
+                    this.waiting.push((id, msg, reply));
                 } else {
                     let mut actor = this.actor.clone();
                     let queue = this.queue.clone();
                     this.handlers.push((
+                        id.clone(),
                         msg.conflicts_with(),
                         reply,
-                        async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
+                        async move { msg.handle_with(id, &mut actor, &queue).await }.boxed(),
                     ))
                 }
             }
@@ -62,29 +69,34 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                 .handlers
                 .iter_mut()
                 .enumerate()
-                .filter_map(|(i, (_, _, f))| match f.poll_unpin(cx) {
+                .filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) {
                     std::task::Poll::Pending => None,
                     std::task::Poll::Ready(res) => Some((i, res)),
                 })
                 .collect::<Vec<_>>();
             for (idx, res) in complete.into_iter().rev() {
                 #[allow(clippy::let_underscore_future)]
-                let (f, reply, _) = this.handlers.swap_remove(idx);
+                let (_, f, reply, _) = this.handlers.swap_remove(idx);
                 let _ = reply.send(res);
                 // TODO: replace with Vec::extract_if once stable
                 if this.shutdown.is_some() {
                     let mut i = 0;
                     while i < this.waiting.len() {
-                        if f(&*this.waiting[i].0)
-                            && !this.handlers.iter().any(|(f, _, _)| f(&*this.waiting[i].0))
+                        if f(&*this.waiting[i].1)
+                            && !this
+                                .handlers
+                                .iter()
+                                .any(|(_, f, _, _)| f(&*this.waiting[i].1))
                         {
-                            let (msg, reply) = this.waiting.remove(i);
+                            let (id, msg, reply) = this.waiting.remove(i);
                             let mut actor = this.actor.clone();
                             let queue = this.queue.clone();
                             this.handlers.push((
+                                id.clone(),
                                 msg.conflicts_with(),
                                 reply,
-                                async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
+                                async move { msg.handle_with(id, &mut actor, &queue).await }
+                                    .boxed(),
                             ));
                             cont = true;
                         } else {
@@ -137,6 +149,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
     /// Message is guaranteed to be queued immediately
     pub fn queue<M: Send + 'static>(
         &self,
+        id: Guid,
         message: M,
     ) -> impl Future<Output = Result<A::Response, Error>>
     where
@@ -150,7 +163,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
         }
         let (reply_send, reply_recv) = oneshot::channel();
         self.messenger
-            .send((Box::new(message), reply_send))
+            .send((id, Box::new(message), reply_send))
             .unwrap();
         futures::future::Either::Right(
             reply_recv
@@ -170,11 +183,11 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
         )
     }
 
-    pub async fn send<M: Send + 'static>(&self, message: M) -> Result<A::Response, Error>
+    pub async fn send<M: Send + 'static>(&self, id: Guid, message: M) -> Result<A::Response, Error>
     where
         A: Handler<M>,
     {
-        self.queue(message).await
+        self.queue(id, message).await
     }
 
     pub async fn shutdown(self, strategy: PendingMessageStrategy) {

core/startos/src/util/actor/mod.rs

@@ -9,6 +9,7 @@ use tokio::sync::oneshot;
 
 #[allow(unused_imports)]
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::BackgroundJobQueue;
 
 pub mod background;
@@ -28,6 +29,7 @@ pub trait Handler<M: Any + Send>: Actor {
     }
     fn handle(
         &mut self,
+        id: Guid,
         msg: M,
         jobs: &BackgroundJobQueue,
     ) -> impl Future<Output = Self::Response> + Send;
@@ -39,6 +41,7 @@ trait Message<A>: Send + Any {
     fn conflicts_with(&self) -> Arc<ConflictFn<A>>;
     fn handle_with<'a>(
         self: Box<Self>,
+        id: Guid,
         actor: &'a mut A,
         jobs: &'a BackgroundJobQueue,
     ) -> BoxFuture<'a, Box<dyn Any + Send>>;
@@ -52,10 +55,11 @@ where
     }
     fn handle_with<'a>(
         self: Box<Self>,
+        id: Guid,
         actor: &'a mut A,
         jobs: &'a BackgroundJobQueue,
     ) -> BoxFuture<'a, Box<dyn Any + Send>> {
-        async move { Box::new(actor.handle(*self, jobs).await) as Box<dyn Any + Send> }.boxed()
+        async move { Box::new(actor.handle(id, *self, jobs).await) as Box<dyn Any + Send> }.boxed()
     }
 }
 impl<A: Actor> dyn Message<A> {
@@ -80,7 +84,11 @@ impl<A: Actor> dyn Message<A> {
     }
 }
 
-type Request<A> = (Box<dyn Message<A>>, oneshot::Sender<Box<dyn Any + Send>>);
+type Request<A> = (
+    Guid,
+    Box<dyn Message<A>>,
+    oneshot::Sender<Box<dyn Any + Send>>,
+);
 
 pub enum PendingMessageStrategy {
     CancelAll,

core/startos/src/util/actor/simple.rs

@@ -7,6 +7,7 @@ use tokio::sync::oneshot::error::TryRecvError;
 use tokio::sync::{mpsc, oneshot};
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::BackgroundJobQueue;
 use crate::util::actor::{Actor, Handler, PendingMessageStrategy, Request};
 
@@ -26,9 +27,9 @@ impl<A: Actor> SimpleActor<A> {
                 tokio::select! {
                     _ = &mut runner => (),
                     msg = messenger_recv.recv() => match msg {
-                        Some((msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
+                        Some((id, msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
                             tokio::select! {
-                                res = msg.handle_with(&mut actor, &queue) => { let _ = reply.send(res); },
+                                res = msg.handle_with(id, &mut actor, &queue) => { let _ = reply.send(res); },
                                 _ = &mut runner => (),
                             }
                         }
@@ -60,7 +61,7 @@ impl<A: Actor> SimpleActor<A> {
         }
         let (reply_send, reply_recv) = oneshot::channel();
         self.messenger
-            .send((Box::new(message), reply_send))
+            .send((Guid::new(), Box::new(message), reply_send))
             .unwrap();
         futures::future::Either::Right(
             reply_recv

core/startos/src/util/io.rs

@@ -681,8 +681,6 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
     }
 }
 
-pub struct TmpFile {}
-
 #[derive(Debug)]
 pub struct TmpDir {
     path: PathBuf,
@@ -707,6 +705,14 @@ impl TmpDir {
         tokio::fs::remove_dir_all(&self.path).await?;
         Ok(())
     }
+
+    pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
+        if let Ok(dir) = Arc::try_unwrap(self) {
+            dir.delete().await
+        } else {
+            Ok(())
+        }
+    }
 }
 impl std::ops::Deref for TmpDir {
     type Target = Path;

core/startos/src/util/mod.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::future::Future;
 use std::marker::PhantomData;
 use std::path::{Path, PathBuf};
@@ -11,6 +11,8 @@ use std::time::Duration;
 use async_trait::async_trait;
 use color_eyre::eyre::{self, eyre};
 use fd_lock_rs::FdLock;
+use futures::future::BoxFuture;
+use futures::FutureExt;
 use helpers::canonicalize;
 pub use helpers::NonDetachingJoinHandle;
 use imbl_value::InternedString;
@@ -19,7 +21,8 @@ pub use models::VersionString;
 use pin_project::pin_project;
 use sha2::Digest;
 use tokio::fs::File;
-use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
+use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
+use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
 use tracing::instrument;
 
 use crate::shutdown::Shutdown;
@@ -62,11 +65,16 @@ pub trait Invoke<'a> {
     where
         Self: 'ext,
         'ext: 'a;
+    fn pipe<'ext: 'a>(
+        &'ext mut self,
+        next: &'ext mut tokio::process::Command,
+    ) -> Self::Extended<'ext>;
     fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext>;
     fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
         &'ext mut self,
         input: Option<&'ext mut Input>,
     ) -> Self::Extended<'ext>;
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext>;
     fn invoke(
         &mut self,
         error_kind: crate::ErrorKind,
@@ -76,7 +84,20 @@ pub trait Invoke<'a> {
 pub struct ExtendedCommand<'a> {
     cmd: &'a mut tokio::process::Command,
     timeout: Option<Duration>,
-    input: Option<&'a mut (dyn tokio::io::AsyncRead + Unpin + Send)>,
+    input: Option<&'a mut (dyn AsyncRead + Unpin + Send)>,
+    pipe: VecDeque<&'a mut tokio::process::Command>,
+    capture: bool,
+}
+impl<'a> From<&'a mut tokio::process::Command> for ExtendedCommand<'a> {
+    fn from(value: &'a mut tokio::process::Command) -> Self {
+        ExtendedCommand {
+            cmd: value,
+            timeout: None,
+            input: None,
+            pipe: VecDeque::new(),
+            capture: true,
+        }
+    }
 }
 impl<'a> std::ops::Deref for ExtendedCommand<'a> {
     type Target = tokio::process::Command;
@@ -95,35 +116,38 @@ impl<'a> Invoke<'a> for tokio::process::Command {
     where
         Self: 'ext,
         'ext: 'a;
-    fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
-        ExtendedCommand {
-            cmd: self,
-            timeout,
-            input: None,
-        }
+    fn pipe<'ext: 'a>(
+        &'ext mut self,
+        next: &'ext mut tokio::process::Command,
+    ) -> Self::Extended<'ext> {
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.pipe.push_back(next);
+        cmd
     }
-    fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
+    fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.timeout = timeout;
+        cmd
+    }
+    fn input<'ext: 'a, Input: AsyncRead + Unpin + Send>(
         &'ext mut self,
         input: Option<&'ext mut Input>,
     ) -> Self::Extended<'ext> {
-        ExtendedCommand {
-            cmd: self,
-            timeout: None,
-            input: if let Some(input) = input {
-                Some(&mut *input)
-            } else {
-                None
-            },
-        }
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.input = if let Some(input) = input {
+            Some(&mut *input)
+        } else {
+            None
+        };
+        cmd
+    }
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.capture = capture;
+        cmd
     }
     async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
-        ExtendedCommand {
-            cmd: self,
-            timeout: None,
-            input: None,
-        }
-        .invoke(error_kind)
-        .await
+        ExtendedCommand::from(self).invoke(error_kind).await
     }
 }
 
@@ -132,6 +156,13 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
     where
         Self: 'ext,
         'ext: 'a;
+    fn pipe<'ext: 'a>(
+        &'ext mut self,
+        next: &'ext mut tokio::process::Command,
+    ) -> Self::Extended<'ext> {
+        self.pipe.push_back(next.kill_on_drop(true));
+        self
+    }
     fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
         self.timeout = timeout;
         self
@@ -147,39 +178,150 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
         };
         self
     }
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
+        self.capture = capture;
+        self
+    }
+    #[instrument(skip_all)]
     async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
+        let cmd_str = self
+            .cmd
+            .as_std()
+            .get_program()
+            .to_string_lossy()
+            .into_owned();
         self.cmd.kill_on_drop(true);
         if self.input.is_some() {
             self.cmd.stdin(Stdio::piped());
         }
-        self.cmd.stdout(Stdio::piped());
-        self.cmd.stderr(Stdio::piped());
-        let mut child = self.cmd.spawn().with_kind(error_kind)?;
-        if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
-            use tokio::io::AsyncWriteExt;
-            tokio::io::copy(input, &mut stdin).await?;
-            stdin.flush().await?;
-            stdin.shutdown().await?;
-            drop(stdin);
+        if self.pipe.is_empty() {
+            if self.capture {
+                self.cmd.stdout(Stdio::piped());
+                self.cmd.stderr(Stdio::piped());
+            }
+            let mut child = self.cmd.spawn().with_ctx(|_| (error_kind, &cmd_str))?;
+            if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
+                use tokio::io::AsyncWriteExt;
+                tokio::io::copy(input, &mut stdin).await?;
+                stdin.flush().await?;
+                stdin.shutdown().await?;
+                drop(stdin);
+            }
+            let res = match self.timeout {
+                None => child
+                    .wait_with_output()
+                    .await
+                    .with_ctx(|_| (error_kind, &cmd_str))?,
+                Some(t) => tokio::time::timeout(t, child.wait_with_output())
+                    .await
+                    .with_kind(ErrorKind::Timeout)?
+                    .with_ctx(|_| (error_kind, &cmd_str))?,
+            };
+            crate::ensure_code!(
+                res.status.success(),
+                error_kind,
+                "{}",
+                Some(&res.stderr)
+                    .filter(|a| !a.is_empty())
+                    .or(Some(&res.stdout))
+                    .filter(|a| !a.is_empty())
+                    .and_then(|a| std::str::from_utf8(a).ok())
+                    .unwrap_or(&format!(
+                        "{} exited with code {}",
+                        self.cmd.as_std().get_program().to_string_lossy(),
+                        res.status
+                    ))
+            );
+            Ok(res.stdout)
+        } else {
+            let mut futures = Vec::<BoxFuture<'_, Result<(), Error>>>::new(); // todo: predict capacity
+
+            let mut cmds = std::mem::take(&mut self.pipe);
+            cmds.push_front(&mut *self.cmd);
+            let len = cmds.len();
+
+            let timeout = self.timeout;
+
+            let mut prev = self
+                .input
+                .take()
+                .map(|i| Box::new(i) as Box<dyn AsyncRead + Unpin + Send>);
+            for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() {
+                let last = idx == len - 1;
+                if self.capture || !last {
+                    cmd.stdout(Stdio::piped());
+                }
+                if self.capture {
+                    cmd.stderr(Stdio::piped());
+                }
+                if prev.is_some() {
+                    cmd.stdin(Stdio::piped());
+                }
+                let mut child = cmd.spawn().with_kind(error_kind)?;
+                let input = std::mem::replace(
+                    &mut prev,
+                    child
+                        .stdout
+                        .take()
+                        .map(|i| Box::new(BufReader::new(i)) as Box<dyn AsyncRead + Unpin + Send>),
+                );
+                futures.push(
+                    async move {
+                        if let (Some(mut stdin), Some(mut input)) = (child.stdin.take(), input) {
+                            use tokio::io::AsyncWriteExt;
+                            tokio::io::copy(&mut input, &mut stdin).await?;
+                            stdin.flush().await?;
+                            stdin.shutdown().await?;
+                            drop(stdin);
+                        }
+                        let res = match timeout {
+                            None => child.wait_with_output().await?,
+                            Some(t) => tokio::time::timeout(t, child.wait_with_output())
+                                .await
+                                .with_kind(ErrorKind::Timeout)??,
+                        };
+                        crate::ensure_code!(
+                            res.status.success(),
+                            error_kind,
+                            "{}",
+                            Some(&res.stderr)
+                                .filter(|a| !a.is_empty())
+                                .or(Some(&res.stdout))
+                                .filter(|a| !a.is_empty())
+                                .and_then(|a| std::str::from_utf8(a).ok())
+                                .unwrap_or(&format!(
+                                    "{} exited with code {}",
+                                    cmd.as_std().get_program().to_string_lossy(),
+                                    res.status
+                                ))
+                        );
+
+                        Ok(())
+                    }
+                    .boxed(),
+                );
+            }
+
+            let (send, recv) = oneshot::channel();
+            futures.push(
+                async move {
+                    if let Some(mut prev) = prev {
+                        let mut res = Vec::new();
+                        prev.read_to_end(&mut res).await?;
+                        send.send(res).unwrap();
+                    } else {
+                        send.send(Vec::new()).unwrap();
+                    }
+
+                    Ok(())
+                }
+                .boxed(),
+            );
+
+            futures::future::try_join_all(futures).await?;
+
+            Ok(recv.await.unwrap())
         }
-        let res = match self.timeout {
-            None => child.wait_with_output().await?,
-            Some(t) => tokio::time::timeout(t, child.wait_with_output())
-                .await
-                .with_kind(ErrorKind::Timeout)??,
-        };
-        crate::ensure_code!(
-            res.status.success(),
-            error_kind,
-            "{}",
-            Some(&res.stderr)
-                .filter(|a| !a.is_empty())
-                .or(Some(&res.stdout))
-                .filter(|a| !a.is_empty())
-                .and_then(|a| std::str::from_utf8(a).ok())
-                .unwrap_or(&format!("Unknown Error ({})", res.status))
-        );
-        Ok(res.stdout)
     }
 }