feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

core/startos/src/service/persistent_container.rs

@@ -6,8 +6,7 @@ use std::time::Duration;
 use futures::future::ready;
 use futures::{Future, FutureExt};
 use helpers::NonDetachingJoinHandle;
-use imbl_value::InternedString;
-use models::{ProcedureName, VolumeId};
+use models::{ImageId, ProcedureName, VolumeId};
 use rpc_toolkit::{Empty, Server, ShutdownHandle};
 use serde::de::DeserializeOwned;
 use tokio::fs::File;
@@ -24,14 +23,15 @@ use crate::disk::mount::filesystem::idmapped::IdMapped;
 use crate::disk::mount::filesystem::loop_dev::LoopDev;
 use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
 use crate::disk::mount::filesystem::{MountType, ReadOnly};
-use crate::disk::mount::guard::MountGuard;
+use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
 use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET};
 use crate::net::net_controller::NetService;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::S9pk;
 use crate::service::start_stop::StartStop;
-use crate::service::{rpc, RunningStatus};
+use crate::service::{rpc, RunningStatus, Service};
 use crate::util::rpc_client::UnixRpcClient;
 use crate::util::Invoke;
 use crate::volume::{asset_dir, data_dir};
@@ -89,7 +89,8 @@ pub struct PersistentContainer {
     js_mount: MountGuard,
     volumes: BTreeMap<VolumeId, MountGuard>,
     assets: BTreeMap<VolumeId, MountGuard>,
-    pub(super) overlays: Arc<Mutex<BTreeMap<InternedString, OverlayGuard>>>,
+    pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
+    pub(super) overlays: Arc<Mutex<BTreeMap<Guid, OverlayGuard<Arc<MountGuard>>>>>,
     pub(super) state: Arc<watch::Sender<ServiceState>>,
     pub(super) net_service: Mutex<NetService>,
     destroyed: bool,
@@ -178,14 +179,62 @@ impl PersistentContainer {
                 .await?,
             );
         }
+
+        let mut images = BTreeMap::new();
         let image_path = lxc_container.rootfs_dir().join("media/startos/images");
         tokio::fs::create_dir_all(&image_path).await?;
-        for image in &s9pk.as_manifest().images {
+        for (image, config) in &s9pk.as_manifest().images {
+            let mut arch = ARCH;
+            let mut sqfs_path = Path::new("images")
+                .join(arch)
+                .join(image)
+                .with_extension("squashfs");
+            if !s9pk
+                .as_archive()
+                .contents()
+                .get_path(&sqfs_path)
+                .and_then(|e| e.as_file())
+                .is_some()
+            {
+                arch = if let Some(arch) = config.emulate_missing_as.as_deref() {
+                    arch
+                } else {
+                    continue;
+                };
+                sqfs_path = Path::new("images")
+                    .join(arch)
+                    .join(image)
+                    .with_extension("squashfs");
+            }
+            let sqfs = s9pk
+                .as_archive()
+                .contents()
+                .get_path(&sqfs_path)
+                .and_then(|e| e.as_file())
+                .or_not_found(sqfs_path.display())?;
+            let mountpoint = image_path.join(image);
+            tokio::fs::create_dir_all(&mountpoint).await?;
+            Command::new("chown")
+                .arg("100000:100000")
+                .arg(&mountpoint)
+                .invoke(ErrorKind::Filesystem)
+                .await?;
+            images.insert(
+                image.clone(),
+                Arc::new(
+                    MountGuard::mount(
+                        &IdMapped::new(LoopDev::from(&**sqfs), 0, 100000, 65536),
+                        &mountpoint,
+                        ReadOnly,
+                    )
+                    .await?,
+                ),
+            );
             let env_filename = Path::new(image.as_ref()).with_extension("env");
             if let Some(env) = s9pk
                 .as_archive()
                 .contents()
-                .get_path(Path::new("images").join(*ARCH).join(&env_filename))
+                .get_path(Path::new("images").join(arch).join(&env_filename))
                 .and_then(|e| e.as_file())
             {
                 env.copy(&mut File::create(image_path.join(&env_filename)).await?)
@@ -195,7 +244,7 @@ impl PersistentContainer {
             if let Some(json) = s9pk
                 .as_archive()
                 .contents()
-                .get_path(Path::new("images").join(*ARCH).join(&json_filename))
+                .get_path(Path::new("images").join(arch).join(&json_filename))
                 .and_then(|e| e.as_file())
             {
                 json.copy(&mut File::create(image_path.join(&json_filename)).await?)
@@ -215,6 +264,7 @@ impl PersistentContainer {
             js_mount,
             volumes,
             assets,
+            images,
             overlays: Arc::new(Mutex::new(BTreeMap::new())),
             state: Arc::new(watch::channel(ServiceState::new(start)).0),
             net_service: Mutex::new(net_service),
@@ -257,7 +307,7 @@ impl PersistentContainer {
     }
 
     #[instrument(skip_all)]
-    pub async fn init(&self, seed: Weak<ServiceActorSeed>) -> Result<(), Error> {
+    pub async fn init(&self, seed: Weak<Service>) -> Result<(), Error> {
         let socket_server_context = EffectContext::new(seed);
         let server = Server::new(
             move || ready(Ok(socket_server_context.clone())),
@@ -330,6 +380,7 @@ impl PersistentContainer {
         let js_mount = self.js_mount.take();
         let volumes = std::mem::take(&mut self.volumes);
         let assets = std::mem::take(&mut self.assets);
+        let images = std::mem::take(&mut self.images);
         let overlays = self.overlays.clone();
         let lxc_container = self.lxc_container.take();
         self.destroyed = true;
@@ -352,6 +403,9 @@ impl PersistentContainer {
                         for (_, overlay) in std::mem::take(&mut *overlays.lock().await) {
                             errs.handle(overlay.unmount(true).await);
                         }
+                        for (_, images) in images {
+                            errs.handle(images.unmount().await);
+                        }
                         errs.handle(js_mount.unmount(true).await);
                         if let Some(lxc_container) = lxc_container {
                             errs.handle(lxc_container.exit().await);
@@ -378,6 +432,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn start(&self) -> Result<(), Error> {
         self.execute(
+            Guid::new(),
             ProcedureName::StartMain,
             Value::Null,
             Some(Duration::from_secs(5)), // TODO
@@ -389,7 +444,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn stop(&self) -> Result<Duration, Error> {
         let timeout: Option<crate::util::serde::Duration> = self
-            .execute(ProcedureName::StopMain, Value::Null, None)
+            .execute(Guid::new(), ProcedureName::StopMain, Value::Null, None)
             .await?;
         Ok(timeout.map(|a| *a).unwrap_or(Duration::from_secs(30)))
     }
@@ -397,6 +452,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn execute<O>(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
@@ -404,7 +460,7 @@ impl PersistentContainer {
     where
         O: DeserializeOwned,
     {
-        self._execute(name, input, timeout)
+        self._execute(id, name, input, timeout)
             .await
             .and_then(from_value)
     }
@@ -412,6 +468,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn sanboxed<O>(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
@@ -419,7 +476,7 @@ impl PersistentContainer {
     where
         O: DeserializeOwned,
     {
-        self._sandboxed(name, input, timeout)
+        self._sandboxed(id, name, input, timeout)
             .await
             .and_then(from_value)
     }
@@ -427,13 +484,15 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     async fn _execute(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
     ) -> Result<Value, Error> {
-        let fut = self
-            .rpc_client
-            .request(rpc::Execute, rpc::ExecuteParams::new(name, input, timeout));
+        let fut = self.rpc_client.request(
+            rpc::Execute,
+            rpc::ExecuteParams::new(id, name, input, timeout),
+        );
 
         Ok(if let Some(timeout) = timeout {
             tokio::time::timeout(timeout, fut)
@@ -447,13 +506,15 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     async fn _sandboxed(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
     ) -> Result<Value, Error> {
-        let fut = self
-            .rpc_client
-            .request(rpc::Sandbox, rpc::ExecuteParams::new(name, input, timeout));
+        let fut = self.rpc_client.request(
+            rpc::Sandbox,
+            rpc::ExecuteParams::new(id, name, input, timeout),
+        );
 
         Ok(if let Some(timeout) = timeout {
             tokio::time::timeout(timeout, fut)