feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

Makefile

@@ -160,6 +160,10 @@ wormhole-deb: results/$(BASENAME).deb
 	@echo "Paste the following command into the shell of your start-os server:"
 	@wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }'
 
+wormhole-cli: core/target/$(ARCH)-unknown-linux-musl/release/start-cli
+	@echo "Paste the following command into the shell of your start-os server:"
+	@wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }'
+
 update: $(ALL_TARGETS)
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
 	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')

container-runtime/deb-install.sh

@@ -6,7 +6,7 @@ mkdir -p /run/systemd/resolve
 echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf
 
 apt-get update
-apt-get install -y curl rsync
+apt-get install -y curl rsync qemu-user-static
 
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
 source ~/.bashrc

container-runtime/package.json

@@ -21,6 +21,7 @@
     "esbuild-plugin-resolve": "^2.0.0",
     "filebrowser": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
+    "lodash": "^4.17.21",
     "node-fetch": "^3.1.0",
     "ts-matches": "^5.5.1",
     "tslib": "^2.5.3",

container-runtime/src/Adapters/HostSystemStartOs.ts

@@ -32,6 +32,8 @@ type RpcError = typeof matchRpcError._TYPE
 const SOCKET_PATH = "/media/startos/rpc/host.sock"
 const MAIN = "/main" as const
 export class HostSystemStartOs implements Effects {
+  procedureId: string | null = null
+
   static of(callbackHolder: CallbackHolder) {
     return new HostSystemStartOs(callbackHolder)
   }
@@ -40,7 +42,7 @@ export class HostSystemStartOs implements Effects {
   id = 0
   rpcRound<K extends keyof Effects | "getStore" | "setStore">(
     method: K,
-    params: unknown,
+    params: Record<string, unknown>,
   ) {
     const id = this.id++
     const client = net.createConnection({ path: SOCKET_PATH }, () => {
@@ -48,7 +50,7 @@ export class HostSystemStartOs implements Effects {
         JSON.stringify({
           id,
           method,
-          params,
+          params: { ...params, procedureId: this.procedureId },
         }) + "\n",
       )
     })
@@ -102,14 +104,14 @@ export class HostSystemStartOs implements Effects {
     }) as ReturnType<T.Effects["bind"]>
   }
   clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
-    return this.rpcRound("clearBindings", null) as ReturnType<
+    return this.rpcRound("clearBindings", {}) as ReturnType<
       T.Effects["clearBindings"]
     >
   }
   clearServiceInterfaces(
     ...[]: Parameters<T.Effects["clearServiceInterfaces"]>
   ) {
-    return this.rpcRound("clearServiceInterfaces", null) as ReturnType<
+    return this.rpcRound("clearServiceInterfaces", {}) as ReturnType<
       T.Effects["clearServiceInterfaces"]
     >
   }
@@ -145,18 +147,20 @@ export class HostSystemStartOs implements Effects {
       T.Effects["exportServiceInterface"]
     >
   }
-  exposeForDependents(...[options]: any) {
+  exposeForDependents(
-    return this.rpcRound("exposeForDependents", null) as ReturnType<
+    ...[options]: Parameters<T.Effects["exposeForDependents"]>
+  ) {
+    return this.rpcRound("exposeForDependents", options) as ReturnType<
       T.Effects["exposeForDependents"]
     >
   }
   getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
-    return this.rpcRound("getConfigured", null) as ReturnType<
+    return this.rpcRound("getConfigured", {}) as ReturnType<
       T.Effects["getConfigured"]
     >
   }
   getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
-    return this.rpcRound("getContainerIp", null) as ReturnType<
+    return this.rpcRound("getContainerIp", {}) as ReturnType<
       T.Effects["getContainerIp"]
     >
   }
@@ -229,7 +233,7 @@ export class HostSystemStartOs implements Effects {
     >
   }
   restart(...[]: Parameters<T.Effects["restart"]>) {
-    return this.rpcRound("restart", null)
+    return this.rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
   }
   running(...[packageId]: Parameters<T.Effects["running"]>) {
     return this.rpcRound("running", { packageId }) as ReturnType<
@@ -262,7 +266,7 @@ export class HostSystemStartOs implements Effects {
     >
   }
   getDependencies(): ReturnType<T.Effects["getDependencies"]> {
-    return this.rpcRound("getDependencies", null) as ReturnType<
+    return this.rpcRound("getDependencies", {}) as ReturnType<
       T.Effects["getDependencies"]
     >
   }
@@ -279,7 +283,7 @@ export class HostSystemStartOs implements Effects {
   }
 
   shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
-    return this.rpcRound("shutdown", null)
+    return this.rpcRound("shutdown", {}) as ReturnType<T.Effects["shutdown"]>
   }
   stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
     return this.rpcRound("stopped", { packageId }) as ReturnType<

container-runtime/src/Adapters/RpcListener.ts

@@ -58,6 +58,7 @@ const runType = object({
   method: literal("execute"),
   params: object(
     {
+      id: string,
       procedure: string,
       input: any,
       timeout: number,
@@ -70,6 +71,7 @@ const sandboxRunType = object({
   method: literal("sandbox"),
   params: object(
     {
+      id: string,
       procedure: string,
       input: any,
       timeout: number,
@@ -195,6 +197,7 @@ export class RpcListener {
         const procedure = jsonPath.unsafeCast(params.procedure)
         return system
           .execute(this.effects, {
+            id: params.id,
             procedure,
             input: params.input,
             timeout: params.timeout,

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -49,7 +49,7 @@ function todo(): never {
 const execFile = promisify(childProcess.execFile)
 
 const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json"
-const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
+export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
 const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig"
 
 const matchSetResult = object(
@@ -199,11 +199,14 @@ export class SystemForEmbassy implements System {
   async execute(
     effects: HostSystemStartOs,
     options: {
+      id: string
       procedure: JsonPath
       input: unknown
       timeout?: number | undefined
     },
   ): Promise<RpcResult> {
+    effects = Object.create(effects)
+    effects.procedureId = options.id
     return this._execute(effects, options)
       .then((x) =>
         matches(x)
@@ -724,7 +727,7 @@ export class SystemForEmbassy implements System {
   private async properties(
     effects: HostSystemStartOs,
     timeoutMs: number | null,
-  ): Promise<ReturnType<T.ExpectedExports.Properties>> {
+  ): Promise<ReturnType<T.ExpectedExports.properties>> {
     // TODO BLU-J set the properties ever so often
     const setConfigValue = this.manifest.properties
     if (!setConfigValue) throw new Error("There is no properties")

container-runtime/src/Adapters/Systems/SystemForStartOs.ts

@@ -1,20 +1,23 @@
 import { ExecuteResult, System } from "../../Interfaces/System"
 import { unNestPath } from "../../Models/JsonPath"
-import { string } from "ts-matches"
+import matches, { any, number, object, string, tuple } from "ts-matches"
 import { HostSystemStartOs } from "../HostSystemStartOs"
 import { Effects } from "../../Models/Effects"
-import { RpcResult } from "../RpcListener"
+import { RpcResult, matchRpcResult } from "../RpcListener"
 import { duration } from "../../Models/Duration"
-const LOCATION = "/usr/lib/startos/package/startos"
+import { T } from "@start9labs/start-sdk"
+import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
+export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js"
 export class SystemForStartOs implements System {
   private onTerm: (() => Promise<void>) | undefined
   static of() {
-    return new SystemForStartOs()
+    return new SystemForStartOs(require(STARTOS_JS_LOCATION))
   }
-  constructor() {}
+  constructor(readonly abi: T.ABI) {}
   async execute(
     effects: HostSystemStartOs,
     options: {
+      id: string
       procedure:
         | "/init"
         | "/uninit"
@@ -33,7 +36,61 @@ export class SystemForStartOs implements System {
       timeout?: number | undefined
     },
   ): Promise<RpcResult> {
-    return { result: await this._execute(effects, options) }
+    effects = Object.create(effects)
+    effects.procedureId = options.id
+    return this._execute(effects, options)
+      .then((x) =>
+        matches(x)
+          .when(
+            object({
+              result: any,
+            }),
+            (x) => x,
+          )
+          .when(
+            object({
+              error: string,
+            }),
+            (x) => ({
+              error: {
+                code: 0,
+                message: x.error,
+              },
+            }),
+          )
+          .when(
+            object({
+              "error-code": tuple(number, string),
+            }),
+            ({ "error-code": [code, message] }) => ({
+              error: {
+                code,
+                message,
+              },
+            }),
+          )
+          .defaultTo({ result: x }),
+      )
+      .catch((error: unknown) => {
+        if (error instanceof Error)
+          return {
+            error: {
+              code: 0,
+              message: error.name,
+              data: {
+                details: error.message,
+                debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`,
+              },
+            },
+          }
+        if (matchRpcResult.test(error)) return error
+        return {
+          error: {
+            code: 0,
+            message: String(error),
+          },
+        }
+      })
   }
   async _execute(
     effects: Effects,
@@ -58,26 +115,27 @@ export class SystemForStartOs implements System {
   ): Promise<unknown> {
     switch (options.procedure) {
       case "/init": {
-        const path = `${LOCATION}/procedures/init`
+        const previousVersion =
-        const procedure: any = await import(path).catch(() => require(path))
+          string.optional().unsafeCast(options.input) || null
-        const previousVersion = string.optional().unsafeCast(options)
+        return this.abi.init({ effects, previousVersion })
-        return procedure.init({ effects, previousVersion })
       }
       case "/uninit": {
-        const path = `${LOCATION}/procedures/init`
+        const nextVersion = string.optional().unsafeCast(options.input) || null
-        const procedure: any = await import(path).catch(() => require(path))
+        return this.abi.uninit({ effects, nextVersion })
-        const nextVersion = string.optional().unsafeCast(options)
-        return procedure.uninit({ effects, nextVersion })
       }
       case "/main/start": {
-        const path = `${LOCATION}/procedures/main`
-        const procedure: any = await import(path).catch(() => require(path))
         const started = async (onTerm: () => Promise<void>) => {
           await effects.setMainStatus({ status: "running" })
           if (this.onTerm) await this.onTerm()
           this.onTerm = onTerm
         }
-        return procedure.main({ effects, started })
+        const daemons = await (
+          await this.abi.main({
+            effects: { ...effects, _type: "main" },
+            started,
+          })
+        ).build()
+        this.onTerm = daemons.term
       }
       case "/main/stop": {
         await effects.setMainStatus({ status: "stopped" })
@@ -86,67 +144,50 @@ export class SystemForStartOs implements System {
         return duration(30, "s")
       }
       case "/config/set": {
-        const path = `${LOCATION}/procedures/config`
+        const input = options.input as any // TODO
-        const procedure: any = await import(path).catch(() => require(path))
+        return this.abi.setConfig({ effects, input })
-        const input = options.input
-        return procedure.setConfig({ effects, input })
       }
       case "/config/get": {
-        const path = `${LOCATION}/procedures/config`
+        return this.abi.getConfig({ effects })
-        const procedure: any = await import(path).catch(() => require(path))
-        return procedure.getConfig({ effects })
       }
       case "/backup/create":
       case "/backup/restore":
         throw new Error("this should be called with the init/unit")
       case "/actions/metadata": {
-        const path = `${LOCATION}/procedures/actions`
+        return this.abi.actionsMetadata({ effects })
-        const procedure: any = await import(path).catch(() => require(path))
-        return procedure.actionsMetadata({ effects })
       }
       default:
         const procedures = unNestPath(options.procedure)
         const id = procedures[2]
         switch (true) {
           case procedures[1] === "actions" && procedures[3] === "get": {
-            const path = `${LOCATION}/procedures/actions`
+            const action = (await this.abi.actions({ effects }))[id]
-            const action: any = (await import(path).catch(() => require(path)))
-              .actions[id]
             if (!action) throw new Error(`Action ${id} not found`)
-            return action.get({ effects })
+            return action.getConfig({ effects })
           }
           case procedures[1] === "actions" && procedures[3] === "run": {
-            const path = `${LOCATION}/procedures/actions`
+            const action = (await this.abi.actions({ effects }))[id]
-            const action: any = (await import(path).catch(() => require(path)))
-              .actions[id]
             if (!action) throw new Error(`Action ${id} not found`)
-            const input = options.input
+            return action.run({ effects, input: options.input as any }) // TODO
-            return action.run({ effects, input })
           }
           case procedures[1] === "dependencies" && procedures[3] === "query": {
-            const path = `${LOCATION}/procedures/dependencies`
+            const dependencyConfig = this.abi.dependencyConfig[id]
-            const dependencyConfig: any = (
-              await import(path).catch(() => require(path))
-            ).dependencyConfig[id]
             if (!dependencyConfig)
               throw new Error(`dependencyConfig ${id} not found`)
             const localConfig = options.input
-            return dependencyConfig.query({ effects, localConfig })
+            return dependencyConfig.query({ effects })
           }
           case procedures[1] === "dependencies" && procedures[3] === "update": {
-            const path = `${LOCATION}/procedures/dependencies`
+            const dependencyConfig = this.abi.dependencyConfig[id]
-            const dependencyConfig: any = (
-              await import(path).catch(() => require(path))
-            ).dependencyConfig[id]
             if (!dependencyConfig)
               throw new Error(`dependencyConfig ${id} not found`)
-            return dependencyConfig.update(options.input)
+            return dependencyConfig.update(options.input as any) // TODO
           }
         }
     }
-    throw new Error("Method not implemented.")
+    throw new Error(`Method ${options.procedure} not implemented.`)
   }
-  exit(effects: Effects): Promise<void> {
+  async exit(effects: Effects): Promise<void> {
-    throw new Error("Method not implemented.")
+    return void null
   }
 }

container-runtime/src/Adapters/Systems/index.ts

@@ -1,6 +1,22 @@
+import * as fs from "node:fs/promises"
 import { System } from "../../Interfaces/System"
-import { SystemForEmbassy } from "./SystemForEmbassy"
+import { EMBASSY_JS_LOCATION, SystemForEmbassy } from "./SystemForEmbassy"
-import { SystemForStartOs } from "./SystemForStartOs"
+import { STARTOS_JS_LOCATION, SystemForStartOs } from "./SystemForStartOs"
 export async function getSystem(): Promise<System> {
-  return SystemForEmbassy.of()
+  if (
+    await fs.access(STARTOS_JS_LOCATION).then(
+      () => true,
+      () => false,
+    )
+  ) {
+    return SystemForStartOs.of()
+  } else if (
+    await fs.access(EMBASSY_JS_LOCATION).then(
+      () => true,
+      () => false,
+    )
+  ) {
+    return SystemForEmbassy.of()
+  }
+  throw new Error(`${STARTOS_JS_LOCATION} not found`)
 }

container-runtime/src/Interfaces/System.ts

@@ -14,6 +14,7 @@ export interface System {
   execute(
     effects: T.Effects,
     options: {
+      id: string
       procedure: JsonPath
       input: unknown
       timeout?: number

core/Cargo.lock

@@ -387,6 +387,22 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "backhand"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f2fc1bc7bb7fd449e02000cc1592cc63dcdcd61710f8b9efe32bab2d1784603"
+dependencies = [
+ "deku",
+ "flate2",
+ "rustc-hash",
+ "thiserror",
+ "tracing",
+ "xz2",
+ "zstd",
+ "zstd-safe",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.71"
@@ -588,6 +604,11 @@ name = "cc"
 version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd"
+dependencies = [
+ "jobserver",
+ "libc",
+ "once_cell",
+]
 
 [[package]]
 name = "cfg-if"
@@ -1058,12 +1079,6 @@ dependencies = [
  "cipher 0.3.0",
 ]
 
-[[package]]
-name = "current_platform"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a74858bcfe44b22016cb49337d7b6f04618c58e5dbfdef61b06b8c434324a0bc"
-
 [[package]]
 name = "curve25519-dalek"
 version = "3.2.0"
@@ -1146,6 +1161,31 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
 
+[[package]]
+name = "deku"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "709ade444d53896e60f6265660eb50480dd08b77bfc822e5dcc233b88b0b2fba"
+dependencies = [
+ "bitvec",
+ "deku_derive",
+ "no_std_io",
+ "rustversion",
+]
+
+[[package]]
+name = "deku_derive"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7534973f93f9de83203e41c8ddd32d230599fa73fa889f3deb1580ccd186913"
+dependencies = [
+ "darling",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.60",
+]
+
 [[package]]
 name = "der"
 version = "0.7.9"
@@ -2405,6 +2445,15 @@ dependencies = [
  "jaq-parse",
 ]
 
+[[package]]
+name = "jobserver"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "josekit"
 version = "0.8.6"
@@ -2561,6 +2610,12 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "libyml"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.13"
@@ -2583,6 +2638,17 @@ version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
+[[package]]
+name = "lzma-sys"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
@@ -2788,6 +2854,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "no_std_io"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fa5f306a6f2c01b4fd172f29bb46195b1764061bf926c75e96ff55df3178208"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -3800,6 +3875,12 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
@@ -3895,9 +3976,9 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.15"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47"
+checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
 
 [[package]]
 name = "rusty-fork"
@@ -3929,9 +4010,9 @@ dependencies = [
 
 [[package]]
 name = "ryu"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
 [[package]]
 name = "same-file"
@@ -4015,9 +4096,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.200"
+version = "1.0.203"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f"
+checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
 dependencies = [
  "serde_derive",
 ]
@@ -4041,9 +4122,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.200"
+version = "1.0.203"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb"
+checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4052,9 +4133,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.116"
+version = "1.0.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813"
+checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
 dependencies = [
  "indexmap 2.2.6",
  "itoa",
@@ -4124,16 +4205,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "serde_yaml"
+name = "serde_yml"
-version = "0.9.34+deprecated"
+version = "0.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f"
 dependencies = [
  "indexmap 2.2.6",
  "itoa",
+ "libyml",
+ "log",
+ "memchr",
  "ryu",
  "serde",
- "unsafe-libyaml",
+ "serde_json",
+ "tempfile",
 ]
 
 [[package]]
@@ -4600,6 +4685,7 @@ dependencies = [
  "async-trait",
  "axum 0.7.5",
  "axum-server",
+ "backhand",
  "base32",
  "base64 0.21.7",
  "base64ct",
@@ -4614,7 +4700,6 @@ dependencies = [
  "console-subscriber",
  "cookie 0.18.1",
  "cookie_store",
- "current_platform",
  "der",
  "digest 0.10.7",
  "divrem",
@@ -4681,7 +4766,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "serde_with",
- "serde_yaml",
+ "serde_yml",
  "sha2 0.10.8",
  "shell-words",
  "simple-logging",
@@ -5537,12 +5622,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -5993,6 +6072,15 @@ dependencies = [
  "rustix",
 ]
 
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "yajrc"
 version = "0.1.3"
@@ -6056,3 +6144,31 @@ dependencies = [
  "quote",
  "syn 2.0.60",
 ]
+
+[[package]]
+name = "zstd"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.10+zstd.1.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
+dependencies = [
+ "cc",
+ "pkg-config",
+]

core/startos/Cargo.toml

@@ -59,6 +59,7 @@ async-stream = "0.3.5"
 async-trait = "0.1.74"
 axum = { version = "0.7.3", features = ["ws"] }
 axum-server = "0.6.0"
+backhand = "0.18.0"
 base32 = "0.4.0"
 base64 = "0.21.4"
 base64ct = "1.6.0"
@@ -72,7 +73,6 @@ console = "0.15.7"
 console-subscriber = { version = "0.2", optional = true }
 cookie = "0.18.0"
 cookie_store = "0.20.0"
-current_platform = "0.2.0"
 der = { version = "0.7.9", features = ["derive", "pem"] }
 digest = "0.10.7"
 divrem = "1.0.0"
@@ -154,7 +154,7 @@ serde_json = "1.0"
 serde_toml = { package = "toml", version = "0.8.2" }
 serde_urlencoded = "0.7"
 serde_with = { version = "3.4.0", features = ["macros", "json"] }
-serde_yaml = "0.9.25"
+serde_yaml = { package = "serde_yml", version = "0.0.10" }
 sha2 = "0.10.2"
 shell-words = "1"
 simple-logging = "2.0.2"

core/startos/src/action.rs

@@ -8,6 +8,7 @@ use ts_rs::TS;
 use crate::config::Config;
 use crate::context::RpcContext;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat};
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -77,6 +78,7 @@ pub async fn action(
         .as_ref()
         .or_not_found(lazy_format!("Manager for {}", package_id))?
         .action(
+            Guid::new(),
             action_id,
             input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(),
         )

core/startos/src/auth.rs

@@ -178,6 +178,7 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(
 #[derive(Deserialize, Serialize, Parser, TS)]
 #[serde(rename_all = "camelCase")]
 #[command(rename_all = "kebab-case")]
+#[ts(export)]
 pub struct LoginParams {
     password: Option<PasswordType>,
     #[ts(skip)]

core/startos/src/backup/restore.rs

@@ -149,7 +149,6 @@ async fn restore_packages(
                 S9pk::open(
                     backup_dir.path().join(&id).with_extension("s9pk"),
                     Some(&id),
-                    true,
                 )
                 .await?,
                 Some(backup_dir),

core/startos/src/config/mod.rs

@@ -16,6 +16,7 @@ use ts_rs::TS;
 
 use crate::context::{CliContext, RpcContext};
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::serde::{HandlerExtSerde, StdinDeserializable};
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -156,7 +157,7 @@ pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result<ConfigRes,
         .await
         .as_ref()
         .or_not_found(lazy_format!("Manager for {id}"))?
-        .get_config()
+        .get_config(Guid::new())
         .await
 }
 
@@ -218,7 +219,7 @@ pub async fn set_impl(
                 ErrorKind::Unknown,
             )
         })?
-        .configure(configure_context)
+        .configure(Guid::new(), configure_context)
         .await?;
     Ok(())
 }

core/startos/src/context/config.rs

@@ -93,26 +93,28 @@ impl ClientConfig {
 #[serde(rename_all = "kebab-case")]
 #[command(rename_all = "kebab-case")]
 pub struct ServerConfig {
-    #[arg(short = 'c', long = "config")]
+    #[arg(short, long)]
     pub config: Option<PathBuf>,
-    #[arg(long = "ethernet-interface")]
+    #[arg(long)]
     pub ethernet_interface: Option<String>,
     #[arg(skip)]
     pub os_partitions: Option<OsPartitionInfo>,
-    #[arg(long = "bind-rpc")]
+    #[arg(long)]
     pub bind_rpc: Option<SocketAddr>,
-    #[arg(long = "tor-control")]
+    #[arg(long)]
     pub tor_control: Option<SocketAddr>,
-    #[arg(long = "tor-socks")]
+    #[arg(long)]
     pub tor_socks: Option<SocketAddr>,
-    #[arg(long = "dns-bind")]
+    #[arg(long)]
     pub dns_bind: Option<Vec<SocketAddr>>,
-    #[arg(long = "revision-cache-size")]
+    #[arg(long)]
     pub revision_cache_size: Option<usize>,
-    #[arg(short = 'd', long = "datadir")]
+    #[arg(short, long)]
     pub datadir: Option<PathBuf>,
-    #[arg(long = "disable-encryption")]
+    #[arg(long)]
     pub disable_encryption: Option<bool>,
+    #[arg(long)]
+    pub multi_arch_s9pks: Option<bool>,
 }
 impl ContextConfig for ServerConfig {
     fn next(&mut self) -> Option<PathBuf> {
@@ -131,6 +133,7 @@ impl ContextConfig for ServerConfig {
             .or(other.revision_cache_size);
         self.datadir = self.datadir.take().or(other.datadir);
         self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
+        self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks);
     }
 }
 

core/startos/src/context/rpc.rs

@@ -43,6 +43,7 @@ pub struct RpcContextSeed {
     pub db: TypedPatchDb<Database>,
     pub account: RwLock<AccountInfo>,
     pub net_controller: Arc<NetController>,
+    pub s9pk_arch: Option<&'static str>,
     pub services: ServiceMap,
     pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
     pub shutdown: broadcast::Sender<Option<Shutdown>>,
@@ -152,6 +153,11 @@ impl RpcContext {
             db,
             account: RwLock::new(account),
             net_controller,
+            s9pk_arch: if config.multi_arch_s9pks.unwrap_or(false) {
+                None
+            } else {
+                Some(crate::ARCH)
+            },
             services,
             metrics_cache,
             shutdown,

core/startos/src/control.rs

@@ -7,6 +7,7 @@ use ts_rs::TS;
 
 use crate::context::RpcContext;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::Error;
 
 #[derive(Deserialize, Serialize, Parser, TS)]
@@ -23,7 +24,7 @@ pub async fn start(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resu
         .await
         .as_ref()
         .or_not_found(lazy_format!("Manager for {id}"))?
-        .start()
+        .start(Guid::new())
         .await?;
 
     Ok(())
@@ -36,7 +37,7 @@ pub async fn stop(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resul
         .await
         .as_ref()
         .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .stop()
+        .stop(Guid::new())
         .await?;
 
     Ok(())
@@ -48,7 +49,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
         .await
         .as_ref()
         .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
-        .restart()
+        .restart(Guid::new())
         .await?;
 
     Ok(())

core/startos/src/dependencies.rs

@@ -13,6 +13,7 @@ use crate::config::{Config, ConfigSpec, ConfigureContext};
 use crate::context::RpcContext;
 use crate::db::model::package::CurrentDependencies;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::Error;
 
 pub fn dependency<C: Context>() -> ParentHandler<C> {
@@ -86,7 +87,7 @@ pub async fn configure_impl(
                 ErrorKind::Unknown,
             )
         })?
-        .configure(configure_context)
+        .configure(Guid::new(), configure_context)
         .await?;
     Ok(())
 }
@@ -103,14 +104,15 @@ pub async fn configure_logic(
     ctx: RpcContext,
     (dependent_id, dependency_id): (PackageId, PackageId),
 ) -> Result<ConfigDryRes, Error> {
+    let procedure_id = Guid::new();
     let dependency_guard = ctx.services.get(&dependency_id).await;
     let dependency = dependency_guard.as_ref().or_not_found(&dependency_id)?;
     let dependent_guard = ctx.services.get(&dependent_id).await;
     let dependent = dependent_guard.as_ref().or_not_found(&dependent_id)?;
-    let config_res = dependency.get_config().await?;
+    let config_res = dependency.get_config(procedure_id.clone()).await?;
     let diff = Value::Object(
         dependent
-            .dependency_config(dependency_id, config_res.config.clone())
+            .dependency_config(procedure_id, dependency_id, config_res.config.clone())
             .await?
             .unwrap_or_default(),
     );
@@ -129,6 +131,7 @@ pub async fn compute_dependency_config_errs(
     id: &PackageId,
     current_dependencies: &mut CurrentDependencies,
 ) -> Result<(), Error> {
+    let procedure_id = Guid::new();
     let service_guard = ctx.services.get(id).await;
     let service = service_guard.as_ref().or_not_found(id)?;
     for (dep_id, dep_info) in current_dependencies.0.iter_mut() {
@@ -137,10 +140,10 @@ pub async fn compute_dependency_config_errs(
             continue;
         };
 
-        let dep_config = dependency.get_config().await?.config;
+        let dep_config = dependency.get_config(procedure_id.clone()).await?.config;
 
         dep_info.config_satisfied = service
-            .dependency_config(dep_id.clone(), dep_config)
+            .dependency_config(procedure_id.clone(), dep_id.clone(), dep_config)
             .await?
             .is_none();
     }

core/startos/src/disk/mount/backup.rs

@@ -178,7 +178,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
         Ok(())
     }
 }
-#[async_trait::async_trait]
 impl<G: GenericMountGuard> GenericMountGuard for BackupMountGuard<G> {
     fn path(&self) -> &Path {
         if let Some(guard) = &self.encrypted_guard {

core/startos/src/disk/mount/filesystem/overlayfs.rs

@@ -6,8 +6,8 @@ use digest::generic_array::GenericArray;
 use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 
-use crate::disk::mount::filesystem::{FileSystem, ReadOnly, ReadWrite};
+use crate::disk::mount::filesystem::{FileSystem, ReadWrite};
-use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
+use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
 use crate::prelude::*;
 use crate::util::io::TmpDir;
 
@@ -94,17 +94,13 @@ impl<
 }
 
 #[derive(Debug)]
-pub struct OverlayGuard {
+pub struct OverlayGuard<G: GenericMountGuard> {
-    lower: Option<TmpMountGuard>,
+    lower: Option<G>,
     upper: Option<TmpDir>,
     inner_guard: MountGuard,
 }
-impl OverlayGuard {
+impl<G: GenericMountGuard> OverlayGuard<G> {
-    pub async fn mount(
+    pub async fn mount(lower: G, mountpoint: impl AsRef<Path>) -> Result<Self, Error> {
-        base: &impl FileSystem,
-        mountpoint: impl AsRef<Path>,
-    ) -> Result<Self, Error> {
-        let lower = TmpMountGuard::mount(base, ReadOnly).await?;
         let upper = TmpDir::new().await?;
         let inner_guard = MountGuard::mount(
             &OverlayFs::new(
@@ -140,16 +136,15 @@ impl OverlayGuard {
         }
     }
 }
-#[async_trait::async_trait]
+impl<G: GenericMountGuard> GenericMountGuard for OverlayGuard<G> {
-impl GenericMountGuard for OverlayGuard {
     fn path(&self) -> &Path {
         self.inner_guard.path()
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         self.unmount(false).await
     }
 }
-impl Drop for OverlayGuard {
+impl<G: GenericMountGuard> Drop for OverlayGuard<G> {
     fn drop(&mut self) {
         let lower = self.lower.take();
         let upper = self.upper.take();

core/startos/src/disk/mount/guard.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Weak};
 
+use futures::Future;
 use lazy_static::lazy_static;
 use models::ResultExt;
 use tokio::sync::Mutex;
@@ -14,23 +15,20 @@ use crate::Error;
 
 pub const TMP_MOUNTPOINT: &'static str = "/media/startos/tmp";
 
-#[async_trait::async_trait]
 pub trait GenericMountGuard: std::fmt::Debug + Send + Sync + 'static {
     fn path(&self) -> &Path;
-    async fn unmount(mut self) -> Result<(), Error>;
+    fn unmount(self) -> impl Future<Output = Result<(), Error>> + Send;
 }
 
-#[async_trait::async_trait]
 impl GenericMountGuard for Never {
     fn path(&self) -> &Path {
         match *self {}
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         match self {}
     }
 }
 
-#[async_trait::async_trait]
 impl<T> GenericMountGuard for Arc<T>
 where
     T: GenericMountGuard,
@@ -38,7 +36,7 @@ where
     fn path(&self) -> &Path {
         (&**self).path()
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         if let Ok(guard) = Arc::try_unwrap(self) {
             guard.unmount().await?;
         }
@@ -102,12 +100,11 @@ impl Drop for MountGuard {
         }
     }
 }
-#[async_trait::async_trait]
 impl GenericMountGuard for MountGuard {
     fn path(&self) -> &Path {
         &self.mountpoint
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         MountGuard::unmount(self, false).await
     }
 }
@@ -165,12 +162,11 @@ impl TmpMountGuard {
         std::mem::replace(self, unmounted)
     }
 }
-#[async_trait::async_trait]
 impl GenericMountGuard for TmpMountGuard {
     fn path(&self) -> &Path {
         self.guard.path()
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         self.guard.unmount().await
     }
 }
@@ -187,12 +183,11 @@ impl<G: GenericMountGuard> SubPath<G> {
         Self { guard, path }
     }
 }
-#[async_trait::async_trait]
 impl<G: GenericMountGuard> GenericMountGuard for SubPath<G> {
     fn path(&self) -> &Path {
         self.path.as_path()
     }
-    async fn unmount(mut self) -> Result<(), Error> {
+    async fn unmount(self) -> Result<(), Error> {
         self.guard.unmount().await
     }
 }

core/startos/src/init.rs

@@ -242,7 +242,7 @@ pub async fn init(cfg: &ServerConfig) -> Result<InitResult, Error> {
 
     let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
         || &*server_info.version < &emver::Version::new(0, 3, 2, 0)
-        || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
+        || (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
 
     let log_dir = cfg.datadir().join("main/logs");
     if tokio::fs::metadata(&log_dir).await.is_err() {

core/startos/src/install/mod.rs

@@ -152,7 +152,6 @@ pub async fn install(
             .await?,
         ),
         None, // TODO
-        true,
     )
     .await?;
 
@@ -262,7 +261,6 @@ pub async fn sideload(ctx: RpcContext) -> Result<SideloadResponse, Error> {
         if let Err(e) = async {
             let s9pk = S9pk::deserialize(
                 &file, None, // TODO
-                true,
             )
             .await?;
             let _ = id_send.send(s9pk.as_manifest().id.clone());

core/startos/src/lib.rs

@@ -4,12 +4,8 @@ pub const CAP_1_KiB: usize = 1024;
 pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB;
 pub const CAP_10_MiB: usize = 10 * CAP_1_MiB;
 pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
-pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
+pub use std::env::consts::ARCH;
 lazy_static::lazy_static! {
-    pub static ref ARCH: &'static str = {
-        let (arch, _) = TARGET.split_once("-").unwrap();
-        arch
-    };
     pub static ref PLATFORM: String = {
         if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") {
             platform

core/startos/src/lxc/mod.rs

@@ -29,7 +29,7 @@ use crate::disk::mount::filesystem::bind::Bind;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::idmapped::IdMapped;
 use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
-use crate::disk::mount::filesystem::{MountType, ReadWrite};
+use crate::disk::mount::filesystem::{MountType, ReadOnly, ReadWrite};
 use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
 use crate::disk::mount::util::unmount;
 use crate::prelude::*;
@@ -153,7 +153,7 @@ impl LxcManager {
 
 pub struct LxcContainer {
     manager: Weak<LxcManager>,
-    rootfs: OverlayGuard,
+    rootfs: OverlayGuard<TmpMountGuard>,
     pub guid: Arc<ContainerId>,
     rpc_bind: TmpMountGuard,
     log_mount: Option<MountGuard>,
@@ -184,12 +184,16 @@ impl LxcContainer {
             .invoke(ErrorKind::Filesystem)
             .await?;
         let rootfs = OverlayGuard::mount(
-            &IdMapped::new(
+            TmpMountGuard::mount(
-                BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
+                &IdMapped::new(
-                0,
+                    BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
-                100000,
+                    0,
-                65536,
+                    100000,
-            ),
+                    65536,
+                ),
+                ReadOnly,
+            )
+            .await?,
             &rootfs_dir,
         )
         .await?;

core/startos/src/os_install/gpt.rs

@@ -87,7 +87,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
             gpt.add_partition(
                 "root",
                 15 * 1024 * 1024 * 1024,
-                match *crate::ARCH {
+                match crate::ARCH {
                     "x86_64" => gpt::partition_types::LINUX_ROOT_X64,
                     "aarch64" => gpt::partition_types::LINUX_ROOT_ARM_64,
                     _ => gpt::partition_types::LINUX_FS,

core/startos/src/os_install/mod.rs

@@ -366,7 +366,7 @@ pub async fn execute<C: Context>(
     if tokio::fs::metadata("/sys/firmware/efi").await.is_err() {
         install.arg("--target=i386-pc");
     } else {
-        match *ARCH {
+        match ARCH {
             "x86_64" => install.arg("--target=x86_64-efi"),
             "aarch64" => install.arg("--target=arm64-efi"),
             _ => &mut install,

core/startos/src/registry/device_info.rs

@@ -134,7 +134,7 @@ pub struct HardwareInfo {
 impl From<&RpcContext> for HardwareInfo {
     fn from(value: &RpcContext) -> Self {
         Self {
-            arch: InternedString::intern(&**crate::ARCH),
+            arch: InternedString::intern(crate::ARCH),
             ram: value.hardware.ram,
             devices: value
                 .hardware

core/startos/src/registry/package/add.rs

@@ -53,7 +53,6 @@ pub async fn add_package(
     let s9pk = S9pk::deserialize(
         &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
         Some(&commitment),
-        false,
     )
     .await?;
 
@@ -109,7 +108,7 @@ pub async fn cli_add_package(
         ..
     }: HandlerArgs<CliContext, CliAddPackageParams>,
 ) -> Result<(), Error> {
-    let s9pk = S9pk::open(&file, None, false).await?;
+    let s9pk = S9pk::open(&file, None).await?;
 
     let mut progress = FullProgressTracker::new();
     let progress_handle = progress.handle();
@@ -143,7 +142,6 @@ pub async fn cli_add_package(
     let mut src = S9pk::deserialize(
         &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
         Some(&commitment),
-        false,
     )
     .await?;
     src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true)

core/startos/src/rpc_continuations.rs

@@ -39,6 +39,11 @@ impl Guid {
         Some(Guid(InternedString::intern(r)))
     }
 }
+impl Default for Guid {
+    fn default() -> Self {
+        Self::new()
+    }
+}
 impl AsRef<str> for Guid {
     fn as_ref(&self) -> &str {
         self.0.as_ref()

core/startos/src/s9pk/merkle_archive/directory_contents.rs

@@ -211,7 +211,10 @@ impl<S: FileSource + Clone> DirectoryContents<S> {
                 if !filter(path) {
                     if v.hash.is_none() {
                         return Err(Error::new(
-                            eyre!("cannot filter out unhashed file, run `update_hashes` first"),
+                            eyre!(
+                                "cannot filter out unhashed file {}, run `update_hashes` first",
+                                path.display()
+                            ),
                             ErrorKind::InvalidRequest,
                         ));
                     }

core/startos/src/s9pk/merkle_archive/expected.rs

@@ -0,0 +1,103 @@
+
+use std::ffi::OsStr;
+use std::path::Path;
+
+use crate::prelude::*;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::FileSource;
+use crate::s9pk::merkle_archive::Entry;
+
+/// An object for tracking the files expected to be in an s9pk 
+pub struct Expected<'a, T> {
+    keep: DirectoryContents<()>,
+    dir: &'a DirectoryContents<T>,
+}
+impl<'a, T> Expected<'a, T> {
+    pub fn new(dir: &'a DirectoryContents<T>,) -> Self {
+        Self {
+            keep: DirectoryContents::new(),
+            dir
+        }
+    }
+}
+impl<'a, T: Clone> Expected<'a, T> {
+    pub fn check_file(&mut self, path: impl AsRef<Path>) -> Result<(), Error> {
+        if self
+            .dir
+            .get_path(path.as_ref())
+            .and_then(|e| e.as_file())
+            .is_some()
+        {
+            self.keep.insert_path(path, Entry::file(()))?;
+            Ok(())
+        } else {
+            Err(Error::new(
+                eyre!("file {} missing from archive", path.as_ref().display()),
+                ErrorKind::ParseS9pk,
+            ))
+        }
+    }
+    pub fn check_stem(
+        &mut self,
+        path: impl AsRef<Path>,
+        mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
+    ) -> Result<(), Error> {
+        let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
+            (
+                self.dir
+                    .get_path(parent)
+                    .and_then(|e| e.as_directory())
+                    .ok_or_else(|| {
+                        Error::new(
+                            eyre!("directory {} missing from archive", parent.display()),
+                            ErrorKind::ParseS9pk,
+                        )
+                    })?,
+                path.as_ref().strip_prefix(parent).unwrap(),
+            )
+        } else {
+            (self.dir, path.as_ref())
+        };
+        let name = dir
+            .with_stem(&stem.as_os_str().to_string_lossy())
+            .filter(|(_, e)| e.as_file().is_some())
+            .try_fold(
+                Err(Error::new(
+                    eyre!(
+                        "file {} with valid extension missing from archive",
+                        path.as_ref().display()
+                    ),
+                    ErrorKind::ParseS9pk,
+                )),
+                |acc, (name, _)| 
+                    if valid_extension(Path::new(&*name).extension()) {
+                        match acc {
+                            Ok(_) => Err(Error::new(
+                                eyre!(
+                                    "more than one file matching {} with valid extension in archive",
+                                    path.as_ref().display()
+                                ),
+                                ErrorKind::ParseS9pk,
+                            )),
+                            Err(_) => Ok(Ok(name))
+                        }
+                    } else {
+                        Ok(acc)
+                    }
+            )??;
+        self.keep
+            .insert_path(path.as_ref().with_file_name(name), Entry::file(()))?;
+        Ok(())
+    }
+    pub fn into_filter(self) -> Filter {
+        Filter(self.keep)
+    }
+}
+
+pub struct Filter(DirectoryContents<()>);
+impl Filter {
+    pub fn keep_checked<T: FileSource + Clone>(&self, dir: &mut DirectoryContents<T>) -> Result<(), Error> {
+        dir.filter(|path| self.0.get_path(path).is_some())
+    }
+}
+

core/startos/src/s9pk/merkle_archive/mod.rs

@@ -19,6 +19,7 @@ use crate::util::serde::Base64;
 use crate::CAP_1_MiB;
 
 pub mod directory_contents;
+pub mod expected;
 pub mod file_contents;
 pub mod hash;
 pub mod sink;
@@ -217,6 +218,9 @@ impl<S> Entry<S> {
     pub fn file(source: S) -> Self {
         Self::new(EntryContents::File(FileContents::new(source)))
     }
+    pub fn directory(directory: DirectoryContents<S>) -> Self {
+        Self::new(EntryContents::Directory(directory))
+    }
     pub fn hash(&self) -> Option<(Hash, u64)> {
         self.hash
     }

core/startos/src/s9pk/merkle_archive/source/mod.rs

@@ -280,3 +280,8 @@ impl<S: ArchiveSource> FileSource for Section<S> {
         self.source.copy_to(self.position, self.size, w).await
     }
 }
+
+pub type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
+pub fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
+    Box::new(r)
+}

core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs

@@ -97,7 +97,7 @@ impl ArchiveSource for MultiCursorFile {
             .ok()
             .map(|m| m.len())
     }
-    async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send, Error> {
+    async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send + 'static, Error> {
         use tokio::io::AsyncSeekExt;
 
         let mut file = self.cursor().await?;

core/startos/src/s9pk/rpc.rs

@@ -1,32 +1,26 @@
-use std::collections::BTreeSet;
+use std::path::PathBuf;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
 
 use clap::Parser;
-use itertools::Itertools;
 use models::ImageId;
 use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 use tokio::fs::File;
-use tokio::process::Command;
 use ts_rs::TS;
 
 use crate::context::CliContext;
 use crate::prelude::*;
 use crate::s9pk::manifest::Manifest;
-use crate::s9pk::merkle_archive::source::DynFileSource;
+use crate::s9pk::v2::pack::ImageConfig;
-use crate::s9pk::merkle_archive::Entry;
-use crate::s9pk::v2::compat::CONTAINER_TOOL;
 use crate::s9pk::v2::SIG_CONTEXT;
 use crate::s9pk::S9pk;
 use crate::util::io::TmpDir;
 use crate::util::serde::{apply_expr, HandlerExtSerde};
-use crate::util::Invoke;
 
 pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
 
 pub fn s9pk() -> ParentHandler<CliContext> {
     ParentHandler::new()
+        .subcommand("pack", from_fn_async(super::v2::pack::pack).no_display())
         .subcommand("edit", edit())
         .subcommand("inspect", inspect())
 }
@@ -77,117 +71,21 @@ fn inspect() -> ParentHandler<CliContext, S9pkPath> {
 #[derive(Deserialize, Serialize, Parser, TS)]
 struct AddImageParams {
     id: ImageId,
-    image: String,
+    #[command(flatten)]
-    arches: Option<Vec<String>>,
+    config: ImageConfig,
 }
 async fn add_image(
     ctx: CliContext,
-    AddImageParams { id, image, arches }: AddImageParams,
+    AddImageParams { id, config }: AddImageParams,
     S9pkPath { s9pk: s9pk_path }: S9pkPath,
 ) -> Result<(), Error> {
-    let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false)
+    let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?)
         .await?
         .into_dyn();
-    let arches: BTreeSet<_> = arches
+    s9pk.as_manifest_mut().images.insert(id, config);
-        .unwrap_or_else(|| vec!["x86_64".to_owned(), "aarch64".to_owned()])
-        .into_iter()
-        .collect();
     let tmpdir = TmpDir::new().await?;
-    for arch in arches {
+    s9pk.load_images(&tmpdir).await?;
-        let sqfs_path = tmpdir.join(format!("image.{arch}.squashfs"));
+    s9pk.validate_and_filter(None)?;
-        let docker_platform = if arch == "x86_64" {
-            "--platform=linux/amd64".to_owned()
-        } else if arch == "aarch64" {
-            "--platform=linux/arm64".to_owned()
-        } else {
-            format!("--platform=linux/{arch}")
-        };
-        let env = String::from_utf8(
-            Command::new(CONTAINER_TOOL)
-                .arg("run")
-                .arg("--rm")
-                .arg(&docker_platform)
-                .arg("--entrypoint")
-                .arg("env")
-                .arg(&image)
-                .invoke(ErrorKind::Docker)
-                .await?,
-        )?
-        .lines()
-        .filter(|l| {
-            l.trim()
-                .split_once("=")
-                .map_or(false, |(v, _)| !SKIP_ENV.contains(&v))
-        })
-        .join("\n")
-            + "\n";
-        let workdir = Path::new(
-            String::from_utf8(
-                Command::new(CONTAINER_TOOL)
-                    .arg("run")
-                    .arg(&docker_platform)
-                    .arg("--rm")
-                    .arg("--entrypoint")
-                    .arg("pwd")
-                    .arg(&image)
-                    .invoke(ErrorKind::Docker)
-                    .await?,
-            )?
-            .trim(),
-        )
-        .to_owned();
-        let container_id = String::from_utf8(
-            Command::new(CONTAINER_TOOL)
-                .arg("create")
-                .arg(&docker_platform)
-                .arg(&image)
-                .invoke(ErrorKind::Docker)
-                .await?,
-        )?;
-        Command::new("bash")
-            .arg("-c")
-            .arg(format!(
-                "{CONTAINER_TOOL} export {container_id} | mksquashfs - {sqfs} -tar",
-                container_id = container_id.trim(),
-                sqfs = sqfs_path.display()
-            ))
-            .invoke(ErrorKind::Docker)
-            .await?;
-        Command::new(CONTAINER_TOOL)
-            .arg("rm")
-            .arg(container_id.trim())
-            .invoke(ErrorKind::Docker)
-            .await?;
-        let archive = s9pk.as_archive_mut();
-        archive.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
-        archive.contents_mut().insert_path(
-            Path::new("images")
-                .join(&arch)
-                .join(&id)
-                .with_extension("squashfs"),
-            Entry::file(DynFileSource::new(sqfs_path)),
-        )?;
-        archive.contents_mut().insert_path(
-            Path::new("images")
-                .join(&arch)
-                .join(&id)
-                .with_extension("env"),
-            Entry::file(DynFileSource::new(Arc::<[u8]>::from(Vec::from(env)))),
-        )?;
-        archive.contents_mut().insert_path(
-            Path::new("images")
-                .join(&arch)
-                .join(&id)
-                .with_extension("json"),
-            Entry::file(DynFileSource::new(Arc::<[u8]>::from(
-                serde_json::to_vec(&serde_json::json!({
-                    "workdir": workdir
-                }))
-                .with_kind(ErrorKind::Serialization)?,
-            ))),
-        )?;
-    }
-    s9pk.as_manifest_mut().images.insert(id);
     let tmp_path = s9pk_path.with_extension("s9pk.tmp");
     let mut tmp_file = File::create(&tmp_path).await?;
     s9pk.serialize(&mut tmp_file, true).await?;
@@ -206,7 +104,7 @@ async fn edit_manifest(
     EditManifestParams { expression }: EditManifestParams,
     S9pkPath { s9pk: s9pk_path }: S9pkPath,
 ) -> Result<Manifest, Error> {
-    let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false).await?;
+    let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?).await?;
     let old = serde_json::to_value(s9pk.as_manifest()).with_kind(ErrorKind::Serialization)?;
     *s9pk.as_manifest_mut() = serde_json::from_value(apply_expr(old.into(), &expression)?.into())
         .with_kind(ErrorKind::Serialization)?;
@@ -227,7 +125,7 @@ async fn file_tree(
     _: Empty,
     S9pkPath { s9pk }: S9pkPath,
 ) -> Result<Vec<PathBuf>, Error> {
-    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
+    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
     Ok(s9pk.as_archive().contents().file_paths(""))
 }
 
@@ -244,7 +142,7 @@ async fn cat(
 ) -> Result<(), Error> {
     use crate::s9pk::merkle_archive::source::FileSource;
 
-    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
+    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
     tokio::io::copy(
         &mut s9pk
             .as_archive()
@@ -266,6 +164,6 @@ async fn inspect_manifest(
     _: Empty,
     S9pkPath { s9pk }: S9pkPath,
 ) -> Result<Manifest, Error> {
-    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
+    let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
     Ok(s9pk.as_manifest().clone())
 }

core/startos/src/s9pk/v2/compat.rs

@@ -1,6 +1,5 @@
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
-use std::io::Cursor;
+use std::path::Path;
-use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -14,49 +13,18 @@ use crate::prelude::*;
 use crate::s9pk::manifest::Manifest;
 use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
-use crate::s9pk::merkle_archive::source::{FileSource, Section};
+use crate::s9pk::merkle_archive::source::Section;
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
 use crate::s9pk::rpc::SKIP_ENV;
 use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure};
 use crate::s9pk::v1::reader::S9pkReader;
+use crate::s9pk::v2::pack::{PackSource, CONTAINER_TOOL};
 use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
 use crate::util::io::TmpDir;
 use crate::util::Invoke;
 
 pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01];
 
-#[cfg(not(feature = "docker"))]
-pub const CONTAINER_TOOL: &str = "podman";
-
-#[cfg(feature = "docker")]
-pub const CONTAINER_TOOL: &str = "docker";
-
-type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
-fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
-    Box::new(r)
-}
-
-#[derive(Clone)]
-enum CompatSource {
-    Buffered(Arc<[u8]>),
-    File(PathBuf),
-}
-impl FileSource for CompatSource {
-    type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
-    async fn size(&self) -> Result<u64, Error> {
-        match self {
-            Self::Buffered(a) => Ok(a.len() as u64),
-            Self::File(f) => Ok(tokio::fs::metadata(f).await?.len()),
-        }
-    }
-    async fn reader(&self) -> Result<Self::Reader, Error> {
-        match self {
-            Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
-            Self::File(f) => Ok(into_dyn_read(File::open(f).await?)),
-        }
-    }
-}
-
 impl S9pk<Section<MultiCursorFile>> {
     #[instrument(skip_all)]
     pub async fn from_v1<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
@@ -66,7 +34,7 @@ impl S9pk<Section<MultiCursorFile>> {
     ) -> Result<Self, Error> {
         let scratch_dir = TmpDir::new().await?;
 
-        let mut archive = DirectoryContents::<CompatSource>::new();
+        let mut archive = DirectoryContents::<PackSource>::new();
 
         // manifest.json
         let manifest_raw = reader.manifest().await?;
@@ -88,21 +56,21 @@ impl S9pk<Section<MultiCursorFile>> {
         let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into();
         archive.insert_path(
             "LICENSE.md",
-            Entry::file(CompatSource::Buffered(license.into())),
+            Entry::file(PackSource::Buffered(license.into())),
         )?;
 
         // instructions.md
         let instructions: Arc<[u8]> = reader.instructions().await?.to_vec().await?.into();
         archive.insert_path(
             "instructions.md",
-            Entry::file(CompatSource::Buffered(instructions.into())),
+            Entry::file(PackSource::Buffered(instructions.into())),
         )?;
 
         // icon.md
         let icon: Arc<[u8]> = reader.icon().await?.to_vec().await?.into();
         archive.insert_path(
             format!("icon.{}", manifest.assets.icon_type()),
-            Entry::file(CompatSource::Buffered(icon.into())),
+            Entry::file(PackSource::Buffered(icon.into())),
         )?;
 
         // images
@@ -122,7 +90,9 @@ impl S9pk<Section<MultiCursorFile>> {
                 .invoke(ErrorKind::Docker)
                 .await?;
             for (image, system) in &images {
-                new_manifest.images.insert(image.clone());
+                let mut image_config = new_manifest.images.remove(image).unwrap_or_default();
+                image_config.arch.insert(arch.as_str().into());
+                new_manifest.images.insert(image.clone(), image_config);
                 let sqfs_path = images_dir.join(image).with_extension("squashfs");
                 let image_name = if *system {
                     format!("start9/{}:latest", image)
@@ -190,21 +160,21 @@ impl S9pk<Section<MultiCursorFile>> {
                         .join(&arch)
                         .join(&image)
                         .with_extension("squashfs"),
-                    Entry::file(CompatSource::File(sqfs_path)),
+                    Entry::file(PackSource::File(sqfs_path)),
                 )?;
                 archive.insert_path(
                     Path::new("images")
                         .join(&arch)
                         .join(&image)
                         .with_extension("env"),
-                    Entry::file(CompatSource::Buffered(Vec::from(env).into())),
+                    Entry::file(PackSource::Buffered(Vec::from(env).into())),
                 )?;
                 archive.insert_path(
                     Path::new("images")
                         .join(&arch)
                         .join(&image)
                         .with_extension("json"),
-                    Entry::file(CompatSource::Buffered(
+                    Entry::file(PackSource::Buffered(
                         serde_json::to_vec(&serde_json::json!({
                             "workdir": workdir
                         }))
@@ -240,7 +210,7 @@ impl S9pk<Section<MultiCursorFile>> {
                 .await?;
             archive.insert_path(
                 Path::new("assets").join(&asset_id),
-                Entry::file(CompatSource::File(sqfs_path)),
+                Entry::file(PackSource::File(sqfs_path)),
             )?;
         }
 
@@ -267,12 +237,12 @@ impl S9pk<Section<MultiCursorFile>> {
             .await?;
         archive.insert_path(
             Path::new("javascript.squashfs"),
-            Entry::file(CompatSource::File(sqfs_path)),
+            Entry::file(PackSource::File(sqfs_path)),
         )?;
 
         archive.insert_path(
             "manifest.json",
-            Entry::file(CompatSource::Buffered(
+            Entry::file(PackSource::Buffered(
                 serde_json::to_vec::<Manifest>(&new_manifest)
                     .with_kind(ErrorKind::Serialization)?
                     .into(),
@@ -289,7 +259,6 @@ impl S9pk<Section<MultiCursorFile>> {
         Ok(S9pk::deserialize(
             &MultiCursorFile::from(File::open(destination.as_ref()).await?),
             None,
-            false,
         )
         .await?)
     }
@@ -310,7 +279,7 @@ impl From<ManifestV1> for Manifest {
             marketing_site: value.marketing_site.unwrap_or_else(|| default_url.clone()),
             donation_url: value.donation_url,
             description: value.description,
-            images: BTreeSet::new(),
+            images: BTreeMap::new(),
             assets: value
                 .volumes
                 .iter()

core/startos/src/s9pk/v2/manifest.rs

@@ -1,10 +1,11 @@
 use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
 
 use color_eyre::eyre::eyre;
 use helpers::const_true;
 use imbl_value::InternedString;
 pub use models::PackageId;
-use models::{ImageId, VolumeId};
+use models::{mime, ImageId, VolumeId};
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
 use url::Url;
@@ -12,6 +13,9 @@ use url::Url;
 use crate::dependencies::Dependencies;
 use crate::prelude::*;
 use crate::s9pk::git_hash::GitHash;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::expected::{Expected, Filter};
+use crate::s9pk::v2::pack::ImageConfig;
 use crate::util::serde::Regex;
 use crate::util::VersionString;
 use crate::version::{Current, VersionT};
@@ -42,7 +46,7 @@ pub struct Manifest {
     #[ts(type = "string | null")]
     pub donation_url: Option<Url>,
     pub description: Description,
-    pub images: BTreeSet<ImageId>,
+    pub images: BTreeMap<ImageId, ImageConfig>,
     pub assets: BTreeSet<VolumeId>, // TODO: AssetsId
     pub volumes: BTreeSet<VolumeId>,
     #[serde(default)]
@@ -59,6 +63,83 @@ pub struct Manifest {
     #[serde(default = "const_true")]
     pub has_config: bool,
 }
+impl Manifest {
+    pub fn validate_for<'a, T: Clone>(
+        &self,
+        arch: Option<&str>,
+        archive: &'a DirectoryContents<T>,
+    ) -> Result<Filter, Error> {
+        let mut expected = Expected::new(archive);
+        expected.check_file("manifest.json")?;
+        expected.check_stem("icon", |ext| {
+            ext.and_then(|e| e.to_str())
+                .and_then(mime)
+                .map_or(false, |mime| mime.starts_with("image/"))
+        })?;
+        expected.check_file("LICENSE.md")?;
+        expected.check_file("instructions.md")?;
+        expected.check_file("javascript.squashfs")?;
+        for assets in &self.assets {
+            expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?;
+        }
+        for (image_id, config) in &self.images {
+            let mut check_arch = |arch: &str| {
+                let mut arch = arch;
+                if let Err(e) = expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("squashfs"),
+                ) {
+                    if let Some(emulate_as) = &config.emulate_missing_as {
+                        expected.check_file(
+                            Path::new("images")
+                                .join(arch)
+                                .join(image_id)
+                                .with_extension("squashfs"),
+                        )?;
+                        arch = &**emulate_as;
+                    } else {
+                        return Err(e);
+                    }
+                }
+                expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("json"),
+                )?;
+                expected.check_file(
+                    Path::new("images")
+                        .join(arch)
+                        .join(image_id)
+                        .with_extension("env"),
+                )?;
+                Ok(())
+            };
+            if let Some(arch) = arch {
+                check_arch(arch)?;
+            } else if let Some(arches) = &self.hardware_requirements.arch {
+                for arch in arches {
+                    check_arch(arch)?;
+                }
+            } else if let Some(arch) = config.emulate_missing_as.as_deref() {
+                if !config.arch.contains(arch) {
+                    return Err(Error::new(
+                        eyre!("`emulateMissingAs` must match an included `arch`"),
+                        ErrorKind::ParseS9pk,
+                    ));
+                }
+                for arch in &config.arch {
+                    check_arch(&arch)?;
+                }
+            } else {
+                return Err(Error::new(eyre!("`emulateMissingAs` required for all images if no `arch` specified in `hardwareRequirements`"), ErrorKind::ParseS9pk));
+            }
+        }
+        Ok(expected.into_filter())
+    }
+}
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
 #[serde(rename_all = "camelCase")]

core/startos/src/s9pk/v2/mod.rs

@@ -14,7 +14,8 @@ use crate::s9pk::merkle_archive::sink::Sink;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
 use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
 use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
-use crate::ARCH;
+use crate::s9pk::v2::pack::{ImageSource, PackSource};
+use crate::util::io::TmpDir;
 
 const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
 
@@ -22,6 +23,7 @@ pub const SIG_CONTEXT: &str = "s9pk";
 
 pub mod compat;
 pub mod manifest;
+pub mod pack;
 
 /**
     /
@@ -34,10 +36,14 @@ pub mod manifest;
     │   └── <id>.squashfs (xN)
     └── images
         └── <arch>
+            ├── <id>.json (xN)
             ├── <id>.env (xN)
             └── <id>.squashfs (xN)
 */
 
+// this sorts the s9pk to optimize such that the parts that are used first appear earlier in the s9pk
+// this is useful for manipulating an s9pk while partially downloaded on a source that does not support
+// random access
 fn priority(s: &str) -> Option<usize> {
     match s {
         "manifest.json" => Some(0),
@@ -51,26 +57,6 @@ fn priority(s: &str) -> Option<usize> {
     }
 }
 
-fn filter(p: &Path) -> bool {
-    match p.iter().count() {
-        1 if p.file_name() == Some(OsStr::new("manifest.json")) => true,
-        1 if p.file_stem() == Some(OsStr::new("icon")) => true,
-        1 if p.file_name() == Some(OsStr::new("LICENSE.md")) => true,
-        1 if p.file_name() == Some(OsStr::new("instructions.md")) => true,
-        1 if p.file_name() == Some(OsStr::new("javascript.squashfs")) => true,
-        1 if p.file_name() == Some(OsStr::new("assets")) => true,
-        1 if p.file_name() == Some(OsStr::new("images")) => true,
-        2 if p.parent() == Some(Path::new("assets")) => {
-            p.extension().map_or(false, |ext| ext == "squashfs")
-        }
-        2 if p.parent() == Some(Path::new("images")) => p.file_name() == Some(OsStr::new(&*ARCH)),
-        3 if p.parent() == Some(&*Path::new("images").join(&*ARCH)) => p
-            .extension()
-            .map_or(false, |ext| ext == "squashfs" || ext == "env"),
-        _ => false,
-    }
-}
-
 #[derive(Clone)]
 pub struct S9pk<S = Section<MultiCursorFile>> {
     pub manifest: Manifest,
@@ -108,6 +94,11 @@ impl<S: FileSource + Clone> S9pk<S> {
         })
     }
 
+    pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> {
+        let filter = self.manifest.validate_for(arch, self.archive.contents())?;
+        filter.keep_checked(self.archive.contents_mut())
+    }
+
     pub async fn icon(&self) -> Result<(InternedString, FileContents<S>), Error> {
         let mut best_icon = None;
         for (path, icon) in self
@@ -174,12 +165,37 @@ impl<S: FileSource + Clone> S9pk<S> {
     }
 }
 
+impl<S: From<PackSource> + FileSource + Clone> S9pk<S> {
+    pub async fn load_images(&mut self, tmpdir: &TmpDir) -> Result<(), Error> {
+        let id = &self.manifest.id;
+        let version = &self.manifest.version;
+        for (image_id, image_config) in &mut self.manifest.images {
+            self.manifest_dirty = true;
+            for arch in &image_config.arch {
+                image_config
+                    .source
+                    .load(
+                        tmpdir,
+                        id,
+                        version,
+                        image_id,
+                        arch,
+                        self.archive.contents_mut(),
+                    )
+                    .await?;
+            }
+            image_config.source = ImageSource::Packed;
+        }
+
+        Ok(())
+    }
+}
+
 impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
     #[instrument(skip_all)]
     pub async fn deserialize(
         source: &S,
         commitment: Option<&MerkleArchiveCommitment>,
-        apply_filter: bool,
     ) -> Result<Self, Error> {
         use tokio::io::AsyncReadExt;
 
@@ -201,10 +217,6 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
         let mut archive =
             MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?;
 
-        if apply_filter {
-            archive.filter(filter)?;
-        }
-
         archive.sort_by(|a, b| match (priority(a), priority(b)) {
             (Some(a), Some(b)) => a.cmp(&b),
             (Some(_), None) => std::cmp::Ordering::Less,
@@ -216,15 +228,11 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
     }
 }
 impl S9pk {
-    pub async fn from_file(file: File, apply_filter: bool) -> Result<Self, Error> {
+    pub async fn from_file(file: File) -> Result<Self, Error> {
-        Self::deserialize(&MultiCursorFile::from(file), None, apply_filter).await
+        Self::deserialize(&MultiCursorFile::from(file), None).await
     }
-    pub async fn open(
+    pub async fn open(path: impl AsRef<Path>, id: Option<&PackageId>) -> Result<Self, Error> {
-        path: impl AsRef<Path>,
+        let res = Self::from_file(tokio::fs::File::open(path).await?).await?;
-        id: Option<&PackageId>,
-        apply_filter: bool,
-    ) -> Result<Self, Error> {
-        let res = Self::from_file(tokio::fs::File::open(path).await?, apply_filter).await?;
         if let Some(id) = id {
             ensure_code!(
                 &res.as_manifest().id == id,

core/startos/src/s9pk/v2/pack.rs

@@ -0,0 +1,536 @@
+use std::collections::BTreeSet;
+use std::ffi::OsStr;
+use std::io::Cursor;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use clap::Parser;
+use futures::future::{ready, BoxFuture};
+use futures::{FutureExt, TryStreamExt};
+use imbl_value::InternedString;
+use models::{ImageId, PackageId, VersionString};
+use serde::{Deserialize, Serialize};
+use tokio::fs::File;
+use tokio::io::AsyncRead;
+use tokio::process::Command;
+use tokio::sync::OnceCell;
+use tokio_stream::wrappers::ReadDirStream;
+use ts_rs::TS;
+
+use crate::context::CliContext;
+use crate::prelude::*;
+use crate::rpc_continuations::Guid;
+use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::s9pk::merkle_archive::source::{
+    into_dyn_read, ArchiveSource, DynFileSource, FileSource,
+};
+use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
+use crate::s9pk::v2::SIG_CONTEXT;
+use crate::s9pk::S9pk;
+use crate::util::io::TmpDir;
+use crate::util::Invoke;
+
+#[cfg(not(feature = "docker"))]
+pub const CONTAINER_TOOL: &str = "podman";
+
+#[cfg(feature = "docker")]
+pub const CONTAINER_TOOL: &str = "docker";
+
+pub struct SqfsDir {
+    path: PathBuf,
+    tmpdir: Arc<TmpDir>,
+    sqfs: OnceCell<MultiCursorFile>,
+}
+impl SqfsDir {
+    pub fn new(path: PathBuf, tmpdir: Arc<TmpDir>) -> Self {
+        Self {
+            path,
+            tmpdir,
+            sqfs: OnceCell::new(),
+        }
+    }
+    async fn file(&self) -> Result<&MultiCursorFile, Error> {
+        self.sqfs
+            .get_or_try_init(|| async move {
+                let guid = Guid::new();
+                let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs");
+                let mut cmd = Command::new("mksquashfs");
+                if self.path.extension().and_then(|s| s.to_str()) == Some("tar") {
+                    cmd.arg("-tar");
+                }
+                cmd.arg(&self.path)
+                    .arg(&path)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+                Ok(MultiCursorFile::from(
+                    File::open(&path)
+                        .await
+                        .with_ctx(|_| (ErrorKind::Filesystem, path.display()))?,
+                ))
+            })
+            .await
+    }
+}
+
+#[derive(Clone)]
+pub enum PackSource {
+    Buffered(Arc<[u8]>),
+    File(PathBuf),
+    Squashfs(Arc<SqfsDir>),
+}
+impl FileSource for PackSource {
+    type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
+    async fn size(&self) -> Result<u64, Error> {
+        match self {
+            Self::Buffered(a) => Ok(a.len() as u64),
+            Self::File(f) => Ok(tokio::fs::metadata(f)
+                .await
+                .with_ctx(|_| (ErrorKind::Filesystem, f.display()))?
+                .len()),
+            Self::Squashfs(dir) => dir
+                .file()
+                .await
+                .with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))?
+                .size()
+                .await
+                .or_not_found("file metadata"),
+        }
+    }
+    async fn reader(&self) -> Result<Self::Reader, Error> {
+        match self {
+            Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
+            Self::File(f) => Ok(into_dyn_read(
+                File::open(f)
+                    .await
+                    .with_ctx(|_| (ErrorKind::Filesystem, f.display()))?,
+            )),
+            Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
+        }
+    }
+}
+impl From<PackSource> for DynFileSource {
+    fn from(value: PackSource) -> Self {
+        DynFileSource::new(value)
+    }
+}
+
+#[derive(Deserialize, Serialize, Parser)]
+pub struct PackParams {
+    pub path: Option<PathBuf>,
+    #[arg(short = 'o', long = "output")]
+    pub output: Option<PathBuf>,
+    #[arg(long = "javascript")]
+    pub javascript: Option<PathBuf>,
+    #[arg(long = "icon")]
+    pub icon: Option<PathBuf>,
+    #[arg(long = "license")]
+    pub license: Option<PathBuf>,
+    #[arg(long = "instructions")]
+    pub instructions: Option<PathBuf>,
+    #[arg(long = "assets")]
+    pub assets: Option<PathBuf>,
+}
+impl PackParams {
+    fn path(&self) -> &Path {
+        self.path.as_deref().unwrap_or(Path::new("."))
+    }
+    fn output(&self, id: &PackageId) -> PathBuf {
+        self.output
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join(id).with_extension("s9pk"))
+    }
+    fn javascript(&self) -> PathBuf {
+        self.javascript
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("javascript"))
+    }
+    async fn icon(&self) -> Result<PathBuf, Error> {
+        if let Some(icon) = &self.icon {
+            Ok(icon.clone())
+        } else {
+            ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
+                Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
+                Err(e) => Ok({
+                    let path = x.path();
+                    if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
+                    Ok(path)
+                    } else {
+                        Err(e)
+                    }
+                })
+            }}).await?
+        }
+    }
+    fn license(&self) -> PathBuf {
+        self.license
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("LICENSE.md"))
+    }
+    fn instructions(&self) -> PathBuf {
+        self.instructions
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("instructions.md"))
+    }
+    fn assets(&self) -> PathBuf {
+        self.assets
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| self.path().join("assets"))
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ImageConfig {
+    pub source: ImageSource,
+    #[ts(type = "string[]")]
+    pub arch: BTreeSet<InternedString>,
+    #[ts(type = "string | null")]
+    pub emulate_missing_as: Option<InternedString>,
+}
+impl Default for ImageConfig {
+    fn default() -> Self {
+        Self {
+            source: ImageSource::Packed,
+            arch: BTreeSet::new(),
+            emulate_missing_as: None,
+        }
+    }
+}
+
+#[derive(Parser)]
+struct CliImageConfig {
+    #[arg(long, conflicts_with("docker-tag"))]
+    docker_build: bool,
+    #[arg(long, requires("docker-build"))]
+    dockerfile: Option<PathBuf>,
+    #[arg(long, requires("docker-build"))]
+    workdir: Option<PathBuf>,
+    #[arg(long, conflicts_with_all(["dockerfile", "workdir"]))]
+    docker_tag: Option<String>,
+    #[arg(long)]
+    arch: Vec<InternedString>,
+    #[arg(long)]
+    emulate_missing_as: Option<InternedString>,
+}
+impl TryFrom<CliImageConfig> for ImageConfig {
+    type Error = clap::Error;
+    fn try_from(value: CliImageConfig) -> Result<Self, Self::Error> {
+        let res = Self {
+            source: if value.docker_build {
+                ImageSource::DockerBuild {
+                    dockerfile: value.dockerfile,
+                    workdir: value.workdir,
+                }
+            } else if let Some(tag) = value.docker_tag {
+                ImageSource::DockerTag(tag)
+            } else {
+                ImageSource::Packed
+            },
+            arch: value.arch.into_iter().collect(),
+            emulate_missing_as: value.emulate_missing_as,
+        };
+        res.emulate_missing_as
+            .as_ref()
+            .map(|a| {
+                if !res.arch.contains(a) {
+                    Err(clap::Error::raw(
+                        clap::error::ErrorKind::InvalidValue,
+                        "`emulate-missing-as` must match one of the provided `arch`es",
+                    ))
+                } else {
+                    Ok(())
+                }
+            })
+            .transpose()?;
+        Ok(res)
+    }
+}
+impl clap::Args for ImageConfig {
+    fn augment_args(cmd: clap::Command) -> clap::Command {
+        CliImageConfig::augment_args(cmd)
+    }
+    fn augment_args_for_update(cmd: clap::Command) -> clap::Command {
+        CliImageConfig::augment_args_for_update(cmd)
+    }
+}
+impl clap::FromArgMatches for ImageConfig {
+    fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
+        Self::try_from(CliImageConfig::from_arg_matches(matches)?)
+    }
+    fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
+        *self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub enum ImageSource {
+    Packed,
+    #[serde(rename_all = "camelCase")]
+    DockerBuild {
+        workdir: Option<PathBuf>,
+        dockerfile: Option<PathBuf>,
+    },
+    DockerTag(String),
+}
+impl ImageSource {
+    #[instrument(skip_all)]
+    pub fn load<'a, S: From<PackSource> + FileSource + Clone>(
+        &'a self,
+        tmpdir: &'a TmpDir,
+        id: &'a PackageId,
+        version: &'a VersionString,
+        image_id: &'a ImageId,
+        arch: &'a str,
+        into: &'a mut DirectoryContents<S>,
+    ) -> BoxFuture<'a, Result<(), Error>> {
+        #[derive(Deserialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct DockerImageConfig {
+            env: Vec<String>,
+            #[serde(default)]
+            working_dir: PathBuf,
+            #[serde(default)]
+            user: String,
+        }
+        async move {
+            match self {
+                ImageSource::Packed => Ok(()),
+                ImageSource::DockerBuild {
+                    workdir,
+                    dockerfile,
+                } => {
+                    let workdir = workdir.as_deref().unwrap_or(Path::new("."));
+                    let dockerfile = dockerfile
+                        .clone()
+                        .unwrap_or_else(|| workdir.join("Dockerfile"));
+                    let docker_platform = if arch == "x86_64" {
+                        "--platform=linux/amd64".to_owned()
+                    } else if arch == "aarch64" {
+                        "--platform=linux/arm64".to_owned()
+                    } else {
+                        format!("--platform=linux/{arch}")
+                    };
+                    // docker buildx build ${path} -o type=image,name=start9/${id}
+                    let tag = format!("start9/{id}/{image_id}:{version}");
+                    Command::new(CONTAINER_TOOL)
+                        .arg("build")
+                        .arg(workdir)
+                        .arg("-f")
+                        .arg(dockerfile)
+                        .arg("-t")
+                        .arg(&tag)
+                        .arg(&docker_platform)
+                        .arg("-o")
+                        .arg("type=image")
+                        .capture(false)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    ImageSource::DockerTag(tag.clone())
+                        .load(tmpdir, id, version, image_id, arch, into)
+                        .await?;
+                    Command::new(CONTAINER_TOOL)
+                        .arg("rmi")
+                        .arg("-f")
+                        .arg(&tag)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    Ok(())
+                }
+                ImageSource::DockerTag(tag) => {
+                    let docker_platform = if arch == "x86_64" {
+                        "--platform=linux/amd64".to_owned()
+                    } else if arch == "aarch64" {
+                        "--platform=linux/arm64".to_owned()
+                    } else {
+                        format!("--platform=linux/{arch}")
+                    };
+                    let mut inspect_cmd = Command::new(CONTAINER_TOOL);
+                    inspect_cmd
+                        .arg("image")
+                        .arg("inspect")
+                        .arg("--format")
+                        .arg("{{json .Config}}")
+                        .arg(&tag);
+                    let inspect_res = match inspect_cmd.invoke(ErrorKind::Docker).await {
+                        Ok(a) => a,
+                        Err(e)
+                            if {
+                                let msg = e.source.to_string();
+                                #[cfg(feature = "docker")]
+                                let matches = msg.contains("No such image:");
+                                #[cfg(not(feature = "docker"))]
+                                let matches = msg.contains(": image not known");
+                                matches
+                            } =>
+                        {
+                            Command::new(CONTAINER_TOOL)
+                                .arg("pull")
+                                .arg(&docker_platform)
+                                .arg(tag)
+                                .capture(false)
+                                .invoke(ErrorKind::Docker)
+                                .await?;
+                            inspect_cmd.invoke(ErrorKind::Docker).await?
+                        }
+                        Err(e) => return Err(e),
+                    };
+                    let config = serde_json::from_slice::<DockerImageConfig>(&inspect_res)
+                        .with_kind(ErrorKind::Deserialization)?;
+                    let base_path = Path::new("images").join(arch).join(image_id);
+                    into.insert_path(
+                        base_path.with_extension("json"),
+                        Entry::file(
+                            PackSource::Buffered(
+                                serde_json::to_vec(&ImageMetadata {
+                                    workdir: if config.working_dir == Path::new("") {
+                                        "/".into()
+                                    } else {
+                                        config.working_dir
+                                    },
+                                    user: if config.user.is_empty() {
+                                        "root".into()
+                                    } else {
+                                        config.user.into()
+                                    },
+                                })
+                                .with_kind(ErrorKind::Serialization)?
+                                .into(),
+                            )
+                            .into(),
+                        ),
+                    )?;
+                    into.insert_path(
+                        base_path.with_extension("env"),
+                        Entry::file(
+                            PackSource::Buffered(config.env.join("\n").into_bytes().into()).into(),
+                        ),
+                    )?;
+                    let dest = tmpdir.join(Guid::new().as_ref()).with_extension("squashfs");
+                    let container = String::from_utf8(
+                        Command::new(CONTAINER_TOOL)
+                            .arg("create")
+                            .arg(&docker_platform)
+                            .arg(&tag)
+                            .invoke(ErrorKind::Docker)
+                            .await?,
+                    )?;
+                    Command::new(CONTAINER_TOOL)
+                        .arg("export")
+                        .arg(container.trim())
+                        .pipe(Command::new("mksquashfs").arg("-").arg(&dest).arg("-tar"))
+                        .capture(false)
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    Command::new(CONTAINER_TOOL)
+                        .arg("rm")
+                        .arg(container.trim())
+                        .invoke(ErrorKind::Docker)
+                        .await?;
+                    into.insert_path(
+                        base_path.with_extension("squashfs"),
+                        Entry::file(PackSource::File(dest).into()),
+                    )?;
+
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export)]
+pub struct ImageMetadata {
+    pub workdir: PathBuf,
+    #[ts(type = "string")]
+    pub user: InternedString,
+}
+
+#[instrument(skip_all)]
+pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
+    let tmpdir = Arc::new(TmpDir::new().await?);
+    let mut files = DirectoryContents::<PackSource>::new();
+    let js_dir = params.javascript();
+    let manifest: Arc<[u8]> = Command::new("node")
+        .arg("-e")
+        .arg(format!(
+            "console.log(JSON.stringify(require('{}/index.js').manifest))",
+            js_dir.display()
+        ))
+        .invoke(ErrorKind::Javascript)
+        .await?
+        .into();
+    files.insert(
+        "manifest.json".into(),
+        Entry::file(PackSource::Buffered(manifest.clone())),
+    );
+    let icon = params.icon().await?;
+    let icon_ext = icon
+        .extension()
+        .or_not_found("icon file extension")?
+        .to_string_lossy();
+    files.insert(
+        InternedString::from_display(&lazy_format!("icon.{}", icon_ext)),
+        Entry::file(PackSource::File(icon)),
+    );
+    files.insert(
+        "LICENSE.md".into(),
+        Entry::file(PackSource::File(params.license())),
+    );
+    files.insert(
+        "instructions.md".into(),
+        Entry::file(PackSource::File(params.instructions())),
+    );
+    files.insert(
+        "javascript.squashfs".into(),
+        Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
+            js_dir,
+            tmpdir.clone(),
+        )))),
+    );
+
+    let mut s9pk = S9pk::new(
+        MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT),
+        None,
+    )
+    .await?;
+
+    let assets_dir = params.assets();
+    for assets in s9pk.as_manifest().assets.clone() {
+        s9pk.as_archive_mut().contents_mut().insert_path(
+            Path::new("assets").join(&assets).with_extension("squashfs"),
+            Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
+                assets_dir.join(&assets),
+                tmpdir.clone(),
+            )))),
+        )?;
+    }
+
+    s9pk.load_images(&*tmpdir).await?;
+
+    s9pk.validate_and_filter(None)?;
+
+    s9pk.serialize(
+        &mut File::create(params.output(&s9pk.as_manifest().id)).await?,
+        false,
+    )
+    .await?;
+
+    drop(s9pk);
+
+    tmpdir.gc().await?;
+
+    Ok(())
+}

core/startos/src/service/action.rs

@@ -4,6 +4,7 @@ use models::{ActionId, ProcedureName};
 
 use crate::action::ActionResult;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::config::GetConfig;
 use crate::service::dependencies::DependencyConfig;
 use crate::service::{Service, ServiceActor};
@@ -23,13 +24,18 @@ impl Handler<Action> for ServiceActor {
     }
     async fn handle(
         &mut self,
-        Action { id, input }: Action,
+        id: Guid,
+        Action {
+            id: action_id,
+            input,
+        }: Action,
         _: &BackgroundJobQueue,
     ) -> Self::Response {
         let container = &self.0.persistent_container;
         container
             .execute::<ActionResult>(
-                ProcedureName::RunAction(id),
+                id,
+                ProcedureName::RunAction(action_id),
                 input,
                 Some(Duration::from_secs(30)),
             )
@@ -39,7 +45,20 @@ impl Handler<Action> for ServiceActor {
 }
 
 impl Service {
-    pub async fn action(&self, id: ActionId, input: Value) -> Result<ActionResult, Error> {
+    pub async fn action(
-        self.actor.send(Action { id, input }).await?
+        &self,
+        id: Guid,
+        action_id: ActionId,
+        input: Value,
+    ) -> Result<ActionResult, Error> {
+        self.actor
+            .send(
+                id,
+                Action {
+                    id: action_id,
+                    input,
+                },
+            )
+            .await?
     }
 }

core/startos/src/service/config.rs

@@ -5,6 +5,7 @@ use models::ProcedureName;
 use crate::config::action::ConfigRes;
 use crate::config::ConfigureContext;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::dependencies::DependencyConfig;
 use crate::service::{Service, ServiceActor};
 use crate::util::actor::background::BackgroundJobQueue;
@@ -19,6 +20,7 @@ impl Handler<Configure> for ServiceActor {
     }
     async fn handle(
         &mut self,
+        id: Guid,
         Configure(ConfigureContext { timeout, config }): Configure,
         _: &BackgroundJobQueue,
     ) -> Self::Response {
@@ -26,7 +28,7 @@ impl Handler<Configure> for ServiceActor {
         let package_id = &self.0.id;
 
         container
-            .execute::<NoOutput>(ProcedureName::SetConfig, to_value(&config)?, timeout)
+            .execute::<NoOutput>(id, ProcedureName::SetConfig, to_value(&config)?, timeout)
             .await
             .with_kind(ErrorKind::ConfigRulesViolation)?;
         self.0
@@ -52,10 +54,11 @@ impl Handler<GetConfig> for ServiceActor {
     fn conflicts_with(_: &GetConfig) -> ConflictBuilder<Self> {
         ConflictBuilder::nothing().except::<Configure>()
     }
-    async fn handle(&mut self, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(&mut self, id: Guid, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
         let container = &self.0.persistent_container;
         container
             .execute::<ConfigRes>(
+                id,
                 ProcedureName::GetConfig,
                 Value::Null,
                 Some(Duration::from_secs(30)), // TODO timeout
@@ -66,10 +69,10 @@ impl Handler<GetConfig> for ServiceActor {
 }
 
 impl Service {
-    pub async fn configure(&self, ctx: ConfigureContext) -> Result<(), Error> {
+    pub async fn configure(&self, id: Guid, ctx: ConfigureContext) -> Result<(), Error> {
-        self.actor.send(Configure(ctx)).await?
+        self.actor.send(id, Configure(ctx)).await?
     }
-    pub async fn get_config(&self) -> Result<ConfigRes, Error> {
+    pub async fn get_config(&self, id: Guid) -> Result<ConfigRes, Error> {
-        self.actor.send(GetConfig).await?
+        self.actor.send(id, GetConfig).await?
     }
 }

core/startos/src/service/control.rs

@@ -1,4 +1,5 @@
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::config::GetConfig;
 use crate::service::dependencies::DependencyConfig;
 use crate::service::start_stop::StartStop;
@@ -15,7 +16,7 @@ impl Handler<Start> for ServiceActor {
             .except::<GetConfig>()
             .except::<DependencyConfig>()
     }
-    async fn handle(&mut self, _: Start, _: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(&mut self, _: Guid, _: Start, _: &BackgroundJobQueue) -> Self::Response {
         self.0.persistent_container.state.send_modify(|x| {
             x.desired_state = StartStop::Start;
         });
@@ -23,8 +24,8 @@ impl Handler<Start> for ServiceActor {
     }
 }
 impl Service {
-    pub async fn start(&self) -> Result<(), Error> {
+    pub async fn start(&self, id: Guid) -> Result<(), Error> {
-        self.actor.send(Start).await
+        self.actor.send(id, Start).await
     }
 }
 
@@ -36,7 +37,7 @@ impl Handler<Stop> for ServiceActor {
             .except::<GetConfig>()
             .except::<DependencyConfig>()
     }
-    async fn handle(&mut self, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(&mut self, _: Guid, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
         let mut transition_state = None;
         self.0.persistent_container.state.send_modify(|x| {
             x.desired_state = StartStop::Stop;
@@ -51,7 +52,7 @@ impl Handler<Stop> for ServiceActor {
     }
 }
 impl Service {
-    pub async fn stop(&self) -> Result<(), Error> {
+    pub async fn stop(&self, id: Guid) -> Result<(), Error> {
-        self.actor.send(Stop).await
+        self.actor.send(id, Stop).await
     }
 }

core/startos/src/service/dependencies.rs

@@ -4,35 +4,28 @@ use imbl_value::json;
 use models::{PackageId, ProcedureName};
 
 use crate::prelude::*;
-use crate::service::{Service, ServiceActor};
+use crate::rpc_continuations::Guid;
+use crate::service::{Service, ServiceActor, ServiceActorSeed};
 use crate::util::actor::background::BackgroundJobQueue;
 use crate::util::actor::{ConflictBuilder, Handler};
 use crate::Config;
 
-pub(super) struct DependencyConfig {
+impl ServiceActorSeed {
-    dependency_id: PackageId,
+    async fn dependency_config(
-    remote_config: Option<Config>,
+        &self,
-}
+        id: Guid,
-impl Handler<DependencyConfig> for ServiceActor {
+        dependency_id: PackageId,
-    type Response = Result<Option<Config>, Error>;
+        remote_config: Option<Config>,
-    fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
+    ) -> Result<Option<Config>, Error> {
-        ConflictBuilder::nothing()
+        let container = &self.persistent_container;
-    }
-    async fn handle(
-        &mut self,
-        DependencyConfig {
-            dependency_id,
-            remote_config,
-        }: DependencyConfig,
-        _: &BackgroundJobQueue,
-    ) -> Self::Response {
-        let container = &self.0.persistent_container;
         container
             .sanboxed::<Option<Config>>(
+                id.clone(),
                 ProcedureName::UpdateDependency(dependency_id.clone()),
                 json!({
                     "queryResults": container
                         .execute::<Value>(
+                            id,
                             ProcedureName::QueryDependency(dependency_id),
                             Value::Null,
                             Some(Duration::from_secs(30)),
@@ -49,17 +42,45 @@ impl Handler<DependencyConfig> for ServiceActor {
     }
 }
 
+pub(super) struct DependencyConfig {
+    dependency_id: PackageId,
+    remote_config: Option<Config>,
+}
+impl Handler<DependencyConfig> for ServiceActor {
+    type Response = Result<Option<Config>, Error>;
+    fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
+        ConflictBuilder::nothing()
+    }
+    async fn handle(
+        &mut self,
+        id: Guid,
+        DependencyConfig {
+            dependency_id,
+            remote_config,
+        }: DependencyConfig,
+        _: &BackgroundJobQueue,
+    ) -> Self::Response {
+        self.0
+            .dependency_config(id, dependency_id, remote_config)
+            .await
+    }
+}
+
 impl Service {
     pub async fn dependency_config(
         &self,
+        id: Guid,
         dependency_id: PackageId,
         remote_config: Option<Config>,
     ) -> Result<Option<Config>, Error> {
         self.actor
-            .send(DependencyConfig {
+            .send(
-                dependency_id,
+                id,
-                remote_config,
+                DependencyConfig {
-            })
+                    dependency_id,
+                    remote_config,
+                },
+            )
             .await?
     }
 }

core/startos/src/service/mod.rs

@@ -1,4 +1,5 @@
-use std::sync::Arc;
+use std::ops::Deref;
+use std::sync::{Arc, Weak};
 use std::time::Duration;
 
 use chrono::{DateTime, Utc};
@@ -68,13 +69,87 @@ pub enum LoadDisposition {
     Undo,
 }
 
+pub struct ServiceRef(Arc<Service>);
+impl ServiceRef {
+    pub fn weak(&self) -> Weak<Service> {
+        Arc::downgrade(&self.0)
+    }
+    pub async fn uninstall(
+        self,
+        target_version: Option<models::VersionString>,
+    ) -> Result<(), Error> {
+        self.seed
+            .persistent_container
+            .execute(
+                Guid::new(),
+                ProcedureName::Uninit,
+                to_value(&target_version)?,
+                None,
+            ) // TODO timeout
+            .await?;
+        let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
+        let ctx = self.seed.ctx.clone();
+        self.shutdown().await?;
+        if target_version.is_none() {
+            ctx.db
+                .mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
+                .await?;
+        }
+        Ok(())
+    }
+    pub async fn shutdown(self) -> Result<(), Error> {
+        if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
+        {
+            self.seed
+                .persistent_container
+                .rpc_client
+                .request(rpc::Exit, Empty {})
+                .await?;
+            shutdown.shutdown();
+            hdl.await.with_kind(ErrorKind::Cancelled)?;
+        }
+        let service = Arc::try_unwrap(self.0).map_err(|_| {
+            Error::new(
+                eyre!("ServiceActor held somewhere after actor shutdown"),
+                ErrorKind::Unknown,
+            )
+        })?;
+        service
+            .actor
+            .shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
+            .await;
+        Arc::try_unwrap(service.seed)
+            .map_err(|_| {
+                Error::new(
+                    eyre!("ServiceActorSeed held somewhere after actor shutdown"),
+                    ErrorKind::Unknown,
+                )
+            })?
+            .persistent_container
+            .exit()
+            .await?;
+        Ok(())
+    }
+}
+impl Deref for ServiceRef {
+    type Target = Service;
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl From<Service> for ServiceRef {
+    fn from(value: Service) -> Self {
+        Self(Arc::new(value))
+    }
+}
+
 pub struct Service {
     actor: ConcurrentActor<ServiceActor>,
     seed: Arc<ServiceActorSeed>,
 }
 impl Service {
     #[instrument(skip_all)]
-    async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<Self, Error> {
+    async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<ServiceRef, Error> {
         let id = s9pk.as_manifest().id.clone();
         let persistent_container = PersistentContainer::new(
             &ctx, s9pk,
@@ -89,13 +164,17 @@ impl Service {
             ctx,
             synchronized: Arc::new(Notify::new()),
         });
-        seed.persistent_container
+        let service: ServiceRef = Self {
-            .init(Arc::downgrade(&seed))
-            .await?;
-        Ok(Self {
             actor: ConcurrentActor::new(ServiceActor(seed.clone())),
             seed,
-        })
+        }
+        .into();
+        service
+            .seed
+            .persistent_container
+            .init(service.weak())
+            .await?;
+        Ok(service)
     }
 
     #[instrument(skip_all)]
@@ -103,7 +182,7 @@ impl Service {
         ctx: &RpcContext,
         id: &PackageId,
         disposition: LoadDisposition,
-    ) -> Result<Option<Self>, Error> {
+    ) -> Result<Option<ServiceRef>, Error> {
         let handle_installed = {
             let ctx = ctx.clone();
             move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
@@ -137,7 +216,7 @@ impl Service {
         match entry.as_state_info().as_match() {
             PackageStateMatchModelRef::Installing(_) => {
                 if disposition == LoadDisposition::Retry {
-                    if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
+                    if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
                         tracing::error!("Error opening s9pk for install: {e}");
                         tracing::debug!("{e:?}")
                     }) {
@@ -170,7 +249,7 @@ impl Service {
                                 && progress == &Progress::Complete(true)
                         })
                 {
-                    if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id), true).await.map_err(|e| {
+                    if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| {
                         tracing::error!("Error opening s9pk for update: {e}");
                         tracing::debug!("{e:?}")
                     }) {
@@ -189,7 +268,7 @@ impl Service {
                         }
                     }
                 }
-                let s9pk = S9pk::open(s9pk_path, Some(id), true).await?;
+                let s9pk = S9pk::open(s9pk_path, Some(id)).await?;
                 ctx.db
                     .mutate({
                         |db| {
@@ -214,7 +293,7 @@ impl Service {
                 handle_installed(s9pk, entry).await
             }
             PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => {
-                if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
+                if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
                     tracing::error!("Error opening s9pk for removal: {e}");
                     tracing::debug!("{e:?}")
                 }) {
@@ -225,7 +304,7 @@ impl Service {
                             tracing::debug!("{e:?}")
                         })
                     {
-                        match service.uninstall(None).await {
+                        match ServiceRef::from(service).uninstall(None).await {
                             Err(e) => {
                                 tracing::error!("Error uninstalling service: {e}");
                                 tracing::debug!("{e:?}")
@@ -242,7 +321,7 @@ impl Service {
                 Ok(None)
             }
             PackageStateMatchModelRef::Installed(_) => {
-                handle_installed(S9pk::open(s9pk_path, Some(id), true).await?, entry).await
+                handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await
             }
             PackageStateMatchModelRef::Error(e) => Err(Error::new(
                 eyre!("Failed to parse PackageDataEntry, found {e:?}"),
@@ -257,7 +336,7 @@ impl Service {
         s9pk: S9pk,
         src_version: Option<models::VersionString>,
         progress: Option<InstallProgressHandles>,
-    ) -> Result<Self, Error> {
+    ) -> Result<ServiceRef, Error> {
         let manifest = s9pk.as_manifest().clone();
         let developer_key = s9pk.as_archive().signer();
         let icon = s9pk.icon_data_url().await?;
@@ -265,7 +344,12 @@ impl Service {
         service
             .seed
             .persistent_container
-            .execute(ProcedureName::Init, to_value(&src_version)?, None) // TODO timeout
+            .execute(
+                Guid::new(),
+                ProcedureName::Init,
+                to_value(&src_version)?,
+                None,
+            ) // TODO timeout
             .await
             .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation
         if let Some(mut progress) = progress {
@@ -301,61 +385,21 @@ impl Service {
         s9pk: S9pk,
         backup_source: impl GenericMountGuard,
         progress: Option<InstallProgressHandles>,
-    ) -> Result<Self, Error> {
+    ) -> Result<ServiceRef, Error> {
         let service = Service::install(ctx.clone(), s9pk, None, progress).await?;
 
         service
             .actor
-            .send(transition::restore::Restore {
+            .send(
-                path: backup_source.path().to_path_buf(),
+                Guid::new(),
-            })
+                transition::restore::Restore {
+                    path: backup_source.path().to_path_buf(),
+                },
+            )
             .await??;
         Ok(service)
     }
 
-    pub async fn shutdown(self) -> Result<(), Error> {
-        self.actor
-            .shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
-            .await;
-        if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
-        {
-            self.seed
-                .persistent_container
-                .rpc_client
-                .request(rpc::Exit, Empty {})
-                .await?;
-            shutdown.shutdown();
-            hdl.await.with_kind(ErrorKind::Cancelled)?;
-        }
-        Arc::try_unwrap(self.seed)
-            .map_err(|_| {
-                Error::new(
-                    eyre!("ServiceActorSeed held somewhere after actor shutdown"),
-                    ErrorKind::Unknown,
-                )
-            })?
-            .persistent_container
-            .exit()
-            .await?;
-        Ok(())
-    }
-
-    pub async fn uninstall(self, target_version: Option<models::VersionString>) -> Result<(), Error> {
-        self.seed
-            .persistent_container
-            .execute(ProcedureName::Uninit, to_value(&target_version)?, None) // TODO timeout
-            .await?;
-        let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
-        let ctx = self.seed.ctx.clone();
-        self.shutdown().await?;
-        if target_version.is_none() {
-            ctx.db
-                .mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
-                .await?;
-        }
-        Ok(())
-    }
-
     #[instrument(skip_all)]
     pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> {
         let id = &self.seed.id;
@@ -368,9 +412,12 @@ impl Service {
             .await?;
         drop(file);
         self.actor
-            .send(transition::backup::Backup {
+            .send(
-                path: guard.path().to_path_buf(),
+                Guid::new(),
-            })
+                transition::backup::Backup {
+                    path: guard.path().to_path_buf(),
+                },
+            )
             .await??;
         Ok(())
     }

core/startos/src/service/persistent_container.rs

@@ -6,8 +6,7 @@ use std::time::Duration;
 use futures::future::ready;
 use futures::{Future, FutureExt};
 use helpers::NonDetachingJoinHandle;
-use imbl_value::InternedString;
+use models::{ImageId, ProcedureName, VolumeId};
-use models::{ProcedureName, VolumeId};
 use rpc_toolkit::{Empty, Server, ShutdownHandle};
 use serde::de::DeserializeOwned;
 use tokio::fs::File;
@@ -24,14 +23,15 @@ use crate::disk::mount::filesystem::idmapped::IdMapped;
 use crate::disk::mount::filesystem::loop_dev::LoopDev;
 use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
 use crate::disk::mount::filesystem::{MountType, ReadOnly};
-use crate::disk::mount::guard::MountGuard;
+use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
 use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET};
 use crate::net::net_controller::NetService;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::S9pk;
 use crate::service::start_stop::StartStop;
-use crate::service::{rpc, RunningStatus};
+use crate::service::{rpc, RunningStatus, Service};
 use crate::util::rpc_client::UnixRpcClient;
 use crate::util::Invoke;
 use crate::volume::{asset_dir, data_dir};
@@ -89,7 +89,8 @@ pub struct PersistentContainer {
     js_mount: MountGuard,
     volumes: BTreeMap<VolumeId, MountGuard>,
     assets: BTreeMap<VolumeId, MountGuard>,
-    pub(super) overlays: Arc<Mutex<BTreeMap<InternedString, OverlayGuard>>>,
+    pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
+    pub(super) overlays: Arc<Mutex<BTreeMap<Guid, OverlayGuard<Arc<MountGuard>>>>>,
     pub(super) state: Arc<watch::Sender<ServiceState>>,
     pub(super) net_service: Mutex<NetService>,
     destroyed: bool,
@@ -178,14 +179,62 @@ impl PersistentContainer {
                 .await?,
             );
         }
+
+        let mut images = BTreeMap::new();
         let image_path = lxc_container.rootfs_dir().join("media/startos/images");
         tokio::fs::create_dir_all(&image_path).await?;
-        for image in &s9pk.as_manifest().images {
+        for (image, config) in &s9pk.as_manifest().images {
+            let mut arch = ARCH;
+            let mut sqfs_path = Path::new("images")
+                .join(arch)
+                .join(image)
+                .with_extension("squashfs");
+            if !s9pk
+                .as_archive()
+                .contents()
+                .get_path(&sqfs_path)
+                .and_then(|e| e.as_file())
+                .is_some()
+            {
+                arch = if let Some(arch) = config.emulate_missing_as.as_deref() {
+                    arch
+                } else {
+                    continue;
+                };
+                sqfs_path = Path::new("images")
+                    .join(arch)
+                    .join(image)
+                    .with_extension("squashfs");
+            }
+            let sqfs = s9pk
+                .as_archive()
+                .contents()
+                .get_path(&sqfs_path)
+                .and_then(|e| e.as_file())
+                .or_not_found(sqfs_path.display())?;
+            let mountpoint = image_path.join(image);
+            tokio::fs::create_dir_all(&mountpoint).await?;
+            Command::new("chown")
+                .arg("100000:100000")
+                .arg(&mountpoint)
+                .invoke(ErrorKind::Filesystem)
+                .await?;
+            images.insert(
+                image.clone(),
+                Arc::new(
+                    MountGuard::mount(
+                        &IdMapped::new(LoopDev::from(&**sqfs), 0, 100000, 65536),
+                        &mountpoint,
+                        ReadOnly,
+                    )
+                    .await?,
+                ),
+            );
             let env_filename = Path::new(image.as_ref()).with_extension("env");
             if let Some(env) = s9pk
                 .as_archive()
                 .contents()
-                .get_path(Path::new("images").join(*ARCH).join(&env_filename))
+                .get_path(Path::new("images").join(arch).join(&env_filename))
                 .and_then(|e| e.as_file())
             {
                 env.copy(&mut File::create(image_path.join(&env_filename)).await?)
@@ -195,7 +244,7 @@ impl PersistentContainer {
             if let Some(json) = s9pk
                 .as_archive()
                 .contents()
-                .get_path(Path::new("images").join(*ARCH).join(&json_filename))
+                .get_path(Path::new("images").join(arch).join(&json_filename))
                 .and_then(|e| e.as_file())
             {
                 json.copy(&mut File::create(image_path.join(&json_filename)).await?)
@@ -215,6 +264,7 @@ impl PersistentContainer {
             js_mount,
             volumes,
             assets,
+            images,
             overlays: Arc::new(Mutex::new(BTreeMap::new())),
             state: Arc::new(watch::channel(ServiceState::new(start)).0),
             net_service: Mutex::new(net_service),
@@ -257,7 +307,7 @@ impl PersistentContainer {
     }
 
     #[instrument(skip_all)]
-    pub async fn init(&self, seed: Weak<ServiceActorSeed>) -> Result<(), Error> {
+    pub async fn init(&self, seed: Weak<Service>) -> Result<(), Error> {
         let socket_server_context = EffectContext::new(seed);
         let server = Server::new(
             move || ready(Ok(socket_server_context.clone())),
@@ -330,6 +380,7 @@ impl PersistentContainer {
         let js_mount = self.js_mount.take();
         let volumes = std::mem::take(&mut self.volumes);
         let assets = std::mem::take(&mut self.assets);
+        let images = std::mem::take(&mut self.images);
         let overlays = self.overlays.clone();
         let lxc_container = self.lxc_container.take();
         self.destroyed = true;
@@ -352,6 +403,9 @@ impl PersistentContainer {
                         for (_, overlay) in std::mem::take(&mut *overlays.lock().await) {
                             errs.handle(overlay.unmount(true).await);
                         }
+                        for (_, images) in images {
+                            errs.handle(images.unmount().await);
+                        }
                         errs.handle(js_mount.unmount(true).await);
                         if let Some(lxc_container) = lxc_container {
                             errs.handle(lxc_container.exit().await);
@@ -378,6 +432,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn start(&self) -> Result<(), Error> {
         self.execute(
+            Guid::new(),
             ProcedureName::StartMain,
             Value::Null,
             Some(Duration::from_secs(5)), // TODO
@@ -389,7 +444,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn stop(&self) -> Result<Duration, Error> {
         let timeout: Option<crate::util::serde::Duration> = self
-            .execute(ProcedureName::StopMain, Value::Null, None)
+            .execute(Guid::new(), ProcedureName::StopMain, Value::Null, None)
             .await?;
         Ok(timeout.map(|a| *a).unwrap_or(Duration::from_secs(30)))
     }
@@ -397,6 +452,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn execute<O>(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
@@ -404,7 +460,7 @@ impl PersistentContainer {
     where
         O: DeserializeOwned,
     {
-        self._execute(name, input, timeout)
+        self._execute(id, name, input, timeout)
             .await
             .and_then(from_value)
     }
@@ -412,6 +468,7 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     pub async fn sanboxed<O>(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
@@ -419,7 +476,7 @@ impl PersistentContainer {
     where
         O: DeserializeOwned,
     {
-        self._sandboxed(name, input, timeout)
+        self._sandboxed(id, name, input, timeout)
             .await
             .and_then(from_value)
     }
@@ -427,13 +484,15 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     async fn _execute(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
     ) -> Result<Value, Error> {
-        let fut = self
+        let fut = self.rpc_client.request(
-            .rpc_client
+            rpc::Execute,
-            .request(rpc::Execute, rpc::ExecuteParams::new(name, input, timeout));
+            rpc::ExecuteParams::new(id, name, input, timeout),
+        );
 
         Ok(if let Some(timeout) = timeout {
             tokio::time::timeout(timeout, fut)
@@ -447,13 +506,15 @@ impl PersistentContainer {
     #[instrument(skip_all)]
     async fn _sandboxed(
         &self,
+        id: Guid,
         name: ProcedureName,
         input: Value,
         timeout: Option<Duration>,
     ) -> Result<Value, Error> {
-        let fut = self
+        let fut = self.rpc_client.request(
-            .rpc_client
+            rpc::Sandbox,
-            .request(rpc::Sandbox, rpc::ExecuteParams::new(name, input, timeout));
+            rpc::ExecuteParams::new(id, name, input, timeout),
+        );
 
         Ok(if let Some(timeout) = timeout {
             tokio::time::timeout(timeout, fut)

core/startos/src/service/properties.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 use models::ProcedureName;
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::Service;
 
 impl Service {
@@ -11,6 +12,7 @@ impl Service {
         let container = &self.seed.persistent_container;
         container
             .execute::<Value>(
+                Guid::new(),
                 ProcedureName::Properties,
                 Value::Null,
                 Some(Duration::from_secs(30)),

core/startos/src/service/rpc.rs

@@ -7,6 +7,7 @@ use rpc_toolkit::Empty;
 use ts_rs::TS;
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 
 #[derive(Clone)]
 pub struct Init;
@@ -46,14 +47,21 @@ impl serde::Serialize for Exit {
 
 #[derive(Clone, serde::Deserialize, serde::Serialize, TS)]
 pub struct ExecuteParams {
+    id: Guid,
     procedure: String,
     #[ts(type = "any")]
     input: Value,
     timeout: Option<u128>,
 }
 impl ExecuteParams {
-    pub fn new(procedure: ProcedureName, input: Value, timeout: Option<Duration>) -> Self {
+    pub fn new(
+        id: Guid,
+        procedure: ProcedureName,
+        input: Value,
+        timeout: Option<Duration>,
+    ) -> Self {
         Self {
+            id,
             procedure: procedure.js_function_name(),
             input,
             timeout: timeout.map(|d| d.as_millis()),

core/startos/src/service/service_effect_handler.rs

@@ -7,9 +7,9 @@ use std::str::FromStr;
 use std::sync::{Arc, Weak};
 
 use clap::builder::ValueParserFactory;
-use clap::Parser;
+use clap::{CommandFactory, FromArgMatches, Parser};
 use emver::VersionRange;
-use imbl_value::{json, InternedString};
+use imbl_value::json;
 use itertools::Itertools;
 use models::{
     ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, ServiceInterfaceId, VolumeId,
@@ -25,35 +25,34 @@ use crate::db::model::package::{
     ActionMetadata, CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind,
     ManifestPreference,
 };
-use crate::disk::mount::filesystem::idmapped::IdMapped;
-use crate::disk::mount::filesystem::loop_dev::LoopDev;
 use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
+use crate::echo;
 use crate::net::host::address::HostAddress;
 use crate::net::host::binding::{BindOptions, LanInfo};
 use crate::net::host::{Host, HostKind};
 use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType};
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::s9pk::merkle_archive::source::http::HttpSource;
 use crate::s9pk::rpc::SKIP_ENV;
 use crate::s9pk::S9pk;
 use crate::service::cli::ContainerCliContext;
-use crate::service::ServiceActorSeed;
+use crate::service::Service;
 use crate::status::health_check::HealthCheckResult;
 use crate::status::MainStatus;
 use crate::util::clap::FromStrParser;
-use crate::util::{new_guid, Invoke};
+use crate::util::Invoke;
-use crate::{echo, ARCH};
 
 #[derive(Clone)]
-pub(super) struct EffectContext(Weak<ServiceActorSeed>);
+pub(super) struct EffectContext(Weak<Service>);
 impl EffectContext {
-    pub fn new(seed: Weak<ServiceActorSeed>) -> Self {
+    pub fn new(service: Weak<Service>) -> Self {
-        Self(seed)
+        Self(service)
     }
 }
 impl Context for EffectContext {}
 impl EffectContext {
-    fn deref(&self) -> Result<Arc<ServiceActorSeed>, Error> {
+    fn deref(&self) -> Result<Arc<Service>, Error> {
         if let Some(seed) = Weak::upgrade(&self.0) {
             Ok(seed)
         } else {
@@ -66,11 +65,55 @@ impl EffectContext {
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-struct RpcData {
+#[serde(rename_all = "camelCase")]
-    id: i64,
+pub struct WithProcedureId<T> {
-    method: String,
+    #[serde(default)]
-    params: Value,
+    procedure_id: Guid,
+    #[serde(flatten)]
+    rest: T,
 }
+impl<T: FromArgMatches> FromArgMatches for WithProcedureId<T> {
+    fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
+        let rest = T::from_arg_matches(matches)?;
+        Ok(Self {
+            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
+            rest,
+        })
+    }
+    fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result<Self, clap::Error> {
+        let rest = T::from_arg_matches_mut(matches)?;
+        Ok(Self {
+            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
+            rest,
+        })
+    }
+    fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
+        self.rest.update_from_arg_matches(matches)?;
+        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
+        Ok(())
+    }
+    fn update_from_arg_matches_mut(
+        &mut self,
+        matches: &mut clap::ArgMatches,
+    ) -> Result<(), clap::Error> {
+        self.rest.update_from_arg_matches_mut(matches)?;
+        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
+        Ok(())
+    }
+}
+impl<T: CommandFactory> CommandFactory for WithProcedureId<T> {
+    fn command() -> clap::Command {
+        T::command_for_update().arg(
+            clap::Arg::new("procedure-id")
+                .action(clap::ArgAction::Set)
+                .value_parser(clap::value_parser!(Guid)),
+        )
+    }
+    fn command_for_update() -> clap::Command {
+        Self::command()
+    }
+}
+
 pub fn service_effect_handler<C: Context>() -> ParentHandler<C> {
     ParentHandler::new()
         .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
@@ -290,6 +333,7 @@ struct MountParams {
 async fn set_system_smtp(context: EffectContext, data: SetSystemSmtpParams) -> Result<(), Error> {
     let context = context.deref()?;
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -304,6 +348,7 @@ async fn get_system_smtp(
 ) -> Result<String, Error> {
     let context = context.deref()?;
     let res = context
+        .seed
         .ctx
         .db
         .peek()
@@ -323,7 +368,7 @@ async fn get_system_smtp(
 }
 async fn get_container_ip(context: EffectContext, _: Empty) -> Result<Ipv4Addr, Error> {
     let context = context.deref()?;
-    let net_service = context.persistent_container.net_service.lock().await;
+    let net_service = context.seed.persistent_container.net_service.lock().await;
     Ok(net_service.get_ip())
 }
 async fn get_service_port_forward(
@@ -333,14 +378,15 @@ async fn get_service_port_forward(
     let internal_port = data.internal_port as u16;
 
     let context = context.deref()?;
-    let net_service = context.persistent_container.net_service.lock().await;
+    let net_service = context.seed.persistent_container.net_service.lock().await;
     net_service.get_ext_port(data.host_id, internal_port)
 }
 async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
 
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -369,7 +415,7 @@ async fn export_service_interface(
     }: ExportServiceInterfaceParams,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
 
     let service_interface = ServiceInterface {
         id: id.clone(),
@@ -384,6 +430,7 @@ async fn export_service_interface(
     let svc_interface_with_host_info = service_interface;
 
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -407,7 +454,7 @@ async fn get_primary_url(
     }: GetPrimaryUrlParams,
 ) -> Result<Option<HostAddress>, Error> {
     let context = context.deref()?;
-    let package_id = package_id.unwrap_or_else(|| context.id.clone());
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
 
     Ok(None) // TODO
 }
@@ -419,9 +466,10 @@ async fn list_service_interfaces(
     }: ListServiceInterfacesParams,
 ) -> Result<BTreeMap<ServiceInterfaceId, ServiceInterface>, Error> {
     let context = context.deref()?;
-    let package_id = package_id.unwrap_or_else(|| context.id.clone());
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
 
     context
+        .seed
         .ctx
         .db
         .peek()
@@ -435,9 +483,10 @@ async fn list_service_interfaces(
 }
 async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
 
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -454,8 +503,9 @@ async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Re
 }
 async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -477,8 +527,9 @@ async fn export_action(context: EffectContext, data: ExportActionParams) -> Resu
 }
 async fn remove_action(context: EffectContext, data: RemoveActionParams) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -514,16 +565,16 @@ struct GetHostInfoParams {
     callback: Callback,
 }
 async fn get_host_info(
-    ctx: EffectContext,
+    context: EffectContext,
     GetHostInfoParams {
         callback,
         package_id,
         host_id,
     }: GetHostInfoParams,
 ) -> Result<Host, Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let db = ctx.ctx.db.peek().await;
+    let db = context.seed.ctx.db.peek().await;
-    let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
 
     db.as_public()
         .as_package_data()
@@ -536,8 +587,8 @@ async fn get_host_info(
 }
 
 async fn clear_bindings(context: EffectContext, _: Empty) -> Result<(), Error> {
-    let ctx = context.deref()?;
+    let context = context.deref()?;
-    let mut svc = ctx.persistent_container.net_service.lock().await;
+    let mut svc = context.seed.persistent_container.net_service.lock().await;
     svc.clear_bindings().await?;
     Ok(())
 }
@@ -559,8 +610,8 @@ async fn bind(context: EffectContext, bind_params: Value) -> Result<(), Error> {
         internal_port,
         options,
     } = from_value(bind_params)?;
-    let ctx = context.deref()?;
+    let context = context.deref()?;
-    let mut svc = ctx.persistent_container.net_service.lock().await;
+    let mut svc = context.seed.persistent_container.net_service.lock().await;
     svc.bind(kind, id, internal_port, options).await
 }
 
@@ -575,16 +626,16 @@ struct GetServiceInterfaceParams {
 }
 
 async fn get_service_interface(
-    ctx: EffectContext,
+    context: EffectContext,
     GetServiceInterfaceParams {
         callback,
         package_id,
         service_interface_id,
     }: GetServiceInterfaceParams,
 ) -> Result<ServiceInterface, Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
+    let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
-    let db = ctx.ctx.db.peek().await;
+    let db = context.seed.ctx.db.peek().await;
 
     let interface = db
         .as_public()
@@ -729,8 +780,8 @@ async fn get_store(
     GetStoreParams { package_id, path }: GetStoreParams,
 ) -> Result<Value, Error> {
     let context = context.deref()?;
-    let peeked = context.ctx.db.peek().await;
+    let peeked = context.seed.ctx.db.peek().await;
-    let package_id = package_id.unwrap_or(context.id.clone());
+    let package_id = package_id.unwrap_or(context.seed.id.clone());
     let value = peeked
         .as_private()
         .as_package_stores()
@@ -758,8 +809,9 @@ async fn set_store(
     SetStoreParams { value, path }: SetStoreParams,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    let package_id = context.id.clone();
+    let package_id = context.seed.id.clone();
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -812,7 +864,7 @@ struct ParamsMaybePackageId {
 
 async fn exists(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
     let context = context.deref()?;
-    let peeked = context.ctx.db.peek().await;
+    let peeked = context.seed.ctx.db.peek().await;
     let package = peeked
         .as_public()
         .as_package_data()
@@ -834,31 +886,30 @@ struct ExecuteAction {
 }
 async fn execute_action(
     context: EffectContext,
-    ExecuteAction {
+    WithProcedureId {
-        action_id,
+        procedure_id,
-        input,
+        rest:
-        service_id,
+            ExecuteAction {
-    }: ExecuteAction,
+                service_id,
+                action_id,
+                input,
+            },
+    }: WithProcedureId<ExecuteAction>,
 ) -> Result<Value, Error> {
     let context = context.deref()?;
-    let package_id = service_id.clone().unwrap_or_else(|| context.id.clone());
+    let package_id = service_id
-    let service = context.ctx.services.get(&package_id).await;
+        .clone()
-    let service = service.as_ref().ok_or_else(|| {
+        .unwrap_or_else(|| context.seed.id.clone());
-        Error::new(
-            eyre!("Could not find package {package_id}"),
-            ErrorKind::Unknown,
-        )
-    })?;
 
-    Ok(json!(service.action(action_id, input).await?))
+    Ok(json!(context.action(procedure_id, action_id, input).await?))
 }
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct FromService {}
 async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error> {
     let context = context.deref()?;
-    let peeked = context.ctx.db.peek().await;
+    let peeked = context.seed.ctx.db.peek().await;
-    let package_id = &context.id;
+    let package_id = &context.seed.id;
     let package = peeked
         .as_public()
         .as_package_data()
@@ -872,8 +923,8 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error
 
 async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result<Value, Error> {
     let context = context.deref()?;
-    let peeked = context.ctx.db.peek().await;
+    let peeked = context.seed.ctx.db.peek().await;
-    let package_id = params.package_id.unwrap_or_else(|| context.id.clone());
+    let package_id = params.package_id.unwrap_or_else(|| context.seed.id.clone());
     let package = peeked
         .as_public()
         .as_package_data()
@@ -887,7 +938,7 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
 async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
     dbg!("Starting the running {params:?}");
     let context = context.deref()?;
-    let peeked = context.ctx.db.peek().await;
+    let peeked = context.seed.ctx.db.peek().await;
     let package_id = params.package_id;
     let package = peeked
         .as_public()
@@ -900,30 +951,24 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Valu
     Ok(json!(matches!(package, MainStatus::Running { .. })))
 }
 
-async fn restart(context: EffectContext, _: Empty) -> Result<Value, Error> {
+async fn restart(
+    context: EffectContext,
+    WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
+) -> Result<(), Error> {
     let context = context.deref()?;
-    let service = context.ctx.services.get(&context.id).await;
+    dbg!("here");
-    let service = service.as_ref().ok_or_else(|| {
+    context.restart(procedure_id).await?;
-        Error::new(
+    dbg!("here");
-            eyre!("Could not find package {}", context.id),
+    Ok(())
-            ErrorKind::Unknown,
-        )
-    })?;
-    service.restart().await?;
-    Ok(json!(()))
 }
 
-async fn shutdown(context: EffectContext, _: Empty) -> Result<Value, Error> {
+async fn shutdown(
+    context: EffectContext,
+    WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
+) -> Result<(), Error> {
     let context = context.deref()?;
-    let service = context.ctx.services.get(&context.id).await;
+    context.stop(procedure_id).await?;
-    let service = service.as_ref().ok_or_else(|| {
+    Ok(())
-        Error::new(
-            eyre!("Could not find package {}", context.id),
-            ErrorKind::Unknown,
-        )
-    })?;
-    service.stop().await?;
-    Ok(json!(()))
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
@@ -935,8 +980,9 @@ struct SetConfigured {
 }
 async fn set_configured(context: EffectContext, params: SetConfigured) -> Result<Value, Error> {
     let context = context.deref()?;
-    let package_id = &context.id;
+    let package_id = &context.seed.id;
     context
+        .seed
         .ctx
         .db
         .mutate(|db| {
@@ -989,9 +1035,9 @@ async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Resul
     dbg!(format!("Status for main will be is {params:?}"));
     let context = context.deref()?;
     match params.status {
-        SetMainStatusStatus::Running => context.started(),
+        SetMainStatusStatus::Running => context.seed.started(),
-        SetMainStatusStatus::Stopped => context.stopped(),
+        SetMainStatusStatus::Stopped => context.seed.stopped(),
-        SetMainStatusStatus::Starting => context.stopped(),
+        SetMainStatusStatus::Starting => context.seed.stopped(),
     }
     Ok(Value::Null)
 }
@@ -1011,8 +1057,9 @@ async fn set_health(
 ) -> Result<Value, Error> {
     let context = context.deref()?;
 
-    let package_id = &context.id;
+    let package_id = &context.seed.id;
     context
+        .seed
         .ctx
         .db
         .mutate(move |db| {
@@ -1041,17 +1088,17 @@ async fn set_health(
 #[command(rename_all = "camelCase")]
 #[ts(export)]
 pub struct DestroyOverlayedImageParams {
-    #[ts(type = "string")]
+    guid: Guid,
-    guid: InternedString,
 }
 
 #[instrument(skip_all)]
 pub async fn destroy_overlayed_image(
-    ctx: EffectContext,
+    context: EffectContext,
     DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
 ) -> Result<(), Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    if ctx
+    if context
+        .seed
         .persistent_container
         .overlays
         .lock()
@@ -1068,30 +1115,25 @@ pub async fn destroy_overlayed_image(
 #[command(rename_all = "camelCase")]
 #[ts(export)]
 pub struct CreateOverlayedImageParams {
-    #[ts(type = "string")]
     image_id: ImageId,
 }
 
 #[instrument(skip_all)]
 pub async fn create_overlayed_image(
-    ctx: EffectContext,
+    context: EffectContext,
     CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams,
-) -> Result<(PathBuf, InternedString), Error> {
+) -> Result<(PathBuf, Guid), Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let path = Path::new("images")
+    if let Some(image) = context
-        .join(*ARCH)
+        .seed
-        .join(&image_id)
-        .with_extension("squashfs");
-    if let Some(image) = ctx
         .persistent_container
-        .s9pk
+        .images
-        .as_archive()
+        .get(&image_id)
-        .contents()
+        .cloned()
-        .get_path(&path)
-        .and_then(|e| e.as_file())
     {
-        let guid = new_guid();
+        let guid = Guid::new();
-        let rootfs_dir = ctx
+        let rootfs_dir = context
+            .seed
             .persistent_container
             .lxc_container
             .get()
@@ -1102,7 +1144,9 @@ pub async fn create_overlayed_image(
                 )
             })?
             .rootfs_dir();
-        let mountpoint = rootfs_dir.join("media/startos/overlays").join(&*guid);
+        let mountpoint = rootfs_dir
+            .join("media/startos/overlays")
+            .join(guid.as_ref());
         tokio::fs::create_dir_all(&mountpoint).await?;
         let container_mountpoint = Path::new("/").join(
             mountpoint
@@ -1110,18 +1154,16 @@ pub async fn create_overlayed_image(
                 .with_kind(ErrorKind::Incoherent)?,
         );
         tracing::info!("Mounting overlay {guid} for {image_id}");
-        let guard = OverlayGuard::mount(
+        let guard = OverlayGuard::mount(image, &mountpoint).await?;
-            &IdMapped::new(LoopDev::from(&**image), 0, 100000, 65536),
-            &mountpoint,
-        )
-        .await?;
         Command::new("chown")
             .arg("100000:100000")
             .arg(&mountpoint)
             .invoke(ErrorKind::Filesystem)
             .await?;
         tracing::info!("Mounted overlay {guid} for {image_id}");
-        ctx.persistent_container
+        context
+            .seed
+            .persistent_container
             .overlays
             .lock()
             .await
@@ -1228,13 +1270,15 @@ struct SetDependenciesParams {
 }
 
 async fn set_dependencies(
-    ctx: EffectContext,
+    context: EffectContext,
-    SetDependenciesParams { dependencies }: SetDependenciesParams,
+    WithProcedureId {
+        procedure_id,
+        rest: SetDependenciesParams { dependencies },
+    }: WithProcedureId<SetDependenciesParams>,
 ) -> Result<(), Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let id = &ctx.id;
+    let id = &context.seed.id;
-    let service_guard = ctx.ctx.services.get(id).await;
+
-    let service = service_guard.as_ref().or_not_found(id)?;
     let mut deps = BTreeMap::new();
     for dependency in dependencies {
         let (dep_id, kind, registry_url, version_spec) = match dependency {
@@ -1264,14 +1308,13 @@ async fn set_dependencies(
             let remote_s9pk = S9pk::deserialize(
                 &Arc::new(
                     HttpSource::new(
-                        ctx.ctx.client.clone(),
+                        context.seed.ctx.client.clone(),
                         registry_url
                             .join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?,
                     )
                     .await?,
                 ),
                 None, // TODO
-                true,
             )
             .await?;
 
@@ -1291,14 +1334,19 @@ async fn set_dependencies(
                 )
             }
         };
-        let config_satisfied = if let Some(dep_service) = &*ctx.ctx.services.get(&dep_id).await {
+        let config_satisfied =
-            service
+            if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await {
-                .dependency_config(dep_id.clone(), dep_service.get_config().await?.config)
+                context
-                .await?
+                    .dependency_config(
-                .is_none()
+                        procedure_id.clone(),
-        } else {
+                        dep_id.clone(),
-            true
+                        dep_service.get_config(procedure_id.clone()).await?.config,
-        };
+                    )
+                    .await?
+                    .is_none()
+            } else {
+                true
+            };
         deps.insert(
             dep_id,
             CurrentDependencyInfo {
@@ -1311,7 +1359,9 @@ async fn set_dependencies(
             },
         );
     }
-    ctx.ctx
+    context
+        .seed
+        .ctx
         .db
         .mutate(|db| {
             db.as_public_mut()
@@ -1324,10 +1374,10 @@ async fn set_dependencies(
         .await
 }
 
-async fn get_dependencies(ctx: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
+async fn get_dependencies(context: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let id = &ctx.id;
+    let id = &context.seed.id;
-    let db = ctx.ctx.db.peek().await;
+    let db = context.seed.ctx.db.peek().await;
     let data = db
         .as_public()
         .as_package_data()
@@ -1384,16 +1434,16 @@ struct CheckDependenciesResult {
 }
 
 async fn check_dependencies(
-    ctx: EffectContext,
+    context: EffectContext,
     CheckDependenciesParam { package_ids }: CheckDependenciesParam,
 ) -> Result<Vec<CheckDependenciesResult>, Error> {
-    let ctx = ctx.deref()?;
+    let context = context.deref()?;
-    let db = ctx.ctx.db.peek().await;
+    let db = context.seed.ctx.db.peek().await;
     let current_dependencies = db
         .as_public()
         .as_package_data()
-        .as_idx(&ctx.id)
+        .as_idx(&context.seed.id)
-        .or_not_found(&ctx.id)?
+        .or_not_found(&context.seed.id)?
         .as_current_dependencies()
         .de()?;
     let package_ids: Vec<_> = package_ids

core/startos/src/service/service_map.rs

@@ -25,7 +25,7 @@ use crate::progress::{
 use crate::s9pk::manifest::PackageId;
 use crate::s9pk::merkle_archive::source::FileSource;
 use crate::s9pk::S9pk;
-use crate::service::{LoadDisposition, Service};
+use crate::service::{LoadDisposition, Service, ServiceRef};
 use crate::status::{MainStatus, Status};
 use crate::util::serde::Pem;
 
@@ -39,23 +39,22 @@ pub struct InstallProgressHandles {
 
 /// This is the structure to contain all the services
 #[derive(Default)]
-pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<Service>>>>>);
+pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<ServiceRef>>>>>);
 impl ServiceMap {
-    async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<Service>>> {
+    async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<ServiceRef>>> {
         let mut lock = self.0.lock().await;
-        dbg!(lock.keys().collect::<Vec<_>>());
         lock.entry(id.clone())
             .or_insert_with(|| Arc::new(RwLock::new(None)))
             .clone()
     }
 
     #[instrument(skip_all)]
-    pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<Service>> {
+    pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<ServiceRef>> {
         self.entry(id).await.read_owned().await
     }
 
     #[instrument(skip_all)]
-    pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<Service>> {
+    pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<ServiceRef>> {
         self.entry(id).await.write_owned().await
     }
 
@@ -83,7 +82,7 @@ impl ServiceMap {
             shutdown_err = service.shutdown().await;
         }
         // TODO: retry on error?
-        *service = Service::load(ctx, id, disposition).await?;
+        *service = Service::load(ctx, id, disposition).await?.map(From::from);
         shutdown_err?;
         Ok(())
     }
@@ -95,6 +94,7 @@ impl ServiceMap {
         mut s9pk: S9pk<S>,
         recovery_source: Option<impl GenericMountGuard>,
     ) -> Result<DownloadInstallFuture, Error> {
+        s9pk.validate_and_filter(ctx.s9pk_arch)?;
         let manifest = s9pk.as_manifest().clone();
         let id = manifest.id.clone();
         let icon = s9pk.icon_data_url().await?;
@@ -128,7 +128,7 @@ impl ServiceMap {
         );
         let restoring = recovery_source.is_some();
 
-        let mut reload_guard = ServiceReloadGuard::new(ctx.clone(), id.clone(), op_name);
+        let mut reload_guard = ServiceRefReloadGuard::new(ctx.clone(), id.clone(), op_name);
 
         reload_guard
             .handle(ctx.db.mutate({
@@ -231,7 +231,7 @@ impl ServiceMap {
             Ok(reload_guard
                 .handle_last(async move {
                     finalization_progress.start();
-                    let s9pk = S9pk::open(&installed_path, Some(&id), true).await?;
+                    let s9pk = S9pk::open(&installed_path, Some(&id)).await?;
                     let prev = if let Some(service) = service.take() {
                         ensure_code!(
                             recovery_source.is_none(),
@@ -264,7 +264,8 @@ impl ServiceMap {
                                     progress_handle,
                                 }),
                             )
-                            .await?,
+                            .await?
+                            .into(),
                         );
                     } else {
                         *service = Some(
@@ -277,7 +278,8 @@ impl ServiceMap {
                                     progress_handle,
                                 }),
                             )
-                            .await?,
+                            .await?
+                            .into(),
                         );
                     }
                     sync_progress_task.await.map_err(|_| {
@@ -295,7 +297,7 @@ impl ServiceMap {
     pub async fn uninstall(&self, ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
         let mut guard = self.get_mut(id).await;
         if let Some(service) = guard.take() {
-            ServiceReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
+            ServiceRefReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
                 .handle_last(async move {
                     let res = service.uninstall(None).await;
                     drop(guard);
@@ -326,17 +328,17 @@ impl ServiceMap {
     }
 }
 
-pub struct ServiceReloadGuard(Option<ServiceReloadInfo>);
+pub struct ServiceRefReloadGuard(Option<ServiceRefReloadInfo>);
-impl Drop for ServiceReloadGuard {
+impl Drop for ServiceRefReloadGuard {
     fn drop(&mut self) {
         if let Some(info) = self.0.take() {
             tokio::spawn(info.reload(None));
         }
     }
 }
-impl ServiceReloadGuard {
+impl ServiceRefReloadGuard {
     pub fn new(ctx: RpcContext, id: PackageId, operation: &'static str) -> Self {
-        Self(Some(ServiceReloadInfo { ctx, id, operation }))
+        Self(Some(ServiceRefReloadInfo { ctx, id, operation }))
     }
 
     pub async fn handle<T>(
@@ -365,12 +367,12 @@ impl ServiceReloadGuard {
     }
 }
 
-struct ServiceReloadInfo {
+struct ServiceRefReloadInfo {
     ctx: RpcContext,
     id: PackageId,
     operation: &'static str,
 }
-impl ServiceReloadInfo {
+impl ServiceRefReloadInfo {
     async fn reload(self, error: Option<Error>) -> Result<(), Error> {
         self.ctx
             .services

core/startos/src/service/transition/backup.rs

@@ -6,6 +6,7 @@ use models::ProcedureName;
 use super::TempDesiredRestore;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::config::GetConfig;
 use crate::service::dependencies::DependencyConfig;
 use crate::service::transition::{TransitionKind, TransitionState};
@@ -24,7 +25,12 @@ impl Handler<Backup> for ServiceActor {
             .except::<GetConfig>()
             .except::<DependencyConfig>()
     }
-    async fn handle(&mut self, backup: Backup, jobs: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(
+        &mut self,
+        id: Guid,
+        backup: Backup,
+        jobs: &BackgroundJobQueue,
+    ) -> Self::Response {
         // So Need a handle to just a single field in the state
         let temp: TempDesiredRestore = TempDesiredRestore::new(&self.0.persistent_container.state);
         let mut current = self.0.persistent_container.state.subscribe();
@@ -45,7 +51,7 @@ impl Handler<Backup> for ServiceActor {
                     .mount_backup(path, ReadWrite)
                     .await?;
                 seed.persistent_container
-                    .execute(ProcedureName::CreateBackup, Value::Null, None)
+                    .execute(id, ProcedureName::CreateBackup, Value::Null, None)
                     .await?;
                 backup_guard.unmount(true).await?;
 

core/startos/src/service/transition/restart.rs

@@ -2,6 +2,7 @@ use futures::FutureExt;
 
 use super::TempDesiredRestore;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::config::GetConfig;
 use crate::service::dependencies::DependencyConfig;
 use crate::service::transition::{TransitionKind, TransitionState};
@@ -18,7 +19,8 @@ impl Handler<Restart> for ServiceActor {
             .except::<GetConfig>()
             .except::<DependencyConfig>()
     }
-    async fn handle(&mut self, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(&mut self, _: Guid, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
+        dbg!("here");
         // So Need a handle to just a single field in the state
         let temp = TempDesiredRestore::new(&self.0.persistent_container.state);
         let mut current = self.0.persistent_container.state.subscribe();
@@ -74,7 +76,8 @@ impl Handler<Restart> for ServiceActor {
 }
 impl Service {
     #[instrument(skip_all)]
-    pub async fn restart(&self) -> Result<(), Error> {
+    pub async fn restart(&self, id: Guid) -> Result<(), Error> {
-        self.actor.send(Restart).await
+        dbg!("here");
+        self.actor.send(id, Restart).await
     }
 }

core/startos/src/service/transition/restore.rs

@@ -5,6 +5,7 @@ use models::ProcedureName;
 
 use crate::disk::mount::filesystem::ReadOnly;
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::service::transition::{TransitionKind, TransitionState};
 use crate::service::ServiceActor;
 use crate::util::actor::background::BackgroundJobQueue;
@@ -19,7 +20,12 @@ impl Handler<Restore> for ServiceActor {
     fn conflicts_with(_: &Restore) -> ConflictBuilder<Self> {
         ConflictBuilder::everything()
     }
-    async fn handle(&mut self, restore: Restore, jobs: &BackgroundJobQueue) -> Self::Response {
+    async fn handle(
+        &mut self,
+        id: Guid,
+        restore: Restore,
+        jobs: &BackgroundJobQueue,
+    ) -> Self::Response {
         // So Need a handle to just a single field in the state
         let path = restore.path.clone();
         let seed = self.0.clone();
@@ -32,7 +38,7 @@ impl Handler<Restore> for ServiceActor {
                     .mount_backup(path, ReadOnly)
                     .await?;
                 seed.persistent_container
-                    .execute(ProcedureName::RestoreBackup, Value::Null, None)
+                    .execute(id, ProcedureName::RestoreBackup, Value::Null, None)
                     .await?;
                 backup_guard.unmount(true).await?;
 

core/startos/src/util/actor/concurrent.rs

@@ -8,6 +8,7 @@ use helpers::NonDetachingJoinHandle;
 use tokio::sync::{mpsc, oneshot};
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner};
 use crate::util::actor::{Actor, ConflictFn, Handler, PendingMessageStrategy, Request};
 
@@ -18,6 +19,7 @@ struct ConcurrentRunner<A> {
     waiting: Vec<Request<A>>,
     recv: mpsc::UnboundedReceiver<Request<A>>,
     handlers: Vec<(
+        Guid,
         Arc<ConflictFn<A>>,
         oneshot::Sender<Box<dyn Any + Send>>,
         BoxFuture<'static, Box<dyn Any + Send>>,
@@ -41,16 +43,21 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
             }
         });
         if this.shutdown.is_some() {
-            while let std::task::Poll::Ready(Some((msg, reply))) = this.recv.poll_recv(cx) {
+            while let std::task::Poll::Ready(Some((id, msg, reply))) = this.recv.poll_recv(cx) {
-                if this.handlers.iter().any(|(f, _, _)| f(&*msg)) {
+                if this
-                    this.waiting.push((msg, reply));
+                    .handlers
+                    .iter()
+                    .any(|(hid, f, _, _)| &id != hid && f(&*msg))
+                {
+                    this.waiting.push((id, msg, reply));
                 } else {
                     let mut actor = this.actor.clone();
                     let queue = this.queue.clone();
                     this.handlers.push((
+                        id.clone(),
                         msg.conflicts_with(),
                         reply,
-                        async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
+                        async move { msg.handle_with(id, &mut actor, &queue).await }.boxed(),
                     ))
                 }
             }
@@ -62,29 +69,34 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
                 .handlers
                 .iter_mut()
                 .enumerate()
-                .filter_map(|(i, (_, _, f))| match f.poll_unpin(cx) {
+                .filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) {
                     std::task::Poll::Pending => None,
                     std::task::Poll::Ready(res) => Some((i, res)),
                 })
                 .collect::<Vec<_>>();
             for (idx, res) in complete.into_iter().rev() {
                 #[allow(clippy::let_underscore_future)]
-                let (f, reply, _) = this.handlers.swap_remove(idx);
+                let (_, f, reply, _) = this.handlers.swap_remove(idx);
                 let _ = reply.send(res);
                 // TODO: replace with Vec::extract_if once stable
                 if this.shutdown.is_some() {
                     let mut i = 0;
                     while i < this.waiting.len() {
-                        if f(&*this.waiting[i].0)
+                        if f(&*this.waiting[i].1)
-                            && !this.handlers.iter().any(|(f, _, _)| f(&*this.waiting[i].0))
+                            && !this
+                                .handlers
+                                .iter()
+                                .any(|(_, f, _, _)| f(&*this.waiting[i].1))
                         {
-                            let (msg, reply) = this.waiting.remove(i);
+                            let (id, msg, reply) = this.waiting.remove(i);
                             let mut actor = this.actor.clone();
                             let queue = this.queue.clone();
                             this.handlers.push((
+                                id.clone(),
                                 msg.conflicts_with(),
                                 reply,
-                                async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
+                                async move { msg.handle_with(id, &mut actor, &queue).await }
+                                    .boxed(),
                             ));
                             cont = true;
                         } else {
@@ -137,6 +149,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
     /// Message is guaranteed to be queued immediately
     pub fn queue<M: Send + 'static>(
         &self,
+        id: Guid,
         message: M,
     ) -> impl Future<Output = Result<A::Response, Error>>
     where
@@ -150,7 +163,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
         }
         let (reply_send, reply_recv) = oneshot::channel();
         self.messenger
-            .send((Box::new(message), reply_send))
+            .send((id, Box::new(message), reply_send))
             .unwrap();
         futures::future::Either::Right(
             reply_recv
@@ -170,11 +183,11 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
         )
     }
 
-    pub async fn send<M: Send + 'static>(&self, message: M) -> Result<A::Response, Error>
+    pub async fn send<M: Send + 'static>(&self, id: Guid, message: M) -> Result<A::Response, Error>
     where
         A: Handler<M>,
     {
-        self.queue(message).await
+        self.queue(id, message).await
     }
 
     pub async fn shutdown(self, strategy: PendingMessageStrategy) {

core/startos/src/util/actor/mod.rs

@@ -9,6 +9,7 @@ use tokio::sync::oneshot;
 
 #[allow(unused_imports)]
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::BackgroundJobQueue;
 
 pub mod background;
@@ -28,6 +29,7 @@ pub trait Handler<M: Any + Send>: Actor {
     }
     fn handle(
         &mut self,
+        id: Guid,
         msg: M,
         jobs: &BackgroundJobQueue,
     ) -> impl Future<Output = Self::Response> + Send;
@@ -39,6 +41,7 @@ trait Message<A>: Send + Any {
     fn conflicts_with(&self) -> Arc<ConflictFn<A>>;
     fn handle_with<'a>(
         self: Box<Self>,
+        id: Guid,
         actor: &'a mut A,
         jobs: &'a BackgroundJobQueue,
     ) -> BoxFuture<'a, Box<dyn Any + Send>>;
@@ -52,10 +55,11 @@ where
     }
     fn handle_with<'a>(
         self: Box<Self>,
+        id: Guid,
         actor: &'a mut A,
         jobs: &'a BackgroundJobQueue,
     ) -> BoxFuture<'a, Box<dyn Any + Send>> {
-        async move { Box::new(actor.handle(*self, jobs).await) as Box<dyn Any + Send> }.boxed()
+        async move { Box::new(actor.handle(id, *self, jobs).await) as Box<dyn Any + Send> }.boxed()
     }
 }
 impl<A: Actor> dyn Message<A> {
@@ -80,7 +84,11 @@ impl<A: Actor> dyn Message<A> {
     }
 }
 
-type Request<A> = (Box<dyn Message<A>>, oneshot::Sender<Box<dyn Any + Send>>);
+type Request<A> = (
+    Guid,
+    Box<dyn Message<A>>,
+    oneshot::Sender<Box<dyn Any + Send>>,
+);
 
 pub enum PendingMessageStrategy {
     CancelAll,

core/startos/src/util/actor/simple.rs

@@ -7,6 +7,7 @@ use tokio::sync::oneshot::error::TryRecvError;
 use tokio::sync::{mpsc, oneshot};
 
 use crate::prelude::*;
+use crate::rpc_continuations::Guid;
 use crate::util::actor::background::BackgroundJobQueue;
 use crate::util::actor::{Actor, Handler, PendingMessageStrategy, Request};
 
@@ -26,9 +27,9 @@ impl<A: Actor> SimpleActor<A> {
                 tokio::select! {
                     _ = &mut runner => (),
                     msg = messenger_recv.recv() => match msg {
-                        Some((msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
+                        Some((id, msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
                             tokio::select! {
-                                res = msg.handle_with(&mut actor, &queue) => { let _ = reply.send(res); },
+                                res = msg.handle_with(id, &mut actor, &queue) => { let _ = reply.send(res); },
                                 _ = &mut runner => (),
                             }
                         }
@@ -60,7 +61,7 @@ impl<A: Actor> SimpleActor<A> {
         }
         let (reply_send, reply_recv) = oneshot::channel();
         self.messenger
-            .send((Box::new(message), reply_send))
+            .send((Guid::new(), Box::new(message), reply_send))
             .unwrap();
         futures::future::Either::Right(
             reply_recv

core/startos/src/util/io.rs

@@ -681,8 +681,6 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
     }
 }
 
-pub struct TmpFile {}
-
 #[derive(Debug)]
 pub struct TmpDir {
     path: PathBuf,
@@ -707,6 +705,14 @@ impl TmpDir {
         tokio::fs::remove_dir_all(&self.path).await?;
         Ok(())
     }
+
+    pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
+        if let Ok(dir) = Arc::try_unwrap(self) {
+            dir.delete().await
+        } else {
+            Ok(())
+        }
+    }
 }
 impl std::ops::Deref for TmpDir {
     type Target = Path;

core/startos/src/util/mod.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::future::Future;
 use std::marker::PhantomData;
 use std::path::{Path, PathBuf};
@@ -11,6 +11,8 @@ use std::time::Duration;
 use async_trait::async_trait;
 use color_eyre::eyre::{self, eyre};
 use fd_lock_rs::FdLock;
+use futures::future::BoxFuture;
+use futures::FutureExt;
 use helpers::canonicalize;
 pub use helpers::NonDetachingJoinHandle;
 use imbl_value::InternedString;
@@ -19,7 +21,8 @@ pub use models::VersionString;
 use pin_project::pin_project;
 use sha2::Digest;
 use tokio::fs::File;
-use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
+use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
+use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
 use tracing::instrument;
 
 use crate::shutdown::Shutdown;
@@ -62,11 +65,16 @@ pub trait Invoke<'a> {
     where
         Self: 'ext,
         'ext: 'a;
+    fn pipe<'ext: 'a>(
+        &'ext mut self,
+        next: &'ext mut tokio::process::Command,
+    ) -> Self::Extended<'ext>;
     fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext>;
     fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
         &'ext mut self,
         input: Option<&'ext mut Input>,
     ) -> Self::Extended<'ext>;
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext>;
     fn invoke(
         &mut self,
         error_kind: crate::ErrorKind,
@@ -76,7 +84,20 @@ pub trait Invoke<'a> {
 pub struct ExtendedCommand<'a> {
     cmd: &'a mut tokio::process::Command,
     timeout: Option<Duration>,
-    input: Option<&'a mut (dyn tokio::io::AsyncRead + Unpin + Send)>,
+    input: Option<&'a mut (dyn AsyncRead + Unpin + Send)>,
+    pipe: VecDeque<&'a mut tokio::process::Command>,
+    capture: bool,
+}
+impl<'a> From<&'a mut tokio::process::Command> for ExtendedCommand<'a> {
+    fn from(value: &'a mut tokio::process::Command) -> Self {
+        ExtendedCommand {
+            cmd: value,
+            timeout: None,
+            input: None,
+            pipe: VecDeque::new(),
+            capture: true,
+        }
+    }
 }
 impl<'a> std::ops::Deref for ExtendedCommand<'a> {
     type Target = tokio::process::Command;
@@ -95,35 +116,38 @@ impl<'a> Invoke<'a> for tokio::process::Command {
     where
         Self: 'ext,
         'ext: 'a;
-    fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
+    fn pipe<'ext: 'a>(
-        ExtendedCommand {
+        &'ext mut self,
-            cmd: self,
+        next: &'ext mut tokio::process::Command,
-            timeout,
+    ) -> Self::Extended<'ext> {
-            input: None,
+        let mut cmd = ExtendedCommand::from(self);
-        }
+        cmd.pipe.push_back(next);
+        cmd
     }
-    fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
+    fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.timeout = timeout;
+        cmd
+    }
+    fn input<'ext: 'a, Input: AsyncRead + Unpin + Send>(
         &'ext mut self,
         input: Option<&'ext mut Input>,
     ) -> Self::Extended<'ext> {
-        ExtendedCommand {
+        let mut cmd = ExtendedCommand::from(self);
-            cmd: self,
+        cmd.input = if let Some(input) = input {
-            timeout: None,
+            Some(&mut *input)
-            input: if let Some(input) = input {
+        } else {
-                Some(&mut *input)
+            None
-            } else {
+        };
-                None
+        cmd
-            },
+    }
-        }
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
+        let mut cmd = ExtendedCommand::from(self);
+        cmd.capture = capture;
+        cmd
     }
     async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
-        ExtendedCommand {
+        ExtendedCommand::from(self).invoke(error_kind).await
-            cmd: self,
-            timeout: None,
-            input: None,
-        }
-        .invoke(error_kind)
-        .await
     }
 }
 
@@ -132,6 +156,13 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
     where
         Self: 'ext,
         'ext: 'a;
+    fn pipe<'ext: 'a>(
+        &'ext mut self,
+        next: &'ext mut tokio::process::Command,
+    ) -> Self::Extended<'ext> {
+        self.pipe.push_back(next.kill_on_drop(true));
+        self
+    }
     fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
         self.timeout = timeout;
         self
@@ -147,39 +178,150 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
         };
         self
     }
+    fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
+        self.capture = capture;
+        self
+    }
+    #[instrument(skip_all)]
     async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
+        let cmd_str = self
+            .cmd
+            .as_std()
+            .get_program()
+            .to_string_lossy()
+            .into_owned();
         self.cmd.kill_on_drop(true);
         if self.input.is_some() {
             self.cmd.stdin(Stdio::piped());
         }
-        self.cmd.stdout(Stdio::piped());
+        if self.pipe.is_empty() {
-        self.cmd.stderr(Stdio::piped());
+            if self.capture {
-        let mut child = self.cmd.spawn().with_kind(error_kind)?;
+                self.cmd.stdout(Stdio::piped());
-        if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
+                self.cmd.stderr(Stdio::piped());
-            use tokio::io::AsyncWriteExt;
+            }
-            tokio::io::copy(input, &mut stdin).await?;
+            let mut child = self.cmd.spawn().with_ctx(|_| (error_kind, &cmd_str))?;
-            stdin.flush().await?;
+            if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
-            stdin.shutdown().await?;
+                use tokio::io::AsyncWriteExt;
-            drop(stdin);
+                tokio::io::copy(input, &mut stdin).await?;
+                stdin.flush().await?;
+                stdin.shutdown().await?;
+                drop(stdin);
+            }
+            let res = match self.timeout {
+                None => child
+                    .wait_with_output()
+                    .await
+                    .with_ctx(|_| (error_kind, &cmd_str))?,
+                Some(t) => tokio::time::timeout(t, child.wait_with_output())
+                    .await
+                    .with_kind(ErrorKind::Timeout)?
+                    .with_ctx(|_| (error_kind, &cmd_str))?,
+            };
+            crate::ensure_code!(
+                res.status.success(),
+                error_kind,
+                "{}",
+                Some(&res.stderr)
+                    .filter(|a| !a.is_empty())
+                    .or(Some(&res.stdout))
+                    .filter(|a| !a.is_empty())
+                    .and_then(|a| std::str::from_utf8(a).ok())
+                    .unwrap_or(&format!(
+                        "{} exited with code {}",
+                        self.cmd.as_std().get_program().to_string_lossy(),
+                        res.status
+                    ))
+            );
+            Ok(res.stdout)
+        } else {
+            let mut futures = Vec::<BoxFuture<'_, Result<(), Error>>>::new(); // todo: predict capacity
+
+            let mut cmds = std::mem::take(&mut self.pipe);
+            cmds.push_front(&mut *self.cmd);
+            let len = cmds.len();
+
+            let timeout = self.timeout;
+
+            let mut prev = self
+                .input
+                .take()
+                .map(|i| Box::new(i) as Box<dyn AsyncRead + Unpin + Send>);
+            for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() {
+                let last = idx == len - 1;
+                if self.capture || !last {
+                    cmd.stdout(Stdio::piped());
+                }
+                if self.capture {
+                    cmd.stderr(Stdio::piped());
+                }
+                if prev.is_some() {
+                    cmd.stdin(Stdio::piped());
+                }
+                let mut child = cmd.spawn().with_kind(error_kind)?;
+                let input = std::mem::replace(
+                    &mut prev,
+                    child
+                        .stdout
+                        .take()
+                        .map(|i| Box::new(BufReader::new(i)) as Box<dyn AsyncRead + Unpin + Send>),
+                );
+                futures.push(
+                    async move {
+                        if let (Some(mut stdin), Some(mut input)) = (child.stdin.take(), input) {
+                            use tokio::io::AsyncWriteExt;
+                            tokio::io::copy(&mut input, &mut stdin).await?;
+                            stdin.flush().await?;
+                            stdin.shutdown().await?;
+                            drop(stdin);
+                        }
+                        let res = match timeout {
+                            None => child.wait_with_output().await?,
+                            Some(t) => tokio::time::timeout(t, child.wait_with_output())
+                                .await
+                                .with_kind(ErrorKind::Timeout)??,
+                        };
+                        crate::ensure_code!(
+                            res.status.success(),
+                            error_kind,
+                            "{}",
+                            Some(&res.stderr)
+                                .filter(|a| !a.is_empty())
+                                .or(Some(&res.stdout))
+                                .filter(|a| !a.is_empty())
+                                .and_then(|a| std::str::from_utf8(a).ok())
+                                .unwrap_or(&format!(
+                                    "{} exited with code {}",
+                                    cmd.as_std().get_program().to_string_lossy(),
+                                    res.status
+                                ))
+                        );
+
+                        Ok(())
+                    }
+                    .boxed(),
+                );
+            }
+
+            let (send, recv) = oneshot::channel();
+            futures.push(
+                async move {
+                    if let Some(mut prev) = prev {
+                        let mut res = Vec::new();
+                        prev.read_to_end(&mut res).await?;
+                        send.send(res).unwrap();
+                    } else {
+                        send.send(Vec::new()).unwrap();
+                    }
+
+                    Ok(())
+                }
+                .boxed(),
+            );
+
+            futures::future::try_join_all(futures).await?;
+
+            Ok(recv.await.unwrap())
         }
-        let res = match self.timeout {
-            None => child.wait_with_output().await?,
-            Some(t) => tokio::time::timeout(t, child.wait_with_output())
-                .await
-                .with_kind(ErrorKind::Timeout)??,
-        };
-        crate::ensure_code!(
-            res.status.success(),
-            error_kind,
-            "{}",
-            Some(&res.stderr)
-                .filter(|a| !a.is_empty())
-                .or(Some(&res.stdout))
-                .filter(|a| !a.is_empty())
-                .and_then(|a| std::str::from_utf8(a).ok())
-                .unwrap_or(&format!("Unknown Error ({})", res.status))
-        );
-        Ok(res.stdout)
     }
 }
 

sdk/lib/StartSdk.ts

@@ -187,7 +187,10 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       nullIfEmpty,
       runCommand: async <A extends string>(
         effects: Effects,
-        image: { id: Manifest["images"][number]; sharedRun?: boolean },
+        image: {
+          id: keyof Manifest["images"] & T.ImageId
+          sharedRun?: boolean
+        },
         command: ValidIfNoStupidEscape<A> | [string, ...string[]],
         options: CommandOptions & {
           mounts?: { path: string; options: MountOptions }[]
@@ -396,7 +399,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       setupProperties:
         (
           fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
-        ): T.ExpectedExports.Properties =>
+        ): T.ExpectedExports.properties =>
         (options) =>
           fn(options).then(nullifyProperties),
       setupUninstall: (fn: UninstallFn<Manifest, Store>) =>
@@ -743,7 +746,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
 
 export async function runCommand<Manifest extends SDKManifest>(
   effects: Effects,
-  image: { id: Manifest["images"][number]; sharedRun?: boolean },
+  image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean },
   command: string | [string, ...string[]],
   options: CommandOptions & {
     mounts?: { path: string; options: MountOptions }[]

sdk/lib/health/HealthCheck.ts

@@ -8,12 +8,13 @@ import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
 import { Overlay } from "../util/Overlay"
 import { object, unknown } from "ts-matches"
+import { T } from ".."
 
 export type HealthCheckParams<Manifest extends SDKManifest> = {
   effects: Effects
   name: string
   image: {
-    id: Manifest["images"][number]
+    id: keyof Manifest["images"] & T.ImageId
     sharedRun?: boolean
   }
   trigger?: Trigger

sdk/lib/interfaces/Host.ts

@@ -69,12 +69,12 @@ type NotProtocolsWithSslVariants = Exclude<
 type BindOptionsByKnownProtocol =
   | {
       protocol: ProtocolsWithSslVariants
-      preferredExternalPort: number
+      preferredExternalPort?: number
       addSsl?: Partial<AddSslOptions>
     }
   | {
       protocol: NotProtocolsWithSslVariants
-      preferredExternalPort: number
+      preferredExternalPort?: number
       addSsl?: AddSslOptions
     }
 export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions

sdk/lib/mainFn/CommandController.ts

@@ -1,6 +1,6 @@
 import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
 import { cpExecFile, cpExec } from "./Daemons"
@@ -15,7 +15,7 @@ export class CommandController {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemon.ts

@@ -1,5 +1,5 @@
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { CommandController } from "./CommandController"
 
@@ -18,7 +18,7 @@ export class Daemon {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemons.ts

@@ -5,7 +5,12 @@ import { SDKManifest } from "../manifest/ManifestTypes"
 import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
-import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types"
+import {
+  DaemonReturned,
+  Effects,
+  ImageId,
+  ValidIfNoStupidEscape,
+} from "../types"
 import { Mounts } from "./Mounts"
 import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
@@ -34,8 +39,8 @@ type DaemonsParams<
   Id extends string,
 > = {
   command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
-  image: { id: Manifest["images"][number]; sharedRun?: boolean }
+  image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean }
-  mounts: { path: string; options: MountOptions }[]
+  mounts: Mounts<Manifest>
   env?: Record<string, string>
   ready: Ready
   requires: Exclude<Ids, Id>[]
@@ -116,12 +121,10 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
     options: DaemonsParams<Manifest, Ids, Command, Id>,
   ) {
     const daemonIndex = this.daemons.length
-    const daemon = Daemon.of()(
+    const daemon = Daemon.of()(this.effects, options.image, options.command, {
-      this.effects,
+      ...options,
-      options.image,
+      mounts: options.mounts.build(),
-      options.command,
+    })
-      options,
-    )
     const healthDaemon = new HealthDaemon(
       daemon,
       daemonIndex,

sdk/lib/manifest/ManifestTypes.ts

@@ -1,5 +1,5 @@
 import { ValidEmVer } from "../emverLite/mod"
-import { ActionMetadata } from "../types"
+import { ActionMetadata, ImageConfig, ImageId } from "../types"
 
 export interface Container {
   /** This should be pointing to a docker container name */
@@ -28,8 +28,6 @@ export type SDKManifest = {
   readonly releaseNotes: string
   /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
   readonly license: string // name of license
-  /** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */
-  readonly replaces: Readonly<string[]>
   /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),
    * any scripts necessary for configuration, backups, actions, or health checks (more below). This key
    * must exist. But could be embedded into the source repository
@@ -52,7 +50,7 @@ export type SDKManifest = {
   }
 
   /** Defines the os images needed to run the container processes */
-  readonly images: string[]
+  readonly images: Record<ImageId, ImageConfig>
   /** This denotes readonly asset directories that should be available to mount to the container.
    * Assuming that there will be three files with names along the lines:
    * icon.* : the icon that will be this packages icon on the ui

sdk/lib/manifest/setupManifest.ts

@@ -1,18 +1,19 @@
+import { ImageConfig, ImageId, VolumeId } from "../osBindings"
 import { SDKManifest, ManifestVersion } from "./ManifestTypes"
 
 export function setupManifest<
   Id extends string,
   Version extends ManifestVersion,
   Dependencies extends Record<string, unknown>,
-  VolumesTypes extends string,
+  VolumesTypes extends VolumeId,
-  AssetTypes extends string,
+  AssetTypes extends VolumeId,
-  ImagesTypes extends string,
+  ImagesTypes extends ImageId,
   Manifest extends SDKManifest & {
     dependencies: Dependencies
     id: Id
     version: Version
     assets: AssetTypes[]
-    images: ImagesTypes[]
+    images: Record<ImagesTypes, ImageConfig>
     volumes: VolumesTypes[]
   },
 >(manifest: Manifest): Manifest {

sdk/lib/osBindings/CreateOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageId } from "./ImageId"
 
-export type CreateOverlayedImageParams = { imageId: string }
+export type CreateOverlayedImageParams = { imageId: ImageId }

sdk/lib/osBindings/DestroyOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
-export type DestroyOverlayedImageParams = { guid: string }
+export type DestroyOverlayedImageParams = { guid: Guid }

sdk/lib/osBindings/ImageConfig.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageSource } from "./ImageSource"
+
+export type ImageConfig = {
+  source: ImageSource
+  arch: string[]
+  emulateMissingAs: string | null
+}

sdk/lib/osBindings/ImageMetadata.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ImageMetadata = { workdir: string; user: string }

sdk/lib/osBindings/ImageSource.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ImageSource =
+  | "packed"
+  | { dockerBuild: { workdir: string | null; dockerfile: string | null } }
+  | { dockerTag: string }

sdk/lib/osBindings/LoginParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { PasswordType } from "./PasswordType"
+
+export type LoginParams = { password: PasswordType | null; metadata: any }

sdk/lib/osBindings/Manifest.ts

@@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts"
 import type { Dependencies } from "./Dependencies"
 import type { Description } from "./Description"
 import type { HardwareRequirements } from "./HardwareRequirements"
+import type { ImageConfig } from "./ImageConfig"
 import type { ImageId } from "./ImageId"
 import type { PackageId } from "./PackageId"
 import type { Version } from "./Version"
@@ -20,7 +21,7 @@ export type Manifest = {
   marketingSite: string
   donationUrl: string | null
   description: Description
-  images: Array<ImageId>
+  images: { [key: ImageId]: ImageConfig }
   assets: Array<VolumeId>
   volumes: Array<VolumeId>
   alerts: Alerts

sdk/lib/osBindings/index.ts

@@ -69,7 +69,10 @@ export { HostKind } from "./HostKind"
 export { HostnameInfo } from "./HostnameInfo"
 export { Hosts } from "./Hosts"
 export { Host } from "./Host"
+export { ImageConfig } from "./ImageConfig"
 export { ImageId } from "./ImageId"
+export { ImageMetadata } from "./ImageMetadata"
+export { ImageSource } from "./ImageSource"
 export { InstalledState } from "./InstalledState"
 export { InstallingInfo } from "./InstallingInfo"
 export { InstallingState } from "./InstallingState"
@@ -78,6 +81,7 @@ export { IpInfo } from "./IpInfo"
 export { LanInfo } from "./LanInfo"
 export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams"
 export { ListVersionSignersParams } from "./ListVersionSignersParams"
+export { LoginParams } from "./LoginParams"
 export { MainStatus } from "./MainStatus"
 export { Manifest } from "./Manifest"
 export { MaybeUtf8String } from "./MaybeUtf8String"

sdk/lib/test/configBuilder.test.ts

@@ -400,7 +400,7 @@ describe("values", () => {
               long: "",
             },
             containers: {},
-            images: [],
+            images: {},
             volumes: [],
             assets: [],
             alerts: {

sdk/lib/test/output.sdk.ts

@@ -21,7 +21,7 @@ export const sdk = StartSdk.of()
         long: "",
       },
       containers: {},
-      images: [],
+      images: {},
       volumes: [],
       assets: [],
       alerts: {

sdk/lib/types.ts

@@ -11,6 +11,7 @@ import {
   GetPrimaryUrlParams,
   LanInfo,
   BindParams,
+  Manifest,
 } from "./osBindings"
 
 import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
@@ -110,9 +111,26 @@ export namespace ExpectedExports {
    */
   export type dependencyConfig = Record<PackageId, DependencyConfig | null>
 
-  export type Properties = (options: {
+  export type properties = (options: {
     effects: Effects
   }) => Promise<PropertiesReturn>
+
+  export type manifest = Manifest
+}
+export type ABI = {
+  setConfig: ExpectedExports.setConfig
+  getConfig: ExpectedExports.getConfig
+  createBackup: ExpectedExports.createBackup
+  restoreBackup: ExpectedExports.restoreBackup
+  actions: ExpectedExports.actions
+  actionsMetadata: ExpectedExports.actionsMetadata
+  main: ExpectedExports.main
+  afterShutdown: ExpectedExports.afterShutdown
+  init: ExpectedExports.init
+  uninit: ExpectedExports.uninit
+  dependencyConfig: ExpectedExports.dependencyConfig
+  properties: ExpectedExports.properties
+  manifest: ExpectedExports.manifest
 }
 export type TimeMs = number
 export type VersionString = string
@@ -453,8 +471,8 @@ export type Effects = {
   /** Exists could be useful during the runtime to know if some service is running, option dep */
   running(options: { packageId: PackageId }): Promise<boolean>
 
-  restart(): void
+  restart(): Promise<void>
-  shutdown(): void
+  shutdown(): Promise<void>
 
   mount(options: {
     location: string

sdk/lib/util/Overlay.ts

@@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
 export class Overlay {
   private constructor(
     readonly effects: T.Effects,
-    readonly imageId: string,
+    readonly imageId: T.ImageId,
     readonly rootfs: string,
-    readonly guid: string,
+    readonly guid: T.Guid,
   ) {}
   static async of(
     effects: T.Effects,
-    image: { id: string; sharedRun?: boolean },
+    image: { id: T.ImageId; sharedRun?: boolean },
   ) {
-    const { id: imageId, sharedRun } = image
+    const { id, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
+    const [rootfs, guid] = await effects.createOverlayedImage({
+      imageId: id as string,
+    })
 
     const shared = ["dev", "sys", "proc"]
     if (!!sharedRun) {
@@ -33,7 +35,7 @@ export class Overlay {
       ])
     }
 
-    return new Overlay(effects, imageId, rootfs, guid)
+    return new Overlay(effects, id, rootfs, guid)
   }
 
   async mount(options: MountOptions, path: string): Promise<Overlay> {
@@ -97,7 +99,7 @@ export class Overlay {
     stdout: string | Buffer
     stderr: string | Buffer
   }> {
-    const imageMeta: any = await fs
+    const imageMeta: T.ImageMetadata = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })

sdk/lib/util/fileHelper.ts

@@ -3,7 +3,7 @@ import * as YAML from "yaml"
 import * as TOML from "@iarna/toml"
 import _ from "lodash"
 import * as T from "../types"
-import * as fs from "fs"
+import * as fs from "node:fs/promises"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
@@ -59,28 +59,24 @@ export class FileHelper<A> {
     readonly readData: (stringValue: string) => A,
   ) {}
   async write(data: A, effects: T.Effects) {
-    if (previousPath.exec(this.path)) {
+    const parent = previousPath.exec(this.path)
-      await new Promise((resolve, reject) =>
+    if (parent) {
-        fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
+      await fs.mkdir(parent[1], { recursive: true })
-      )
     }
 
-    await new Promise((resolve, reject) =>
+    await fs.writeFile(this.path, this.writeData(data))
-      fs.writeFile(this.path, this.writeData(data), (err: any) =>
-        !err ? resolve(null) : reject(err),
-      ),
-    )
   }
   async read(effects: T.Effects) {
-    if (!fs.existsSync(this.path)) {
+    if (
+      !(await fs.access(this.path).then(
+        () => true,
+        () => false,
+      ))
+    ) {
       return null
     }
     return this.readData(
-      await new Promise((resolve, reject) =>
+      await fs.readFile(this.path).then((data) => data.toString("utf-8")),
-        fs.readFile(this.path, (err: any, data: any) =>
-          !err ? resolve(data.toString("utf-8")) : reject(err),
-        ),
-      ),
     )
   }
 
@@ -142,7 +138,7 @@ export class FileHelper<A> {
     return new FileHelper<A>(
       path,
       (inData) => {
-        return JSON.stringify(inData, null, 2)
+        return YAML.stringify(inData, null, 2)
       },
       (inString) => {
         return shape.unsafeCast(YAML.parse(inString))

sdk/package-lock.json

@@ -10,7 +10,7 @@
       "license": "MIT",
       "dependencies": {
         "isomorphic-fetch": "^3.0.0",
-        "lodash": "4.*.*",
+        "lodash": "^4.17.21",
         "ts-matches": "^5.4.1"
       },
       "devDependencies": {

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.3.6-alpha1",
+  "version": "0.3.6-alpha5",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./cjs/lib/index.js",
   "types": "./cjs/lib/index.d.ts",
@@ -31,8 +31,10 @@
   "homepage": "https://github.com/Start9Labs/start-sdk#readme",
   "dependencies": {
     "isomorphic-fetch": "^3.0.0",
-    "lodash": "4.*.*",
+    "lodash": "^4.17.21",
-    "ts-matches": "^5.4.1"
+    "ts-matches": "^5.4.1",
+    "yaml": "^2.2.2",
+    "@iarna/toml": "^2.2.5"
   },
   "prettier": {
     "trailingComma": "all",
@@ -41,7 +43,6 @@
     "singleQuote": false
   },
   "devDependencies": {
-    "@iarna/toml": "^2.2.5",
     "@types/jest": "^29.4.0",
     "@types/lodash": "^4.17.5",
     "jest": "^29.4.3",
@@ -49,7 +50,6 @@
     "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1",
     "tsx": "^4.7.1",
-    "typescript": "^5.0.4",
+    "typescript": "^5.0.4"
-    "yaml": "^2.2.2"
   }
 }

web/projects/ui/src/app/services/api/api.fixures.ts

@@ -69,7 +69,13 @@ export module Mock {
     osVersion: '0.2.12',
     dependencies: {},
     hasConfig: true,
-    images: ['main'],
+    images: {
+      main: {
+        source: 'packed',
+        arch: ['x86_64', 'aarch64'],
+        emulateMissingAs: 'aarch64',
+      },
+    },
     assets: [],
     volumes: ['main'],
     hardwareRequirements: {
@@ -116,7 +122,13 @@ export module Mock {
       },
     },
     hasConfig: true,
-    images: ['main'],
+    images: {
+      main: {
+        source: 'packed',
+        arch: ['x86_64', 'aarch64'],
+        emulateMissingAs: 'aarch64',
+      },
+    },
     assets: [],
     volumes: ['main'],
     hardwareRequirements: {
@@ -157,7 +169,13 @@ export module Mock {
       },
     },
     hasConfig: false,
-    images: ['main'],
+    images: {
+      main: {
+        source: 'packed',
+        arch: ['x86_64', 'aarch64'],
+        emulateMissingAs: 'aarch64',
+      },
+    },
     assets: [],
     volumes: ['main'],
     hardwareRequirements: {