feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

container-runtime/deb-install.sh

@@ -6,7 +6,7 @@ mkdir -p /run/systemd/resolve
 echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf
 
 apt-get update
-apt-get install -y curl rsync
+apt-get install -y curl rsync qemu-user-static
 
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
 source ~/.bashrc

container-runtime/package.json

@@ -21,6 +21,7 @@
     "esbuild-plugin-resolve": "^2.0.0",
     "filebrowser": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
+    "lodash": "^4.17.21",
     "node-fetch": "^3.1.0",
     "ts-matches": "^5.5.1",
     "tslib": "^2.5.3",

container-runtime/src/Adapters/HostSystemStartOs.ts

@@ -32,6 +32,8 @@ type RpcError = typeof matchRpcError._TYPE
 const SOCKET_PATH = "/media/startos/rpc/host.sock"
 const MAIN = "/main" as const
 export class HostSystemStartOs implements Effects {
+  procedureId: string | null = null
+
   static of(callbackHolder: CallbackHolder) {
     return new HostSystemStartOs(callbackHolder)
   }
@@ -40,7 +42,7 @@ export class HostSystemStartOs implements Effects {
   id = 0
   rpcRound<K extends keyof Effects | "getStore" | "setStore">(
     method: K,
-    params: unknown,
+    params: Record<string, unknown>,
   ) {
     const id = this.id++
     const client = net.createConnection({ path: SOCKET_PATH }, () => {
@@ -48,7 +50,7 @@ export class HostSystemStartOs implements Effects {
         JSON.stringify({
           id,
           method,
-          params,
+          params: { ...params, procedureId: this.procedureId },
         }) + "\n",
       )
     })
@@ -102,14 +104,14 @@ export class HostSystemStartOs implements Effects {
     }) as ReturnType<T.Effects["bind"]>
   }
   clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
-    return this.rpcRound("clearBindings", null) as ReturnType<
+    return this.rpcRound("clearBindings", {}) as ReturnType<
       T.Effects["clearBindings"]
     >
   }
   clearServiceInterfaces(
     ...[]: Parameters<T.Effects["clearServiceInterfaces"]>
   ) {
-    return this.rpcRound("clearServiceInterfaces", null) as ReturnType<
+    return this.rpcRound("clearServiceInterfaces", {}) as ReturnType<
       T.Effects["clearServiceInterfaces"]
     >
   }
@@ -145,18 +147,20 @@ export class HostSystemStartOs implements Effects {
       T.Effects["exportServiceInterface"]
     >
   }
-  exposeForDependents(...[options]: any) {
-    return this.rpcRound("exposeForDependents", null) as ReturnType<
+  exposeForDependents(
+    ...[options]: Parameters<T.Effects["exposeForDependents"]>
+  ) {
+    return this.rpcRound("exposeForDependents", options) as ReturnType<
       T.Effects["exposeForDependents"]
     >
   }
   getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
-    return this.rpcRound("getConfigured", null) as ReturnType<
+    return this.rpcRound("getConfigured", {}) as ReturnType<
       T.Effects["getConfigured"]
     >
   }
   getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
-    return this.rpcRound("getContainerIp", null) as ReturnType<
+    return this.rpcRound("getContainerIp", {}) as ReturnType<
       T.Effects["getContainerIp"]
     >
   }
@@ -229,7 +233,7 @@ export class HostSystemStartOs implements Effects {
     >
   }
   restart(...[]: Parameters<T.Effects["restart"]>) {
-    return this.rpcRound("restart", null)
+    return this.rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
   }
   running(...[packageId]: Parameters<T.Effects["running"]>) {
     return this.rpcRound("running", { packageId }) as ReturnType<
@@ -262,7 +266,7 @@ export class HostSystemStartOs implements Effects {
     >
   }
   getDependencies(): ReturnType<T.Effects["getDependencies"]> {
-    return this.rpcRound("getDependencies", null) as ReturnType<
+    return this.rpcRound("getDependencies", {}) as ReturnType<
       T.Effects["getDependencies"]
     >
   }
@@ -279,7 +283,7 @@ export class HostSystemStartOs implements Effects {
   }
 
   shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
-    return this.rpcRound("shutdown", null)
+    return this.rpcRound("shutdown", {}) as ReturnType<T.Effects["shutdown"]>
   }
   stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
     return this.rpcRound("stopped", { packageId }) as ReturnType<

container-runtime/src/Adapters/RpcListener.ts

@@ -58,6 +58,7 @@ const runType = object({
   method: literal("execute"),
   params: object(
     {
+      id: string,
       procedure: string,
       input: any,
       timeout: number,
@@ -70,6 +71,7 @@ const sandboxRunType = object({
   method: literal("sandbox"),
   params: object(
     {
+      id: string,
       procedure: string,
       input: any,
       timeout: number,
@@ -195,6 +197,7 @@ export class RpcListener {
         const procedure = jsonPath.unsafeCast(params.procedure)
         return system
           .execute(this.effects, {
+            id: params.id,
             procedure,
             input: params.input,
             timeout: params.timeout,

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -49,7 +49,7 @@ function todo(): never {
 const execFile = promisify(childProcess.execFile)
 
 const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json"
-const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
+export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
 const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig"
 
 const matchSetResult = object(
@@ -199,11 +199,14 @@ export class SystemForEmbassy implements System {
   async execute(
     effects: HostSystemStartOs,
     options: {
+      id: string
       procedure: JsonPath
       input: unknown
       timeout?: number | undefined
     },
   ): Promise<RpcResult> {
+    effects = Object.create(effects)
+    effects.procedureId = options.id
     return this._execute(effects, options)
       .then((x) =>
         matches(x)
@@ -724,7 +727,7 @@ export class SystemForEmbassy implements System {
   private async properties(
     effects: HostSystemStartOs,
     timeoutMs: number | null,
-  ): Promise<ReturnType<T.ExpectedExports.Properties>> {
+  ): Promise<ReturnType<T.ExpectedExports.properties>> {
     // TODO BLU-J set the properties ever so often
     const setConfigValue = this.manifest.properties
     if (!setConfigValue) throw new Error("There is no properties")

container-runtime/src/Adapters/Systems/SystemForStartOs.ts

@@ -1,20 +1,23 @@
 import { ExecuteResult, System } from "../../Interfaces/System"
 import { unNestPath } from "../../Models/JsonPath"
-import { string } from "ts-matches"
+import matches, { any, number, object, string, tuple } from "ts-matches"
 import { HostSystemStartOs } from "../HostSystemStartOs"
 import { Effects } from "../../Models/Effects"
-import { RpcResult } from "../RpcListener"
+import { RpcResult, matchRpcResult } from "../RpcListener"
 import { duration } from "../../Models/Duration"
-const LOCATION = "/usr/lib/startos/package/startos"
+import { T } from "@start9labs/start-sdk"
+import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
+export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js"
 export class SystemForStartOs implements System {
   private onTerm: (() => Promise<void>) | undefined
   static of() {
-    return new SystemForStartOs()
+    return new SystemForStartOs(require(STARTOS_JS_LOCATION))
   }
-  constructor() {}
+  constructor(readonly abi: T.ABI) {}
   async execute(
     effects: HostSystemStartOs,
     options: {
+      id: string
       procedure:
         | "/init"
         | "/uninit"
@@ -33,7 +36,61 @@ export class SystemForStartOs implements System {
       timeout?: number | undefined
     },
   ): Promise<RpcResult> {
-    return { result: await this._execute(effects, options) }
+    effects = Object.create(effects)
+    effects.procedureId = options.id
+    return this._execute(effects, options)
+      .then((x) =>
+        matches(x)
+          .when(
+            object({
+              result: any,
+            }),
+            (x) => x,
+          )
+          .when(
+            object({
+              error: string,
+            }),
+            (x) => ({
+              error: {
+                code: 0,
+                message: x.error,
+              },
+            }),
+          )
+          .when(
+            object({
+              "error-code": tuple(number, string),
+            }),
+            ({ "error-code": [code, message] }) => ({
+              error: {
+                code,
+                message,
+              },
+            }),
+          )
+          .defaultTo({ result: x }),
+      )
+      .catch((error: unknown) => {
+        if (error instanceof Error)
+          return {
+            error: {
+              code: 0,
+              message: error.name,
+              data: {
+                details: error.message,
+                debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`,
+              },
+            },
+          }
+        if (matchRpcResult.test(error)) return error
+        return {
+          error: {
+            code: 0,
+            message: String(error),
+          },
+        }
+      })
   }
   async _execute(
     effects: Effects,
@@ -58,26 +115,27 @@ export class SystemForStartOs implements System {
   ): Promise<unknown> {
     switch (options.procedure) {
       case "/init": {
-        const path = `${LOCATION}/procedures/init`
-        const procedure: any = await import(path).catch(() => require(path))
-        const previousVersion = string.optional().unsafeCast(options)
-        return procedure.init({ effects, previousVersion })
+        const previousVersion =
+          string.optional().unsafeCast(options.input) || null
+        return this.abi.init({ effects, previousVersion })
       }
       case "/uninit": {
-        const path = `${LOCATION}/procedures/init`
-        const procedure: any = await import(path).catch(() => require(path))
-        const nextVersion = string.optional().unsafeCast(options)
-        return procedure.uninit({ effects, nextVersion })
+        const nextVersion = string.optional().unsafeCast(options.input) || null
+        return this.abi.uninit({ effects, nextVersion })
       }
       case "/main/start": {
-        const path = `${LOCATION}/procedures/main`
-        const procedure: any = await import(path).catch(() => require(path))
         const started = async (onTerm: () => Promise<void>) => {
           await effects.setMainStatus({ status: "running" })
           if (this.onTerm) await this.onTerm()
           this.onTerm = onTerm
         }
-        return procedure.main({ effects, started })
+        const daemons = await (
+          await this.abi.main({
+            effects: { ...effects, _type: "main" },
+            started,
+          })
+        ).build()
+        this.onTerm = daemons.term
       }
       case "/main/stop": {
         await effects.setMainStatus({ status: "stopped" })
@@ -86,67 +144,50 @@ export class SystemForStartOs implements System {
         return duration(30, "s")
       }
       case "/config/set": {
-        const path = `${LOCATION}/procedures/config`
-        const procedure: any = await import(path).catch(() => require(path))
-        const input = options.input
-        return procedure.setConfig({ effects, input })
+        const input = options.input as any // TODO
+        return this.abi.setConfig({ effects, input })
       }
       case "/config/get": {
-        const path = `${LOCATION}/procedures/config`
-        const procedure: any = await import(path).catch(() => require(path))
-        return procedure.getConfig({ effects })
+        return this.abi.getConfig({ effects })
       }
       case "/backup/create":
       case "/backup/restore":
         throw new Error("this should be called with the init/unit")
       case "/actions/metadata": {
-        const path = `${LOCATION}/procedures/actions`
-        const procedure: any = await import(path).catch(() => require(path))
-        return procedure.actionsMetadata({ effects })
+        return this.abi.actionsMetadata({ effects })
       }
       default:
         const procedures = unNestPath(options.procedure)
         const id = procedures[2]
         switch (true) {
           case procedures[1] === "actions" && procedures[3] === "get": {
-            const path = `${LOCATION}/procedures/actions`
-            const action: any = (await import(path).catch(() => require(path)))
-              .actions[id]
+            const action = (await this.abi.actions({ effects }))[id]
             if (!action) throw new Error(`Action ${id} not found`)
-            return action.get({ effects })
+            return action.getConfig({ effects })
           }
           case procedures[1] === "actions" && procedures[3] === "run": {
-            const path = `${LOCATION}/procedures/actions`
-            const action: any = (await import(path).catch(() => require(path)))
-              .actions[id]
+            const action = (await this.abi.actions({ effects }))[id]
             if (!action) throw new Error(`Action ${id} not found`)
-            const input = options.input
-            return action.run({ effects, input })
+            return action.run({ effects, input: options.input as any }) // TODO
           }
           case procedures[1] === "dependencies" && procedures[3] === "query": {
-            const path = `${LOCATION}/procedures/dependencies`
-            const dependencyConfig: any = (
-              await import(path).catch(() => require(path))
-            ).dependencyConfig[id]
+            const dependencyConfig = this.abi.dependencyConfig[id]
             if (!dependencyConfig)
               throw new Error(`dependencyConfig ${id} not found`)
             const localConfig = options.input
-            return dependencyConfig.query({ effects, localConfig })
+            return dependencyConfig.query({ effects })
           }
           case procedures[1] === "dependencies" && procedures[3] === "update": {
-            const path = `${LOCATION}/procedures/dependencies`
-            const dependencyConfig: any = (
-              await import(path).catch(() => require(path))
-            ).dependencyConfig[id]
+            const dependencyConfig = this.abi.dependencyConfig[id]
             if (!dependencyConfig)
               throw new Error(`dependencyConfig ${id} not found`)
-            return dependencyConfig.update(options.input)
+            return dependencyConfig.update(options.input as any) // TODO
           }
         }
     }
-    throw new Error("Method not implemented.")
+    throw new Error(`Method ${options.procedure} not implemented.`)
   }
-  exit(effects: Effects): Promise<void> {
-    throw new Error("Method not implemented.")
+  async exit(effects: Effects): Promise<void> {
+    return void null
   }
 }

container-runtime/src/Adapters/Systems/index.ts

@@ -1,6 +1,22 @@
+import * as fs from "node:fs/promises"
 import { System } from "../../Interfaces/System"
-import { SystemForEmbassy } from "./SystemForEmbassy"
-import { SystemForStartOs } from "./SystemForStartOs"
+import { EMBASSY_JS_LOCATION, SystemForEmbassy } from "./SystemForEmbassy"
+import { STARTOS_JS_LOCATION, SystemForStartOs } from "./SystemForStartOs"
 export async function getSystem(): Promise<System> {
-  return SystemForEmbassy.of()
+  if (
+    await fs.access(STARTOS_JS_LOCATION).then(
+      () => true,
+      () => false,
+    )
+  ) {
+    return SystemForStartOs.of()
+  } else if (
+    await fs.access(EMBASSY_JS_LOCATION).then(
+      () => true,
+      () => false,
+    )
+  ) {
+    return SystemForEmbassy.of()
+  }
+  throw new Error(`${STARTOS_JS_LOCATION} not found`)
 }

container-runtime/src/Interfaces/System.ts

@@ -14,6 +14,7 @@ export interface System {
   execute(
     effects: T.Effects,
     options: {
+      id: string
       procedure: JsonPath
       input: unknown
       timeout?: number