feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

sdk/lib/util/Overlay.ts

@@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
 export class Overlay {
   private constructor(
     readonly effects: T.Effects,
-    readonly imageId: string,
+    readonly imageId: T.ImageId,
     readonly rootfs: string,
-    readonly guid: string,
+    readonly guid: T.Guid,
   ) {}
   static async of(
     effects: T.Effects,
-    image: { id: string; sharedRun?: boolean },
+    image: { id: T.ImageId; sharedRun?: boolean },
   ) {
-    const { id: imageId, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
+    const { id, sharedRun } = image
+    const [rootfs, guid] = await effects.createOverlayedImage({
+      imageId: id as string,
+    })
 
     const shared = ["dev", "sys", "proc"]
     if (!!sharedRun) {
@@ -33,7 +35,7 @@ export class Overlay {
       ])
     }
 
-    return new Overlay(effects, imageId, rootfs, guid)
+    return new Overlay(effects, id, rootfs, guid)
   }
 
   async mount(options: MountOptions, path: string): Promise<Overlay> {
@@ -97,7 +99,7 @@ export class Overlay {
     stdout: string | Buffer
     stderr: string | Buffer
   }> {
-    const imageMeta: any = await fs
+    const imageMeta: T.ImageMetadata = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })

sdk/lib/util/fileHelper.ts

@@ -3,7 +3,7 @@ import * as YAML from "yaml"
 import * as TOML from "@iarna/toml"
 import _ from "lodash"
 import * as T from "../types"
-import * as fs from "fs"
+import * as fs from "node:fs/promises"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
@@ -59,28 +59,24 @@ export class FileHelper<A> {
     readonly readData: (stringValue: string) => A,
   ) {}
   async write(data: A, effects: T.Effects) {
-    if (previousPath.exec(this.path)) {
-      await new Promise((resolve, reject) =>
-        fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
-      )
+    const parent = previousPath.exec(this.path)
+    if (parent) {
+      await fs.mkdir(parent[1], { recursive: true })
     }
 
-    await new Promise((resolve, reject) =>
-      fs.writeFile(this.path, this.writeData(data), (err: any) =>
-        !err ? resolve(null) : reject(err),
-      ),
-    )
+    await fs.writeFile(this.path, this.writeData(data))
   }
   async read(effects: T.Effects) {
-    if (!fs.existsSync(this.path)) {
+    if (
+      !(await fs.access(this.path).then(
+        () => true,
+        () => false,
+      ))
+    ) {
       return null
     }
     return this.readData(
-      await new Promise((resolve, reject) =>
-        fs.readFile(this.path, (err: any, data: any) =>
-          !err ? resolve(data.toString("utf-8")) : reject(err),
-        ),
-      ),
+      await fs.readFile(this.path).then((data) => data.toString("utf-8")),
     )
   }
 
@@ -142,7 +138,7 @@ export class FileHelper<A> {
     return new FileHelper<A>(
       path,
       (inData) => {
-        return JSON.stringify(inData, null, 2)
+        return YAML.stringify(inData, null, 2)
       },
       (inString) => {
         return shape.unsafeCast(YAML.parse(inString))