feature: pack s9pk (#2642)

* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

sdk/lib/StartSdk.ts

@@ -187,7 +187,10 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       nullIfEmpty,
       runCommand: async <A extends string>(
         effects: Effects,
-        image: { id: Manifest["images"][number]; sharedRun?: boolean },
+        image: {
+          id: keyof Manifest["images"] & T.ImageId
+          sharedRun?: boolean
+        },
         command: ValidIfNoStupidEscape<A> | [string, ...string[]],
         options: CommandOptions & {
           mounts?: { path: string; options: MountOptions }[]
@@ -396,7 +399,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       setupProperties:
         (
           fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
-        ): T.ExpectedExports.Properties =>
+        ): T.ExpectedExports.properties =>
         (options) =>
           fn(options).then(nullifyProperties),
       setupUninstall: (fn: UninstallFn<Manifest, Store>) =>
@@ -743,7 +746,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
 
 export async function runCommand<Manifest extends SDKManifest>(
   effects: Effects,
-  image: { id: Manifest["images"][number]; sharedRun?: boolean },
+  image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean },
   command: string | [string, ...string[]],
   options: CommandOptions & {
     mounts?: { path: string; options: MountOptions }[]

sdk/lib/health/HealthCheck.ts

@@ -8,12 +8,13 @@ import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
 import { Overlay } from "../util/Overlay"
 import { object, unknown } from "ts-matches"
+import { T } from ".."
 
 export type HealthCheckParams<Manifest extends SDKManifest> = {
   effects: Effects
   name: string
   image: {
-    id: Manifest["images"][number]
+    id: keyof Manifest["images"] & T.ImageId
     sharedRun?: boolean
   }
   trigger?: Trigger

sdk/lib/interfaces/Host.ts

@@ -69,12 +69,12 @@ type NotProtocolsWithSslVariants = Exclude<
 type BindOptionsByKnownProtocol =
   | {
       protocol: ProtocolsWithSslVariants
-      preferredExternalPort: number
+      preferredExternalPort?: number
       addSsl?: Partial<AddSslOptions>
     }
   | {
       protocol: NotProtocolsWithSslVariants
-      preferredExternalPort: number
+      preferredExternalPort?: number
       addSsl?: AddSslOptions
     }
 export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions

sdk/lib/mainFn/CommandController.ts

@@ -1,6 +1,6 @@
 import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
 import { cpExecFile, cpExec } from "./Daemons"
@@ -15,7 +15,7 @@ export class CommandController {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemon.ts

@@ -1,5 +1,5 @@
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { CommandController } from "./CommandController"
 
@@ -18,7 +18,7 @@ export class Daemon {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemons.ts

@@ -5,7 +5,12 @@ import { SDKManifest } from "../manifest/ManifestTypes"
 import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
-import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types"
+import {
+  DaemonReturned,
+  Effects,
+  ImageId,
+  ValidIfNoStupidEscape,
+} from "../types"
 import { Mounts } from "./Mounts"
 import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
@@ -34,8 +39,8 @@ type DaemonsParams<
   Id extends string,
 > = {
   command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
-  image: { id: Manifest["images"][number]; sharedRun?: boolean }
-  mounts: { path: string; options: MountOptions }[]
+  image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean }
+  mounts: Mounts<Manifest>
   env?: Record<string, string>
   ready: Ready
   requires: Exclude<Ids, Id>[]
@@ -116,12 +121,10 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
     options: DaemonsParams<Manifest, Ids, Command, Id>,
   ) {
     const daemonIndex = this.daemons.length
-    const daemon = Daemon.of()(
-      this.effects,
-      options.image,
-      options.command,
-      options,
-    )
+    const daemon = Daemon.of()(this.effects, options.image, options.command, {
+      ...options,
+      mounts: options.mounts.build(),
+    })
     const healthDaemon = new HealthDaemon(
       daemon,
       daemonIndex,

sdk/lib/manifest/ManifestTypes.ts

@@ -1,5 +1,5 @@
 import { ValidEmVer } from "../emverLite/mod"
-import { ActionMetadata } from "../types"
+import { ActionMetadata, ImageConfig, ImageId } from "../types"
 
 export interface Container {
   /** This should be pointing to a docker container name */
@@ -28,8 +28,6 @@ export type SDKManifest = {
   readonly releaseNotes: string
   /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
   readonly license: string // name of license
-  /** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */
-  readonly replaces: Readonly<string[]>
   /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),
    * any scripts necessary for configuration, backups, actions, or health checks (more below). This key
    * must exist. But could be embedded into the source repository
@@ -52,7 +50,7 @@ export type SDKManifest = {
   }
 
   /** Defines the os images needed to run the container processes */
-  readonly images: string[]
+  readonly images: Record<ImageId, ImageConfig>
   /** This denotes readonly asset directories that should be available to mount to the container.
    * Assuming that there will be three files with names along the lines:
    * icon.* : the icon that will be this packages icon on the ui

sdk/lib/manifest/setupManifest.ts

@@ -1,18 +1,19 @@
+import { ImageConfig, ImageId, VolumeId } from "../osBindings"
 import { SDKManifest, ManifestVersion } from "./ManifestTypes"
 
 export function setupManifest<
   Id extends string,
   Version extends ManifestVersion,
   Dependencies extends Record<string, unknown>,
-  VolumesTypes extends string,
-  AssetTypes extends string,
-  ImagesTypes extends string,
+  VolumesTypes extends VolumeId,
+  AssetTypes extends VolumeId,
+  ImagesTypes extends ImageId,
   Manifest extends SDKManifest & {
     dependencies: Dependencies
     id: Id
     version: Version
     assets: AssetTypes[]
-    images: ImagesTypes[]
+    images: Record<ImagesTypes, ImageConfig>
     volumes: VolumesTypes[]
   },
 >(manifest: Manifest): Manifest {

sdk/lib/osBindings/CreateOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageId } from "./ImageId"
 
-export type CreateOverlayedImageParams = { imageId: string }
+export type CreateOverlayedImageParams = { imageId: ImageId }

sdk/lib/osBindings/DestroyOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
-export type DestroyOverlayedImageParams = { guid: string }
+export type DestroyOverlayedImageParams = { guid: Guid }

sdk/lib/osBindings/ImageConfig.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageSource } from "./ImageSource"
+
+export type ImageConfig = {
+  source: ImageSource
+  arch: string[]
+  emulateMissingAs: string | null
+}

sdk/lib/osBindings/ImageMetadata.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ImageMetadata = { workdir: string; user: string }

sdk/lib/osBindings/ImageSource.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ImageSource =
+  | "packed"
+  | { dockerBuild: { workdir: string | null; dockerfile: string | null } }
+  | { dockerTag: string }

sdk/lib/osBindings/LoginParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { PasswordType } from "./PasswordType"
+
+export type LoginParams = { password: PasswordType | null; metadata: any }

sdk/lib/osBindings/Manifest.ts

@@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts"
 import type { Dependencies } from "./Dependencies"
 import type { Description } from "./Description"
 import type { HardwareRequirements } from "./HardwareRequirements"
+import type { ImageConfig } from "./ImageConfig"
 import type { ImageId } from "./ImageId"
 import type { PackageId } from "./PackageId"
 import type { Version } from "./Version"
@@ -20,7 +21,7 @@ export type Manifest = {
   marketingSite: string
   donationUrl: string | null
   description: Description
-  images: Array<ImageId>
+  images: { [key: ImageId]: ImageConfig }
   assets: Array<VolumeId>
   volumes: Array<VolumeId>
   alerts: Alerts

sdk/lib/osBindings/index.ts

@@ -69,7 +69,10 @@ export { HostKind } from "./HostKind"
 export { HostnameInfo } from "./HostnameInfo"
 export { Hosts } from "./Hosts"
 export { Host } from "./Host"
+export { ImageConfig } from "./ImageConfig"
 export { ImageId } from "./ImageId"
+export { ImageMetadata } from "./ImageMetadata"
+export { ImageSource } from "./ImageSource"
 export { InstalledState } from "./InstalledState"
 export { InstallingInfo } from "./InstallingInfo"
 export { InstallingState } from "./InstallingState"
@@ -78,6 +81,7 @@ export { IpInfo } from "./IpInfo"
 export { LanInfo } from "./LanInfo"
 export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams"
 export { ListVersionSignersParams } from "./ListVersionSignersParams"
+export { LoginParams } from "./LoginParams"
 export { MainStatus } from "./MainStatus"
 export { Manifest } from "./Manifest"
 export { MaybeUtf8String } from "./MaybeUtf8String"

sdk/lib/test/configBuilder.test.ts

@@ -400,7 +400,7 @@ describe("values", () => {
               long: "",
             },
             containers: {},
-            images: [],
+            images: {},
             volumes: [],
             assets: [],
             alerts: {

sdk/lib/test/output.sdk.ts

@@ -21,7 +21,7 @@ export const sdk = StartSdk.of()
         long: "",
       },
       containers: {},
-      images: [],
+      images: {},
       volumes: [],
       assets: [],
       alerts: {

sdk/lib/types.ts

@@ -11,6 +11,7 @@ import {
   GetPrimaryUrlParams,
   LanInfo,
   BindParams,
+  Manifest,
 } from "./osBindings"
 
 import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
@@ -110,9 +111,26 @@ export namespace ExpectedExports {
    */
   export type dependencyConfig = Record<PackageId, DependencyConfig | null>
 
-  export type Properties = (options: {
+  export type properties = (options: {
     effects: Effects
   }) => Promise<PropertiesReturn>
+
+  export type manifest = Manifest
+}
+export type ABI = {
+  setConfig: ExpectedExports.setConfig
+  getConfig: ExpectedExports.getConfig
+  createBackup: ExpectedExports.createBackup
+  restoreBackup: ExpectedExports.restoreBackup
+  actions: ExpectedExports.actions
+  actionsMetadata: ExpectedExports.actionsMetadata
+  main: ExpectedExports.main
+  afterShutdown: ExpectedExports.afterShutdown
+  init: ExpectedExports.init
+  uninit: ExpectedExports.uninit
+  dependencyConfig: ExpectedExports.dependencyConfig
+  properties: ExpectedExports.properties
+  manifest: ExpectedExports.manifest
 }
 export type TimeMs = number
 export type VersionString = string
@@ -453,8 +471,8 @@ export type Effects = {
   /** Exists could be useful during the runtime to know if some service is running, option dep */
   running(options: { packageId: PackageId }): Promise<boolean>
 
-  restart(): void
-  shutdown(): void
+  restart(): Promise<void>
+  shutdown(): Promise<void>
 
   mount(options: {
     location: string

sdk/lib/util/Overlay.ts

@@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
 export class Overlay {
   private constructor(
     readonly effects: T.Effects,
-    readonly imageId: string,
+    readonly imageId: T.ImageId,
     readonly rootfs: string,
-    readonly guid: string,
+    readonly guid: T.Guid,
   ) {}
   static async of(
     effects: T.Effects,
-    image: { id: string; sharedRun?: boolean },
+    image: { id: T.ImageId; sharedRun?: boolean },
   ) {
-    const { id: imageId, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
+    const { id, sharedRun } = image
+    const [rootfs, guid] = await effects.createOverlayedImage({
+      imageId: id as string,
+    })
 
     const shared = ["dev", "sys", "proc"]
     if (!!sharedRun) {
@@ -33,7 +35,7 @@ export class Overlay {
       ])
     }
 
-    return new Overlay(effects, imageId, rootfs, guid)
+    return new Overlay(effects, id, rootfs, guid)
   }
 
   async mount(options: MountOptions, path: string): Promise<Overlay> {
@@ -97,7 +99,7 @@ export class Overlay {
     stdout: string | Buffer
     stderr: string | Buffer
   }> {
-    const imageMeta: any = await fs
+    const imageMeta: T.ImageMetadata = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })

sdk/lib/util/fileHelper.ts

@@ -3,7 +3,7 @@ import * as YAML from "yaml"
 import * as TOML from "@iarna/toml"
 import _ from "lodash"
 import * as T from "../types"
-import * as fs from "fs"
+import * as fs from "node:fs/promises"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
@@ -59,28 +59,24 @@ export class FileHelper<A> {
     readonly readData: (stringValue: string) => A,
   ) {}
   async write(data: A, effects: T.Effects) {
-    if (previousPath.exec(this.path)) {
-      await new Promise((resolve, reject) =>
-        fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
-      )
+    const parent = previousPath.exec(this.path)
+    if (parent) {
+      await fs.mkdir(parent[1], { recursive: true })
     }
 
-    await new Promise((resolve, reject) =>
-      fs.writeFile(this.path, this.writeData(data), (err: any) =>
-        !err ? resolve(null) : reject(err),
-      ),
-    )
+    await fs.writeFile(this.path, this.writeData(data))
   }
   async read(effects: T.Effects) {
-    if (!fs.existsSync(this.path)) {
+    if (
+      !(await fs.access(this.path).then(
+        () => true,
+        () => false,
+      ))
+    ) {
       return null
     }
     return this.readData(
-      await new Promise((resolve, reject) =>
-        fs.readFile(this.path, (err: any, data: any) =>
-          !err ? resolve(data.toString("utf-8")) : reject(err),
-        ),
-      ),
+      await fs.readFile(this.path).then((data) => data.toString("utf-8")),
     )
   }
 
@@ -142,7 +138,7 @@ export class FileHelper<A> {
     return new FileHelper<A>(
       path,
       (inData) => {
-        return JSON.stringify(inData, null, 2)
+        return YAML.stringify(inData, null, 2)
       },
       (inString) => {
         return shape.unsafeCast(YAML.parse(inString))

sdk/package-lock.json

@@ -10,7 +10,7 @@
       "license": "MIT",
       "dependencies": {
         "isomorphic-fetch": "^3.0.0",
-        "lodash": "4.*.*",
+        "lodash": "^4.17.21",
         "ts-matches": "^5.4.1"
       },
       "devDependencies": {

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.3.6-alpha1",
+  "version": "0.3.6-alpha5",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./cjs/lib/index.js",
   "types": "./cjs/lib/index.d.ts",
@@ -31,8 +31,10 @@
   "homepage": "https://github.com/Start9Labs/start-sdk#readme",
   "dependencies": {
     "isomorphic-fetch": "^3.0.0",
-    "lodash": "4.*.*",
-    "ts-matches": "^5.4.1"
+    "lodash": "^4.17.21",
+    "ts-matches": "^5.4.1",
+    "yaml": "^2.2.2",
+    "@iarna/toml": "^2.2.5"
   },
   "prettier": {
     "trailingComma": "all",
@@ -41,7 +43,6 @@
     "singleQuote": false
   },
   "devDependencies": {
-    "@iarna/toml": "^2.2.5",
     "@types/jest": "^29.4.0",
     "@types/lodash": "^4.17.5",
     "jest": "^29.4.3",
@@ -49,7 +50,6 @@
     "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1",
     "tsx": "^4.7.1",
-    "typescript": "^5.0.4",
-    "yaml": "^2.2.2"
+    "typescript": "^5.0.4"
   }
 }