Feature/subcontainers (#2720)

* wip: subcontainers

* wip: subcontainer infra

* rename NonDestroyableOverlay to SubContainerHandle

* chore: Changes to the container and other things

* wip:

* wip: fixes

* fix launch & exec

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* tweak apis

* misc fixes

* don't treat sigterm as error

* handle health check set during starting

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <Blu-J@users.noreply.github.com>

container-runtime/src/Adapters/EffectCreator.ts

@@ -40,7 +40,7 @@ export type EffectContext = {
 
 const rpcRoundFor =
   (procedureId: string | null) =>
-  <K extends keyof Effects | "getStore" | "setStore" | "clearCallbacks">(
+  <K extends T.EffectMethod | "clearCallbacks">(
     method: K,
     params: Record<string, unknown>,
   ) => {
@@ -110,65 +110,65 @@ function makeEffects(context: EffectContext): Effects {
       }) as ReturnType<T.Effects["bind"]>
     },
     clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
-      return rpcRound("clearBindings", {}) as ReturnType<
+      return rpcRound("clear-bindings", {}) as ReturnType<
         T.Effects["clearBindings"]
       >
     },
     clearServiceInterfaces(
       ...[]: Parameters<T.Effects["clearServiceInterfaces"]>
     ) {
-      return rpcRound("clearServiceInterfaces", {}) as ReturnType<
+      return rpcRound("clear-service-interfaces", {}) as ReturnType<
         T.Effects["clearServiceInterfaces"]
       >
     },
     getInstalledPackages(...[]: Parameters<T.Effects["getInstalledPackages"]>) {
-      return rpcRound("getInstalledPackages", {}) as ReturnType<
+      return rpcRound("get-installed-packages", {}) as ReturnType<
         T.Effects["getInstalledPackages"]
       >
     },
-    createOverlayedImage(options: {
-      imageId: string
-    }): Promise<[string, string]> {
-      return rpcRound("createOverlayedImage", options) as ReturnType<
-        T.Effects["createOverlayedImage"]
-      >
-    },
-    destroyOverlayedImage(options: { guid: string }): Promise<void> {
-      return rpcRound("destroyOverlayedImage", options) as ReturnType<
-        T.Effects["destroyOverlayedImage"]
-      >
+    subcontainer: {
+      createFs(options: { imageId: string }) {
+        return rpcRound("subcontainer.create-fs", options) as ReturnType<
+          T.Effects["subcontainer"]["createFs"]
+        >
+      },
+      destroyFs(options: { guid: string }): Promise<void> {
+        return rpcRound("subcontainer.destroy-fs", options) as ReturnType<
+          T.Effects["subcontainer"]["destroyFs"]
+        >
+      },
     },
     executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
-      return rpcRound("executeAction", options) as ReturnType<
+      return rpcRound("execute-action", options) as ReturnType<
         T.Effects["executeAction"]
       >
     },
     exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
-      return rpcRound("exportAction", options) as ReturnType<
+      return rpcRound("export-action", options) as ReturnType<
         T.Effects["exportAction"]
       >
     },
     exportServiceInterface: ((
       ...[options]: Parameters<Effects["exportServiceInterface"]>
     ) => {
-      return rpcRound("exportServiceInterface", options) as ReturnType<
+      return rpcRound("export-service-interface", options) as ReturnType<
         T.Effects["exportServiceInterface"]
       >
     }) as Effects["exportServiceInterface"],
     exposeForDependents(
       ...[options]: Parameters<T.Effects["exposeForDependents"]>
     ) {
-      return rpcRound("exposeForDependents", options) as ReturnType<
+      return rpcRound("expose-for-dependents", options) as ReturnType<
         T.Effects["exposeForDependents"]
       >
     },
     getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
-      return rpcRound("getConfigured", {}) as ReturnType<
+      return rpcRound("get-configured", {}) as ReturnType<
         T.Effects["getConfigured"]
       >
     },
     getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
-      return rpcRound("getContainerIp", {}) as ReturnType<
+      return rpcRound("get-container-ip", {}) as ReturnType<
         T.Effects["getContainerIp"]
       >
     },
@@ -177,21 +177,21 @@ function makeEffects(context: EffectContext): Effects {
         ...allOptions,
         callback: context.callbacks?.addCallback(allOptions.callback) || null,
       }
-      return rpcRound("getHostInfo", options) as ReturnType<
+      return rpcRound("get-host-info", options) as ReturnType<
         T.Effects["getHostInfo"]
       > as any
     }) as Effects["getHostInfo"],
     getServiceInterface(
       ...[options]: Parameters<T.Effects["getServiceInterface"]>
     ) {
-      return rpcRound("getServiceInterface", {
+      return rpcRound("get-service-interface", {
         ...options,
         callback: context.callbacks?.addCallback(options.callback) || null,
       }) as ReturnType<T.Effects["getServiceInterface"]>
     },
 
     getPrimaryUrl(...[options]: Parameters<T.Effects["getPrimaryUrl"]>) {
-      return rpcRound("getPrimaryUrl", {
+      return rpcRound("get-primary-url", {
         ...options,
         callback: context.callbacks?.addCallback(options.callback) || null,
       }) as ReturnType<T.Effects["getPrimaryUrl"]>
@@ -199,22 +199,22 @@ function makeEffects(context: EffectContext): Effects {
     getServicePortForward(
       ...[options]: Parameters<T.Effects["getServicePortForward"]>
     ) {
-      return rpcRound("getServicePortForward", options) as ReturnType<
+      return rpcRound("get-service-port-forward", options) as ReturnType<
         T.Effects["getServicePortForward"]
       >
     },
     getSslCertificate(options: Parameters<T.Effects["getSslCertificate"]>[0]) {
-      return rpcRound("getSslCertificate", options) as ReturnType<
+      return rpcRound("get-ssl-certificate", options) as ReturnType<
         T.Effects["getSslCertificate"]
       >
     },
     getSslKey(options: Parameters<T.Effects["getSslKey"]>[0]) {
-      return rpcRound("getSslKey", options) as ReturnType<
+      return rpcRound("get-ssl-key", options) as ReturnType<
         T.Effects["getSslKey"]
       >
     },
     getSystemSmtp(...[options]: Parameters<T.Effects["getSystemSmtp"]>) {
-      return rpcRound("getSystemSmtp", {
+      return rpcRound("get-system-smtp", {
         ...options,
         callback: context.callbacks?.addCallback(options.callback) || null,
       }) as ReturnType<T.Effects["getSystemSmtp"]>
@@ -222,7 +222,7 @@ function makeEffects(context: EffectContext): Effects {
     listServiceInterfaces(
       ...[options]: Parameters<T.Effects["listServiceInterfaces"]>
     ) {
-      return rpcRound("listServiceInterfaces", {
+      return rpcRound("list-service-interfaces", {
         ...options,
         callback: context.callbacks?.addCallback(options.callback) || null,
       }) as ReturnType<T.Effects["listServiceInterfaces"]>
@@ -231,7 +231,7 @@ function makeEffects(context: EffectContext): Effects {
       return rpcRound("mount", options) as ReturnType<T.Effects["mount"]>
     },
     clearActions(...[]: Parameters<T.Effects["clearActions"]>) {
-      return rpcRound("clearActions", {}) as ReturnType<
+      return rpcRound("clear-actions", {}) as ReturnType<
         T.Effects["clearActions"]
       >
     },
@@ -239,37 +239,39 @@ function makeEffects(context: EffectContext): Effects {
       return rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
     },
     setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
-      return rpcRound("setConfigured", { configured }) as ReturnType<
+      return rpcRound("set-configured", { configured }) as ReturnType<
         T.Effects["setConfigured"]
       >
     },
     setDependencies(
       dependencies: Parameters<T.Effects["setDependencies"]>[0],
     ): ReturnType<T.Effects["setDependencies"]> {
-      return rpcRound("setDependencies", dependencies) as ReturnType<
+      return rpcRound("set-dependencies", dependencies) as ReturnType<
         T.Effects["setDependencies"]
       >
     },
     checkDependencies(
       options: Parameters<T.Effects["checkDependencies"]>[0],
     ): ReturnType<T.Effects["checkDependencies"]> {
-      return rpcRound("checkDependencies", options) as ReturnType<
+      return rpcRound("check-dependencies", options) as ReturnType<
         T.Effects["checkDependencies"]
       >
     },
     getDependencies(): ReturnType<T.Effects["getDependencies"]> {
-      return rpcRound("getDependencies", {}) as ReturnType<
+      return rpcRound("get-dependencies", {}) as ReturnType<
         T.Effects["getDependencies"]
       >
     },
     setHealth(...[options]: Parameters<T.Effects["setHealth"]>) {
-      return rpcRound("setHealth", options) as ReturnType<
+      return rpcRound("set-health", options) as ReturnType<
         T.Effects["setHealth"]
       >
     },
 
     setMainStatus(o: { status: "running" | "stopped" }): Promise<void> {
-      return rpcRound("setMainStatus", o) as ReturnType<T.Effects["setHealth"]>
+      return rpcRound("set-main-status", o) as ReturnType<
+        T.Effects["setHealth"]
+      >
     },
 
     shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
@@ -277,20 +279,20 @@ function makeEffects(context: EffectContext): Effects {
     },
     store: {
       get: async (options: any) =>
-        rpcRound("getStore", {
+        rpcRound("store.get", {
           ...options,
           callback: context.callbacks?.addCallback(options.callback) || null,
         }) as any,
       set: async (options: any) =>
-        rpcRound("setStore", options) as ReturnType<T.Effects["store"]["set"]>,
+        rpcRound("store.set", options) as ReturnType<T.Effects["store"]["set"]>,
     } as T.Effects["store"],
     getDataVersion() {
-      return rpcRound("getDataVersion", {}) as ReturnType<
+      return rpcRound("get-data-version", {}) as ReturnType<
         T.Effects["getDataVersion"]
       >
     },
     setDataVersion(...[options]: Parameters<T.Effects["setDataVersion"]>) {
-      return rpcRound("setDataVersion", options) as ReturnType<
+      return rpcRound("set-data-version", options) as ReturnType<
         T.Effects["setDataVersion"]
       >
     },

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -1,56 +1,60 @@
 import * as fs from "fs/promises"
 import * as cp from "child_process"
-import { Overlay, types as T } from "@start9labs/start-sdk"
+import { SubContainer, types as T } from "@start9labs/start-sdk"
 import { promisify } from "util"
 import { DockerProcedure, VolumeId } from "../../../Models/DockerProcedure"
 import { Volume } from "./matchVolume"
-import { ExecSpawnable } from "@start9labs/start-sdk/cjs/lib/util/Overlay"
+import {
+  CommandOptions,
+  ExecOptions,
+  ExecSpawnable,
+} from "@start9labs/start-sdk/cjs/lib/util/SubContainer"
 export const exec = promisify(cp.exec)
 export const execFile = promisify(cp.execFile)
 
 export class DockerProcedureContainer {
-  private constructor(private readonly overlay: ExecSpawnable) {}
+  private constructor(private readonly subcontainer: ExecSpawnable) {}
 
   static async of(
     effects: T.Effects,
     packageId: string,
     data: DockerProcedure,
     volumes: { [id: VolumeId]: Volume },
-    options: { overlay?: ExecSpawnable } = {},
+    options: { subcontainer?: ExecSpawnable } = {},
   ) {
-    const overlay =
-      options?.overlay ??
-      (await DockerProcedureContainer.createOverlay(
+    const subcontainer =
+      options?.subcontainer ??
+      (await DockerProcedureContainer.createSubContainer(
         effects,
         packageId,
         data,
         volumes,
       ))
-    return new DockerProcedureContainer(overlay)
+    return new DockerProcedureContainer(subcontainer)
   }
-  static async createOverlay(
+  static async createSubContainer(
     effects: T.Effects,
     packageId: string,
     data: DockerProcedure,
     volumes: { [id: VolumeId]: Volume },
   ) {
-    const overlay = await Overlay.of(effects, { id: data.image })
+    const subcontainer = await SubContainer.of(effects, { id: data.image })
 
     if (data.mounts) {
       const mounts = data.mounts
       for (const mount in mounts) {
         const path = mounts[mount].startsWith("/")
-          ? `${overlay.rootfs}${mounts[mount]}`
-          : `${overlay.rootfs}/${mounts[mount]}`
+          ? `${subcontainer.rootfs}${mounts[mount]}`
+          : `${subcontainer.rootfs}/${mounts[mount]}`
         await fs.mkdir(path, { recursive: true })
         const volumeMount = volumes[mount]
         if (volumeMount.type === "data") {
-          await overlay.mount(
+          await subcontainer.mount(
             { type: "volume", id: mount, subpath: null, readonly: false },
             mounts[mount],
           )
         } else if (volumeMount.type === "assets") {
-          await overlay.mount(
+          await subcontainer.mount(
             { type: "assets", id: mount, subpath: null },
             mounts[mount],
           )
@@ -96,24 +100,35 @@ export class DockerProcedureContainer {
             })
             .catch(console.warn)
         } else if (volumeMount.type === "backup") {
-          await overlay.mount({ type: "backup", subpath: null }, mounts[mount])
+          await subcontainer.mount(
+            { type: "backup", subpath: null },
+            mounts[mount],
+          )
         }
       }
     }
-    return overlay
+    return subcontainer
   }
 
-  async exec(commands: string[], {} = {}) {
+  async exec(
+    commands: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs?: number | null,
+  ) {
     try {
-      return await this.overlay.exec(commands)
+      return await this.subcontainer.exec(commands, options, timeoutMs)
     } finally {
-      await this.overlay.destroy?.()
+      await this.subcontainer.destroy?.()
     }
   }
 
-  async execFail(commands: string[], timeoutMs: number | null, {} = {}) {
+  async execFail(
+    commands: string[],
+    timeoutMs: number | null,
+    options?: CommandOptions & ExecOptions,
+  ) {
     try {
-      const res = await this.overlay.exec(commands, {}, timeoutMs)
+      const res = await this.subcontainer.exec(commands, options, timeoutMs)
       if (res.exitCode !== 0) {
         const codeOrSignal =
           res.exitCode !== null
@@ -125,11 +140,11 @@ export class DockerProcedureContainer {
       }
       return res
     } finally {
-      await this.overlay.destroy?.()
+      await this.subcontainer.destroy?.()
     }
   }
 
   async spawn(commands: string[]): Promise<cp.ChildProcessWithoutNullStreams> {
-    return await this.overlay.spawn(commands)
+    return await this.subcontainer.spawn(commands)
   }
 }

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -6,6 +6,7 @@ import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon"
 import { Effects } from "../../../Models/Effects"
 import { off } from "node:process"
 import { CommandController } from "@start9labs/start-sdk/cjs/lib/mainFn/CommandController"
+import { asError } from "@start9labs/start-sdk/cjs/lib/util"
 
 const EMBASSY_HEALTH_INTERVAL = 15 * 1000
 const EMBASSY_PROPERTIES_LOOP = 30 * 1000
@@ -15,8 +16,8 @@ const EMBASSY_PROPERTIES_LOOP = 30 * 1000
  * Also, this has an ability to clean itself up too if need be.
  */
 export class MainLoop {
-  get mainOverlay() {
-    return this.mainEvent?.daemon?.overlay
+  get mainSubContainerHandle() {
+    return this.mainEvent?.daemon?.subContainerHandle
   }
   private healthLoops?: {
     name: string
@@ -56,7 +57,7 @@ export class MainLoop {
       throw new Error("Unreachable")
     }
     const daemon = new Daemon(async () => {
-      const overlay = await DockerProcedureContainer.createOverlay(
+      const subcontainer = await DockerProcedureContainer.createSubContainer(
         effects,
         this.system.manifest.id,
         this.system.manifest.main,
@@ -64,11 +65,10 @@ export class MainLoop {
       )
       return CommandController.of()(
         this.effects,
-
-        { id: this.system.manifest.main.image },
+        subcontainer,
         currentCommand,
         {
-          overlay,
+          runAsInit: true,
           env: {
             TINI_SUBREAPER: "true",
           },
@@ -147,12 +147,20 @@ export class MainLoop {
     const start = Date.now()
     return Object.entries(manifest["health-checks"]).map(
       ([healthId, value]) => {
+        effects
+          .setHealth({
+            id: healthId,
+            name: value.name,
+            result: "starting",
+            message: null,
+          })
+          .catch((e) => console.error(asError(e)))
         const interval = setInterval(async () => {
           const actionProcedure = value
           const timeChanged = Date.now() - start
           if (actionProcedure.type === "docker") {
-            const overlay = actionProcedure.inject
-              ? this.mainOverlay
+            const subcontainer = actionProcedure.inject
+              ? this.mainSubContainerHandle
               : undefined
             // prettier-ignore
             const container = 
@@ -162,16 +170,12 @@ export class MainLoop {
                 actionProcedure,
                 manifest.volumes,
                 {
-                  overlay,
+                  subcontainer,
                 }
               )
             const executed = await container.exec(
-              [
-                actionProcedure.entrypoint,
-                ...actionProcedure.args,
-                JSON.stringify(timeChanged),
-              ],
-              {},
+              [actionProcedure.entrypoint, ...actionProcedure.args],
+              { input: JSON.stringify(timeChanged) },
             )
             if (executed.exitCode === 0) {
               await effects.setHealth({

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -747,11 +747,17 @@ export class SystemForEmbassy implements System {
     })
     if (!actionProcedure) throw Error("Action not found")
     if (actionProcedure.type === "docker") {
+      const subcontainer = actionProcedure.inject
+        ? this.currentRunning?.mainSubContainerHandle
+        : undefined
       const container = await DockerProcedureContainer.of(
         effects,
         this.manifest.id,
         actionProcedure,
         this.manifest.volumes,
+        {
+          subcontainer,
+        },
       )
       return toActionResult(
         JSON.parse(

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -124,20 +124,18 @@ export const polyfillEffects = (
       wait(): Promise<oet.ResultType<string>>
       term(): Promise<void>
     } {
-      const promiseOverlay = DockerProcedureContainer.createOverlay(
+      const promiseSubcontainer = DockerProcedureContainer.createSubContainer(
         effects,
         manifest.id,
         manifest.main,
         manifest.volumes,
       )
-      const daemon = promiseOverlay.then((overlay) =>
+      const daemon = promiseSubcontainer.then((subcontainer) =>
         daemons.runCommand()(
           effects,
-          { id: manifest.main.image },
+          subcontainer,
           [input.command, ...(input.args || [])],
-          {
-            overlay,
-          },
+          {},
         ),
       )
       return {