Fix/overlay destroy (#2707)

* feature: Make all errors in console.error be including an error for that stack tract

* feature: Make all errors in console.error be including an error for that stack tract

* fix: Add the tinisubreaper for the subreapers to know they are not the reaper

* fix: overlay always destroyed

* chore: Move the style of destroy to just private

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -4,19 +4,35 @@ import { Overlay, types as T } from "@start9labs/start-sdk"
 import { promisify } from "util"
 import { DockerProcedure, VolumeId } from "../../../Models/DockerProcedure"
 import { Volume } from "./matchVolume"
+import { ExecSpawnable } from "@start9labs/start-sdk/cjs/lib/util/Overlay"
 export const exec = promisify(cp.exec)
 export const execFile = promisify(cp.execFile)
 
 export class DockerProcedureContainer {
-  private constructor(readonly overlay: Overlay) {}
-  // static async readonlyOf(data: DockerProcedure) {
-  //   return DockerProcedureContainer.of(data, ["-o", "ro"])
-  // }
+  private constructor(private readonly overlay: ExecSpawnable) {}
+
   static async of(
     effects: T.Effects,
     packageId: string,
     data: DockerProcedure,
     volumes: { [id: VolumeId]: Volume },
+    options: { overlay?: ExecSpawnable } = {},
+  ) {
+    const overlay =
+      options?.overlay ??
+      (await DockerProcedureContainer.createOverlay(
+        effects,
+        packageId,
+        data,
+        volumes,
+      ))
+    return new DockerProcedureContainer(overlay)
+  }
+  static async createOverlay(
+    effects: T.Effects,
+    packageId: string,
+    data: DockerProcedure,
+    volumes: { [id: VolumeId]: Volume },
   ) {
     const overlay = await Overlay.of(effects, { id: data.image })
 
@@ -84,23 +100,18 @@ export class DockerProcedureContainer {
         }
       }
     }
-
-    return new DockerProcedureContainer(overlay)
+    return overlay
   }
 
-  async exec(commands: string[], { destroy = true } = {}) {
+  async exec(commands: string[], {} = {}) {
     try {
       return await this.overlay.exec(commands)
     } finally {
-      if (destroy) await this.overlay.destroy()
+      await this.overlay.destroy?.()
     }
   }
 
-  async execFail(
-    commands: string[],
-    timeoutMs: number | null,
-    { destroy = true } = {},
-  ) {
+  async execFail(commands: string[], timeoutMs: number | null, {} = {}) {
     try {
       const res = await this.overlay.exec(commands, {}, timeoutMs)
       if (res.exitCode !== 0) {
@@ -114,7 +125,7 @@ export class DockerProcedureContainer {
       }
       return res
     } finally {
-      if (destroy) await this.overlay.destroy()
+      await this.overlay.destroy?.()
     }
   }
 

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -5,6 +5,7 @@ import { T, utils } from "@start9labs/start-sdk"
 import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon"
 import { Effects } from "../../../Models/Effects"
 import { off } from "node:process"
+import { CommandController } from "@start9labs/start-sdk/cjs/lib/mainFn/CommandController"
 
 const EMBASSY_HEALTH_INTERVAL = 15 * 1000
 const EMBASSY_PROPERTIES_LOOP = 30 * 1000
@@ -14,9 +15,8 @@ const EMBASSY_PROPERTIES_LOOP = 30 * 1000
  * Also, this has an ability to clean itself up too if need be.
  */
 export class MainLoop {
-  private _mainDockerContainer?: DockerProcedureContainer
-  get mainDockerContainer() {
-    return this._mainDockerContainer
+  get mainOverlay() {
+    return this.mainEvent?.daemon?.overlay
   }
   private healthLoops?: {
     name: string
@@ -52,27 +52,33 @@ export class MainLoop {
     await this.setupInterfaces(effects)
     await effects.setMainStatus({ status: "running" })
     const jsMain = (this.system.moduleCode as any)?.jsMain
-    const dockerProcedureContainer = await DockerProcedureContainer.of(
-      effects,
-      this.system.manifest.id,
-      this.system.manifest.main,
-      this.system.manifest.volumes,
-    )
-    this._mainDockerContainer = dockerProcedureContainer
     if (jsMain) {
       throw new Error("Unreachable")
     }
-    const daemon = await Daemon.of()(
-      this.effects,
-      { id: this.system.manifest.main.image },
-      currentCommand,
-      {
-        overlay: dockerProcedureContainer.overlay,
-        sigtermTimeout: utils.inMs(
-          this.system.manifest.main["sigterm-timeout"],
-        ),
-      },
-    )
+    const daemon = new Daemon(async () => {
+      const overlay = await DockerProcedureContainer.createOverlay(
+        effects,
+        this.system.manifest.id,
+        this.system.manifest.main,
+        this.system.manifest.volumes,
+      )
+      return CommandController.of()(
+        this.effects,
+
+        { id: this.system.manifest.main.image },
+        currentCommand,
+        {
+          overlay,
+          env: {
+            TINI_SUBREAPER: "true",
+          },
+          sigtermTimeout: utils.inMs(
+            this.system.manifest.main["sigterm-timeout"],
+          ),
+        },
+      )
+    })
+
     daemon.start()
     return {
       daemon,
@@ -128,10 +134,11 @@ export class MainLoop {
     const main = await mainEvent
     delete this.mainEvent
     delete this.healthLoops
-    await main?.daemon.stop().catch((e) => console.error(e))
+    await main?.daemon
+      .stop()
+      .catch((e) => console.error(`Main loop error`, utils.asError(e)))
     this.effects.setMainStatus({ status: "stopped" })
     if (healthLoops) healthLoops.forEach((x) => clearInterval(x.interval))
-    delete this._mainDockerContainer
   }
 
   private constructHealthLoops() {
@@ -144,24 +151,27 @@ export class MainLoop {
           const actionProcedure = value
           const timeChanged = Date.now() - start
           if (actionProcedure.type === "docker") {
+            const overlay = actionProcedure.inject
+              ? this.mainOverlay
+              : undefined
             // prettier-ignore
             const container = 
-              actionProcedure.inject && this._mainDockerContainer ?
-              this._mainDockerContainer :
               await DockerProcedureContainer.of(
                 effects,
                 manifest.id,
                 actionProcedure,
                 manifest.volumes,
+                {
+                  overlay,
+                }
               )
-            const shouldDestroy = container !== this._mainDockerContainer
             const executed = await container.exec(
               [
                 actionProcedure.entrypoint,
                 ...actionProcedure.args,
                 JSON.stringify(timeChanged),
               ],
-              { destroy: shouldDestroy },
+              {},
             )
             if (executed.exitCode === 0) {
               await effects.setHealth({

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -194,7 +194,7 @@ export class SystemForEmbassy implements System {
     const moduleCode = await import(EMBASSY_JS_LOCATION)
       .catch((_) => require(EMBASSY_JS_LOCATION))
       .catch(async (_) => {
-        console.error("Could not load the js")
+        console.error(utils.asError("Could not load the js"))
         console.error({
           exists: await fs.stat(EMBASSY_JS_LOCATION),
         })
@@ -798,17 +798,18 @@ export class SystemForEmbassy implements System {
     const actionProcedure = this.manifest.actions?.[actionId]?.implementation
     if (!actionProcedure) return { message: "Action not found", value: null }
     if (actionProcedure.type === "docker") {
-      const container =
-        actionProcedure.inject && this.currentRunning?.mainDockerContainer
-          ? this.currentRunning?.mainDockerContainer
-          : await DockerProcedureContainer.of(
-              effects,
-              this.manifest.id,
-              actionProcedure,
-              this.manifest.volumes,
-            )
-      const shouldDestroy =
-        container !== this.currentRunning?.mainDockerContainer
+      const overlay = actionProcedure.inject
+        ? this.currentRunning?.mainOverlay
+        : undefined
+      const container = await DockerProcedureContainer.of(
+        effects,
+        this.manifest.id,
+        actionProcedure,
+        this.manifest.volumes,
+        {
+          overlay,
+        },
+      )
       return JSON.parse(
         (
           await container.execFail(
@@ -818,7 +819,6 @@ export class SystemForEmbassy implements System {
               JSON.stringify(formData),
             ],
             timeoutMs,
-            { destroy: shouldDestroy },
           )
         ).stdout.toString(),
       )
@@ -987,7 +987,10 @@ async function updateConfig(
           })
           .once()
           .catch((x) => {
-            console.error("Could not get the service interface", x)
+            console.error(
+              "Could not get the service interface",
+              utils.asError(x),
+            )
             return null
           })
         const catchFn = <X>(fn: () => X) => {

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -3,7 +3,7 @@ import * as oet from "./oldEmbassyTypes"
 import { Volume } from "../../../Models/Volume"
 import * as child_process from "child_process"
 import { promisify } from "util"
-import { daemons, startSdk, T } from "@start9labs/start-sdk"
+import { daemons, startSdk, T, utils } from "@start9labs/start-sdk"
 import "isomorphic-fetch"
 import { Manifest } from "./matchManifest"
 import { DockerProcedureContainer } from "./DockerProcedureContainer"
@@ -124,19 +124,19 @@ export const polyfillEffects = (
       wait(): Promise<oet.ResultType<string>>
       term(): Promise<void>
     } {
-      const dockerProcedureContainer = DockerProcedureContainer.of(
+      const promiseOverlay = DockerProcedureContainer.createOverlay(
         effects,
         manifest.id,
         manifest.main,
         manifest.volumes,
       )
-      const daemon = dockerProcedureContainer.then((dockerProcedureContainer) =>
+      const daemon = promiseOverlay.then((overlay) =>
         daemons.runCommand()(
           effects,
           { id: manifest.main.image },
           [input.command, ...(input.args || [])],
           {
-            overlay: dockerProcedureContainer.overlay,
+            overlay,
           },
         ),
       )
@@ -224,16 +224,16 @@ export const polyfillEffects = (
       return new Promise((resolve) => setTimeout(resolve, timeMs))
     },
     trace(whatToPrint: string): void {
-      console.trace(whatToPrint)
+      console.trace(utils.asError(whatToPrint))
     },
     warn(whatToPrint: string): void {
-      console.warn(whatToPrint)
+      console.warn(utils.asError(whatToPrint))
     },
     error(whatToPrint: string): void {
-      console.error(whatToPrint)
+      console.error(utils.asError(whatToPrint))
     },
     debug(whatToPrint: string): void {
-      console.debug(whatToPrint)
+      console.debug(utils.asError(whatToPrint))
     },
     info(whatToPrint: string): void {
       console.log(false)
@@ -357,7 +357,7 @@ export const polyfillEffects = (
       })
 
       spawned.stderr.on("data", (data: unknown) => {
-        console.error(String(data))
+        console.error(`polyfill.runAsync`, utils.asError(data))
       })
 
       const id = async () => {

container-runtime/src/Adapters/Systems/SystemForStartOs.ts

@@ -4,7 +4,7 @@ import matches, { any, number, object, string, tuple } from "ts-matches"
 import { Effects } from "../../Models/Effects"
 import { RpcResult, matchRpcResult } from "../RpcListener"
 import { duration } from "../../Models/Duration"
-import { T } from "@start9labs/start-sdk"
+import { T, utils } from "@start9labs/start-sdk"
 import { Volume } from "../../Models/Volume"
 import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
 import { CallbackHolder } from "../../Models/CallbackHolder"
@@ -57,7 +57,9 @@ export class SystemForStartOs implements System {
     if (this.runningMain) {
       this.runningMain.callbacks
         .callCallback(callback, args)
-        .catch((error) => console.error(`callback ${callback} failed`, error))
+        .catch((error) =>
+          console.error(`callback ${callback} failed`, utils.asError(error)),
+        )
     } else {
       console.warn(`callback ${callback} ignored because system is not running`)
     }