Fix/overlay destroy (#2707)

* feature: Make all errors in console.error be including an error for that stack tract

* feature: Make all errors in console.error be including an error for that stack tract

* fix: Add the tinisubreaper for the subreapers to know they are not the reaper

* fix: overlay always destroyed

* chore: Move the style of destroy to just private

sdk/lib/backup/Backups.ts

@@ -2,6 +2,7 @@ import * as T from "../types"
 
 import * as child_process from "child_process"
 import { promises as fsPromises } from "fs"
+import { asError } from "../util"
 
 export type BACKUP = "BACKUP"
 export const DEFAULT_OPTIONS: T.BackupOptions = {
@@ -183,7 +184,7 @@ async function runRsync(
   })
 
   spawned.stderr.on("data", (data: unknown) => {
-    console.error(String(data))
+    console.error(`Backups.runAsync`, asError(data))
   })
 
   const id = async () => {

sdk/lib/config/setupConfig.ts

@@ -57,7 +57,9 @@ export function setupConfig<
   return {
     setConfig: (async ({ effects, input }) => {
       if (!validator.test(input)) {
-        await console.error(String(validator.errorMessage(input)))
+        await console.error(
+          new Error(validator.errorMessage(input)?.toString()),
+        )
         return { error: "Set config type error for config" }
       }
       await effects.clearBindings()

sdk/lib/health/HealthCheck.ts

@@ -8,6 +8,7 @@ import { once } from "../util/once"
 import { Overlay } from "../util/Overlay"
 import { object, unknown } from "ts-matches"
 import * as T from "../types"
+import { asError } from "../util/asError"
 
 export type HealthCheckParams = {
   effects: Effects
@@ -44,7 +45,7 @@ export function healthCheck(o: HealthCheckParams) {
         })
         currentValue.lastResult = result
         await triggerFirstSuccess().catch((err) => {
-          console.error(err)
+          console.error(asError(err))
         })
       } catch (e) {
         await o.effects.setHealth({

sdk/lib/health/checkFns/checkWebUrl.ts

@@ -1,4 +1,5 @@
 import { Effects } from "../../types"
+import { asError } from "../../util/asError"
 import { HealthCheckResult } from "./HealthCheckResult"
 import { timeoutPromise } from "./index"
 import "isomorphic-fetch"
@@ -29,7 +30,7 @@ export const checkWebUrl = async (
     .catch((e) => {
       console.warn(`Error while fetching URL: ${url}`)
       console.error(JSON.stringify(e))
-      console.error(e.toString())
+      console.error(asError(e))
       return { result: "failure" as const, message: errorMessage }
     })
 }

sdk/lib/mainFn/CommandController.ts

@@ -2,14 +2,20 @@ import { DEFAULT_SIGTERM_TIMEOUT } from "."
 import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
 
 import * as T from "../types"
-import { MountOptions, Overlay } from "../util/Overlay"
+import { asError } from "../util/asError"
+import {
+  ExecSpawnable,
+  MountOptions,
+  NonDestroyableOverlay,
+  Overlay,
+} from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
 import { cpExecFile, cpExec } from "./Daemons"
 
 export class CommandController {
   private constructor(
     readonly runningAnswer: Promise<unknown>,
-    readonly overlay: Overlay,
+    private readonly overlay: ExecSpawnable,
     readonly pid: number | undefined,
     readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
   ) {}
@@ -25,7 +31,7 @@ export class CommandController {
         // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
         sigtermTimeout?: number
         mounts?: { path: string; options: MountOptions }[]
-        overlay?: Overlay
+        overlay?: ExecSpawnable
         env?:
           | {
               [variable: string]: string
@@ -38,10 +44,15 @@ export class CommandController {
       },
     ) => {
       const commands = splitCommand(command)
-      const overlay = options.overlay || (await Overlay.of(effects, imageId))
-      for (let mount of options.mounts || []) {
-        await overlay.mount(mount.options, mount.path)
-      }
+      const overlay =
+        options.overlay ||
+        (await (async () => {
+          const overlay = await Overlay.of(effects, imageId)
+          for (let mount of options.mounts || []) {
+            await overlay.mount(mount.options, mount.path)
+          }
+          return overlay
+        })())
       const childProcess = await overlay.spawn(commands, {
         env: options.env,
       })
@@ -57,7 +68,7 @@ export class CommandController {
           "data",
           options.onStderr ??
             ((data: any) => {
-              console.error(data.toString())
+              console.error(asError(data))
             }),
         )
 
@@ -74,7 +85,10 @@ export class CommandController {
       return new CommandController(answer, overlay, pid, options.sigtermTimeout)
     }
   }
-  async wait(timeout: number = NO_TIMEOUT) {
+  get nonDestroyableOverlay() {
+    return new NonDestroyableOverlay(this.overlay)
+  }
+  async wait({ timeout = NO_TIMEOUT } = {}) {
     if (timeout > 0)
       setTimeout(() => {
         this.term()
@@ -87,7 +101,7 @@ export class CommandController {
           (_) => {},
         )
       }
-      await this.overlay.destroy().catch((_) => {})
+      await this.overlay.destroy?.().catch((_) => {})
     }
   }
   async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
@@ -106,7 +120,7 @@ export class CommandController {
         )
       }
     } finally {
-      await this.overlay.destroy()
+      await this.overlay.destroy?.()
     }
   }
 }

sdk/lib/mainFn/Daemon.ts

@@ -1,5 +1,6 @@
 import * as T from "../types"
-import { MountOptions, Overlay } from "../util/Overlay"
+import { asError } from "../util/asError"
+import { ExecSpawnable, MountOptions, Overlay } from "../util/Overlay"
 import { CommandController } from "./CommandController"
 
 const TIMEOUT_INCREMENT_MS = 1000
@@ -12,7 +13,10 @@ const MAX_TIMEOUT_MS = 30000
 export class Daemon {
   private commandController: CommandController | null = null
   private shouldBeRunning = false
-  private constructor(private startCommand: () => Promise<CommandController>) {}
+  constructor(private startCommand: () => Promise<CommandController>) {}
+  get overlay(): undefined | ExecSpawnable {
+    return this.commandController?.nonDestroyableOverlay
+  }
   static of<Manifest extends T.Manifest>() {
     return async <A extends string>(
       effects: T.Effects,
@@ -41,7 +45,6 @@ export class Daemon {
       return new Daemon(startCommand)
     }
   }
-
   async start() {
     if (this.commandController) {
       return
@@ -57,7 +60,7 @@ export class Daemon {
         timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
       }
     }).catch((err) => {
-      console.error(err)
+      console.error(asError(err))
     })
   }
   async term(termOptions?: {
@@ -72,8 +75,8 @@ export class Daemon {
   }) {
     this.shouldBeRunning = false
     await this.commandController
-      ?.term(termOptions)
-      .catch((e) => console.error(e))
+      ?.term({ ...termOptions })
+      .catch((e) => console.error(asError(e)))
     this.commandController = null
   }
 }

sdk/lib/mainFn/HealthDaemon.ts

@@ -4,6 +4,7 @@ import { Ready } from "./Daemons"
 import { Daemon } from "./Daemon"
 import { Effects, SetHealth } from "../types"
 import { DEFAULT_SIGTERM_TIMEOUT } from "."
+import { asError } from "../util/asError"
 
 const oncePromise = <T>() => {
   let resolve: (value: T) => void
@@ -21,13 +22,13 @@ const oncePromise = <T>() => {
  *
  */
 export class HealthDaemon {
-  #health: HealthCheckResult = { result: "starting", message: null }
-  #healthWatchers: Array<() => unknown> = []
-  #running = false
+  private _health: HealthCheckResult = { result: "starting", message: null }
+  private healthWatchers: Array<() => unknown> = []
+  private running = false
   constructor(
-    readonly daemon: Promise<Daemon>,
+    private readonly daemon: Promise<Daemon>,
     readonly daemonIndex: number,
-    readonly dependencies: HealthDaemon[],
+    private readonly dependencies: HealthDaemon[],
     readonly id: string,
     readonly ids: string[],
     readonly ready: Ready,
@@ -43,12 +44,12 @@ export class HealthDaemon {
     signal?: NodeJS.Signals | undefined
     timeout?: number | undefined
   }) {
-    this.#healthWatchers = []
-    this.#running = false
-    this.#healthCheckCleanup?.()
+    this.healthWatchers = []
+    this.running = false
+    this.healthCheckCleanup?.()
 
     await this.daemon.then((d) =>
-      d.stop({
+      d.term({
         timeout: this.sigtermTimeout,
         ...termOptions,
       }),
@@ -57,17 +58,17 @@ export class HealthDaemon {
 
   /** Want to add another notifier that the health might have changed */
   addWatcher(watcher: () => unknown) {
-    this.#healthWatchers.push(watcher)
+    this.healthWatchers.push(watcher)
   }
 
   get health() {
-    return Object.freeze(this.#health)
+    return Object.freeze(this._health)
   }
 
   private async changeRunning(newStatus: boolean) {
-    if (this.#running === newStatus) return
+    if (this.running === newStatus) return
 
-    this.#running = newStatus
+    this.running = newStatus
 
     if (newStatus) {
       ;(await this.daemon).start()
@@ -80,14 +81,14 @@ export class HealthDaemon {
     }
   }
 
-  #healthCheckCleanup: (() => void) | null = null
+  private healthCheckCleanup: (() => void) | null = null
   private turnOffHealthCheck() {
-    this.#healthCheckCleanup?.()
+    this.healthCheckCleanup?.()
   }
   private async setupHealthCheck() {
-    if (this.#healthCheckCleanup) return
+    if (this.healthCheckCleanup) return
     const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({
-      lastResult: this.#health.result,
+      lastResult: this._health.result,
     }))
 
     const { promise: status, resolve: setStatus } = oncePromise<{
@@ -102,7 +103,7 @@ export class HealthDaemon {
         const response: HealthCheckResult = await Promise.resolve(
           this.ready.fn(),
         ).catch((err) => {
-          console.error(err)
+          console.error(asError(err))
           return {
             result: "failure",
             message: "message" in err ? err.message : String(err),
@@ -112,15 +113,15 @@ export class HealthDaemon {
       }
     }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
 
-    this.#healthCheckCleanup = () => {
+    this.healthCheckCleanup = () => {
       setStatus({ done: true })
-      this.#healthCheckCleanup = null
+      this.healthCheckCleanup = null
     }
   }
 
   private async setHealth(health: HealthCheckResult) {
-    this.#health = health
-    this.#healthWatchers.forEach((watcher) => watcher())
+    this._health = health
+    this.healthWatchers.forEach((watcher) => watcher())
     const display = this.ready.display
     const result = health.result
     if (!display) {
@@ -134,7 +135,7 @@ export class HealthDaemon {
   }
 
   private async updateStatus() {
-    const healths = this.dependencies.map((d) => d.#health)
+    const healths = this.dependencies.map((d) => d._health)
     this.changeRunning(healths.every((x) => x.result === "success"))
   }
 }

sdk/lib/s9pk/merkleArchive/varint.ts

@@ -1,3 +1,5 @@
+import { asError } from "../../util/asError"
+
 const msb = 0x80
 const dropMsb = 0x7f
 const maxSize = Math.floor((8 * 8 + 7) / 7)
@@ -38,7 +40,7 @@ export class VarIntProcessor {
     if (success) {
       return result
     } else {
-      console.error(this.buf)
+      console.error(asError(this.buf))
       return null
     }
   }

sdk/lib/util/Overlay.ts

@@ -5,7 +5,40 @@ import { promisify } from "util"
 import { Buffer } from "node:buffer"
 export const execFile = promisify(cp.execFile)
 const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
-export class Overlay {
+
+type ExecResults = {
+  exitCode: number | null
+  exitSignal: NodeJS.Signals | null
+  stdout: string | Buffer
+  stderr: string | Buffer
+}
+
+/**
+ * This is the type that is going to describe what an overlay could do. The main point of the
+ * overlay is to have commands that run in a chrooted environment. This is useful for running
+ * commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the
+ * case where the overlay isn't owned by the process, the overlay shouldn't be destroyed.
+ */
+export interface ExecSpawnable {
+  get destroy(): undefined | (() => Promise<void>)
+  exec(
+    command: string[],
+    options?: CommandOptions,
+    timeoutMs?: number | null,
+  ): Promise<ExecResults>
+  spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams>
+}
+/**
+ * Want to limit what we can do in a container, so we want to launch a container with a specific image and the mounts.
+ *
+ * Implements:
+ * @see {@link ExecSpawnable}
+ */
+export class Overlay implements ExecSpawnable {
+  private destroyed = false
   private constructor(
     readonly effects: T.Effects,
     readonly imageId: T.ImageId,
@@ -39,23 +72,6 @@ export class Overlay {
     return new Overlay(effects, id, rootfs, guid)
   }
 
-  static async with<T>(
-    effects: T.Effects,
-    image: { id: T.ImageId; sharedRun?: boolean },
-    mounts: { options: MountOptions; path: string }[],
-    fn: (overlay: Overlay) => Promise<T>,
-  ): Promise<T> {
-    const overlay = await Overlay.of(effects, image)
-    try {
-      for (let mount of mounts) {
-        await overlay.mount(mount.options, mount.path)
-      }
-      return await fn(overlay)
-    } finally {
-      await overlay.destroy()
-    }
-  }
-
   async mount(options: MountOptions, path: string): Promise<Overlay> {
     path = path.startsWith("/")
       ? `${this.rootfs}${path}`
@@ -70,7 +86,7 @@ export class Overlay {
 
       await fs.mkdir(from, { recursive: true })
       await fs.mkdir(path, { recursive: true })
-      await await execFile("mount", ["--bind", from, path])
+      await execFile("mount", ["--bind", from, path])
     } else if (options.type === "assets") {
       const subpath = options.subpath
         ? options.subpath.startsWith("/")
@@ -101,10 +117,14 @@ export class Overlay {
     return this
   }
 
-  async destroy() {
-    const imageId = this.imageId
-    const guid = this.guid
-    await this.effects.destroyOverlayedImage({ guid })
+  get destroy() {
+    return async () => {
+      if (this.destroyed) return
+      this.destroyed = true
+      const imageId = this.imageId
+      const guid = this.guid
+      await this.effects.destroyOverlayedImage({ guid })
+    }
   }
 
   async exec(
@@ -218,6 +238,32 @@ export class Overlay {
   }
 }
 
+/**
+ * Take an overlay but remove the ability to add the mounts and the destroy function.
+ * Lets other functions, like health checks, to not destroy the parents.
+ *
+ */
+export class NonDestroyableOverlay implements ExecSpawnable {
+  constructor(private overlay: ExecSpawnable) {}
+  get destroy() {
+    return undefined
+  }
+
+  exec(
+    command: string[],
+    options?: CommandOptions,
+    timeoutMs?: number | null,
+  ): Promise<ExecResults> {
+    return this.overlay.exec(command, options, timeoutMs)
+  }
+  spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    return this.overlay.spawn(command, options)
+  }
+}
+
 export type CommandOptions = {
   env?: { [variable: string]: string }
   cwd?: string

sdk/lib/util/asError.ts

@@ -0,0 +1,6 @@
+export const asError = (e: unknown) => {
+  if (e instanceof Error) {
+    return new Error(e as any)
+  }
+  return new Error(`${e}`)
+}

sdk/lib/util/index.ts

@@ -7,6 +7,7 @@ import "./Overlay"
 import "./once"
 
 export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
+export { asError } from "./asError"
 export { getServiceInterfaces } from "./getServiceInterfaces"
 export { addressHostToUrl } from "./getServiceInterface"
 export { hostnameInfoToAddress } from "./Hostname"