Feature/subcontainers (#2720)

* wip: subcontainers * wip: subcontainer infra * rename NonDestroyableOverlay to SubContainerHandle * chore: Changes to the container and other things * wip: * wip: fixes * fix launch & exec Co-authored-by: Jade <Blu-J@users.noreply.github.com> * tweak apis * misc fixes * don't treat sigterm as error * handle health check set during starting --------- Co-authored-by: J H <dragondef@gmail.com> Co-authored-by: Jade <Blu-J@users.noreply.github.com>
2026-03-30 20:14:49 +00:00 · 2024-08-22 21:45:54 -06:00
parent 72898d897c
commit 4defec194f
37 changed files with 1212 additions and 566 deletions
--- a/sdk/lib/StartSdk.ts
+++ b/sdk/lib/StartSdk.ts
@@ -61,7 +61,7 @@ import {
 } from "./util/getServiceInterface"
 import { getServiceInterfaces } from "./util/getServiceInterfaces"
 import { getStore } from "./store/getStore"
-import { CommandOptions, MountOptions, Overlay } from "./util/Overlay"
+import { CommandOptions, MountOptions, SubContainer } from "./util/SubContainer"
 import { splitCommand } from "./util/splitCommand"
 import { Mounts } from "./mainFn/Mounts"
 import { Dependency } from "./Dependency"
@@ -734,8 +734,11 @@ export async function runCommand<Manifest extends T.Manifest>(
  },
 ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
  const commands = splitCommand(command)
-  return Overlay.with(effects, image, options.mounts || [], (overlay) =>
-    overlay.exec(commands),
+  return SubContainer.with(
+    effects,
+    image,
+    options.mounts || [],
+    (subcontainer) => subcontainer.exec(commands),
  )
 }
 function nullifyProperties(value: T.SdkPropertiesReturn): T.PropertiesReturn {
--- a/sdk/lib/health/HealthCheck.ts
+++ b/sdk/lib/health/HealthCheck.ts
@@ -5,7 +5,7 @@ import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
-import { Overlay } from "../util/Overlay"
+import { SubContainer } from "../util/SubContainer"
 import { object, unknown } from "ts-matches"
 import * as T from "../types"
 import { asError } from "../util/asError"
--- a/sdk/lib/health/checkFns/runHealthScript.ts
+++ b/sdk/lib/health/checkFns/runHealthScript.ts
@@ -1,5 +1,5 @@
 import { Effects } from "../../types"
-import { Overlay } from "../../util/Overlay"
+import { SubContainer } from "../../util/SubContainer"
 import { stringFromStdErrOut } from "../../util/stringFromStdErrOut"
 import { HealthCheckResult } from "./HealthCheckResult"
 import { timeoutPromise } from "./index"
@@ -13,7 +13,7 @@ import { timeoutPromise } from "./index"
 */
 export const runHealthScript = async (
  runCommand: string[],
-  overlay: Overlay,
+  subcontainer: SubContainer,
  {
    timeout = 30000,
    errorMessage = `Error while running command: ${runCommand}`,
@@ -22,7 +22,7 @@ export const runHealthScript = async (
  } = {},
 ): Promise<HealthCheckResult> => {
  const res = await Promise.race([
-    overlay.exec(runCommand),
+    subcontainer.exec(runCommand),
    timeoutPromise(timeout),
  ]).catch((e) => {
    console.warn(errorMessage)
--- a/sdk/lib/index.ts
+++ b/sdk/lib/index.ts
@@ -1,5 +1,5 @@
 export { Daemons } from "./mainFn/Daemons"
-export { Overlay } from "./util/Overlay"
+export { SubContainer } from "./util/SubContainer"
 export { StartSdk } from "./StartSdk"
 export { setupManifest } from "./manifest/setupManifest"
 export { FileHelper } from "./util/fileHelper"
--- a/sdk/lib/mainFn/CommandController.ts
+++ b/sdk/lib/mainFn/CommandController.ts
@@ -6,32 +6,35 @@ import { asError } from "../util/asError"
 import {
  ExecSpawnable,
  MountOptions,
-  NonDestroyableOverlay,
-  Overlay,
-} from "../util/Overlay"
+  SubContainerHandle,
+  SubContainer,
+} from "../util/SubContainer"
 import { splitCommand } from "../util/splitCommand"
-import { cpExecFile, cpExec } from "./Daemons"
+import * as cp from "child_process"

 export class CommandController {
  private constructor(
    readonly runningAnswer: Promise<unknown>,
-    private readonly overlay: ExecSpawnable,
-    readonly pid: number | undefined,
+    private state: { exited: boolean },
+    private readonly subcontainer: SubContainer,
+    private process: cp.ChildProcessWithoutNullStreams,
    readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
  ) {}
  static of<Manifest extends T.Manifest>() {
    return async <A extends string>(
      effects: T.Effects,
-      imageId: {
-        id: keyof Manifest["images"] & T.ImageId
-        sharedRun?: boolean
-      },
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
      command: T.CommandType,
      options: {
        // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
        sigtermTimeout?: number
        mounts?: { path: string; options: MountOptions }[]
-        overlay?: ExecSpawnable
+        runAsInit?: boolean
        env?:
          | {
              [variable: string]: string
@@ -44,49 +47,60 @@ export class CommandController {
      },
    ) => {
      const commands = splitCommand(command)
-      const overlay =
-        options.overlay ||
-        (await (async () => {
-          const overlay = await Overlay.of(effects, imageId)
-          for (let mount of options.mounts || []) {
-            await overlay.mount(mount.options, mount.path)
-          }
-          return overlay
-        })())
-      const childProcess = await overlay.spawn(commands, {
-        env: options.env,
-      })
+      const subc =
+        subcontainer instanceof SubContainer
+          ? subcontainer
+          : await (async () => {
+              const subc = await SubContainer.of(effects, subcontainer)
+              for (let mount of options.mounts || []) {
+                await subc.mount(mount.options, mount.path)
+              }
+              return subc
+            })()
+      let childProcess: cp.ChildProcessWithoutNullStreams
+      if (options.runAsInit) {
+        childProcess = await subc.launch(commands, {
+          env: options.env,
+        })
+      } else {
+        childProcess = await subc.spawn(commands, {
+          env: options.env,
+        })
+      }
+      const state = { exited: false }
      const answer = new Promise<null>((resolve, reject) => {
-        childProcess.stdout.on(
-          "data",
-          options.onStdout ??
-            ((data: any) => {
-              console.log(data.toString())
-            }),
-        )
-        childProcess.stderr.on(
-          "data",
-          options.onStderr ??
-            ((data: any) => {
-              console.error(asError(data))
-            }),
-        )
-
-        childProcess.on("exit", (code: any) => {
-          if (code === 0) {
+        childProcess.on("exit", (code) => {
+          state.exited = true
+          if (
+            code === 0 ||
+            code === 143 ||
+            (code === null && childProcess.signalCode == "SIGTERM")
+          ) {
            return resolve(null)
          }
-          return reject(new Error(`${commands[0]} exited with code ${code}`))
+          if (code) {
+            return reject(new Error(`${commands[0]} exited with code ${code}`))
+          } else {
+            return reject(
+              new Error(
+                `${commands[0]} exited with signal ${childProcess.signalCode}`,
+              ),
+            )
+          }
        })
      })

-      const pid = childProcess.pid
-
-      return new CommandController(answer, overlay, pid, options.sigtermTimeout)
+      return new CommandController(
+        answer,
+        state,
+        subc,
+        childProcess,
+        options.sigtermTimeout,
+      )
    }
  }
-  get nonDestroyableOverlay() {
-    return new NonDestroyableOverlay(this.overlay)
+  get subContainerHandle() {
+    return new SubContainerHandle(this.subcontainer)
  }
  async wait({ timeout = NO_TIMEOUT } = {}) {
    if (timeout > 0)
@@ -96,75 +110,30 @@ export class CommandController {
    try {
      return await this.runningAnswer
    } finally {
-      if (this.pid !== undefined) {
-        await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch(
-          (_) => {},
-        )
+      if (!this.state.exited) {
+        this.process.kill("SIGKILL")
      }
-      await this.overlay.destroy?.().catch((_) => {})
+      await this.subcontainer.destroy?.().catch((_) => {})
    }
  }
  async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
-    if (this.pid === undefined) return
    try {
-      await cpExecFile("pkill", [
-        `-${signal.replace("SIG", "")}`,
-        "-s",
-        String(this.pid),
-      ])
-
-      const didTimeout = await waitSession(this.pid, timeout)
-      if (didTimeout) {
-        await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch(
-          (_) => {},
-        )
+      if (!this.state.exited) {
+        if (!this.process.kill(signal)) {
+          console.error(
+            `failed to send signal ${signal} to pid ${this.process.pid}`,
+          )
+        }
      }
-    } finally {
-      await this.overlay.destroy?.()
-    }
-  }
-}

-function waitSession(
-  sid: number,
-  timeout = NO_TIMEOUT,
-  interval = 100,
-): Promise<boolean> {
-  let nextInterval = interval * 2
-  if (timeout >= 0 && timeout < nextInterval) {
-    nextInterval = timeout
-  }
-  let nextTimeout = timeout
-  if (timeout > 0) {
-    if (timeout >= interval) {
-      nextTimeout -= interval
-    } else {
-      nextTimeout = 0
+      if (signal !== "SIGKILL") {
+        setTimeout(() => {
+          this.process.kill("SIGKILL")
+        }, timeout)
+      }
+      await this.runningAnswer
+    } finally {
+      await this.subcontainer.destroy?.()
    }
  }
-  return new Promise((resolve, reject) => {
-    let next: NodeJS.Timeout | null = null
-    if (timeout !== 0) {
-      next = setTimeout(() => {
-        waitSession(sid, nextTimeout, nextInterval).then(resolve, reject)
-      }, interval)
-    }
-    cpExecFile("ps", [`--sid=${sid}`, "-o", "--pid="]).then(
-      (_) => {
-        if (timeout === 0) {
-          resolve(true)
-        }
-      },
-      (e) => {
-        if (next) {
-          clearTimeout(next)
-        }
-        if (typeof e === "object" && e && "code" in e && e.code) {
-          resolve(false)
-        } else {
-          reject(e)
-        }
-      },
-    )
-  })
 }
--- a/sdk/lib/mainFn/Daemon.ts
+++ b/sdk/lib/mainFn/Daemon.ts
@@ -1,6 +1,6 @@
 import * as T from "../types"
 import { asError } from "../util/asError"
-import { ExecSpawnable, MountOptions, Overlay } from "../util/Overlay"
+import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer"
 import { CommandController } from "./CommandController"

 const TIMEOUT_INCREMENT_MS = 1000
@@ -14,20 +14,21 @@ export class Daemon {
  private commandController: CommandController | null = null
  private shouldBeRunning = false
  constructor(private startCommand: () => Promise<CommandController>) {}
-  get overlay(): undefined | ExecSpawnable {
-    return this.commandController?.nonDestroyableOverlay
+  get subContainerHandle(): undefined | ExecSpawnable {
+    return this.commandController?.subContainerHandle
  }
  static of<Manifest extends T.Manifest>() {
    return async <A extends string>(
      effects: T.Effects,
-      imageId: {
-        id: keyof Manifest["images"] & T.ImageId
-        sharedRun?: boolean
-      },
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
      command: T.CommandType,
      options: {
        mounts?: { path: string; options: MountOptions }[]
-        overlay?: Overlay
        env?:
          | {
              [variable: string]: string
@@ -41,7 +42,12 @@ export class Daemon {
      },
    ) => {
      const startCommand = () =>
-        CommandController.of<Manifest>()(effects, imageId, command, options)
+        CommandController.of<Manifest>()(
+          effects,
+          subcontainer,
+          command,
+          options,
+        )
      return new Daemon(startCommand)
    }
  }
--- a/sdk/lib/mainFn/Daemons.ts
+++ b/sdk/lib/mainFn/Daemons.ts
@@ -7,7 +7,12 @@ import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import * as T from "../types"
 import { Mounts } from "./Mounts"
-import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
+import {
+  CommandOptions,
+  ExecSpawnable,
+  MountOptions,
+  SubContainer,
+} from "../util/SubContainer"
 import { splitCommand } from "../util/splitCommand"

 import { promisify } from "node:util"
@@ -23,7 +28,9 @@ export const cpExec = promisify(CP.exec)
 export const cpExecFile = promisify(CP.execFile)
 export type Ready = {
  display: string | null
-  fn: () => Promise<HealthCheckResult> | HealthCheckResult
+  fn: (
+    spawnable: ExecSpawnable,
+  ) => Promise<HealthCheckResult> | HealthCheckResult
  trigger?: Trigger
 }

--- a/sdk/lib/mainFn/HealthDaemon.ts
+++ b/sdk/lib/mainFn/HealthDaemon.ts
@@ -100,16 +100,25 @@ export class HealthDaemon {
        !res.done;
        res = await Promise.race([status, trigger.next()])
      ) {
-        const response: HealthCheckResult = await Promise.resolve(
-          this.ready.fn(),
-        ).catch((err) => {
-          console.error(asError(err))
-          return {
+        const handle = (await this.daemon).subContainerHandle
+
+        if (handle) {
+          const response: HealthCheckResult = await Promise.resolve(
+            this.ready.fn(handle),
+          ).catch((err) => {
+            console.error(asError(err))
+            return {
+              result: "failure",
+              message: "message" in err ? err.message : String(err),
+            }
+          })
+          await this.setHealth(response)
+        } else {
+          await this.setHealth({
            result: "failure",
-            message: "message" in err ? err.message : String(err),
-          }
-        })
-        await this.setHealth(response)
+            message: "Daemon not running",
+          })
+        }
      }
    }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))

--- a/sdk/lib/mainFn/Mounts.ts
+++ b/sdk/lib/mainFn/Mounts.ts
@@ -1,5 +1,5 @@
 import * as T from "../types"
-import { MountOptions } from "../util/Overlay"
+import { MountOptions } from "../util/SubContainer"

 type MountArray = { path: string; options: MountOptions }[]

--- a/sdk/lib/osBindings/CreateSubcontainerFsParams.ts
+++ b/sdk/lib/osBindings/CreateSubcontainerFsParams.ts
@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ImageId } from "./ImageId"

-export type CreateOverlayedImageParams = { imageId: ImageId }
+export type CreateSubcontainerFsParams = { imageId: ImageId }
--- a/sdk/lib/osBindings/DestroySubcontainerFsParams.ts
+++ b/sdk/lib/osBindings/DestroySubcontainerFsParams.ts
@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Guid } from "./Guid"

-export type DestroyOverlayedImageParams = { guid: Guid }
+export type DestroySubcontainerFsParams = { guid: Guid }
--- a/sdk/lib/osBindings/MainStatus.ts
+++ b/sdk/lib/osBindings/MainStatus.ts
@@ -1,20 +1,20 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { HealthCheckId } from "./HealthCheckId"
 import type { NamedHealthCheckResult } from "./NamedHealthCheckResult"
+import type { StartStop } from "./StartStop"

 export type MainStatus =
  | { status: "stopped" }
  | { status: "restarting" }
  | { status: "restoring" }
  | { status: "stopping" }
-  | { status: "starting" }
+  | {
+      status: "starting"
+      health: { [key: HealthCheckId]: NamedHealthCheckResult }
+    }
  | {
      status: "running"
      started: string
      health: { [key: HealthCheckId]: NamedHealthCheckResult }
    }
-  | {
-      status: "backingUp"
-      started: string | null
-      health: { [key: HealthCheckId]: NamedHealthCheckResult }
-    }
+  | { status: "backingUp"; onComplete: StartStop }
--- a/sdk/lib/osBindings/StartStop.ts
+++ b/sdk/lib/osBindings/StartStop.ts
@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type StartStop = "start" | "stop"
--- a/sdk/lib/osBindings/index.ts
+++ b/sdk/lib/osBindings/index.ts
@@ -31,7 +31,7 @@ export { CheckDependenciesParam } from "./CheckDependenciesParam"
 export { CheckDependenciesResult } from "./CheckDependenciesResult"
 export { Cifs } from "./Cifs"
 export { ContactInfo } from "./ContactInfo"
-export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams"
+export { CreateSubcontainerFsParams } from "./CreateSubcontainerFsParams"
 export { CurrentDependencies } from "./CurrentDependencies"
 export { CurrentDependencyInfo } from "./CurrentDependencyInfo"
 export { DataUrl } from "./DataUrl"
@@ -41,7 +41,7 @@ export { DependencyMetadata } from "./DependencyMetadata"
 export { DependencyRequirement } from "./DependencyRequirement"
 export { DepInfo } from "./DepInfo"
 export { Description } from "./Description"
-export { DestroyOverlayedImageParams } from "./DestroyOverlayedImageParams"
+export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams"
 export { Duration } from "./Duration"
 export { EchoParams } from "./EchoParams"
 export { EncryptedWire } from "./EncryptedWire"
@@ -145,6 +145,7 @@ export { SetupStatusRes } from "./SetupStatusRes"
 export { SignAssetParams } from "./SignAssetParams"
 export { SignerInfo } from "./SignerInfo"
 export { SmtpValue } from "./SmtpValue"
+export { StartStop } from "./StartStop"
 export { Status } from "./Status"
 export { UpdatingState } from "./UpdatingState"
 export { VerifyCifsParams } from "./VerifyCifsParams"
--- a/sdk/lib/test/startosTypeValidation.test.ts
+++ b/sdk/lib/test/startosTypeValidation.test.ts
@@ -3,11 +3,13 @@ import {
  CheckDependenciesParam,
  ExecuteAction,
  GetConfiguredParams,
+  GetStoreParams,
  SetDataVersionParams,
  SetMainStatus,
+  SetStoreParams,
 } from ".././osBindings"
-import { CreateOverlayedImageParams } from ".././osBindings"
-import { DestroyOverlayedImageParams } from ".././osBindings"
+import { CreateSubcontainerFsParams } from ".././osBindings"
+import { DestroySubcontainerFsParams } from ".././osBindings"
 import { BindParams } from ".././osBindings"
 import { GetHostInfoParams } from ".././osBindings"
 import { SetConfigured } from ".././osBindings"
@@ -24,21 +26,28 @@ import { GetPrimaryUrlParams } from ".././osBindings"
 import { ListServiceInterfacesParams } from ".././osBindings"
 import { ExportActionParams } from ".././osBindings"
 import { MountParams } from ".././osBindings"
+import { StringObject } from "../util"
 function typeEquality<ExpectedType>(_a: ExpectedType) {}

 type WithCallback<T> = Omit<T, "callback"> & { callback: () => void }

+type EffectsTypeChecker<T extends StringObject = Effects> = {
+  [K in keyof T]: T[K] extends (args: infer A) => any
+    ? A
+    : T[K] extends StringObject
+      ? EffectsTypeChecker<T[K]>
+      : never
+}
+
 describe("startosTypeValidation ", () => {
  test(`checking the params match`, () => {
    const testInput: any = {}
-    typeEquality<{
-      [K in keyof Effects]: Effects[K] extends (args: infer A) => any
-        ? A
-        : never
-    }>({
+    typeEquality<EffectsTypeChecker>({
      executeAction: {} as ExecuteAction,
-      createOverlayedImage: {} as CreateOverlayedImageParams,
-      destroyOverlayedImage: {} as DestroyOverlayedImageParams,
+      subcontainer: {
+        createFs: {} as CreateSubcontainerFsParams,
+        destroyFs: {} as DestroySubcontainerFsParams,
+      },
      clearBindings: undefined,
      getInstalledPackages: undefined,
      bind: {} as BindParams,
@@ -55,7 +64,10 @@ describe("startosTypeValidation ", () => {
      getSslKey: {} as GetSslKeyParams,
      getServiceInterface: {} as WithCallback<GetServiceInterfaceParams>,
      setDependencies: {} as SetDependenciesParams,
-      store: {} as never,
+      store: {
+        get: {} as any, // as GetStoreParams,
+        set: {} as any, // as SetStoreParams,
+      },
      getSystemSmtp: {} as WithCallback<GetSystemSmtpParams>,
      getContainerIp: undefined,
      getServicePortForward: {} as GetServicePortForwardParams,
--- a/sdk/lib/trigger/index.ts
+++ b/sdk/lib/trigger/index.ts
@@ -1,3 +1,4 @@
+import { ExecSpawnable } from "../util/SubContainer"
 import { TriggerInput } from "./TriggerInput"
 export { changeOnFirstSuccess } from "./changeOnFirstSuccess"
 export { cooldownTrigger } from "./cooldownTrigger"
--- a/sdk/lib/types.ts
+++ b/sdk/lib/types.ts
@@ -25,6 +25,7 @@ import { Daemons } from "./mainFn/Daemons"
 import { StorePath } from "./store/PathBuilder"
 import { ExposedStorePaths } from "./store/setupExposeStore"
 import { UrlString } from "./util/getServiceInterface"
+import { StringObject, ToKebab } from "./util"
 export * from "./osBindings"
 export { SDKManifest } from "./manifest/ManifestTypes"
 export { HealthReceipt } from "./health/HealthReceipt"
@@ -286,6 +287,16 @@ export type PropertiesReturn = {
  [key: string]: PropertiesValue
 }

+export type EffectMethod<T extends StringObject = Effects> = {
+  [K in keyof T]-?: K extends string
+    ? T[K] extends Function
+      ? ToKebab<K>
+      : T[K] extends StringObject
+        ? `${ToKebab<K>}.${EffectMethod<T[K]>}`
+        : never
+    : never
+}[keyof T]
+
 /** Used to reach out from the pure js runtime */
 export type Effects = {
  // action
@@ -352,12 +363,13 @@ export type Effects = {
  /** sets the result of a health check */
  setHealth(o: SetHealth): Promise<void>

-  // image
-
-  /** A low level api used by Overlay */
-  createOverlayedImage(options: { imageId: string }): Promise<[string, string]>
-  /** A low level api used by Overlay */
-  destroyOverlayedImage(options: { guid: string }): Promise<void>
+  // subcontainer
+  subcontainer: {
+    /** A low level api used by SubContainer */
+    createFs(options: { imageId: string }): Promise<[string, string]>
+    /** A low level api used by SubContainer */
+    destroyFs(options: { guid: string }): Promise<void>
+  }

  // net

--- a/sdk/lib/util/SubContainer.ts
+++ b/sdk/lib/util/SubContainer.ts
@@ -13,17 +13,21 @@ type ExecResults = {
  stderr: string | Buffer
 }

+export type ExecOptions = {
+  input?: string | Buffer
+}
+
 /**
- * This is the type that is going to describe what an overlay could do. The main point of the
- * overlay is to have commands that run in a chrooted environment. This is useful for running
+ * This is the type that is going to describe what an subcontainer could do. The main point of the
+ * subcontainer is to have commands that run in a chrooted environment. This is useful for running
 * commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the
- * case where the overlay isn't owned by the process, the overlay shouldn't be destroyed.
+ * case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed.
 */
 export interface ExecSpawnable {
  get destroy(): undefined | (() => Promise<void>)
  exec(
    command: string[],
-    options?: CommandOptions,
+    options?: CommandOptions & ExecOptions,
    timeoutMs?: number | null,
  ): Promise<ExecResults>
  spawn(
@@ -37,24 +41,34 @@ export interface ExecSpawnable {
 * Implements:
 * @see {@link ExecSpawnable}
 */
-export class Overlay implements ExecSpawnable {
-  private destroyed = false
+export class SubContainer implements ExecSpawnable {
+  private leader: cp.ChildProcess
+  private leaderExited: boolean = false
  private constructor(
    readonly effects: T.Effects,
    readonly imageId: T.ImageId,
    readonly rootfs: string,
    readonly guid: T.Guid,
-  ) {}
+  ) {
+    this.leaderExited = false
+    this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], {
+      killSignal: "SIGKILL",
+      stdio: "ignore",
+    })
+    this.leader.on("exit", () => {
+      this.leaderExited = true
+    })
+  }
  static async of(
    effects: T.Effects,
    image: { id: T.ImageId; sharedRun?: boolean },
  ) {
    const { id, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({
+    const [rootfs, guid] = await effects.subcontainer.createFs({
      imageId: id as string,
    })

-    const shared = ["dev", "sys", "proc"]
+    const shared = ["dev", "sys"]
    if (!!sharedRun) {
      shared.push("run")
    }
@@ -69,27 +83,27 @@ export class Overlay implements ExecSpawnable {
      await execFile("mount", ["--rbind", from, to])
    }

-    return new Overlay(effects, id, rootfs, guid)
+    return new SubContainer(effects, id, rootfs, guid)
  }

  static async with<T>(
    effects: T.Effects,
    image: { id: T.ImageId; sharedRun?: boolean },
    mounts: { options: MountOptions; path: string }[],
-    fn: (overlay: Overlay) => Promise<T>,
+    fn: (subContainer: SubContainer) => Promise<T>,
  ): Promise<T> {
-    const overlay = await Overlay.of(effects, image)
+    const subContainer = await SubContainer.of(effects, image)
    try {
      for (let mount of mounts) {
-        await overlay.mount(mount.options, mount.path)
+        await subContainer.mount(mount.options, mount.path)
      }
-      return await fn(overlay)
+      return await fn(subContainer)
    } finally {
-      await overlay.destroy()
+      await subContainer.destroy()
    }
  }

-  async mount(options: MountOptions, path: string): Promise<Overlay> {
+  async mount(options: MountOptions, path: string): Promise<SubContainer> {
    path = path.startsWith("/")
      ? `${this.rootfs}${path}`
      : `${this.rootfs}/${path}`
@@ -134,19 +148,35 @@ export class Overlay implements ExecSpawnable {
    return this
  }

+  private async killLeader() {
+    if (this.leaderExited) {
+      return
+    }
+    return new Promise<void>((resolve, reject) => {
+      try {
+        this.leader.on("exit", () => {
+          resolve()
+        })
+        if (!this.leader.kill("SIGKILL")) {
+          reject(new Error("kill(2) failed"))
+        }
+      } catch (e) {
+        reject(e)
+      }
+    })
+  }
+
  get destroy() {
    return async () => {
-      if (this.destroyed) return
-      this.destroyed = true
-      const imageId = this.imageId
      const guid = this.guid
-      await this.effects.destroyOverlayedImage({ guid })
+      await this.killLeader()
+      await this.effects.subcontainer.destroyFs({ guid })
    }
  }

  async exec(
    command: string[],
-    options?: CommandOptions,
+    options?: CommandOptions & ExecOptions,
    timeoutMs: number | null = 30000,
  ): Promise<{
    exitCode: number | null
@@ -173,7 +203,8 @@ export class Overlay implements ExecSpawnable {
    const child = cp.spawn(
      "start-cli",
      [
-        "chroot",
+        "subcontainer",
+        "exec",
        `--env=/media/startos/images/${this.imageId}.env`,
        `--workdir=${workdir}`,
        ...extra,
@@ -182,6 +213,18 @@ export class Overlay implements ExecSpawnable {
      ],
      options || {},
    )
+    if (options?.input) {
+      await new Promise<void>((resolve, reject) =>
+        child.stdin.write(options.input, (e) => {
+          if (e) {
+            reject(e)
+          } else {
+            resolve()
+          }
+        }),
+      )
+      await new Promise<void>((resolve) => child.stdin.end(resolve))
+    }
    const pid = child.pid
    const stdout = { data: "" as string | Buffer }
    const stderr = { data: "" as string | Buffer }
@@ -201,25 +244,65 @@ export class Overlay implements ExecSpawnable {
      }
    return new Promise((resolve, reject) => {
      child.on("error", reject)
-      if (timeoutMs !== null && pid) {
-        setTimeout(
-          () => execFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}),
-          timeoutMs,
-        )
+      let killTimeout: NodeJS.Timeout | undefined
+      if (timeoutMs !== null && child.pid) {
+        killTimeout = setTimeout(() => child.kill("SIGKILL"), timeoutMs)
      }
      child.stdout.on("data", appendData(stdout))
      child.stderr.on("data", appendData(stderr))
-      child.on("exit", (code, signal) =>
+      child.on("exit", (code, signal) => {
+        clearTimeout(killTimeout)
        resolve({
          exitCode: code,
          exitSignal: signal,
          stdout: stdout.data,
          stderr: stderr.data,
-        }),
-      )
+        })
+      })
    })
  }

+  async launch(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    const imageMeta: any = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    await this.killLeader()
+    this.leaderExited = false
+    this.leader = cp.spawn(
+      "start-cli",
+      [
+        "subcontainer",
+        "launch",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      { ...options, stdio: "inherit" },
+    )
+    this.leader.on("exit", () => {
+      this.leaderExited = true
+    })
+    return this.leader as cp.ChildProcessWithoutNullStreams
+  }
+
  async spawn(
    command: string[],
    options?: CommandOptions,
@@ -243,7 +326,8 @@ export class Overlay implements ExecSpawnable {
    return cp.spawn(
      "start-cli",
      [
-        "chroot",
+        "subcontainer",
+        "exec",
        `--env=/media/startos/images/${this.imageId}.env`,
        `--workdir=${workdir}`,
        ...extra,
@@ -256,12 +340,12 @@ export class Overlay implements ExecSpawnable {
 }

 /**
- * Take an overlay but remove the ability to add the mounts and the destroy function.
+ * Take an subcontainer but remove the ability to add the mounts and the destroy function.
 * Lets other functions, like health checks, to not destroy the parents.
 *
 */
-export class NonDestroyableOverlay implements ExecSpawnable {
-  constructor(private overlay: ExecSpawnable) {}
+export class SubContainerHandle implements ExecSpawnable {
+  constructor(private subContainer: ExecSpawnable) {}
  get destroy() {
    return undefined
  }
@@ -271,13 +355,13 @@ export class NonDestroyableOverlay implements ExecSpawnable {
    options?: CommandOptions,
    timeoutMs?: number | null,
  ): Promise<ExecResults> {
-    return this.overlay.exec(command, options, timeoutMs)
+    return this.subContainer.exec(command, options, timeoutMs)
  }
  spawn(
    command: string[],
    options?: CommandOptions,
  ): Promise<cp.ChildProcessWithoutNullStreams> {
-    return this.overlay.spawn(command, options)
+    return this.subContainer.spawn(command, options)
  }
 }

--- a/sdk/lib/util/index.ts
+++ b/sdk/lib/util/index.ts
@@ -3,7 +3,7 @@ import "./fileHelper"
 import "../store/getStore"
 import "./deepEqual"
 import "./deepMerge"
-import "./Overlay"
+import "./SubContainer"
 import "./once"

 export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
--- a/sdk/lib/util/typeHelpers.ts
+++ b/sdk/lib/util/typeHelpers.ts
@@ -21,3 +21,96 @@ export type NoAny<A> = NeverPossible extends A
    ? never
    : A
  : A
+
+type CapitalLetters =
+  | "A"
+  | "B"
+  | "C"
+  | "D"
+  | "E"
+  | "F"
+  | "G"
+  | "H"
+  | "I"
+  | "J"
+  | "K"
+  | "L"
+  | "M"
+  | "N"
+  | "O"
+  | "P"
+  | "Q"
+  | "R"
+  | "S"
+  | "T"
+  | "U"
+  | "V"
+  | "W"
+  | "X"
+  | "Y"
+  | "Z"
+
+type Numbers = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+
+type CapitalChars = CapitalLetters | Numbers
+
+export type ToKebab<S extends string> = S extends string
+  ? S extends `${infer Head}${CapitalChars}${infer Tail}` // string has a capital char somewhere
+    ? Head extends "" // there is a capital char in the first position
+      ? Tail extends ""
+        ? Lowercase<S> /*  'A' */
+        : S extends `${infer Caps}${Tail}` // tail exists, has capital characters
+          ? Caps extends CapitalChars
+            ? Tail extends CapitalLetters
+              ? `${Lowercase<Caps>}-${Lowercase<Tail>}` /* 'AB' */
+              : Tail extends `${CapitalLetters}${string}`
+                ? `${ToKebab<Caps>}-${ToKebab<Tail>}` /* first tail char is upper? 'ABcd' */
+                : `${ToKebab<Caps>}${ToKebab<Tail>}` /* 'AbCD','AbcD',  */ /* TODO: if tail is only numbers, append without underscore */
+            : never /* never reached, used for inference of caps */
+          : never
+      : Tail extends "" /* 'aB' 'abCD' 'ABCD' 'AB' */
+        ? S extends `${Head}${infer Caps}`
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* 'abcD' */
+              ? Caps extends Numbers
+                ? // Head exists and is lowercase, tail does not, Caps is a number, we may be in a sub-select
+                  // if head ends with number, don't split head an Caps, keep contiguous numbers together
+                  Head extends `${string}${Numbers}`
+                  ? never
+                  : // head does not end in number, safe to split. 'abc2' -> 'abc-2'
+                    `${ToKebab<Head>}-${Caps}`
+                : `${ToKebab<Head>}-${ToKebab<Caps>}` /* 'abcD' 'abc25' */
+              : never /* stop union type forming */
+            : never
+          : never /* never reached, used for inference of caps */
+        : S extends `${Head}${infer Caps}${Tail}` /* 'abCd' 'ABCD' 'AbCd' 'ABcD' */
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* is 'abCd' 'abCD' ? */
+              ? Tail extends CapitalLetters /* is 'abCD' where Caps = 'C' */
+                ? `${ToKebab<Head>}-${ToKebab<Caps>}-${Lowercase<Tail>}` /* aBCD Tail = 'D', Head = 'aB' */
+                : Tail extends `${CapitalLetters}${string}` /* is 'aBCd' where Caps = 'B' */
+                  ? Head extends Numbers
+                    ? never /* stop union type forming */
+                    : Head extends `${string}${Numbers}`
+                      ? never /* stop union type forming */
+                      : `${Head}-${ToKebab<Caps>}-${ToKebab<Tail>}` /* 'aBCd' => `${'a'}-${Lowercase<'B'>}-${ToSnake<'Cd'>}` */
+                  : `${ToKebab<Head>}-${Lowercase<Caps>}${ToKebab<Tail>}` /* 'aBcD' where Caps = 'B' tail starts as lowercase */
+              : never
+            : never
+          : never
+    : S /* 'abc'  */
+  : never
+
+export type StringObject = Record<string, unknown>
+
+function test() {
+  // prettier-ignore
+  const t = <A, B>(a: (
+    A extends B ? (
+      B extends A ? null : never
+    ) : never
+  )) =>{ }
+  t<"foo-bar", ToKebab<"FooBar">>(null)
+  // @ts-expect-error
+  t<"foo-3ar", ToKebab<"FooBar">>(null)
+}