feat: Add in overlay

lib/StartSdk.ts

@@ -89,7 +89,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
         },
         fn: (options: {
           effects: Effects
-          utils: Utils<Store>
+          utils: Utils<Manifest, Store>
           input: Type
         }) => Promise<ActionResult>,
       ) => {
@@ -105,11 +105,11 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       >(
         metaData: (options: {
           effects: Effects
-          utils: Utils<Store>
+          utils: Utils<Manifest, Store>
         }) => MaybePromise<Omit<ActionMetadata, "input">>,
         fn: (options: {
           effects: Effects
-          utils: Utils<Store>
+          utils: Utils<Manifest, Store>
           input: Type
         }) => Promise<ActionResult>,
         input: Config<Type, Store> | Config<Type, never>,
@@ -136,7 +136,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       >(
         spec: ConfigType,
         write: Save<Store, Type, Manifest>,
-        read: Read<Store, Type>,
+        read: Read<Manifest, Store, Type>,
       ) => setupConfig<Store, ConfigType, Manifest, Type>(spec, write, read),
       setupConfigRead: <
         ConfigSpec extends
@@ -144,7 +144,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
           | Config<Record<string, never>, never>,
       >(
         _configSpec: ConfigSpec,
-        fn: Read<Store, ConfigSpec>,
+        fn: Read<Manifest, Store, ConfigSpec>,
       ) => fn,
       setupConfigSave: <
         ConfigSpec extends
@@ -158,6 +158,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
         config: Config<Input, Store> | Config<Input, never>,
         autoConfigs: {
           [K in keyof Manifest["dependencies"]]: DependencyConfig<
+            Manifest,
             Store,
             Input,
             any
@@ -192,9 +193,9 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
         fn: (o: {
           effects: Effects
           started(onTerm: () => void): null
-          utils: Utils<Store, {}>
+          utils: Utils<Manifest, Store, {}>
         }) => Promise<Daemons<any>>,
-      ) => setupMain<Store>(fn),
+      ) => setupMain<Manifest, Store>(fn),
       setupMigrations: <Migrations extends Array<Migration<Store, any>>>(
         ...migrations: EnsureUniqueId<Migrations>
       ) => setupMigrations<Store, Migrations>(this.manifest, ...migrations),
@@ -238,14 +239,16 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
           dependencyConfig: (options: {
             effects: Effects
             localConfig: LocalConfig
-            utils: Utils<Store>
+            utils: Utils<Manifest, Store>
           }) => Promise<void | DeepPartial<RemoteConfig>>
           update?: Update<void | DeepPartial<RemoteConfig>, RemoteConfig>
         }) {
-          return new DependencyConfig<Store, LocalConfig, RemoteConfig>(
+          return new DependencyConfig<
-            dependencyConfig,
+            Manifest,
-            update,
+            Store,
-          )
+            LocalConfig,
+            RemoteConfig
+          >(dependencyConfig, update)
         },
       },
       List: {
@@ -320,10 +323,13 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       Migration: {
         of: <Version extends ManifestVersion>(options: {
           version: Version
-          up: (opts: { effects: Effects; utils: Utils<Store> }) => Promise<void>
+          up: (opts: {
+            effects: Effects
+            utils: Utils<Manifest, Store>
+          }) => Promise<void>
           down: (opts: {
             effects: Effects
-            utils: Utils<Store>
+            utils: Utils<Manifest, Store>
           }) => Promise<void>
         }) => Migration.of<Store, Version>(options),
       },

lib/config/setupConfig.ts

@@ -22,7 +22,7 @@ export type Save<
 > = (options: {
   effects: Effects
   input: ExtractConfigType<A> & Record<string, any>
-  utils: Utils<Store>
+  utils: Utils<Manifest, Store>
   dependencies: D.ConfigDependencies<Manifest>
 }) => Promise<{
   dependenciesReceipt: DependenciesReceipt
@@ -30,6 +30,7 @@ export type Save<
   restart: boolean
 }>
 export type Read<
+  Manifest extends SDKManifest,
   Store,
   A extends
     | Record<string, any>
@@ -37,7 +38,7 @@ export type Read<
     | Config<Record<string, any>, never>,
 > = (options: {
   effects: Effects
-  utils: Utils<Store>
+  utils: Utils<Manifest, Store>
 }) => Promise<void | (ExtractConfigType<A> & Record<string, any>)>
 /**
  * We want to setup a config export with a get and set, this
@@ -57,7 +58,7 @@ export function setupConfig<
 >(
   spec: Config<Type, Store> | Config<Type, never>,
   write: Save<Store, Type, Manifest>,
-  read: Read<Store, Type>,
+  read: Read<Manifest, Store, Type>,
 ) {
   const validator = spec.validator
   return {
@@ -79,7 +80,7 @@ export function setupConfig<
       }
     }) as ExpectedExports.setConfig,
     getConfig: (async ({ effects }) => {
-      const myUtils = utils<Store>(effects)
+      const myUtils = utils<Manifest, Store>(effects)
       const configValue = nullIfEmpty(
         (await read({ effects, utils: myUtils })) || null,
       )

lib/dependencyConfig/DependencyConfig.ts

@@ -6,6 +6,7 @@ import {
 import { Utils, utils } from "../util/utils"
 import { deepEqual } from "../util/deepEqual"
 import { deepMerge } from "../util/deepMerge"
+import { SDKManifest } from "../manifest/ManifestTypes"
 
 export type Update<QueryResults, RemoteConfig> = (options: {
   remoteConfig: RemoteConfig
@@ -13,6 +14,7 @@ export type Update<QueryResults, RemoteConfig> = (options: {
 }) => Promise<RemoteConfig>
 
 export class DependencyConfig<
+  Manifest extends SDKManifest,
   Store,
   Input extends Record<string, any>,
   RemoteConfig extends Record<string, any>,
@@ -27,7 +29,7 @@ export class DependencyConfig<
     readonly dependencyConfig: (options: {
       effects: Effects
       localConfig: Input
-      utils: Utils<Store>
+      utils: Utils<Manifest, Store>
     }) => Promise<void | DeepPartial<RemoteConfig>>,
     readonly update: Update<
       void | DeepPartial<RemoteConfig>,
@@ -39,7 +41,7 @@ export class DependencyConfig<
     return this.dependencyConfig({
       localConfig: options.localConfig as Input,
       effects: options.effects,
-      utils: utils<Store>(options.effects),
+      utils: utils<Manifest, Store>(options.effects),
     })
   }
 }

lib/dependencyConfig/setupDependencyConfig.ts

@@ -11,6 +11,7 @@ export function setupDependencyConfig<
   _config: Config<Input, Store> | Config<Input, never>,
   autoConfigs: {
     [key in keyof Manifest["dependencies"] & string]: DependencyConfig<
+      Manifest,
       Store,
       Input,
       any

lib/mainFn/index.ts

@@ -6,6 +6,7 @@ import "../interfaces/NetworkInterfaceBuilder"
 import "../interfaces/Origin"
 
 import "./Daemons"
+import { SDKManifest } from "../manifest/ManifestTypes"
 
 /**
  * Used to ensure that the main function is running with the valid proofs.
@@ -17,17 +18,17 @@ import "./Daemons"
  * @param fn
  * @returns
  */
-export const setupMain = <Store>(
+export const setupMain = <Manifest extends SDKManifest, Store>(
   fn: (o: {
     effects: Effects
     started(onTerm: () => void): null
-    utils: Utils<Store, {}>
+    utils: Utils<Manifest, Store, {}>
   }) => Promise<Daemons<any>>,
 ): ExpectedExports.main => {
   return async (options) => {
     const result = await fn({
       ...options,
-      utils: createMainUtils<Store>(options.effects),
+      utils: createMainUtils<Manifest, Store>(options.effects),
     })
     await result.build().then((x) => x.wait())
   }

lib/test/store.test.ts

@@ -7,6 +7,7 @@ type Store = {
     someValue: "a" | "b"
   }
 }
+type Manifest = any
 const todo = <A>(): A => {
   throw new Error("not implemented")
 }
@@ -14,14 +15,17 @@ const noop = () => {}
 describe("Store", () => {
   test("types", async () => {
     ;async () => {
-      utils<Store>(todo<Effects>()).store.setOwn("/config", {
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn("/config", {
         someValue: "a",
       })
-      utils<Store>(todo<Effects>()).store.setOwn("/config/someValue", "b")
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn(
-      utils<Store>(todo<Effects>()).store.setOwn("", {
+        "/config/someValue",
+        "b",
+      )
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn("", {
         config: { someValue: "b" },
       })
-      utils<Store>(todo<Effects>()).store.setOwn(
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn(
         "/config/someValue",
 
         // @ts-expect-error Type is wrong for the setting value
@@ -48,10 +52,10 @@ describe("Store", () => {
         path: "/config/some2Value",
         value: "a",
       })
-      ;(await createMainUtils<Store>(todo<Effects>())
+      ;(await createMainUtils<Manifest, Store>(todo<Effects>())
         .store.getOwn("/config/someValue")
         .const()) satisfies string
-      ;(await createMainUtils<Store>(todo<Effects>())
+      ;(await createMainUtils<Manifest, Store>(todo<Effects>())
         .store.getOwn("/config")
         .const()) satisfies Store["config"]
       await createMainUtils(todo<Effects>())
@@ -60,31 +64,31 @@ describe("Store", () => {
         .const()
       ///  ----------------- ERRORS -----------------
 
-      utils<Store>(todo<Effects>()).store.setOwn("", {
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn("", {
         // @ts-expect-error Type is wrong for the setting value
         config: { someValue: "notInAOrB" },
       })
-      utils<Store>(todo<Effects>()).store.setOwn(
+      utils<Manifest, Store>(todo<Effects>()).store.setOwn(
         "/config/someValue",
         // @ts-expect-error Type is wrong for the setting value
         "notInAOrB",
       )
-      ;(await utils<Store>(todo<Effects>())
+      ;(await utils<Manifest, Store>(todo<Effects>())
         .store.getOwn("/config/someValue")
         // @ts-expect-error Const should normally not be callable
         .const()) satisfies string
-      ;(await utils<Store>(todo<Effects>())
+      ;(await utils<Manifest, Store>(todo<Effects>())
         .store.getOwn("/config")
         // @ts-expect-error Const should normally not be callable
         .const()) satisfies Store["config"]
-      await utils<Store>(todo<Effects>())
+      await utils<Manifest, Store>(todo<Effects>())
         // @ts-expect-error Path is wrong
         .store.getOwn("/config/somdsfeValue")
         // @ts-expect-error Const should normally not be callable
         .const()
 
       ///
-      ;(await utils<Store>(todo<Effects>())
+      ;(await utils<Manifest, Store>(todo<Effects>())
         .store.getOwn("/config/someValue")
         // @ts-expect-error satisfies type is wrong
         .const()) satisfies number

lib/types.ts

@@ -2,6 +2,7 @@ export * as configTypes from "./config/configTypes"
 import { InputSpec } from "./config/configTypes"
 import { DependenciesReceipt } from "./config/setupConfig"
 import { PortOptions } from "./interfaces/Host"
+import { Overlay } from "./util/Overlay"
 import { UrlString } from "./util/getNetworkInterface"
 import { NetworkInterfaceType, Signals } from "./util/utils"
 
@@ -225,7 +226,7 @@ export type Effects = {
     input: Input
   }): Promise<unknown>
 
-  /** The idea is that we want to create a sub image. This would be useful for things like creating a ro mode for sandbox. */
+  /** A low level api used by makeOverlay */
   createOverlayedImage(options: { imageId: string }): Promise<string>
 
   /** Removes all network bindings */

lib/util/Overlay.ts

@@ -0,0 +1,92 @@
+import fs from "fs/promises"
+import * as T from "../types"
+import cp from "child_process"
+import { promisify } from "util"
+import { Buffer } from "node:buffer"
+export const execFile = promisify(cp.execFile)
+
+export class Overlay {
+  private constructor(readonly effects: T.Effects, readonly rootfs: string) {}
+  static async of(effects: T.Effects, imageId: string) {
+    const rootfs = await effects.createOverlayedImage({ imageId })
+
+    for (const dirPart of ["dev", "sys", "proc", "run"] as const) {
+      const dir = await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true })
+      if (!dir) break
+      await execFile("mount", ["--bind", `/${dirPart}`, dir])
+    }
+
+    return new Overlay(effects, rootfs)
+  }
+
+  async mount(options: MountOptions, path: string): Promise<Overlay> {
+    path = path.startsWith("/")
+      ? `${this.rootfs}${path}`
+      : `${this.rootfs}/${path}`
+    if (options.type === "volume") {
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/volumes/${options.id}`,
+        path,
+      ])
+    } else if (options.type === "assets") {
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/assets/${options.id}`,
+        path,
+      ])
+    } else if (options.type === "pointer") {
+      await this.effects.mount({ location: path, target: options })
+    } else {
+      throw new Error(`unknown type ${(options as any).type}`)
+    }
+    return this
+  }
+
+  async destroy() {
+    await execFile("umount", ["-R", this.rootfs])
+    await fs.rm(this.rootfs, { recursive: true, force: true })
+  }
+
+  async exec(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
+    return await execFile("chroot", [this.rootfs, ...command], options)
+  }
+
+  spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): cp.ChildProcessWithoutNullStreams {
+    return cp.spawn("chroot", [this.rootfs, ...command], options)
+  }
+}
+
+export type CommandOptions = {
+  env?: { [variable: string]: string }
+  cwd?: string
+}
+
+export type MountOptions =
+  | MountOptionsVolume
+  | MountOptionsAssets
+  | MountOptionsPointer
+
+export type MountOptionsVolume = {
+  type: "volume"
+  id: string
+}
+
+export type MountOptionsAssets = {
+  type: "assets"
+  id: string
+}
+
+export type MountOptionsPointer = {
+  type: "pointer"
+  packageId: string
+  volumeId: string
+  path: string
+  readonly: boolean
+}

lib/util/index.ts

@@ -5,8 +5,10 @@ import "./fileHelper"
 import "../store/getStore"
 import "./deepEqual"
 import "./deepMerge"
+import "./Overlay"
 import "./once"
 import { utils } from "./utils"
+import { SDKManifest } from "../manifest/ManifestTypes"
 
 // prettier-ignore
 export type FlattenIntersection<T> = 
@@ -22,8 +24,9 @@ export const isKnownError = (e: unknown): e is T.KnownError =>
 declare const affine: unique symbol
 
 export const createUtils = utils
-export const createMainUtils = <Store>(effects: T.Effects) =>
+export const createMainUtils = <Manifest extends SDKManifest, Store>(
-  createUtils<Store, {}>(effects)
+  effects: T.Effects,
+) => createUtils<Manifest, Store, {}>(effects)
 
 type NeverPossible = { [affine]: string }
 export type NoAny<A> = NeverPossible extends A

lib/util/utils.ts

@@ -37,6 +37,8 @@ import {
 import * as CP from "node:child_process"
 import { promisify } from "node:util"
 import { splitCommand } from "./splitCommand"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { MountOptions, Overlay, CommandOptions } from "./Overlay"
 export type Signals = NodeJS.Signals
 
 export const SIGTERM: Signals = "SIGTERM"
@@ -50,7 +52,11 @@ const childProcess = {
 
 export type NetworkInterfaceType = "ui" | "p2p" | "api" | "other"
 
-export type Utils<Store, WrapperOverWrite = { const: never }> = {
+export type Utils<
+  Manifest extends SDKManifest,
+  Store,
+  WrapperOverWrite = { const: never },
+> = {
   checkPortListening(
     port: number,
     options: {
@@ -107,9 +113,19 @@ export type Utils<Store, WrapperOverWrite = { const: never }> = {
     }) => GetNetworkInterfaces & WrapperOverWrite
   }
   nullIfEmpty: typeof nullIfEmpty
-  runDaemon: <A extends string>(
+  runCommand: <A extends string>(
+    imageId: Manifest["images"][number],
     command: ValidIfNoStupidEscape<A> | [string, ...string[]],
-    options: { env?: Record<string, string> },
+    options: CommandOptions & {
+      mounts?: { path: string; options: MountOptions }[]
+    },
+  ) => Promise<{ stdout: string | Buffer; stderr: string | Buffer }>
+  runDaemon: <A extends string>(
+    imageId: Manifest["images"][number],
+    command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+    options: CommandOptions & {
+      mounts?: { path: string; options: MountOptions }[]
+    },
   ) => Promise<DaemonReturned>
   store: {
     get: <Path extends string>(
@@ -125,9 +141,13 @@ export type Utils<Store, WrapperOverWrite = { const: never }> = {
     ) => Promise<void>
   }
 }
-export const utils = <Store = never, WrapperOverWrite = { const: never }>(
+export const utils = <
+  Manifest extends SDKManifest,
+  Store = never,
+  WrapperOverWrite = { const: never },
+>(
   effects: Effects,
-): Utils<Store, WrapperOverWrite> => {
+): Utils<Manifest, Store, WrapperOverWrite> => {
   return {
     createInterface: (options: {
       name: string
@@ -181,12 +201,39 @@ export const utils = <Store = never, WrapperOverWrite = { const: never }>(
       ) => effects.store.set<Store, Path>({ value, path: path as any }),
     },
 
-    runDaemon: async <A extends string>(
+    runCommand: async <A extends string>(
+      imageId: Manifest["images"][number],
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],
-      options: { env?: Record<string, string> },
+      options: CommandOptions & {
+        mounts?: { path: string; options: MountOptions }[]
+      },
+    ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => {
+      const commands = splitCommand(command)
+      const overlay = await Overlay.of(effects, imageId)
+      try {
+        for (let mount of options.mounts || []) {
+          await overlay.mount(mount.options, mount.path)
+        }
+        return await overlay.exec(commands)
+      } finally {
+        await overlay.destroy()
+      }
+    },
+    runDaemon: async <A extends string>(
+      imageId: Manifest["images"][number],
+      command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+      options: CommandOptions & {
+        mounts?: { path: string; options: MountOptions }[]
+      },
     ): Promise<DaemonReturned> => {
       const commands = splitCommand(command)
-      const childProcess = CP.spawn(commands[0], commands.slice(1), options)
+      const overlay = await Overlay.of(effects, imageId)
+      for (let mount of options.mounts || []) {
+        await overlay.mount(mount.options, mount.path)
+      }
+      const childProcess = overlay.spawn(commands, {
+        env: options.env,
+      })
       const answer = new Promise<string>((resolve, reject) => {
         const output: string[] = []
         childProcess.stdout.on("data", (data) => {
@@ -210,18 +257,22 @@ export const utils = <Store = never, WrapperOverWrite = { const: never }>(
           return answer
         },
         async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) {
-          childProcess.kill(signal)
+          try {
+            childProcess.kill(signal)
 
-          if (timeout <= NO_TIMEOUT) {
+            if (timeout <= NO_TIMEOUT) {
-            const didTimeout = await Promise.race([
+              const didTimeout = await Promise.race([
-              new Promise((resolve) => setTimeout(resolve, timeout)).then(
+                new Promise((resolve) => setTimeout(resolve, timeout)).then(
-                () => true,
+                  () => true,
-              ),
+                ),
-              answer.then(() => false),
+                answer.then(() => false),
-            ])
+              ])
-            if (didTimeout) childProcess.kill(SIGKILL)
+              if (didTimeout) childProcess.kill(SIGKILL)
+            }
+            await answer
+          } finally {
+            await overlay.destroy()
           }
-          await answer
         },
       }
     },