diff --git a/lib/StartSdk.ts b/lib/StartSdk.ts index 636bcff..72e6cf2 100644 --- a/lib/StartSdk.ts +++ b/lib/StartSdk.ts @@ -89,7 +89,7 @@ export class StartSdk { }, fn: (options: { effects: Effects - utils: Utils + utils: Utils input: Type }) => Promise, ) => { @@ -105,11 +105,11 @@ export class StartSdk { >( metaData: (options: { effects: Effects - utils: Utils + utils: Utils }) => MaybePromise>, fn: (options: { effects: Effects - utils: Utils + utils: Utils input: Type }) => Promise, input: Config | Config, @@ -136,7 +136,7 @@ export class StartSdk { >( spec: ConfigType, write: Save, - read: Read, + read: Read, ) => setupConfig(spec, write, read), setupConfigRead: < ConfigSpec extends @@ -144,7 +144,7 @@ export class StartSdk { | Config, never>, >( _configSpec: ConfigSpec, - fn: Read, + fn: Read, ) => fn, setupConfigSave: < ConfigSpec extends @@ -158,6 +158,7 @@ export class StartSdk { config: Config | Config, autoConfigs: { [K in keyof Manifest["dependencies"]]: DependencyConfig< + Manifest, Store, Input, any @@ -192,9 +193,9 @@ export class StartSdk { fn: (o: { effects: Effects started(onTerm: () => void): null - utils: Utils + utils: Utils }) => Promise>, - ) => setupMain(fn), + ) => setupMain(fn), setupMigrations: >>( ...migrations: EnsureUniqueId ) => setupMigrations(this.manifest, ...migrations), @@ -238,14 +239,16 @@ export class StartSdk { dependencyConfig: (options: { effects: Effects localConfig: LocalConfig - utils: Utils + utils: Utils }) => Promise> update?: Update, RemoteConfig> }) { - return new DependencyConfig( - dependencyConfig, - update, - ) + return new DependencyConfig< + Manifest, + Store, + LocalConfig, + RemoteConfig + >(dependencyConfig, update) }, }, List: { @@ -320,10 +323,13 @@ export class StartSdk { Migration: { of: (options: { version: Version - up: (opts: { effects: Effects; utils: Utils }) => Promise + up: (opts: { + effects: Effects + utils: Utils + }) => Promise down: (opts: { effects: Effects - utils: Utils + utils: Utils }) => Promise }) => Migration.of(options), }, diff --git a/lib/config/setupConfig.ts b/lib/config/setupConfig.ts index 1e8c2c4..92680a5 100644 --- a/lib/config/setupConfig.ts +++ b/lib/config/setupConfig.ts @@ -22,7 +22,7 @@ export type Save< > = (options: { effects: Effects input: ExtractConfigType & Record - utils: Utils + utils: Utils dependencies: D.ConfigDependencies }) => Promise<{ dependenciesReceipt: DependenciesReceipt @@ -30,6 +30,7 @@ export type Save< restart: boolean }> export type Read< + Manifest extends SDKManifest, Store, A extends | Record @@ -37,7 +38,7 @@ export type Read< | Config, never>, > = (options: { effects: Effects - utils: Utils + utils: Utils }) => Promise & Record)> /** * We want to setup a config export with a get and set, this @@ -57,7 +58,7 @@ export function setupConfig< >( spec: Config | Config, write: Save, - read: Read, + read: Read, ) { const validator = spec.validator return { @@ -79,7 +80,7 @@ export function setupConfig< } }) as ExpectedExports.setConfig, getConfig: (async ({ effects }) => { - const myUtils = utils(effects) + const myUtils = utils(effects) const configValue = nullIfEmpty( (await read({ effects, utils: myUtils })) || null, ) diff --git a/lib/dependencyConfig/DependencyConfig.ts b/lib/dependencyConfig/DependencyConfig.ts index 7d936bc..dc01058 100644 --- a/lib/dependencyConfig/DependencyConfig.ts +++ b/lib/dependencyConfig/DependencyConfig.ts @@ -6,6 +6,7 @@ import { import { Utils, utils } from "../util/utils" import { deepEqual } from "../util/deepEqual" import { deepMerge } from "../util/deepMerge" +import { SDKManifest } from "../manifest/ManifestTypes" export type Update = (options: { remoteConfig: RemoteConfig @@ -13,6 +14,7 @@ export type Update = (options: { }) => Promise export class DependencyConfig< + Manifest extends SDKManifest, Store, Input extends Record, RemoteConfig extends Record, @@ -27,7 +29,7 @@ export class DependencyConfig< readonly dependencyConfig: (options: { effects: Effects localConfig: Input - utils: Utils + utils: Utils }) => Promise>, readonly update: Update< void | DeepPartial, @@ -39,7 +41,7 @@ export class DependencyConfig< return this.dependencyConfig({ localConfig: options.localConfig as Input, effects: options.effects, - utils: utils(options.effects), + utils: utils(options.effects), }) } } diff --git a/lib/dependencyConfig/setupDependencyConfig.ts b/lib/dependencyConfig/setupDependencyConfig.ts index 1039524..3b67769 100644 --- a/lib/dependencyConfig/setupDependencyConfig.ts +++ b/lib/dependencyConfig/setupDependencyConfig.ts @@ -11,6 +11,7 @@ export function setupDependencyConfig< _config: Config | Config, autoConfigs: { [key in keyof Manifest["dependencies"] & string]: DependencyConfig< + Manifest, Store, Input, any diff --git a/lib/mainFn/index.ts b/lib/mainFn/index.ts index 58e09b7..96b5642 100644 --- a/lib/mainFn/index.ts +++ b/lib/mainFn/index.ts @@ -6,6 +6,7 @@ import "../interfaces/NetworkInterfaceBuilder" import "../interfaces/Origin" import "./Daemons" +import { SDKManifest } from "../manifest/ManifestTypes" /** * Used to ensure that the main function is running with the valid proofs. @@ -17,17 +18,17 @@ import "./Daemons" * @param fn * @returns */ -export const setupMain = ( +export const setupMain = ( fn: (o: { effects: Effects started(onTerm: () => void): null - utils: Utils + utils: Utils }) => Promise>, ): ExpectedExports.main => { return async (options) => { const result = await fn({ ...options, - utils: createMainUtils(options.effects), + utils: createMainUtils(options.effects), }) await result.build().then((x) => x.wait()) } diff --git a/lib/test/store.test.ts b/lib/test/store.test.ts index 0db8757..9291c31 100644 --- a/lib/test/store.test.ts +++ b/lib/test/store.test.ts @@ -7,6 +7,7 @@ type Store = { someValue: "a" | "b" } } +type Manifest = any const todo = (): A => { throw new Error("not implemented") } @@ -14,14 +15,17 @@ const noop = () => {} describe("Store", () => { test("types", async () => { ;async () => { - utils(todo()).store.setOwn("/config", { + utils(todo()).store.setOwn("/config", { someValue: "a", }) - utils(todo()).store.setOwn("/config/someValue", "b") - utils(todo()).store.setOwn("", { + utils(todo()).store.setOwn( + "/config/someValue", + "b", + ) + utils(todo()).store.setOwn("", { config: { someValue: "b" }, }) - utils(todo()).store.setOwn( + utils(todo()).store.setOwn( "/config/someValue", // @ts-expect-error Type is wrong for the setting value @@ -48,10 +52,10 @@ describe("Store", () => { path: "/config/some2Value", value: "a", }) - ;(await createMainUtils(todo()) + ;(await createMainUtils(todo()) .store.getOwn("/config/someValue") .const()) satisfies string - ;(await createMainUtils(todo()) + ;(await createMainUtils(todo()) .store.getOwn("/config") .const()) satisfies Store["config"] await createMainUtils(todo()) @@ -60,31 +64,31 @@ describe("Store", () => { .const() /// ----------------- ERRORS ----------------- - utils(todo()).store.setOwn("", { + utils(todo()).store.setOwn("", { // @ts-expect-error Type is wrong for the setting value config: { someValue: "notInAOrB" }, }) - utils(todo()).store.setOwn( + utils(todo()).store.setOwn( "/config/someValue", // @ts-expect-error Type is wrong for the setting value "notInAOrB", ) - ;(await utils(todo()) + ;(await utils(todo()) .store.getOwn("/config/someValue") // @ts-expect-error Const should normally not be callable .const()) satisfies string - ;(await utils(todo()) + ;(await utils(todo()) .store.getOwn("/config") // @ts-expect-error Const should normally not be callable .const()) satisfies Store["config"] - await utils(todo()) + await utils(todo()) // @ts-expect-error Path is wrong .store.getOwn("/config/somdsfeValue") // @ts-expect-error Const should normally not be callable .const() /// - ;(await utils(todo()) + ;(await utils(todo()) .store.getOwn("/config/someValue") // @ts-expect-error satisfies type is wrong .const()) satisfies number diff --git a/lib/types.ts b/lib/types.ts index 2189523..c57d70d 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -2,6 +2,7 @@ export * as configTypes from "./config/configTypes" import { InputSpec } from "./config/configTypes" import { DependenciesReceipt } from "./config/setupConfig" import { PortOptions } from "./interfaces/Host" +import { Overlay } from "./util/Overlay" import { UrlString } from "./util/getNetworkInterface" import { NetworkInterfaceType, Signals } from "./util/utils" @@ -225,7 +226,7 @@ export type Effects = { input: Input }): Promise - /** The idea is that we want to create a sub image. This would be useful for things like creating a ro mode for sandbox. */ + /** A low level api used by makeOverlay */ createOverlayedImage(options: { imageId: string }): Promise /** Removes all network bindings */ diff --git a/lib/util/Overlay.ts b/lib/util/Overlay.ts new file mode 100644 index 0000000..2b82f4d --- /dev/null +++ b/lib/util/Overlay.ts @@ -0,0 +1,92 @@ +import fs from "fs/promises" +import * as T from "../types" +import cp from "child_process" +import { promisify } from "util" +import { Buffer } from "node:buffer" +export const execFile = promisify(cp.execFile) + +export class Overlay { + private constructor(readonly effects: T.Effects, readonly rootfs: string) {} + static async of(effects: T.Effects, imageId: string) { + const rootfs = await effects.createOverlayedImage({ imageId }) + + for (const dirPart of ["dev", "sys", "proc", "run"] as const) { + const dir = await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true }) + if (!dir) break + await execFile("mount", ["--bind", `/${dirPart}`, dir]) + } + + return new Overlay(effects, rootfs) + } + + async mount(options: MountOptions, path: string): Promise { + path = path.startsWith("/") + ? `${this.rootfs}${path}` + : `${this.rootfs}/${path}` + if (options.type === "volume") { + await execFile("mount", [ + "--bind", + `/media/startos/volumes/${options.id}`, + path, + ]) + } else if (options.type === "assets") { + await execFile("mount", [ + "--bind", + `/media/startos/assets/${options.id}`, + path, + ]) + } else if (options.type === "pointer") { + await this.effects.mount({ location: path, target: options }) + } else { + throw new Error(`unknown type ${(options as any).type}`) + } + return this + } + + async destroy() { + await execFile("umount", ["-R", this.rootfs]) + await fs.rm(this.rootfs, { recursive: true, force: true }) + } + + async exec( + command: string[], + options?: CommandOptions, + ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> { + return await execFile("chroot", [this.rootfs, ...command], options) + } + + spawn( + command: string[], + options?: CommandOptions, + ): cp.ChildProcessWithoutNullStreams { + return cp.spawn("chroot", [this.rootfs, ...command], options) + } +} + +export type CommandOptions = { + env?: { [variable: string]: string } + cwd?: string +} + +export type MountOptions = + | MountOptionsVolume + | MountOptionsAssets + | MountOptionsPointer + +export type MountOptionsVolume = { + type: "volume" + id: string +} + +export type MountOptionsAssets = { + type: "assets" + id: string +} + +export type MountOptionsPointer = { + type: "pointer" + packageId: string + volumeId: string + path: string + readonly: boolean +} diff --git a/lib/util/index.ts b/lib/util/index.ts index ab8ee08..7ed65b1 100644 --- a/lib/util/index.ts +++ b/lib/util/index.ts @@ -5,8 +5,10 @@ import "./fileHelper" import "../store/getStore" import "./deepEqual" import "./deepMerge" +import "./Overlay" import "./once" import { utils } from "./utils" +import { SDKManifest } from "../manifest/ManifestTypes" // prettier-ignore export type FlattenIntersection = @@ -22,8 +24,9 @@ export const isKnownError = (e: unknown): e is T.KnownError => declare const affine: unique symbol export const createUtils = utils -export const createMainUtils = (effects: T.Effects) => - createUtils(effects) +export const createMainUtils = ( + effects: T.Effects, +) => createUtils(effects) type NeverPossible = { [affine]: string } export type NoAny = NeverPossible extends A diff --git a/lib/util/utils.ts b/lib/util/utils.ts index 75a9512..50540ea 100644 --- a/lib/util/utils.ts +++ b/lib/util/utils.ts @@ -37,6 +37,8 @@ import { import * as CP from "node:child_process" import { promisify } from "node:util" import { splitCommand } from "./splitCommand" +import { SDKManifest } from "../manifest/ManifestTypes" +import { MountOptions, Overlay, CommandOptions } from "./Overlay" export type Signals = NodeJS.Signals export const SIGTERM: Signals = "SIGTERM" @@ -50,7 +52,11 @@ const childProcess = { export type NetworkInterfaceType = "ui" | "p2p" | "api" | "other" -export type Utils = { +export type Utils< + Manifest extends SDKManifest, + Store, + WrapperOverWrite = { const: never }, +> = { checkPortListening( port: number, options: { @@ -107,9 +113,19 @@ export type Utils = { }) => GetNetworkInterfaces & WrapperOverWrite } nullIfEmpty: typeof nullIfEmpty - runDaemon: ( + runCommand: ( + imageId: Manifest["images"][number], command: ValidIfNoStupidEscape | [string, ...string[]], - options: { env?: Record }, + options: CommandOptions & { + mounts?: { path: string; options: MountOptions }[] + }, + ) => Promise<{ stdout: string | Buffer; stderr: string | Buffer }> + runDaemon: ( + imageId: Manifest["images"][number], + command: ValidIfNoStupidEscape | [string, ...string[]], + options: CommandOptions & { + mounts?: { path: string; options: MountOptions }[] + }, ) => Promise store: { get: ( @@ -125,9 +141,13 @@ export type Utils = { ) => Promise } } -export const utils = ( +export const utils = < + Manifest extends SDKManifest, + Store = never, + WrapperOverWrite = { const: never }, +>( effects: Effects, -): Utils => { +): Utils => { return { createInterface: (options: { name: string @@ -181,12 +201,39 @@ export const utils = ( ) => effects.store.set({ value, path: path as any }), }, - runDaemon: async ( + runCommand: async ( + imageId: Manifest["images"][number], command: ValidIfNoStupidEscape | [string, ...string[]], - options: { env?: Record }, + options: CommandOptions & { + mounts?: { path: string; options: MountOptions }[] + }, + ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => { + const commands = splitCommand(command) + const overlay = await Overlay.of(effects, imageId) + try { + for (let mount of options.mounts || []) { + await overlay.mount(mount.options, mount.path) + } + return await overlay.exec(commands) + } finally { + await overlay.destroy() + } + }, + runDaemon: async ( + imageId: Manifest["images"][number], + command: ValidIfNoStupidEscape | [string, ...string[]], + options: CommandOptions & { + mounts?: { path: string; options: MountOptions }[] + }, ): Promise => { const commands = splitCommand(command) - const childProcess = CP.spawn(commands[0], commands.slice(1), options) + const overlay = await Overlay.of(effects, imageId) + for (let mount of options.mounts || []) { + await overlay.mount(mount.options, mount.path) + } + const childProcess = overlay.spawn(commands, { + env: options.env, + }) const answer = new Promise((resolve, reject) => { const output: string[] = [] childProcess.stdout.on("data", (data) => { @@ -210,18 +257,22 @@ export const utils = ( return answer }, async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) { - childProcess.kill(signal) + try { + childProcess.kill(signal) - if (timeout <= NO_TIMEOUT) { - const didTimeout = await Promise.race([ - new Promise((resolve) => setTimeout(resolve, timeout)).then( - () => true, - ), - answer.then(() => false), - ]) - if (didTimeout) childProcess.kill(SIGKILL) + if (timeout <= NO_TIMEOUT) { + const didTimeout = await Promise.race([ + new Promise((resolve) => setTimeout(resolve, timeout)).then( + () => true, + ), + answer.then(() => false), + ]) + if (didTimeout) childProcess.kill(SIGKILL) + } + await answer + } finally { + await overlay.destroy() } - await answer }, } },