Refactor/actions (#2733)

* store, properties, manifest * interfaces * init and backups * fix init and backups * file models * more versions * dependencies * config except dynamic types * clean up config * remove disabled from non-dynamic vaues * actions * standardize example code block formats * wip: actions refactor Co-authored-by: Jade <Blu-J@users.noreply.github.com> * commit types * fix types * update types * update action request type * update apis * add description to actionrequest * clean up imports * revert package json * chore: Remove the recursive to the index * chore: Remove the other thing I was testing * flatten action requests * update container runtime with new config paradigm * new actions strategy * seems to be working * misc backend fixes * fix fe bugs * only show breakages if breakages * only show success modal if result * don't panic on failed removal * hide config from actions page * polyfill autoconfig * use metadata strategy for actions instead of prev * misc fixes * chore: split the sdk into 2 libs (#2736) * follow sideload progress (#2718) * follow sideload progress * small bugfix * shareReplay with no refcount false * don't wrap sideload progress in RPCResult * dont present toast --------- Co-authored-by: Aiden McClelland <me@drbonez.dev> * chore: Add the initial of the creation of the two sdk * chore: Add in the baseDist * chore: Add in the baseDist * chore: Get the web and the runtime-container running * chore: Remove the empty file * chore: Fix it so the container-runtime works --------- Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com> Co-authored-by: Aiden McClelland <me@drbonez.dev> * misc fixes * update todos * minor clean up * fix link script * update node version in CI test * fix node version syntax in ci build * wip: fixing callbacks * fix sdk makefile dependencies * add support for const outside of main * update apis * don't panic! * Chore: Capture weird case on rpc, and log that * fix procedure id issue * pass input value for dep auto config * handle disabled and warning for actions * chore: Fix for link not having node_modules * sdk fixes * fix build * fix build * fix build --------- Co-authored-by: Matt Hill <mattnine@protonmail.com> Co-authored-by: Jade <Blu-J@users.noreply.github.com> Co-authored-by: J H <dragondef@gmail.com> Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com> Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>
2026-04-04 22:39:46 +00:00 · 2024-09-25 16:12:52 -06:00
parent eec5cf6b65
commit db0695126f
469 changed files with 16218 additions and 10485 deletions
--- a/sdk/package/lib/mainFn/CommandController.ts
+++ b/sdk/package/lib/mainFn/CommandController.ts
@@ -0,0 +1,144 @@
+import { DEFAULT_SIGTERM_TIMEOUT } from "."
+import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../../../base/lib/types"
+
+import * as T from "../../../base/lib/types"
+import { asError } from "../../../base/lib/util/asError"
+import {
+  ExecSpawnable,
+  MountOptions,
+  SubContainerHandle,
+  SubContainer,
+} from "../util/SubContainer"
+import { splitCommand } from "../util"
+import * as cp from "child_process"
+
+export class CommandController {
+  private constructor(
+    readonly runningAnswer: Promise<unknown>,
+    private state: { exited: boolean },
+    private readonly subcontainer: SubContainer,
+    private process: cp.ChildProcessWithoutNullStreams,
+    readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
+  ) {}
+  static of<Manifest extends T.Manifest>() {
+    return async <A extends string>(
+      effects: T.Effects,
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
+      command: T.CommandType,
+      options: {
+        subcontainerName?: string
+        // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
+        sigtermTimeout?: number
+        mounts?: { path: string; options: MountOptions }[]
+        runAsInit?: boolean
+        env?:
+          | {
+              [variable: string]: string
+            }
+          | undefined
+        cwd?: string | undefined
+        user?: string | undefined
+        onStdout?: (x: Buffer) => void
+        onStderr?: (x: Buffer) => void
+      },
+    ) => {
+      const commands = splitCommand(command)
+      const subc =
+        subcontainer instanceof SubContainer
+          ? subcontainer
+          : await (async () => {
+              const subc = await SubContainer.of(
+                effects,
+                subcontainer,
+                options?.subcontainerName || commands.join(" "),
+              )
+              for (let mount of options.mounts || []) {
+                await subc.mount(mount.options, mount.path)
+              }
+              return subc
+            })()
+      let childProcess: cp.ChildProcessWithoutNullStreams
+      if (options.runAsInit) {
+        childProcess = await subc.launch(commands, {
+          env: options.env,
+        })
+      } else {
+        childProcess = await subc.spawn(commands, {
+          env: options.env,
+        })
+      }
+      const state = { exited: false }
+      const answer = new Promise<null>((resolve, reject) => {
+        childProcess.on("exit", (code) => {
+          state.exited = true
+          if (
+            code === 0 ||
+            code === 143 ||
+            (code === null && childProcess.signalCode == "SIGTERM")
+          ) {
+            return resolve(null)
+          }
+          if (code) {
+            return reject(new Error(`${commands[0]} exited with code ${code}`))
+          } else {
+            return reject(
+              new Error(
+                `${commands[0]} exited with signal ${childProcess.signalCode}`,
+              ),
+            )
+          }
+        })
+      })
+
+      return new CommandController(
+        answer,
+        state,
+        subc,
+        childProcess,
+        options.sigtermTimeout,
+      )
+    }
+  }
+  get subContainerHandle() {
+    return new SubContainerHandle(this.subcontainer)
+  }
+  async wait({ timeout = NO_TIMEOUT } = {}) {
+    if (timeout > 0)
+      setTimeout(() => {
+        this.term()
+      }, timeout)
+    try {
+      return await this.runningAnswer
+    } finally {
+      if (!this.state.exited) {
+        this.process.kill("SIGKILL")
+      }
+      await this.subcontainer.destroy?.().catch((_) => {})
+    }
+  }
+  async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
+    try {
+      if (!this.state.exited) {
+        if (signal !== "SIGKILL") {
+          setTimeout(() => {
+            if (!this.state.exited) this.process.kill("SIGKILL")
+          }, timeout)
+        }
+        if (!this.process.kill(signal)) {
+          console.error(
+            `failed to send signal ${signal} to pid ${this.process.pid}`,
+          )
+        }
+      }
+
+      await this.runningAnswer
+    } finally {
+      await this.subcontainer.destroy?.()
+    }
+  }
+}
--- a/sdk/package/lib/mainFn/Daemon.ts
+++ b/sdk/package/lib/mainFn/Daemon.ts
@@ -0,0 +1,89 @@
+import * as T from "../../../base/lib/types"
+import { asError } from "../../../base/lib/util/asError"
+import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer"
+import { CommandController } from "./CommandController"
+
+const TIMEOUT_INCREMENT_MS = 1000
+const MAX_TIMEOUT_MS = 30000
+/**
+ * This is a wrapper around CommandController that has a state of off, where the command shouldn't be running
+ * and the others state of running, where it will keep a living running command
+ */
+
+export class Daemon {
+  private commandController: CommandController | null = null
+  private shouldBeRunning = false
+  constructor(private startCommand: () => Promise<CommandController>) {}
+  get subContainerHandle(): undefined | ExecSpawnable {
+    return this.commandController?.subContainerHandle
+  }
+  static of<Manifest extends T.Manifest>() {
+    return async <A extends string>(
+      effects: T.Effects,
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
+      command: T.CommandType,
+      options: {
+        subcontainerName?: string
+        mounts?: { path: string; options: MountOptions }[]
+        env?:
+          | {
+              [variable: string]: string
+            }
+          | undefined
+        cwd?: string | undefined
+        user?: string | undefined
+        onStdout?: (x: Buffer) => void
+        onStderr?: (x: Buffer) => void
+        sigtermTimeout?: number
+      },
+    ) => {
+      const startCommand = () =>
+        CommandController.of<Manifest>()(
+          effects,
+          subcontainer,
+          command,
+          options,
+        )
+      return new Daemon(startCommand)
+    }
+  }
+  async start() {
+    if (this.commandController) {
+      return
+    }
+    this.shouldBeRunning = true
+    let timeoutCounter = 0
+    new Promise(async () => {
+      while (this.shouldBeRunning) {
+        this.commandController = await this.startCommand()
+        await this.commandController.wait().catch((err) => console.error(err))
+        await new Promise((resolve) => setTimeout(resolve, timeoutCounter))
+        timeoutCounter += TIMEOUT_INCREMENT_MS
+        timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
+      }
+    }).catch((err) => {
+      console.error(asError(err))
+    })
+  }
+  async term(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    return this.stop(termOptions)
+  }
+  async stop(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    this.shouldBeRunning = false
+    await this.commandController
+      ?.term({ ...termOptions })
+      .catch((e) => console.error(asError(e)))
+    this.commandController = null
+  }
+}
--- a/sdk/package/lib/mainFn/Daemons.ts
+++ b/sdk/package/lib/mainFn/Daemons.ts
@@ -0,0 +1,172 @@
+import { HealthReceipt, Signals } from "../../../base/lib/types"
+
+import { HealthCheckResult } from "../health/checkFns"
+
+import { Trigger } from "../trigger"
+import * as T from "../../../base/lib/types"
+import { Mounts } from "./Mounts"
+import { ExecSpawnable, MountOptions } from "../util/SubContainer"
+
+import { promisify } from "node:util"
+import * as CP from "node:child_process"
+
+export { Daemon } from "./Daemon"
+export { CommandController } from "./CommandController"
+import { HealthDaemon } from "./HealthDaemon"
+import { Daemon } from "./Daemon"
+import { CommandController } from "./CommandController"
+
+export const cpExec = promisify(CP.exec)
+export const cpExecFile = promisify(CP.execFile)
+export type Ready = {
+  display: string | null
+  fn: (
+    spawnable: ExecSpawnable,
+  ) => Promise<HealthCheckResult> | HealthCheckResult
+  trigger?: Trigger
+}
+
+type DaemonsParams<
+  Manifest extends T.Manifest,
+  Ids extends string,
+  Command extends string,
+  Id extends string,
+> = {
+  command: T.CommandType
+  image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean }
+  mounts: Mounts<Manifest>
+  env?: Record<string, string>
+  ready: Ready
+  requires: Exclude<Ids, Id>[]
+  sigtermTimeout?: number
+}
+
+type ErrorDuplicateId<Id extends string> = `The id '${Id}' is already used`
+
+export const runCommand = <Manifest extends T.Manifest>() =>
+  CommandController.of<Manifest>()
+
+/**
+ * A class for defining and controlling the service daemons
+```ts
+Daemons.of({
+  effects,
+  started,
+  interfaceReceipt, // Provide the interfaceReceipt to prove it was completed
+  healthReceipts, // Provide the healthReceipts or [] to prove they were at least considered
+}).addDaemon('webui', {
+  command: 'hello-world', // The command to start the daemon
+  ready: {
+    display: 'Web Interface',
+    // The function to run to determine the health status of the daemon
+    fn: () =>
+      checkPortListening(effects, 80, {
+        successMessage: 'The web interface is ready',
+        errorMessage: 'The web interface is not ready',
+      }),
+  },
+  requires: [],
+})
+```
+ */
+export class Daemons<Manifest extends T.Manifest, Ids extends string>
+  implements T.DaemonBuildable
+{
+  private constructor(
+    readonly effects: T.Effects,
+    readonly started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>,
+    readonly daemons: Promise<Daemon>[],
+    readonly ids: Ids[],
+    readonly healthDaemons: HealthDaemon[],
+  ) {}
+  /**
+   * Returns an empty new Daemons class with the provided inputSpec.
+   *
+   * Call .addDaemon() on the returned class to add a daemon.
+   *
+   * Daemons run in the order they are defined, with latter daemons being capable of
+   * depending on prior daemons
+   * @param inputSpec
+   * @returns
+   */
+  static of<Manifest extends T.Manifest>(inputSpec: {
+    effects: T.Effects
+    started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>
+    healthReceipts: HealthReceipt[]
+  }) {
+    return new Daemons<Manifest, never>(
+      inputSpec.effects,
+      inputSpec.started,
+      [],
+      [],
+      [],
+    )
+  }
+  /**
+   * Returns the complete list of daemons, including the one defined here
+   * @param id
+   * @param newDaemon
+   * @returns
+   */
+  addDaemon<Id extends string, Command extends string>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    options: DaemonsParams<Manifest, Ids, Command, Id>,
+  ) {
+    const daemonIndex = this.daemons.length
+    const daemon = Daemon.of()(this.effects, options.image, options.command, {
+      ...options,
+      mounts: options.mounts.build(),
+      subcontainerName: id,
+    })
+    const healthDaemon = new HealthDaemon(
+      daemon,
+      daemonIndex,
+      options.requires
+        .map((x) => this.ids.indexOf(id as any))
+        .filter((x) => x >= 0)
+        .map((id) => this.healthDaemons[id]),
+      id,
+      this.ids,
+      options.ready,
+      this.effects,
+      options.sigtermTimeout,
+    )
+    const daemons = this.daemons.concat(daemon)
+    const ids = [...this.ids, id] as (Ids | Id)[]
+    const healthDaemons = [...this.healthDaemons, healthDaemon]
+    return new Daemons<Manifest, Ids | Id>(
+      this.effects,
+      this.started,
+      daemons,
+      ids,
+      healthDaemons,
+    )
+  }
+
+  async build() {
+    this.updateMainHealth()
+    this.healthDaemons.forEach((x) =>
+      x.addWatcher(() => this.updateMainHealth()),
+    )
+    const built = {
+      term: async (options?: { signal?: Signals; timeout?: number }) => {
+        try {
+          await Promise.all(this.healthDaemons.map((x) => x.term(options)))
+        } finally {
+          this.effects.setMainStatus({ status: "stopped" })
+        }
+      },
+    }
+    this.started(() => built.term())
+    return built
+  }
+
+  private updateMainHealth() {
+    this.effects.setMainStatus({ status: "running" })
+  }
+}
--- a/sdk/package/lib/mainFn/HealthDaemon.ts
+++ b/sdk/package/lib/mainFn/HealthDaemon.ts
@@ -0,0 +1,150 @@
+import { HealthCheckResult } from "../health/checkFns"
+import { defaultTrigger } from "../trigger/defaultTrigger"
+import { Ready } from "./Daemons"
+import { Daemon } from "./Daemon"
+import { SetHealth, Effects } from "../../../base/lib/types"
+import { DEFAULT_SIGTERM_TIMEOUT } from "."
+import { asError } from "../../../base/lib/util/asError"
+
+const oncePromise = <T>() => {
+  let resolve: (value: T) => void
+  const promise = new Promise<T>((res) => {
+    resolve = res
+  })
+  return { resolve: resolve!, promise }
+}
+
+/**
+ * Wanted a structure that deals with controlling daemons by their health status
+ * States:
+ * -- Waiting for dependencies to be success
+ * -- Running: Daemon is running and the status is in the health
+ *
+ */
+export class HealthDaemon {
+  private _health: HealthCheckResult = { result: "starting", message: null }
+  private healthWatchers: Array<() => unknown> = []
+  private running = false
+  constructor(
+    private readonly daemon: Promise<Daemon>,
+    readonly daemonIndex: number,
+    private readonly dependencies: HealthDaemon[],
+    readonly id: string,
+    readonly ids: string[],
+    readonly ready: Ready,
+    readonly effects: Effects,
+    readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
+  ) {
+    this.updateStatus()
+    this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus()))
+  }
+
+  /** Run after we want to do cleanup */
+  async term(termOptions?: {
+    signal?: NodeJS.Signals | undefined
+    timeout?: number | undefined
+  }) {
+    this.healthWatchers = []
+    this.running = false
+    this.healthCheckCleanup?.()
+
+    await this.daemon.then((d) =>
+      d.term({
+        timeout: this.sigtermTimeout,
+        ...termOptions,
+      }),
+    )
+  }
+
+  /** Want to add another notifier that the health might have changed */
+  addWatcher(watcher: () => unknown) {
+    this.healthWatchers.push(watcher)
+  }
+
+  get health() {
+    return Object.freeze(this._health)
+  }
+
+  private async changeRunning(newStatus: boolean) {
+    if (this.running === newStatus) return
+
+    this.running = newStatus
+
+    if (newStatus) {
+      ;(await this.daemon).start()
+      this.setupHealthCheck()
+    } else {
+      ;(await this.daemon).stop()
+      this.turnOffHealthCheck()
+
+      this.setHealth({ result: "starting", message: null })
+    }
+  }
+
+  private healthCheckCleanup: (() => void) | null = null
+  private turnOffHealthCheck() {
+    this.healthCheckCleanup?.()
+  }
+  private async setupHealthCheck() {
+    if (this.healthCheckCleanup) return
+    const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({
+      lastResult: this._health.result,
+    }))
+
+    const { promise: status, resolve: setStatus } = oncePromise<{
+      done: true
+    }>()
+    new Promise(async () => {
+      for (
+        let res = await Promise.race([status, trigger.next()]);
+        !res.done;
+        res = await Promise.race([status, trigger.next()])
+      ) {
+        const handle = (await this.daemon).subContainerHandle
+
+        if (handle) {
+          const response: HealthCheckResult = await Promise.resolve(
+            this.ready.fn(handle),
+          ).catch((err) => {
+            console.error(asError(err))
+            return {
+              result: "failure",
+              message: "message" in err ? err.message : String(err),
+            }
+          })
+          await this.setHealth(response)
+        } else {
+          await this.setHealth({
+            result: "failure",
+            message: "Daemon not running",
+          })
+        }
+      }
+    }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
+
+    this.healthCheckCleanup = () => {
+      setStatus({ done: true })
+      this.healthCheckCleanup = null
+    }
+  }
+
+  private async setHealth(health: HealthCheckResult) {
+    this._health = health
+    this.healthWatchers.forEach((watcher) => watcher())
+    const display = this.ready.display
+    const result = health.result
+    if (!display) {
+      return
+    }
+    await this.effects.setHealth({
+      ...health,
+      id: this.id,
+      name: display,
+    } as SetHealth)
+  }
+
+  private async updateStatus() {
+    const healths = this.dependencies.map((d) => d._health)
+    this.changeRunning(healths.every((x) => x.result === "success"))
+  }
+}
--- a/sdk/package/lib/mainFn/Mounts.ts
+++ b/sdk/package/lib/mainFn/Mounts.ts
@@ -0,0 +1,125 @@
+import * as T from "../../../base/lib/types"
+import { MountOptions } from "../util/SubContainer"
+
+type MountArray = { path: string; options: MountOptions }[]
+
+export class Mounts<Manifest extends T.Manifest> {
+  private constructor(
+    readonly volumes: {
+      id: Manifest["volumes"][number]
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+    readonly assets: {
+      id: Manifest["assets"][number]
+      subpath: string | null
+      mountpoint: string
+    }[],
+    readonly dependencies: {
+      dependencyId: string
+      volumeId: string
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+  ) {}
+
+  static of<Manifest extends T.Manifest>() {
+    return new Mounts<Manifest>([], [], [])
+  }
+
+  addVolume(
+    id: Manifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.volumes.push({
+      id,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  addAssets(
+    id: Manifest["assets"][number],
+    subpath: string | null,
+    mountpoint: string,
+  ) {
+    this.assets.push({
+      id,
+      subpath,
+      mountpoint,
+    })
+    return this
+  }
+
+  addDependency<DependencyManifest extends T.Manifest>(
+    dependencyId: keyof Manifest["dependencies"] & string,
+    volumeId: DependencyManifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.dependencies.push({
+      dependencyId,
+      volumeId,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  build(): MountArray {
+    const mountpoints = new Set()
+    for (let mountpoint of this.volumes
+      .map((v) => v.mountpoint)
+      .concat(this.assets.map((a) => a.mountpoint))
+      .concat(this.dependencies.map((d) => d.mountpoint))) {
+      if (mountpoints.has(mountpoint)) {
+        throw new Error(
+          `cannot mount more than once to mountpoint ${mountpoint}`,
+        )
+      }
+      mountpoints.add(mountpoint)
+    }
+    return ([] as MountArray)
+      .concat(
+        this.volumes.map((v) => ({
+          path: v.mountpoint,
+          options: {
+            type: "volume",
+            id: v.id,
+            subpath: v.subpath,
+            readonly: v.readonly,
+          },
+        })),
+      )
+      .concat(
+        this.assets.map((a) => ({
+          path: a.mountpoint,
+          options: {
+            type: "assets",
+            id: a.id,
+            subpath: a.subpath,
+          },
+        })),
+      )
+      .concat(
+        this.dependencies.map((d) => ({
+          path: d.mountpoint,
+          options: {
+            type: "pointer",
+            packageId: d.dependencyId,
+            volumeId: d.volumeId,
+            subpath: d.subpath,
+            readonly: d.readonly,
+          },
+        })),
+      )
+  }
+}
--- a/sdk/package/lib/mainFn/index.ts
+++ b/sdk/package/lib/mainFn/index.ts
@@ -0,0 +1,27 @@
+import * as T from "../../../base/lib/types"
+import { Daemons } from "./Daemons"
+import "../../../base/lib/interfaces/ServiceInterfaceBuilder"
+import "../../../base/lib/interfaces/Origin"
+
+export const DEFAULT_SIGTERM_TIMEOUT = 30_000
+/**
+ * Used to ensure that the main function is running with the valid proofs.
+ * We first do the folowing order of things
+ * 1. We get the interfaces
+ * 2. We setup all the commands to setup the system
+ * 3. We create the health checks
+ * 4. We setup the daemons init system
+ * @param fn
+ * @returns
+ */
+export const setupMain = <Manifest extends T.Manifest, Store>(
+  fn: (o: {
+    effects: T.Effects
+    started(onTerm: () => PromiseLike<void>): PromiseLike<void>
+  }) => Promise<Daemons<Manifest, any>>,
+): T.ExpectedExports.main => {
+  return async (options) => {
+    const result = await fn(options)
+    return result
+  }
+}