Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 04:01:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

addHealthCheck instead of additionalHealthChecks for Daemons (#2962)

Browse Source 
* addHealthCheck on Daemons

* fix bug that prevents domains without protocols from being deleted

* fixes from testing

* version bump

* add sdk version to UI

* fix useEntrypoint

* fix dependency health check error display

* minor fixes

* beta.29

* fixes from testing

* beta.30

* set /etc/os-release (#2918)

* remove check-monitor from kiosk (#2059)

* add units for progress (#2693)

* use new progress type

* alpha.7

* fix up pwa stuff

* fix wormhole-squashfs and prune boot (#2964)

* don't exit on expected errors

* use bash

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
This commit is contained in:
Aiden McClelland
2025-06-17 23:50:01 +00:00
committed by GitHub
parent f5688e077a
commit 3ec4db0225
100 changed files with 846 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
sdk/package/lib/mainFn/CommandController.ts
View File

@@ -2,44 +2,49 @@ import { DEFAULT_SIGTERM_TIMEOUT } from "."
import { NO_TIMEOUT, SIGTERM } from "../../../base/lib/types"
import * as T from "../../../base/lib/types"
import { MountOptions, SubContainer } from "../util/SubContainer"
import { SubContainer } from "../util/SubContainer"
import { Drop, splitCommand } from "../util"
import * as cp from "child_process"
import * as fs from "node:fs/promises"
import { Mounts } from "./Mounts"
import { DaemonCommandType } from "./Daemons"
import { DaemonCommandType, ExecCommandOptions, ExecFnOptions } from "./Daemons"
export class CommandController<Manifest extends T.SDKManifest> extends Drop {
export class CommandController<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null,
> extends Drop {
private constructor(
readonly runningAnswer: Promise<null>,
private state: { exited: boolean },
private readonly subcontainer: SubContainer<Manifest>,
private readonly subcontainer: C,
private process: cp.ChildProcess | AbortController,
readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
) {
super()
}
static of<Manifest extends T.SDKManifest>() {
static of<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null,
>() {
return async (
effects: T.Effects,
subcontainer: SubContainer<Manifest>,
exec: DaemonCommandType,
subcontainer: C,
exec: DaemonCommandType<Manifest, C>,
) => {
try {
if ("fn" in exec) {
const abort = new AbortController()
const cell: { ctrl: CommandController<Manifest> } = {
ctrl: new CommandController(
const cell: { ctrl: CommandController<Manifest, C> } = {
ctrl: new CommandController<Manifest, C>(
exec.fn(subcontainer, abort).then(async (command) => {
if (command && !abort.signal.aborted) {
Object.assign(
cell.ctrl,
await CommandController.of<Manifest>()(
effects,
subcontainer,
command,
),
)
if (subcontainer && command && !abort.signal.aborted) {
const newCtrl = (
await CommandController.of<
Manifest,
SubContainer<Manifest>
>()(effects, subcontainer, command as ExecCommandOptions)
).leak()
Object.assign(cell.ctrl, newCtrl)
return await cell.ctrl.runningAnswer
} else {
cell.ctrl.state.exited = true
@@ -57,7 +62,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
let commands: string[]
if (T.isUseEntrypoint(exec.command)) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
.readFile(`/media/startos/images/${subcontainer!.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
@@ -70,11 +75,11 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
let childProcess: cp.ChildProcess
if (exec.runAsInit) {
childProcess = await subcontainer.launch(commands, {
childProcess = await subcontainer!.launch(commands, {
env: exec.env,
})
} else {
childProcess = await subcontainer.spawn(commands, {
childProcess = await subcontainer!.spawn(commands, {
env: exec.env,
stdio: exec.onStdout || exec.onStderr ? "pipe" : "inherit",
})
@@ -108,7 +113,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
})
})
return new CommandController(
return new CommandController<Manifest, C>(
answer,
state,
subcontainer,
@@ -116,7 +121,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
exec.sigtermTimeout,
)
} catch (e) {
await subcontainer.destroy()
await subcontainer?.destroy()
throw e
}
}
@@ -144,7 +149,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
if (this.process instanceof AbortController) this.process.abort()
else this.process.kill("SIGKILL")
}
await this.subcontainer.destroy()
await this.subcontainer?.destroy()
}
}
async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
@@ -178,7 +183,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
])
else await this.runningAnswer
} finally {
await this.subcontainer.destroy()
await this.subcontainer?.destroy()
}
}
onDrop(): void {

40
sdk/package/lib/mainFn/Daemon.ts
View File

@@ -17,14 +17,17 @@ const MAX_TIMEOUT_MS = 30000
* and the others state of running, where it will keep a living running command
*/
export class Daemon<Manifest extends T.SDKManifest> extends Drop {
private commandController: CommandController<Manifest> | null = null
export class Daemon<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null = SubContainer<Manifest> | null,
> extends Drop {
private commandController: CommandController<Manifest, C> | null = null
private shouldBeRunning = false
protected exitedSuccess = false
private onExitFns: ((success: boolean) => void)[] = []
protected constructor(
private subcontainer: SubContainer<Manifest>,
private startCommand: (() => Promise<CommandController<Manifest>>) | null,
private subcontainer: C,
private startCommand: () => Promise<CommandController<Manifest, C>>,
readonly oneshot: boolean = false,
) {
super()
@@ -33,17 +36,20 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
return this.oneshot
}
static of<Manifest extends T.SDKManifest>() {
return async (
return async <C extends SubContainer<Manifest> | null>(
effects: T.Effects,
subcontainer: SubContainer<Manifest>,
exec: DaemonCommandType | null,
subcontainer: C,
exec: DaemonCommandType<Manifest, C>,
) => {
if (subcontainer.isOwned()) subcontainer = subcontainer.rc()
const startCommand = exec
? () =>
CommandController.of<Manifest>()(effects, subcontainer.rc(), exec)
: null
return new Daemon(subcontainer, startCommand)
let subc: SubContainer<Manifest> | null = subcontainer
if (subcontainer && subcontainer.isOwned()) subc = subcontainer.rc()
const startCommand = () =>
CommandController.of<Manifest, C>()(
effects,
(subc?.rc() ?? null) as C,
exec,
)
return new Daemon(subc, startCommand)
}
}
async start() {
@@ -53,7 +59,7 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
this.shouldBeRunning = true
let timeoutCounter = 0
;(async () => {
while (this.startCommand && this.shouldBeRunning) {
while (this.shouldBeRunning) {
if (this.commandController)
await this.commandController
.term({})
@@ -106,10 +112,10 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
.catch((e) => console.error(asError(e)))
this.commandController = null
this.onExitFns = []
await this.subcontainer.destroy()
await this.subcontainer?.destroy()
}
subcontainerRc(): SubContainerRc<Manifest> {
return this.subcontainer.rc()
subcontainerRc(): SubContainerRc<Manifest> | null {
return this.subcontainer?.rc() ?? null
}
onExit(fn: (success: boolean) => void) {
this.onExitFns.push(fn)

172
sdk/package/lib/mainFn/Daemons.ts
View File

@@ -37,9 +37,7 @@ export type Ready = {
})
* ```
*/
fn: (
subcontainer: SubContainer<Manifest>,
) => Promise<HealthCheckResult> | HealthCheckResult
fn: () => Promise<HealthCheckResult> | HealthCheckResult
/**
* A duration in milliseconds to treat a failing health check as "starting"
*
@@ -65,30 +63,40 @@ export type ExecCommandOptions = {
onStderr?: (chunk: Buffer | string | any) => void
}
export type ExecFnOptions = {
export type ExecFnOptions<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null,
> = {
fn: (
subcontainer: SubContainer<Manifest>,
subcontainer: C,
abort: AbortController,
) => Promise<ExecCommandOptions | null>
) => Promise<C extends null ? null : ExecCommandOptions | null>
// Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
sigtermTimeout?: number
}
export type DaemonCommandType = ExecCommandOptions | ExecFnOptions
export type DaemonCommandType<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null,
> = ExecFnOptions<Manifest, C> | (C extends null ? never : ExecCommandOptions)
type NewDaemonParams<Manifest extends T.SDKManifest> = {
type NewDaemonParams<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null,
> = {
/** What to run as the daemon: either an async fn or a commandline command to run in the subcontainer */
exec: DaemonCommandType | null
/** Information about the subcontainer in which the daemon runs */
subcontainer: SubContainer<Manifest>
exec: DaemonCommandType<Manifest, C>
/** The subcontainer in which the daemon runs */
subcontainer: C
}
type AddDaemonParams<
Manifest extends T.SDKManifest,
Ids extends string,
Id extends string,
C extends SubContainer<Manifest> | null,
> = (
| NewDaemonParams<Manifest>
| NewDaemonParams<Manifest, C>
| {
daemon: Daemon<Manifest>
}
@@ -102,8 +110,15 @@ type AddOneshotParams<
Manifest extends T.SDKManifest,
Ids extends string,
Id extends string,
> = NewDaemonParams<Manifest> & {
exec: DaemonCommandType
C extends SubContainer<Manifest> | null,
> = NewDaemonParams<Manifest, C> & {
exec: DaemonCommandType<Manifest, C>
/** An array of IDs of prior daemons whose successful initializations are required before this daemon will initialize */
requires: Exclude<Ids, Id>[]
}
type AddHealthCheckParams<Ids extends string, Id extends string> = {
ready: Ready
/** An array of IDs of prior daemons whose successful initializations are required before this daemon will initialize */
requires: Exclude<Ids, Id>[]
}
@@ -111,7 +126,7 @@ type AddOneshotParams<
type ErrorDuplicateId<Id extends string> = `The id '${Id}' is already used`
export const runCommand = <Manifest extends T.SDKManifest>() =>
CommandController.of<Manifest>()
CommandController.of<Manifest, SubContainer<Manifest>>()
/**
* A class for defining and controlling the service daemons
@@ -141,11 +156,12 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
{
private constructor(
readonly effects: T.Effects,
readonly started: (onTerm: () => PromiseLike<void>) => PromiseLike<null>,
readonly started:
| ((onTerm: () => PromiseLike<void>) => PromiseLike<null>)
| null,
readonly daemons: Promise<Daemon<Manifest>>[],
readonly ids: Ids[],
readonly healthDaemons: HealthDaemon<Manifest>[],
readonly healthChecks: HealthCheck[],
) {}
/**
* Returns an empty new Daemons class with the provided inputSpec.
@@ -154,13 +170,18 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
*
* Daemons run in the order they are defined, with latter daemons being capable of
* depending on prior daemons
* @param options
*
* @param effects
*
* @param started
* @returns
*/
static of<Manifest extends T.SDKManifest>(options: {
effects: T.Effects
started: (onTerm: () => PromiseLike<void>) => PromiseLike<null>
healthChecks: HealthCheck[]
/**
* A closure to run once the system is launched. If you are in main, provide the `started` argument you receive from the function arguments
*/
started: ((onTerm: () => PromiseLike<void>) => PromiseLike<null>) | null
}) {
return new Daemons<Manifest, never>(
options.effects,
@@ -168,7 +189,6 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
[],
[],
[],
options.healthChecks,
)
}
/**
@@ -177,19 +197,19 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
* @param options
* @returns a new Daemons object
*/
addDaemon<Id extends string>(
addDaemon<Id extends string, C extends SubContainer<Manifest> | null>(
// prettier-ignore
id:
"" extends Id ? never :
ErrorDuplicateId<Id> extends Id ? never :
Id extends Ids ? ErrorDuplicateId<Id> :
Id,
options: AddDaemonParams<Manifest, Ids, Id>,
options: AddDaemonParams<Manifest, Ids, Id, C>,
) {
const daemon =
"daemon" in options
? Promise.resolve(options.daemon)
: Daemon.of<Manifest>()(
: Daemon.of<Manifest>()<C>(
this.effects,
options.subcontainer,
options.exec,
@@ -201,11 +221,10 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
.filter((x) => x >= 0)
.map((id) => this.healthDaemons[id]),
id,
this.ids,
options.ready,
this.effects,
)
const daemons = this.daemons.concat(daemon)
const daemons = [...this.daemons, daemon]
const ids = [...this.ids, id] as (Ids | Id)[]
const healthDaemons = [...this.healthDaemons, healthDaemon]
return new Daemons<Manifest, Ids | Id>(
@@ -214,7 +233,6 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
daemons,
ids,
healthDaemons,
this.healthChecks,
)
}
@@ -225,7 +243,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
* @param options
* @returns a new Daemons object
*/
addOneshot<Id extends string>(
addOneshot<Id extends string, C extends SubContainer<Manifest> | null>(
id: "" extends Id
? never
: ErrorDuplicateId<Id> extends Id
@@ -233,9 +251,9 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
: Id extends Ids
? ErrorDuplicateId<Id>
: Id,
options: AddOneshotParams<Manifest, Ids, Id>,
options: AddOneshotParams<Manifest, Ids, Id, C>,
) {
const daemon = Oneshot.of<Manifest>()(
const daemon = Oneshot.of<Manifest>()<C>(
this.effects,
options.subcontainer,
options.exec,
@@ -247,11 +265,10 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
.filter((x) => x >= 0)
.map((id) => this.healthDaemons[id]),
id,
this.ids,
"EXIT_SUCCESS",
this.effects,
)
const daemons = this.daemons.concat(daemon)
const daemons = [...this.daemons, daemon]
const ids = [...this.ids, id] as (Ids | Id)[]
const healthDaemons = [...this.healthDaemons, healthDaemon]
return new Daemons<Manifest, Ids | Id>(
@@ -260,13 +277,95 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
daemons,
ids,
healthDaemons,
this.healthChecks,
)
}
/**
* Returns the complete list of daemons, including a new HealthCheck defined here
* @param id
* @param options
* @returns a new Daemons object
*/
addHealthCheck<Id extends string>(
id: "" extends Id
? never
: ErrorDuplicateId<Id> extends Id
? never
: Id extends Ids
? ErrorDuplicateId<Id>
: Id,
options: AddHealthCheckParams<Ids, Id>,
) {
const healthDaemon = new HealthDaemon<Manifest>(
null,
options.requires
.map((x) => this.ids.indexOf(x))
.filter((x) => x >= 0)
.map((id) => this.healthDaemons[id]),
id,
options.ready,
this.effects,
)
const daemons = [...this.daemons]
const ids = [...this.ids, id] as (Ids | Id)[]
const healthDaemons = [...this.healthDaemons, healthDaemon]
return new Daemons<Manifest, Ids | Id>(
this.effects,
this.started,
daemons,
ids,
healthDaemons,
)
}
/**
* Runs the entire system until all daemons have returned `ready`.
* @param id
* @param options
* @returns a new Daemons object
*/
async runUntilSuccess(timeout: number | null) {
let resolve = (_: void) => {}
const res = new Promise<void>((res, rej) => {
resolve = res
if (timeout)
setTimeout(() => {
const notReady = this.healthDaemons
.filter((d) => !d.isReady)
.map((d) => d.id)
rej(new Error(`Timed out waiting for ${notReady}`))
}, timeout)
})
const daemon = Oneshot.of()(this.effects, null, {
fn: async () => {
resolve()
return null
},
})
const healthDaemon = new HealthDaemon<Manifest>(
daemon,
[...this.healthDaemons],
"__RUN_UNTIL_SUCCESS",
"EXIT_SUCCESS",
this.effects,
)
const daemons = await new Daemons<Manifest, Ids>(
this.effects,
this.started,
[...this.daemons, daemon],
this.ids,
[...this.healthDaemons, healthDaemon],
).build()
try {
await res
} finally {
await daemons.term()
}
return null
}
async term() {
try {
this.healthChecks.forEach((health) => health.stop())
for (let result of await Promise.allSettled(
this.healthDaemons.map((x) => x.term()),
)) {
@@ -283,10 +382,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
for (const daemon of this.healthDaemons) {
await daemon.init()
}
for (const health of this.healthChecks) {
health.start()
}
this.started(() => this.term())
this.started?.(() => this.term())
return this
}
}

89
sdk/package/lib/mainFn/HealthDaemon.ts
View File

@@ -6,6 +6,7 @@ import { SetHealth, Effects, SDKManifest } from "../../../base/lib/types"
import { DEFAULT_SIGTERM_TIMEOUT } from "."
import { asError } from "../../../base/lib/util/asError"
import { Oneshot } from "./Oneshot"
import { SubContainer } from "../util/SubContainer"
const oncePromise = <T>() => {
let resolve: (value: T) => void
@@ -30,16 +31,22 @@ export class HealthDaemon<Manifest extends SDKManifest> {
private running = false
private started?: number
private resolveReady: (() => void) | undefined
private resolvedReady: boolean = false
private readyPromise: Promise<void>
constructor(
private readonly daemon: Promise<Daemon<Manifest>>,
private readonly daemon: Promise<Daemon<Manifest>> | null,
private readonly dependencies: HealthDaemon<Manifest>[],
readonly id: string,
readonly ids: string[],
readonly ready: Ready | typeof EXIT_SUCCESS,
readonly effects: Effects,
) {
this.readyPromise = new Promise((resolve) => (this.resolveReady = resolve))
this.readyPromise = new Promise(
(resolve) =>
(this.resolveReady = () => {
resolve()
this.resolvedReady = true
}),
)
this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus()))
}
@@ -52,7 +59,7 @@ export class HealthDaemon<Manifest extends SDKManifest> {
this.running = false
this.healthCheckCleanup?.()
await this.daemon.then((d) =>
await this.daemon?.then((d) =>
d.term({
...termOptions,
}),
@@ -74,11 +81,13 @@ export class HealthDaemon<Manifest extends SDKManifest> {
this.running = newStatus
if (newStatus) {
;(await this.daemon).start()
this.started = performance.now()
console.debug(`Launching ${this.id}...`)
this.setupHealthCheck()
;(await this.daemon)?.start()
this.started = performance.now()
} else {
;(await this.daemon).stop()
console.debug(`Stopping ${this.id}...`)
;(await this.daemon)?.stop()
this.turnOffHealthCheck()
this.setHealth({ result: "starting", message: null })
@@ -88,10 +97,19 @@ export class HealthDaemon<Manifest extends SDKManifest> {
private healthCheckCleanup: (() => null) | null = null
private turnOffHealthCheck() {
this.healthCheckCleanup?.()
this.resolvedReady = false
this.readyPromise = new Promise(
(resolve) =>
(this.resolveReady = () => {
resolve()
this.resolvedReady = true
}),
)
}
private async setupHealthCheck() {
const daemon = await this.daemon
daemon.onExit((success) => {
daemon?.onExit((success) => {
if (success && this.ready === "EXIT_SUCCESS") {
this.setHealth({ result: "success", message: null })
} else if (!success) {
@@ -122,28 +140,17 @@ export class HealthDaemon<Manifest extends SDKManifest> {
!res.done;
res = await Promise.race([status, trigger.next()])
) {
const handle = (await this.daemon).subcontainerRc()
try {
const response: HealthCheckResult = await Promise.resolve(
this.ready.fn(handle),
).catch((err) => {
console.error(asError(err))
return {
result: "failure",
message: "message" in err ? err.message : String(err),
}
})
if (
this.resolveReady &&
(response.result === "success" || response.result === "disabled")
) {
this.resolveReady()
const response: HealthCheckResult = await Promise.resolve(
this.ready.fn(),
).catch((err) => {
console.error(asError(err))
return {
result: "failure",
message: "message" in err ? err.message : String(err),
}
await this.setHealth(response)
} finally {
await handle.destroy()
}
})
await this.setHealth(response)
}
}).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
@@ -158,10 +165,18 @@ export class HealthDaemon<Manifest extends SDKManifest> {
return this.readyPromise
}
get isReady() {
return this.resolvedReady
}
private async setHealth(health: HealthCheckResult) {
const changed = this._health.result !== health.result
this._health = health
if (this.resolveReady && health.result === "success") {
this.resolveReady()
}
if (changed) this.healthWatchers.forEach((watcher) => watcher())
if (this.ready === "EXIT_SUCCESS") return
this.healthWatchers.forEach((watcher) => watcher())
const display = this.ready.display
if (!display) {
return
@@ -182,8 +197,18 @@ export class HealthDaemon<Manifest extends SDKManifest> {
}
async updateStatus() {
const healths = this.dependencies.map((d) => d.running && d._health)
this.changeRunning(healths.every((x) => x && x.result === "success"))
const healths = this.dependencies.map((d) => ({
health: d.running && d._health,
id: d.id,
}))
const waitingOn = healths.filter(
(h) => !h.health || h.health.result !== "success",
)
if (waitingOn.length)
console.debug(
`daemon ${this.id} waiting on ${waitingOn.map((w) => w.id)}`,
)
this.changeRunning(!waitingOn.length)
}
async init() {

26
sdk/package/lib/mainFn/Oneshot.ts
View File

@@ -10,19 +10,25 @@ import { DaemonCommandType } from "./Daemons"
* unlike Daemon, does not restart on success
*/
export class Oneshot<Manifest extends T.SDKManifest> extends Daemon<Manifest> {
export class Oneshot<
Manifest extends T.SDKManifest,
C extends SubContainer<Manifest> | null = SubContainer<Manifest> | null,
> extends Daemon<Manifest, C> {
static of<Manifest extends T.SDKManifest>() {
return async (
return async <C extends SubContainer<Manifest> | null>(
effects: T.Effects,
subcontainer: SubContainer<Manifest>,
exec: DaemonCommandType | null,
subcontainer: C,
exec: DaemonCommandType<Manifest, C>,
) => {
if (subcontainer.isOwned()) subcontainer = subcontainer.rc()
const startCommand = exec
? () =>
CommandController.of<Manifest>()(effects, subcontainer.rc(), exec)
: null
return new Oneshot(subcontainer, startCommand, true)
let subc: SubContainer<Manifest> | null = subcontainer
if (subcontainer && subcontainer.isOwned()) subc = subcontainer.rc()
const startCommand = () =>
CommandController.of<Manifest, C>()(
effects,
(subc?.rc() ?? null) as C,
exec,
)
return new Oneshot<Manifest, C>(subcontainer, startCommand, true)
}
}
}