Refactor/sdk init (#2947)

* fixes for main

* refactor package initialization

* fixes from testing

* more fixes

* beta.21

* do not use instanceof

* closes #2921

* beta22

* allow disabling kiosk

* migration

* fix /etc/shadow

* actionRequest -> task

* beta.23
This commit is contained in:
Aiden McClelland
2025-05-21 10:24:37 -06:00
committed by GitHub
parent 46fd01c264
commit 44560c8da8
237 changed files with 1827 additions and 98800 deletions

View File

@@ -1,7 +1,11 @@
import { Value } from "../../base/lib/actions/input/builder/value"
import { InputSpec } from "../../base/lib/actions/input/builder/inputSpec"
import { Variants } from "../../base/lib/actions/input/builder/variants"
import { Action, Actions } from "../../base/lib/actions/setupActions"
import {
Action,
ActionInfo,
Actions,
} from "../../base/lib/actions/setupActions"
import {
SyncOptions,
ServiceInterfaceId,
@@ -17,9 +21,7 @@ import { HealthCheck } from "./health/HealthCheck"
import { checkPortListening } from "./health/checkFns/checkPortListening"
import { checkWebUrl, runHealthScript } from "./health/checkFns"
import { List } from "../../base/lib/actions/input/builder/list"
import { InstallFn, PostInstall, PreInstall } from "./inits/setupInstall"
import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
import { UninstallFn, setupUninstall } from "./inits/setupUninstall"
import { setupMain } from "./mainFn"
import { defaultTrigger } from "./trigger/defaultTrigger"
import { changeOnFirstSuccess, cooldownTrigger } from "./trigger"
@@ -33,24 +35,40 @@ import { ServiceInterfaceBuilder } from "../../base/lib/interfaces/ServiceInterf
import { GetSystemSmtp } from "./util"
import { nullIfEmpty } from "./util"
import { getServiceInterface, getServiceInterfaces } from "./util"
import { CommandOptions, ExitError, SubContainer } from "./util/SubContainer"
import {
CommandOptions,
ExitError,
SubContainer,
SubContainerOwned,
} from "./util/SubContainer"
import { splitCommand } from "./util"
import { Mounts } from "./mainFn/Mounts"
import { setupDependencies } from "../../base/lib/dependencies/setupDependencies"
import * as T from "../../base/lib/types"
import { testTypeVersion } from "../../base/lib/exver"
import {
ExtendedVersion,
testTypeVersion,
VersionRange,
} from "../../base/lib/exver"
import {
CheckDependencies,
checkDependencies,
} from "../../base/lib/dependencies/dependencies"
import { GetSslCertificate } from "./util"
import { VersionGraph } from "./version"
import { getDataVersion, setDataVersion, VersionGraph } from "./version"
import { MaybeFn } from "../../base/lib/actions/setupActions"
import { GetInput } from "../../base/lib/actions/setupActions"
import { Run } from "../../base/lib/actions/setupActions"
import * as actions from "../../base/lib/actions"
import { setupInit } from "./inits/setupInit"
import * as fs from "node:fs/promises"
import {
setupInit,
setupUninit,
setupOnInstall,
setupOnUpdate,
setupOnInstallOrUpdate,
setupOnInit,
} from "../../base/lib/inits"
export const OSVersion = testTypeVersion("0.4.0-alpha.4")
@@ -90,6 +108,8 @@ export class StartSdk<Manifest extends T.SDKManifest> {
| "getSslCertificate"
| "getSystemSmtp"
| "getContainerIp"
| "getDataVersion"
| "setDataVersion"
// prettier-ignore
type StartSdkEffectWrapper = {
@@ -108,8 +128,6 @@ export class StartSdk<Manifest extends T.SDKManifest> {
clearBindings: (effects, ...args) => effects.clearBindings(...args),
getOsIp: (effects, ...args) => effects.getOsIp(...args),
getSslKey: (effects, ...args) => effects.getSslKey(...args),
setDataVersion: (effects, ...args) => effects.setDataVersion(...args),
getDataVersion: (effects, ...args) => effects.getDataVersion(...args),
shutdown: (effects, ...args) => effects.shutdown(...args),
getDependencies: (effects, ...args) => effects.getDependencies(...args),
getStatus: (effects, ...args) => effects.getStatus(...args),
@@ -118,37 +136,39 @@ export class StartSdk<Manifest extends T.SDKManifest> {
return {
manifest: this.manifest,
...startSdkEffectWrapper,
setDataVersion,
getDataVersion,
action: {
run: actions.runAction,
request: <T extends Action<T.ActionId, any>>(
createTask: <T extends ActionInfo<T.ActionId, any>>(
effects: T.Effects,
packageId: T.PackageId,
action: T,
severity: T.ActionSeverity,
options?: actions.ActionRequestOptions<T>,
severity: T.TaskSeverity,
options?: actions.TaskOptions<T>,
) =>
actions.requestAction({
actions.createTask({
effects,
packageId,
action,
severity,
options: options,
}),
requestOwn: <T extends Action<T.ActionId, any>>(
createOwnTask: <T extends ActionInfo<T.ActionId, any>>(
effects: T.Effects,
action: T,
severity: T.ActionSeverity,
options?: actions.ActionRequestOptions<T>,
severity: T.TaskSeverity,
options?: actions.TaskOptions<T>,
) =>
actions.requestAction({
actions.createTask({
effects,
packageId: this.manifest.id,
action,
severity,
options: options,
}),
clearRequest: (effects: T.Effects, ...replayIds: string[]) =>
effects.action.clearRequests({ only: replayIds }),
clearTask: (effects: T.Effects, ...replayIds: string[]) =>
effects.action.clearTasks({ only: replayIds }),
},
checkDependencies: checkDependencies as <
DependencyId extends keyof Manifest["dependencies"] &
@@ -379,7 +399,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
schemeOverride: null,
username: null,
path: '',
search: {},
query: {},
})
* ```
*/
@@ -399,7 +419,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
/** (optional) appends the provided path to all URLs. */
path: string
/** (optional) appends the provided query params to all URLs. */
search: Record<string, string>
query: Record<string, string>
/** (optional) overrides the protocol prefix provided by the bind function.
*
* @example `ftp://`
@@ -485,33 +505,61 @@ export class StartSdk<Manifest extends T.SDKManifest> {
* ```
*/
setupDependencies: setupDependencies<Manifest>,
setupInit: setupInit<Manifest>,
/**
* @description Use this function to execute arbitrary logic *once*, on initial install *before* interfaces, actions, and dependencies are updated.
* @description Use this function to create an InitScript that runs every time the service initializes
*/
setupOnInit,
/**
* @description Use this function to create an InitScript that runs only when the service is freshly installed
*/
setupOnInstall,
/**
* @description Use this function to create an InitScript that runs only when the service is updated
*/
setupOnUpdate,
/**
* @description Use this function to create an InitScript that runs only when the service is installed or updated
*/
setupOnInstallOrUpdate,
/**
* @description Use this function to setup what happens when the service initializes.
*
* This happens when the server boots, or a service is installed, updated, or restored
*
* Not every init script does something on every initialization. For example, versions only does something on install or update
*
* These scripts are run in the order they are supplied
* @example
* In the this example, we initialize a config file
*
* ```
const preInstall = sdk.setupPreInstall(async ({ effects }) => {
await configFile.write(effects, { name: 'World' })
})
export const init = sdk.setupInit(
restoreInit,
versions,
setDependencies,
setInterfaces,
actions,
postInstall,
)
* ```
*/
setupPreInstall: (fn: InstallFn<Manifest>) => PreInstall.of(fn),
setupInit: setupInit,
/**
* @description Use this function to execute arbitrary logic *once*, on initial install *after* interfaces, actions, and dependencies are updated.
* @description Use this function to setup what happens when the service uninitializes.
*
* This happens when the server shuts down, or a service is uninstalled or updated
*
* Not every uninit script does something on every uninitialization. For example, versions only does something on uninstall or update
*
* These scripts are run in the order they are supplied
* @example
* In the this example, we create a task for the user to perform.
*
* ```
const postInstall = sdk.setupPostInstall(async ({ effects }) => {
await sdk.action.requestOwn(effects, showSecretPhrase, 'important', {
reason: 'Check out your secret phrase!',
})
})
export const uninit = sdk.setupUninit(
versions,
)
* ```
*/
setupPostInstall: (fn: InstallFn<Manifest>) => PostInstall.of(fn),
setupUninit: setupUninit,
/**
* @description Use this function to determine how this service will be hosted and served. The function executes on service install, service update, and inputSpec save.
* @param inputSpec - The inputSpec spec of this service as exported from /inputSpec/spec.
@@ -537,7 +585,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
schemeOverride: null,
username: null,
path: '',
search: {},
query: {},
})
// Admin UI
const adminUi = sdk.createInterface(effects, {
@@ -549,7 +597,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
schemeOverride: null,
username: null,
path: '/admin',
search: {},
query: {},
})
// UI receipt
const uiReceipt = await uiMultiOrigin.export([primaryUi, adminUi])
@@ -569,7 +617,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
schemeOverride: null,
username: null,
path: '',
search: {},
query: {},
})
// API receipt
const apiReceipt = await apiMultiOrigin.export([api])
@@ -587,11 +635,6 @@ export class StartSdk<Manifest extends T.SDKManifest> {
started(onTerm: () => PromiseLike<void>): PromiseLike<null>
}) => Promise<Daemons<Manifest, any>>,
) => setupMain<Manifest>(fn),
/**
* Use this function to execute arbitrary logic *once*, on uninstall only. Most services will not use this.
*/
setupUninstall: (fn: UninstallFn<Manifest>) =>
setupUninstall<Manifest>(fn),
trigger: {
defaultTrigger,
cooldownTrigger,
@@ -675,7 +718,12 @@ export class StartSdk<Manifest extends T.SDKManifest> {
mounts: Mounts<Manifest> | null,
name: string,
) {
return SubContainer.of(effects, image, mounts, name)
return SubContainerOwned.of<Manifest, Effects>(
effects,
image,
mounts,
name,
)
},
/**
* @description Run a function with a temporary SubContainer
@@ -694,7 +742,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
name: string,
fn: (subContainer: SubContainer<Manifest>) => Promise<T>,
): Promise<T> {
return SubContainer.withTemp(effects, image, mounts, name, fn)
return SubContainerOwned.withTemp(effects, image, mounts, name, fn)
},
},
List,
@@ -714,7 +762,7 @@ export async function runCommand<Manifest extends T.SDKManifest>(
name?: string,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
let commands: string[]
if (command instanceof T.UseEntrypoint) {
if (T.isUseEntrypoint(command)) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${image.imageId}.json`, {
encoding: "utf8",
@@ -724,7 +772,7 @@ export async function runCommand<Manifest extends T.SDKManifest>(
commands = imageMeta.entrypoint ?? []
commands = commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
} else commands = splitCommand(command)
return SubContainer.withTemp(
return SubContainerOwned.withTemp(
effects,
image,
options.mounts,

View File

@@ -2,6 +2,8 @@ import * as T from "../../../base/lib/types"
import * as child_process from "child_process"
import * as fs from "fs/promises"
import { Affine, asError } from "../util"
import { ExtendedVersion, VersionRange } from "../../../base/lib"
import { InitKind, InitScript } from "../../../base/lib/inits"
export const DEFAULT_OPTIONS: T.SyncOptions = {
delete: true,
@@ -17,7 +19,7 @@ export type BackupSync<Volumes extends string> = {
export type BackupEffects = T.Effects & Affine<"Backups">
export class Backups<M extends T.SDKManifest> {
export class Backups<M extends T.SDKManifest> implements InitScript {
private constructor(
private options = DEFAULT_OPTIONS,
private restoreOptions: Partial<T.SyncOptions> = {},
@@ -35,7 +37,7 @@ export class Backups<M extends T.SDKManifest> {
return Backups.withSyncs(
...volumeNames.map((srcVolume) => ({
dataPath: `/media/startos/volumes/${srcVolume}/` as const,
backupPath: `/media/startos/backup/${srcVolume}/` as const,
backupPath: `/media/startos/backup/volumes/${srcVolume}/` as const,
})),
)
}
@@ -96,7 +98,7 @@ export class Backups<M extends T.SDKManifest> {
return this
}
mountVolume(
addVolume(
volume: M["volumes"][number],
options?: Partial<{
options: T.SyncOptions
@@ -106,7 +108,7 @@ export class Backups<M extends T.SDKManifest> {
) {
return this.addSync({
dataPath: `/media/startos/volumes/${volume}/` as const,
backupPath: `/media/startos/backup/${volume}/` as const,
backupPath: `/media/startos/backup/volumes/${volume}/` as const,
...options,
})
}
@@ -143,6 +145,12 @@ export class Backups<M extends T.SDKManifest> {
return
}
async init(effects: T.Effects, kind: InitKind): Promise<void> {
if (kind === "restore") {
await this.restoreBackup(effects)
}
}
async restoreBackup(effects: T.Effects) {
this.preRestore(effects as BackupEffects)

View File

@@ -1,6 +1,7 @@
import { Backups } from "./Backups"
import * as T from "../../../base/lib/types"
import { _ } from "../util"
import { InitScript } from "../../../base/lib/inits"
export type SetupBackupsParams<M extends T.SDKManifest> =
| M["volumes"][number][]
@@ -8,7 +9,7 @@ export type SetupBackupsParams<M extends T.SDKManifest> =
type SetupBackupsRes = {
createBackup: T.ExpectedExports.createBackup
restoreBackup: T.ExpectedExports.restoreBackup
restoreInit: InitScript
}
export function setupBackups<M extends T.SDKManifest>(
@@ -20,19 +21,18 @@ export function setupBackups<M extends T.SDKManifest>(
} else {
backupsFactory = async () => Backups.withVolumes(...options)
}
const answer: {
createBackup: T.ExpectedExports.createBackup
restoreBackup: T.ExpectedExports.restoreBackup
} = {
const answer: SetupBackupsRes = {
get createBackup() {
return (async (options) => {
return (await backupsFactory(options)).createBackup(options.effects)
}) as T.ExpectedExports.createBackup
},
get restoreBackup() {
return (async (options) => {
return (await backupsFactory(options)).restoreBackup(options.effects)
}) as T.ExpectedExports.restoreBackup
get restoreInit(): InitScript {
return {
init: async (effects, kind) => {
return (await backupsFactory({ effects })).init(effects, kind)
},
}
},
}
return answer

View File

@@ -35,7 +35,6 @@ export * as backup from "./backup"
export * as daemons from "./mainFn/Daemons"
export * as health from "./health"
export * as healthFns from "./health/checkFns"
export * as inits from "./inits"
export * as mainFn from "./mainFn"
export * as toml from "@iarna/toml"
export * as yaml from "yaml"

View File

@@ -1,3 +0,0 @@
import "./setupInit"
import "./setupUninstall"
import "./setupInstall"

View File

@@ -1,66 +0,0 @@
import { Actions } from "../../../base/lib/actions/setupActions"
import { ExtendedVersion } from "../../../base/lib/exver"
import { UpdateServiceInterfaces } from "../../../base/lib/interfaces/setupInterfaces"
import * as T from "../../../base/lib/types"
import { VersionGraph } from "../version/VersionGraph"
import { PostInstall, PreInstall } from "./setupInstall"
import { Uninstall } from "./setupUninstall"
export function setupInit<Manifest extends T.SDKManifest>(
versions: VersionGraph<string>,
preInstall: PreInstall<Manifest>,
postInstall: PostInstall<Manifest>,
uninstall: Uninstall<Manifest>,
setServiceInterfaces: UpdateServiceInterfaces<any>,
setDependencies: (options: {
effects: T.Effects
}) => Promise<null | void | undefined>,
actions: Actions<any>,
): {
packageInit: T.ExpectedExports.packageInit
packageUninit: T.ExpectedExports.packageUninit
containerInit: T.ExpectedExports.containerInit
} {
return {
packageInit: async (opts) => {
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: versions.currentVersion(),
})
} else {
await postInstall.postInstall(opts)
await opts.effects.setDataVersion({
version: versions.current.options.version,
})
}
},
packageUninit: async (opts) => {
if (opts.nextVersion) {
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: ExtendedVersion.parse(opts.nextVersion),
})
}
} else {
await uninstall.uninstall(opts)
}
},
containerInit: async (opts) => {
const prev = await opts.effects.getDataVersion()
if (!prev) {
await preInstall.preInstall(opts)
}
await setServiceInterfaces({
...opts,
})
await actions.update({ effects: opts.effects })
await setDependencies({ effects: opts.effects })
},
}
}

View File

@@ -1,54 +0,0 @@
import * as T from "../../../base/lib/types"
export type InstallFn<Manifest extends T.SDKManifest> = (opts: {
effects: T.Effects
}) => Promise<null | void | undefined>
export class Install<Manifest extends T.SDKManifest> {
protected constructor(readonly fn: InstallFn<Manifest>) {}
}
export class PreInstall<
Manifest extends T.SDKManifest,
> extends Install<Manifest> {
private constructor(fn: InstallFn<Manifest>) {
super(fn)
}
static of<Manifest extends T.SDKManifest>(fn: InstallFn<Manifest>) {
return new PreInstall(fn)
}
async preInstall({ effects }: Parameters<T.ExpectedExports.packageInit>[0]) {
await this.fn({
effects,
})
}
}
export function setupPreInstall<Manifest extends T.SDKManifest>(
fn: InstallFn<Manifest>,
) {
return PreInstall.of(fn)
}
export class PostInstall<
Manifest extends T.SDKManifest,
> extends Install<Manifest> {
private constructor(fn: InstallFn<Manifest>) {
super(fn)
}
static of<Manifest extends T.SDKManifest>(fn: InstallFn<Manifest>) {
return new PostInstall(fn)
}
async postInstall({ effects }: Parameters<T.ExpectedExports.packageInit>[0]) {
await this.fn({
effects,
})
}
}
export function setupPostInstall<Manifest extends T.SDKManifest>(
fn: InstallFn<Manifest>,
) {
return PostInstall.of(fn)
}

View File

@@ -1,27 +0,0 @@
import * as T from "../../../base/lib/types"
export type UninstallFn<Manifest extends T.SDKManifest> = (opts: {
effects: T.Effects
}) => Promise<null | void | undefined>
export class Uninstall<Manifest extends T.SDKManifest> {
private constructor(readonly fn: UninstallFn<Manifest>) {}
static of<Manifest extends T.SDKManifest>(fn: UninstallFn<Manifest>) {
return new Uninstall(fn)
}
async uninstall({
effects,
nextVersion,
}: Parameters<T.ExpectedExports.packageUninit>[0]) {
if (!nextVersion)
await this.fn({
effects,
})
}
}
export function setupUninstall<Manifest extends T.SDKManifest>(
fn: UninstallFn<Manifest>,
) {
return Uninstall.of(fn)
}

View File

@@ -2,11 +2,7 @@ import { DEFAULT_SIGTERM_TIMEOUT } from "."
import { NO_TIMEOUT, SIGTERM } from "../../../base/lib/types"
import * as T from "../../../base/lib/types"
import {
MountOptions,
SubContainerHandle,
SubContainer,
} from "../util/SubContainer"
import { MountOptions, SubContainer } from "../util/SubContainer"
import { Drop, splitCommand } from "../util"
import * as cp from "child_process"
import * as fs from "node:fs/promises"
@@ -44,7 +40,7 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
) => {
try {
let commands: string[]
if (command instanceof T.UseEntrypoint) {
if (T.isUseEntrypoint(command)) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
encoding: "utf8",
@@ -110,13 +106,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
}
}
}
get subContainerHandle() {
return new SubContainerHandle(this.subcontainer)
}
async wait({ timeout = NO_TIMEOUT, keepSubcontainer = false } = {}) {
async wait({ timeout = NO_TIMEOUT } = {}) {
if (timeout > 0)
setTimeout(() => {
this.term({ keepSubcontainer })
this.term()
}, timeout)
try {
return await this.runningAnswer
@@ -124,14 +117,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
if (!this.state.exited) {
this.process.kill("SIGKILL")
}
if (!keepSubcontainer) await this.subcontainer.destroy()
await this.subcontainer.destroy()
}
}
async term({
signal = SIGTERM,
timeout = this.sigtermTimeout,
keepSubcontainer = false,
} = {}) {
async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
try {
if (!this.state.exited) {
if (signal !== "SIGKILL") {
@@ -148,10 +137,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
await this.runningAnswer
} finally {
if (!keepSubcontainer) await this.subcontainer.destroy()
await this.subcontainer.destroy()
}
}
onDrop(): void {
this.term({ keepSubcontainer: true }).catch(console.error)
this.term().catch(console.error)
}
}

View File

@@ -1,8 +1,13 @@
import * as T from "../../../base/lib/types"
import { asError } from "../../../base/lib/util/asError"
import { Drop } from "../util"
import { ExecSpawnable, SubContainer } from "../util/SubContainer"
import {
SubContainer,
SubContainerOwned,
SubContainerRc,
} from "../util/SubContainer"
import { CommandController } from "./CommandController"
import { Oneshot } from "./Oneshot"
const TIMEOUT_INCREMENT_MS = 1000
const MAX_TIMEOUT_MS = 30000
@@ -16,14 +21,15 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
private shouldBeRunning = false
protected exitedSuccess = false
protected constructor(
private subcontainer: SubContainer<Manifest>,
private startCommand: () => Promise<CommandController<Manifest>>,
readonly oneshot: boolean = false,
protected onExitSuccessFns: (() => void)[] = [],
) {
super()
}
get subContainerHandle(): undefined | ExecSpawnable {
return this.commandController?.subContainerHandle
isOneshot(): this is Oneshot<Manifest> {
return this.oneshot
}
static of<Manifest extends T.SDKManifest>() {
return async (
@@ -44,14 +50,15 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
sigtermTimeout?: number
},
) => {
if (subcontainer.isOwned()) subcontainer = subcontainer.rc()
const startCommand = () =>
CommandController.of<Manifest>()(
effects,
subcontainer,
subcontainer.rc(),
command,
options,
)
return new Daemon(startCommand)
return new Daemon(subcontainer, startCommand)
}
}
async start() {
@@ -65,11 +72,11 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
while (this.shouldBeRunning) {
if (this.commandController)
await this.commandController
.term({ keepSubcontainer: true })
.term({})
.catch((err) => console.error(err))
this.commandController = await this.startCommand()
if (
(await this.commandController.wait({ keepSubcontainer: true }).then(
(await this.commandController.wait().then(
(_) => true,
(err) => {
console.error(err)
@@ -112,6 +119,10 @@ export class Daemon<Manifest extends T.SDKManifest> extends Drop {
?.term({ ...termOptions })
.catch((e) => console.error(asError(e)))
this.commandController = null
await this.subcontainer.destroy()
}
subcontainerRc(): SubContainerRc<Manifest> {
return this.subcontainer.rc()
}
onDrop(): void {
this.stop().catch((e) => console.error(asError(e)))

View File

@@ -5,7 +5,7 @@ import { HealthCheckResult } from "../health/checkFns"
import { Trigger } from "../trigger"
import * as T from "../../../base/lib/types"
import { Mounts } from "./Mounts"
import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer"
import { MountOptions, SubContainer } from "../util/SubContainer"
import { promisify } from "node:util"
import * as CP from "node:child_process"
@@ -17,6 +17,7 @@ import { Daemon } from "./Daemon"
import { CommandController } from "./CommandController"
import { HealthCheck } from "../health/HealthCheck"
import { Oneshot } from "./Oneshot"
import { Manifest } from "../test/output.sdk"
export const cpExec = promisify(CP.exec)
export const cpExecFile = promisify(CP.execFile)
@@ -38,7 +39,7 @@ export type Ready = {
* ```
*/
fn: (
spawnable: ExecSpawnable,
subcontainer: SubContainer<Manifest>,
) => Promise<HealthCheckResult> | HealthCheckResult
/**
* A duration in milliseconds to treat a failing health check as "starting"
@@ -168,9 +169,14 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
const daemon =
"daemon" in options
? Promise.resolve(options.daemon)
: Daemon.of()(this.effects, options.subcontainer, options.command, {
...options,
})
: Daemon.of<Manifest>()(
this.effects,
options.subcontainer,
options.command,
{
...options,
},
)
const healthDaemon = new HealthDaemon(
daemon,
options.requires
@@ -212,7 +218,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
: Id,
options: AddOneshotParams<Manifest, Ids, Id>,
) {
const daemon = Oneshot.of()(
const daemon = Oneshot.of<Manifest>()(
this.effects,
options.subcontainer,
options.command,
@@ -220,7 +226,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
...options,
},
)
const healthDaemon = new HealthDaemon(
const healthDaemon = new HealthDaemon<Manifest>(
daemon,
options.requires
.map((x) => this.ids.indexOf(x))

View File

@@ -91,8 +91,9 @@ export class HealthDaemon<Manifest extends SDKManifest> {
}
private async setupHealthCheck() {
if (this.ready === "EXIT_SUCCESS") {
if (this.daemon instanceof Oneshot) {
this.daemon.onExitSuccess(() =>
const daemon = await this.daemon
if (daemon.isOneshot()) {
daemon.onExitSuccess(() =>
this.setHealth({ result: "success", message: null }),
)
}
@@ -113,9 +114,9 @@ export class HealthDaemon<Manifest extends SDKManifest> {
!res.done;
res = await Promise.race([status, trigger.next()])
) {
const handle = (await this.daemon).subContainerHandle
const handle = (await this.daemon).subcontainerRc()
if (handle) {
try {
const response: HealthCheckResult = await Promise.resolve(
this.ready.fn(handle),
).catch((err) => {
@@ -132,11 +133,8 @@ export class HealthDaemon<Manifest extends SDKManifest> {
this.resolveReady()
}
await this.setHealth(response)
} else {
await this.setHealth({
result: "failure",
message: "Daemon not running",
})
} finally {
await handle.destroy()
}
}
}).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
@@ -164,7 +162,7 @@ export class HealthDaemon<Manifest extends SDKManifest> {
if (
result === "failure" &&
this.started &&
performance.now() - this.started <= (this.ready.gracePeriod ?? 5000)
performance.now() - this.started <= (this.ready.gracePeriod ?? 10_000)
)
result = "starting"
await this.effects.setHealth({

View File

@@ -1,5 +1,5 @@
import * as T from "../../../base/lib/types"
import { SubContainer } from "../util/SubContainer"
import { SubContainer, SubContainerOwned } from "../util/SubContainer"
import { CommandController } from "./CommandController"
import { Daemon } from "./Daemon"
@@ -28,14 +28,15 @@ export class Oneshot<Manifest extends T.SDKManifest> extends Daemon<Manifest> {
sigtermTimeout?: number
},
) => {
if (subcontainer.isOwned()) subcontainer = subcontainer.rc()
const startCommand = () =>
CommandController.of<Manifest>()(
effects,
subcontainer,
subcontainer.rc(),
command,
options,
)
return new Oneshot(startCommand, true, [])
return new Oneshot(subcontainer, startCommand, true, [])
}
}

View File

@@ -18,7 +18,7 @@ describe("host", () => {
type: "ui",
username: "bar",
path: "/baz",
search: { qux: "yes" },
query: { qux: "yes" },
schemeOverride: null,
masked: false,
})

View File

@@ -536,14 +536,14 @@ describe("values", () => {
})
describe("filtering", () => {
test("union", async () => {
const value = Value.filteredUnion(
() => ["a", "c"],
{
const value = Value.dynamicUnion(
() => ({
name: "Testing",
default: "a",
description: null,
warning: null,
},
disabled: ["a", "c"],
}),
Variants.of({
a: {
name: "a",

View File

@@ -59,50 +59,97 @@ async function bind(
await execFile("mount", ["--bind", from, to])
}
/**
* This is the type that is going to describe what an subcontainer could do. The main point of the
* subcontainer is to have commands that run in a chrooted environment. This is useful for running
* commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the
* case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed.
*/
export interface ExecSpawnable {
get destroy(): undefined | (() => Promise<null>)
export interface SubContainer<
Manifest extends T.SDKManifest,
Effects extends T.Effects = T.Effects,
> extends Drop {
readonly imageId: keyof Manifest["images"] & T.ImageId
readonly rootfs: string
readonly guid: T.Guid
mount(
mounts: Effects extends BackupEffects
? Mounts<
Manifest,
{
subpath: string | null
mountpoint: string
}
>
: Mounts<Manifest, never>,
): Promise<this>
destroy: () => Promise<null>
/**
* @description run a command inside this subcontainer
* DOES NOT THROW ON NONZERO EXIT CODE (see execFail)
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
* @returns
*/
exec(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<ExecResults>
): Promise<{
throw: () => { stdout: string | Buffer; stderr: string | Buffer }
exitCode: number | null
exitSignal: NodeJS.Signals | null
stdout: string | Buffer
stderr: string | Buffer
}>
/**
* @description run a command inside this subcontainer, throwing on non-zero exit status
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
* @returns
*/
execFail(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }>
): Promise<{
stdout: string | Buffer
stderr: string | Buffer
}>
launch(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams>
spawn(
command: string[],
options?: CommandOptions & StdioOptions,
): Promise<cp.ChildProcess>
rc(): SubContainerRc<Manifest, Effects>
isOwned(): this is SubContainerOwned<Manifest, Effects>
}
/**
* Want to limit what we can do in a container, so we want to launch a container with a specific image and the mounts.
*
* Implements:
* @see {@link ExecSpawnable}
*/
export class SubContainer<
export class SubContainerOwned<
Manifest extends T.SDKManifest,
Effects extends T.Effects = T.Effects,
>
extends Drop
implements ExecSpawnable
implements SubContainer<Manifest, Effects>
{
private destroyed = false
public rcs = 0
private leader: cp.ChildProcess
private leaderExited: boolean = false
private waitProc: () => Promise<null>
private constructor(
readonly effects: Effects,
readonly imageId: T.ImageId,
readonly imageId: keyof Manifest["images"] & T.ImageId,
readonly rootfs: string,
readonly guid: T.Guid,
) {
@@ -156,14 +203,14 @@ export class SubContainer<
: Mounts<Manifest, never>)
| null,
name: string,
) {
): Promise<SubContainerOwned<Manifest, Effects>> {
const { imageId, sharedRun } = image
const [rootfs, guid] = await effects.subcontainer.createFs({
imageId,
name,
})
const res = new SubContainer(effects, imageId, rootfs, guid)
const res = new SubContainerOwned(effects, imageId, rootfs, guid)
try {
if (mounts) {
@@ -216,7 +263,12 @@ export class SubContainer<
name: string,
fn: (subContainer: SubContainer<Manifest, Effects>) => Promise<T>,
): Promise<T> {
const subContainer = await SubContainer.of(effects, image, mounts, name)
const subContainer = await SubContainerOwned.of(
effects,
image,
mounts,
name,
)
try {
return await fn(subContainer)
} finally {
@@ -234,7 +286,7 @@ export class SubContainer<
}
>
: Mounts<Manifest, never>,
): Promise<SubContainer<Manifest, Effects>> {
): Promise<this> {
for (let mount of mounts.build()) {
let { options, mountpoint } = mount
const path = mountpoint.startsWith("/")
@@ -526,40 +578,188 @@ export class SubContainer<
options,
)
}
rc(): SubContainerRc<Manifest, Effects> {
return new SubContainerRc(this)
}
isOwned(): this is SubContainerOwned<Manifest, Effects> {
return true
}
}
/**
* Take an subcontainer but remove the ability to add the mounts and the destroy function.
* Lets other functions, like health checks, to not destroy the parents.
*
*/
export class SubContainerHandle implements ExecSpawnable {
constructor(private subContainer: ExecSpawnable) {}
export class SubContainerRc<
Manifest extends T.SDKManifest,
Effects extends T.Effects = T.Effects,
>
extends Drop
implements SubContainer<Manifest, Effects>
{
get imageId() {
return this.subcontainer.imageId
}
get rootfs() {
return this.subcontainer.rootfs
}
get guid() {
return this.subcontainer.guid
}
private destroyed = false
public constructor(
private readonly subcontainer: SubContainerOwned<Manifest, Effects>,
) {
subcontainer.rcs++
super()
}
static async of<Manifest extends T.SDKManifest, Effects extends T.Effects>(
effects: Effects,
image: {
imageId: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
},
mounts:
| (Effects extends BackupEffects
? Mounts<
Manifest,
{
subpath: string | null
mountpoint: string
}
>
: Mounts<Manifest, never>)
| null,
name: string,
) {
return new SubContainerRc(
await SubContainerOwned.of(effects, image, mounts, name),
)
}
static async withTemp<
Manifest extends T.SDKManifest,
T,
Effects extends T.Effects,
>(
effects: Effects,
image: {
imageId: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
},
mounts:
| (Effects extends BackupEffects
? Mounts<
Manifest,
{
subpath: string | null
mountpoint: string
}
>
: Mounts<Manifest, never>)
| null,
name: string,
fn: (subContainer: SubContainer<Manifest, Effects>) => Promise<T>,
): Promise<T> {
const subContainer = await SubContainerRc.of(effects, image, mounts, name)
try {
return await fn(subContainer)
} finally {
await subContainer.destroy()
}
}
async mount(
mounts: Effects extends BackupEffects
? Mounts<
Manifest,
{
subpath: string | null
mountpoint: string
}
>
: Mounts<Manifest, never>,
): Promise<this> {
await this.subcontainer.mount(mounts)
return this
}
get destroy() {
return undefined
return async () => {
if (!this.destroyed) {
const rcs = --this.subcontainer.rcs
if (rcs <= 0) {
await this.subcontainer.destroy()
if (rcs < 0) console.error(new Error("UNREACHABLE: rcs < 0").stack)
}
this.destroyed = true
}
return null
}
}
exec(
command: string[],
options?: CommandOptions,
timeoutMs?: number | null,
): Promise<ExecResults> {
return this.subContainer.exec(command, options, timeoutMs)
onDrop(): void {
this.destroy()
}
execFail(
/**
* @description run a command inside this subcontainer
* DOES NOT THROW ON NONZERO EXIT CODE (see execFail)
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
* @returns
*/
async exec(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
return this.subContainer.execFail(command, options, timeoutMs)
timeoutMs: number | null = 30000,
): Promise<{
throw: () => { stdout: string | Buffer; stderr: string | Buffer }
exitCode: number | null
exitSignal: NodeJS.Signals | null
stdout: string | Buffer
stderr: string | Buffer
}> {
return this.subcontainer.exec(command, options, timeoutMs)
}
spawn(
/**
* @description run a command inside this subcontainer, throwing on non-zero exit status
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
* @returns
*/
async execFail(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs: number | null = 30000,
): Promise<{
stdout: string | Buffer
stderr: string | Buffer
}> {
return this.subcontainer.execFail(command, options, timeoutMs)
}
async launch(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams> {
return this.subcontainer.launch(command, options)
}
async spawn(
command: string[],
options: CommandOptions & StdioOptions = { stdio: "inherit" },
): Promise<cp.ChildProcess> {
return this.subContainer.spawn(command, options)
return this.subcontainer.spawn(command, options)
}
rc(): SubContainerRc<Manifest, Effects> {
return this.subcontainer.rc()
}
isOwned(): this is SubContainerOwned<Manifest, Effects> {
return false
}
}

View File

@@ -1,9 +1,58 @@
import { ExtendedVersion, VersionRange } from "../../../base/lib/exver"
import * as T from "../../../base/lib/types"
import {
InitFn,
InitKind,
InitScript,
InitScriptOrFn,
UninitFn,
UninitScript,
} from "../../../base/lib/inits"
import { Graph, Vertex, once } from "../util"
import { IMPOSSIBLE, VersionInfo } from "./VersionInfo"
export class VersionGraph<CurrentVersion extends string> {
export async function getDataVersion(effects: T.Effects) {
const versionStr = await effects.getDataVersion()
if (!versionStr) return null
try {
return ExtendedVersion.parse(versionStr)
} catch (_) {
return VersionRange.parse(versionStr)
}
}
export async function setDataVersion(
effects: T.Effects,
version: ExtendedVersion | VersionRange | null,
) {
return effects.setDataVersion({ version: version?.toString() || null })
}
function isExver(v: ExtendedVersion | VersionRange): v is ExtendedVersion {
return "satisfies" in v
}
function isRange(v: ExtendedVersion | VersionRange): v is VersionRange {
return "satisfiedBy" in v
}
export function overlaps(
a: ExtendedVersion | VersionRange,
b: ExtendedVersion | VersionRange,
) {
return (
(isRange(a) && isRange(b) && a.intersects(b)) ||
(isRange(a) && isExver(b) && a.satisfiedBy(b)) ||
(isExver(a) && isRange(b) && a.satisfies(b)) ||
(isExver(a) && isExver(b) && a.equals(b))
)
}
export class VersionGraph<CurrentVersion extends string>
implements InitScript, UninitScript
{
protected initFn = this.init.bind(this)
protected uninitFn = this.uninit.bind(this)
private readonly graph: () => Graph<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
@@ -11,6 +60,8 @@ export class VersionGraph<CurrentVersion extends string> {
private constructor(
readonly current: VersionInfo<CurrentVersion>,
versions: Array<VersionInfo<any>>,
private readonly preInstall?: InitScriptOrFn<"install">,
private readonly uninstall?: UninitScript | UninitFn,
) {
this.graph = once(() => {
const graph = new Graph<
@@ -88,9 +139,7 @@ export class VersionGraph<CurrentVersion extends string> {
vertex,
)
for (let matching of graph.findVertex(
(v) =>
v.metadata instanceof ExtendedVersion &&
v.metadata.satisfies(range),
(v) => isExver(v.metadata) && v.metadata.satisfies(range),
)) {
graph.addEdge(
version.options.migrations.other[rangeStr],
@@ -116,11 +165,24 @@ export class VersionGraph<CurrentVersion extends string> {
static of<
CurrentVersion extends string,
OtherVersions extends Array<VersionInfo<any>>,
>(
currentVersion: VersionInfo<CurrentVersion>,
...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
) {
return new VersionGraph(currentVersion, other as Array<VersionInfo<any>>)
>(options: {
current: VersionInfo<CurrentVersion>
other: OtherVersions
/**
* A script to run only on fresh install
*/
preInstall?: InitScript | InitFn
/**
* A script to run only on uninstall
*/
uninstall?: UninitScript | UninitFn
}) {
return new VersionGraph(
options.current,
options.other,
options.preInstall,
options.uninstall,
)
}
async migrate({
effects,
@@ -128,46 +190,42 @@ export class VersionGraph<CurrentVersion extends string> {
to,
}: {
effects: T.Effects
from: ExtendedVersion
to: ExtendedVersion
}) {
from: ExtendedVersion | VersionRange
to: ExtendedVersion | VersionRange
}): Promise<ExtendedVersion | VersionRange> {
if (overlaps(from, to)) return from
const graph = this.graph()
if (from && to) {
const path = graph.shortestPath(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(from)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(from)),
(v) =>
(v.metadata instanceof VersionRange && v.metadata.satisfiedBy(to)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(to)),
(v) => overlaps(v.metadata, from),
(v) => overlaps(v.metadata, to),
)
if (path) {
let dataVersion = from
for (let edge of path) {
if (edge.metadata) {
await edge.metadata({ effects })
}
await effects.setDataVersion({ version: edge.to.metadata.toString() })
dataVersion = edge.to.metadata
await setDataVersion(effects, edge.to.metadata)
}
return
return dataVersion
}
}
throw new Error()
throw new Error(
`cannot migrate from ${from.toString()} to ${to.toString()}`,
)
}
canMigrateFrom = once(() =>
Array.from(
this.graph().reverseBreadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
this.graph().reverseBreadthFirstSearch((v) =>
overlaps(v.metadata, this.currentVersion()),
),
)
.reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
isRange(x.metadata)
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
@@ -177,18 +235,14 @@ export class VersionGraph<CurrentVersion extends string> {
)
canMigrateTo = once(() =>
Array.from(
this.graph().breadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
this.graph().breadthFirstSearch((v) =>
overlaps(v.metadata, this.currentVersion()),
),
)
.reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
isRange(x.metadata)
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
@@ -196,6 +250,45 @@ export class VersionGraph<CurrentVersion extends string> {
)
.normalize(),
)
async init(effects: T.Effects, kind: InitKind): Promise<void> {
const from = await getDataVersion(effects)
if (from) {
await this.migrate({
effects,
from,
to: this.currentVersion(),
})
} else {
kind = "install" // implied by !dataVersion
if (this.preInstall)
if ("init" in this.preInstall) await this.preInstall.init(effects, kind)
else await this.preInstall(effects, kind)
await effects.setDataVersion({ version: this.current.options.version })
}
}
async uninit(
effects: T.Effects,
target: VersionRange | ExtendedVersion | null,
): Promise<void> {
if (target) {
const from = await getDataVersion(effects)
if (from) {
target = await this.migrate({
effects,
from,
to: target,
})
}
} else {
if (this.uninstall)
if ("uninit" in this.uninstall)
await this.uninstall.uninit(effects, target)
else await this.uninstall(effects, target)
}
await setDataVersion(effects, target)
}
}
// prettier-ignore

View File

@@ -1,12 +1,12 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.20",
"version": "0.4.0-beta.23",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.20",
"version": "0.4.0-beta.23",
"license": "MIT",
"dependencies": {
"@iarna/toml": "^3.0.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.20",
"version": "0.4.0-beta.23",
"description": "Software development kit to facilitate packaging services for StartOS",
"main": "./package/lib/index.js",
"types": "./package/lib/index.d.ts",