Files
start-os/sdk/package/lib/StartSdk.ts
Aiden McClelland 2fbaaebf44 Bugfixes for alpha.12 (#3049)
* squashfs-wip

* sdk fixes

* misc fixes

* bump sdk

* Include StartTunnel installation command

Added installation instructions for StartTunnel.

* CA instead of leaf for StartTunnel (#3046)

* updated docs for CA instead of cert

* generate ca instead of self-signed in start-tunnel

* Fix formatting in START-TUNNEL.md installation instructions

* Fix formatting in START-TUNNEL.md

* fix infinite loop

* add success message to install

* hide loopback and bridge gateways

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>

* prevent gateways from getting stuck empty

* fix set-password

* misc networking fixes

* build and efi fixes

* efi fixes

* alpha.13

* remove cross

* fix tests

* provide path to upgrade

* fix networkmanager issues

* remove squashfs before creating

---------

Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>
2025-11-15 22:33:03 -07:00

797 lines
28 KiB
TypeScript

import { Value } from "../../base/lib/actions/input/builder/value"
import { InputSpec } from "../../base/lib/actions/input/builder/inputSpec"
import { Variants } from "../../base/lib/actions/input/builder/variants"
import {
Action,
ActionInfo,
Actions,
} from "../../base/lib/actions/setupActions"
import {
SyncOptions,
ServiceInterfaceId,
PackageId,
ServiceInterfaceType,
Effects,
} from "../../base/lib/types"
import * as patterns from "../../base/lib/util/patterns"
import { BackupSync, Backups } from "./backup/Backups"
import { smtpInputSpec } from "../../base/lib/actions/input/inputSpecConstants"
import { Daemon, Daemons } from "./mainFn/Daemons"
import { checkPortListening } from "./health/checkFns/checkPortListening"
import { checkWebUrl, runHealthScript } from "./health/checkFns"
import { List } from "../../base/lib/actions/input/builder/list"
import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
import { setupMain } from "./mainFn"
import { defaultTrigger } from "./trigger/defaultTrigger"
import { changeOnFirstSuccess, cooldownTrigger } from "./trigger"
import { setupServiceInterfaces } from "../../base/lib/interfaces/setupInterfaces"
import { successFailure } from "./trigger/successFailure"
import { MultiHost, Scheme } from "../../base/lib/interfaces/Host"
import { ServiceInterfaceBuilder } from "../../base/lib/interfaces/ServiceInterfaceBuilder"
import { GetSystemSmtp } from "./util"
import { nullIfEmpty } from "./util"
import { getServiceInterface, getServiceInterfaces } from "./util"
import {
CommandOptions,
ExitError,
SubContainer,
SubContainerOwned,
} from "./util/SubContainer"
import { splitCommand } from "./util"
import { Mounts } from "./mainFn/Mounts"
import { setupDependencies } from "../../base/lib/dependencies/setupDependencies"
import * as T from "../../base/lib/types"
import { testTypeVersion } from "../../base/lib/exver"
import {
CheckDependencies,
checkDependencies,
} from "../../base/lib/dependencies/dependencies"
import { GetSslCertificate } from "./util"
import { getDataVersion, setDataVersion } from "./version"
import { MaybeFn } from "../../base/lib/actions/setupActions"
import { GetInput } from "../../base/lib/actions/setupActions"
import { Run } from "../../base/lib/actions/setupActions"
import * as actions from "../../base/lib/actions"
import * as fs from "node:fs/promises"
import {
setupInit,
setupUninit,
setupOnInit,
setupOnUninit,
} from "../../base/lib/inits"
import { DropGenerator } from "../../base/lib/util/Drop"
export const OSVersion = testTypeVersion("0.4.0-alpha.13")
// prettier-ignore
type AnyNeverCond<T extends any[], Then, Else> =
T extends [] ? Else :
T extends [never, ...Array<any>] ? Then :
T extends [any, ...infer U] ? AnyNeverCond<U,Then, Else> :
never
export class StartSdk<Manifest extends T.SDKManifest> {
private constructor(readonly manifest: Manifest) {}
static of() {
return new StartSdk<never>(null as never)
}
withManifest<Manifest extends T.SDKManifest = never>(manifest: Manifest) {
return new StartSdk<Manifest>(manifest)
}
build(isReady: AnyNeverCond<[Manifest], "Build not ready", true>) {
type NestedEffects = "subcontainer" | "store" | "action"
type InterfaceEffects =
| "getServiceInterface"
| "listServiceInterfaces"
| "exportServiceInterface"
| "clearServiceInterfaces"
| "bind"
| "getHostInfo"
type MainUsedEffects = "setMainStatus"
type CallbackEffects =
| "child"
| "constRetry"
| "isInContext"
| "onLeaveContext"
| "clearCallbacks"
type AlreadyExposed =
| "getSslCertificate"
| "getSystemSmtp"
| "getContainerIp"
| "getDataVersion"
| "setDataVersion"
// prettier-ignore
type StartSdkEffectWrapper = {
[K in keyof Omit<Effects, "eventId" | NestedEffects | InterfaceEffects | MainUsedEffects | CallbackEffects | AlreadyExposed>]: (effects: Effects, ...args: Parameters<Effects[K]>) => ReturnType<Effects[K]>
}
const startSdkEffectWrapper: StartSdkEffectWrapper = {
restart: (effects, ...args) => effects.restart(...args),
setDependencies: (effects, ...args) => effects.setDependencies(...args),
checkDependencies: (effects, ...args) =>
effects.checkDependencies(...args),
mount: (effects, ...args) => effects.mount(...args),
getInstalledPackages: (effects, ...args) =>
effects.getInstalledPackages(...args),
getServicePortForward: (effects, ...args) =>
effects.getServicePortForward(...args),
clearBindings: (effects, ...args) => effects.clearBindings(...args),
getOsIp: (effects, ...args) => effects.getOsIp(...args),
getSslKey: (effects, ...args) => effects.getSslKey(...args),
shutdown: (effects, ...args) => effects.shutdown(...args),
getDependencies: (effects, ...args) => effects.getDependencies(...args),
getStatus: (effects, ...args) => effects.getStatus(...args),
setHealth: (effects, ...args) => effects.setHealth(...args),
}
return {
manifest: this.manifest,
...startSdkEffectWrapper,
setDataVersion,
getDataVersion,
action: {
run: actions.runAction,
createTask: <T extends ActionInfo<T.ActionId, any>>(
effects: T.Effects,
packageId: T.PackageId,
action: T,
severity: T.TaskSeverity,
options?: actions.TaskOptions<T>,
) =>
actions.createTask({
effects,
packageId,
action,
severity,
options: options,
}),
createOwnTask: <T extends ActionInfo<T.ActionId, any>>(
effects: T.Effects,
action: T,
severity: T.TaskSeverity,
options?: actions.TaskOptions<T>,
) =>
actions.createTask({
effects,
packageId: this.manifest.id,
action,
severity,
options: options,
}),
clearTask: (effects: T.Effects, ...replayIds: string[]) =>
effects.action.clearTasks({ only: replayIds }),
},
checkDependencies: checkDependencies as <
DependencyId extends keyof Manifest["dependencies"] &
PackageId = keyof Manifest["dependencies"] & PackageId,
>(
effects: Effects,
packageIds?: DependencyId[],
) => Promise<CheckDependencies<DependencyId>>,
serviceInterface: {
getOwn: <E extends Effects>(effects: E, id: ServiceInterfaceId) =>
getServiceInterface(effects, {
id,
}),
get: <E extends Effects>(
effects: E,
opts: { id: ServiceInterfaceId; packageId: PackageId },
) => getServiceInterface(effects, opts),
getAllOwn: <E extends Effects>(effects: E) =>
getServiceInterfaces(effects, {}),
getAll: <E extends Effects>(
effects: E,
opts: { packageId: PackageId },
) => getServiceInterfaces(effects, opts),
},
getContainerIp: (
effects: T.Effects,
options: Omit<
Parameters<T.Effects["getContainerIp"]>[0],
"callback"
> = {},
) => {
async function* watch(abort?: AbortSignal) {
const resolveCell = { resolve: () => {} }
effects.onLeaveContext(() => {
resolveCell.resolve()
})
abort?.addEventListener("abort", () => resolveCell.resolve())
while (effects.isInContext && !abort?.aborted) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
resolveCell.resolve = resolve
})
yield await effects.getContainerIp({ ...options, callback })
await waitForNext
}
}
return {
const: () =>
effects.getContainerIp({
...options,
callback:
effects.constRetry &&
(() => effects.constRetry && effects.constRetry()),
}),
once: () => effects.getContainerIp(options),
watch: (abort?: AbortSignal) => {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
return DropGenerator.of(watch(ctrl.signal), () => ctrl.abort())
},
onChange: (
callback: (
value: string | null,
error?: Error,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) => {
;(async () => {
const ctrl = new AbortController()
for await (const value of watch(ctrl.signal)) {
try {
const res = await callback(value)
if (res.cancel) {
ctrl.abort()
break
}
} catch (e) {
console.error(
"callback function threw an error @ getContainerIp.onChange",
e,
)
}
}
})()
.catch((e) => callback(null, e))
.catch((e) =>
console.error(
"callback function threw an error @ getContainerIp.onChange",
e,
),
)
},
waitFor: async (pred: (value: string | null) => boolean) => {
const resolveCell = { resolve: () => {} }
effects.onLeaveContext(() => {
resolveCell.resolve()
})
while (effects.isInContext) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
resolveCell.resolve = resolve
})
const res = await effects.getContainerIp({ ...options, callback })
if (pred(res)) {
resolveCell.resolve()
return res
}
await waitForNext
}
return null
},
}
},
MultiHost: {
of: (effects: Effects, id: string) => new MultiHost({ id, effects }),
},
nullIfEmpty,
useEntrypoint: (overrideCmd?: string[]) =>
new T.UseEntrypoint(overrideCmd),
/**
* @description Use this class to create an Action. By convention, each Action should receive its own file.
*
*/
Action: {
/**
* @description Use this function to create an action that accepts form input
* @param id - a unique ID for this action
* @param metadata - information describing the action and its availability
* @param inputSpec - define the form input using the InputSpec and Value classes
* @param prefillFn - optionally fetch data from the file system to pre-fill the input form. Must returns a deep partial of the input spec
* @param executionFn - execute the action. Optionally return data for the user to view. Must be in the structure of an ActionResult, version "1"
* @example
* In this example, we create an action for a user to provide their name.
* We prefill the input form with their existing name from the service's yaml file.
* The new name is saved to the yaml file, and we return nothing to the user, which
* means they will receive a generic success message.
*
* ```
import { sdk } from '../sdk'
import { yamlFile } from '../file-models/config.yml'
const { InputSpec, Value } = sdk
export const inputSpec = InputSpec.of({
name: Value.text({
name: 'Name',
description:
'When you launch the Hello World UI, it will display "Hello [Name]"',
required: true,
default: 'World',
}),
})
export const setName = sdk.Action.withInput(
// id
'set-name',
// metadata
async ({ effects }) => ({
name: 'Set Name',
description: 'Set your name so Hello World can say hello to you',
warning: null,
allowedStatuses: 'any',
group: null,
visibility: 'enabled',
}),
// form input specification
inputSpec,
// optionally pre-fill the input form
async ({ effects }) => {
const name = await yamlFile.read.const(effects)?.name
return { name }
},
// the execution function
async ({ effects, input }) => yamlFile.merge(input)
)
* ```
*/
withInput: Action.withInput,
/**
* @description Use this function to create an action that does not accept form input
* @param id - a unique ID for this action
* @param metadata - information describing the action and its availability
* @param executionFn - execute the action. Optionally return data for the user to view. Must be in the structure of an ActionResult, version "1"
* @example
* In this example, we create an action that returns a secret phrase for the user to see.
*
* ```
import { store } from '../file-models/store.json'
import { sdk } from '../sdk'
export const showSecretPhrase = sdk.Action.withoutInput(
// id
'show-secret-phrase',
// metadata
async ({ effects }) => ({
name: 'Show Secret Phrase',
description: 'Reveal the secret phrase for Hello World',
warning: null,
allowedStatuses: 'any',
group: null,
visibility: 'enabled',
}),
// the execution function
async ({ effects }) => ({
version: '1',
title: 'Secret Phrase',
message:
'Below is your secret phrase. Use it to gain access to extraordinary places',
result: {
type: 'single',
value: (await store.read.once())?.secretPhrase,
copyable: true,
qr: true,
masked: true,
},
}),
)
* ```
*/
withoutInput: <Id extends T.ActionId>(
id: Id,
metadata: MaybeFn<Omit<T.ActionMetadata, "hasInput">>,
run: Run<{}>,
) => Action.withoutInput(id, metadata, run),
},
inputSpecConstants: { smtpInputSpec },
/**
* @description Use this function to create a service interface.
* @param effects
* @param options
* @example
* In this example, we create a standard web UI
*
* ```
const ui = sdk.createInterface(effects, {
name: 'Web UI',
id: 'ui',
description: 'The primary web app for this service.',
type: 'ui',
masked: false,
schemeOverride: null,
username: null,
path: '',
query: {},
})
* ```
*/
createInterface: (
effects: Effects,
options: {
/** The human readable name of this service interface. */
name: string
/** A unique ID for this service interface. */
id: string
/** The human readable description. */
description: string
/** Affects how the interface appears to the user. One of: 'ui', 'api', 'p2p'. If 'ui', the user will see an option to open the UI in a new tab */
type: ServiceInterfaceType
/** (optional) prepends the provided username to all URLs. */
username: null | string
/** (optional) appends the provided path to all URLs. */
path: string
/** (optional) appends the provided query params to all URLs. */
query: Record<string, string>
/** (optional) overrides the protocol prefix provided by the bind function.
*
* @example `ftp://`
*/
schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
/** TODO Aiden how would someone include a password in the URL? Whether or not to mask the URLs on the screen, for example, when they contain a password */
masked: boolean
},
) => new ServiceInterfaceBuilder({ ...options, effects }),
getSystemSmtp: <E extends Effects>(effects: E) =>
new GetSystemSmtp(effects),
getSslCerificate: <E extends Effects>(
effects: E,
hostnames: string[],
algorithm?: T.Algorithm,
) => new GetSslCertificate(effects, hostnames, algorithm),
healthCheck: {
checkPortListening,
checkWebUrl,
runHealthScript,
},
patterns,
/**
* @description Use this function to list every Action offered by the service. Actions will be displayed in the provided order.
*
* By convention, each Action should receive its own file in the "actions" directory.
* @example
*
* ```
import { sdk } from '../sdk'
import { config } from './config'
import { nameToLogs } from './nameToLogs'
export const actions = sdk.Actions.of().addAction(config).addAction(nameToLogs)
* ```
*/
Actions: Actions<{}>,
/**
* @description Use this function to determine which volumes are backed up when a user creates a backup, including advanced options.
* @example
* In this example, we back up the entire "main" volume and nothing else.
*
* ```
import { sdk } from './sdk'
export const { createBackup, restoreBackup } = sdk.setupBackups(
async ({ effects }) => sdk.Backups.volumes('main'),
)
* ```
* @example
* In this example, we back up the "main" volume, but exclude hypothetical directory "excludedDir".
*
* ```
import { sdk } from './sdk'
export const { createBackup, restoreBackup } = sdk.setupBackups(async () =>
sdk.Backups.volumes('main').setOptions({
exclude: ['excludedDir'],
}),
)
* ```
*/
setupBackups: (options: SetupBackupsParams<Manifest>) =>
setupBackups<Manifest>(options),
/**
* @description Use this function to set dependency information.
* @example
* In this example, we create a dependency on Hello World >=1.0.0:0, where Hello World must be running and passing its "primary" health check.
*
* ```
export const setDependencies = sdk.setupDependencies(
async ({ effects }) => {
return {
'hello-world': {
kind: 'running',
versionRange: '>=1.0.0',
healthChecks: ['primary'],
},
}
},
)
* ```
*/
setupDependencies: setupDependencies<Manifest>,
/**
* @description Use this function to create an InitScript that runs every time the service initializes (install, update, restore, rebuild, and server bootup)
*/
setupOnInit,
/**
* @description Use this function to create an UninitScript that runs every time the service uninitializes (update, uninstall, and server shutdown)
*/
setupOnUninit,
/**
* @description Use this function to setup what happens when the service initializes.
*
* This happens when the server boots, or a service is installed, updated, or restored
*
* Not every init script does something on every initialization. For example, versions only does something on install or update
*
* These scripts are run in the order they are supplied
* @example
*
* ```
export const init = sdk.setupInit(
restoreInit,
versions,
setDependencies,
setInterfaces,
actions,
postInstall,
)
* ```
*/
setupInit: setupInit,
/**
* @description Use this function to setup what happens when the service uninitializes.
*
* This happens when the server shuts down, or a service is uninstalled or updated
*
* Not every uninit script does something on every uninitialization. For example, versions only does something on uninstall or update
*
* These scripts are run in the order they are supplied
* @example
*
* ```
export const uninit = sdk.setupUninit(
versions,
)
* ```
*/
setupUninit: setupUninit,
/**
* @description Use this function to determine how this service will be hosted and served. The function executes on service install, service update, and inputSpec save.
* @param inputSpec - The inputSpec spec of this service as exported from /inputSpec/spec.
* @param fn - an async function that returns an array of interface receipts. The function always has access to `effects`; it has access to `input` only after inputSpec save, otherwise `input` will be null.
* @example
* In this example, we create two UIs from one multi-host, and one API from another multi-host.
*
* ```
export const setInterfaces = sdk.setupInterfaces(
async ({ effects }) => {
// ** UI multi-host **
const uiMulti = sdk.MultiHost.of(effects, 'ui-multi')
const uiMultiOrigin = await uiMulti.bindPort(80, {
protocol: 'http',
})
// Primary UI
const primaryUi = sdk.createInterface(effects, {
name: 'Primary UI',
id: 'primary-ui',
description: 'The primary web app for this service.',
type: 'ui',
masked: false,
schemeOverride: null,
username: null,
path: '',
query: {},
})
// Admin UI
const adminUi = sdk.createInterface(effects, {
name: 'Admin UI',
id: 'admin-ui',
description: 'The admin web app for this service.',
type: 'ui',
masked: false,
schemeOverride: null,
username: null,
path: '/admin',
query: {},
})
// UI receipt
const uiReceipt = await uiMultiOrigin.export([primaryUi, adminUi])
// ** API multi-host **
const apiMulti = sdk.MultiHost.of(effects, 'api-multi')
const apiMultiOrigin = await apiMulti.bindPort(5959, {
protocol: 'http',
})
// API
const api = sdk.createInterface(effects, {
name: 'Admin API',
id: 'api',
description: 'The advanced API for this service.',
type: 'api',
masked: false,
schemeOverride: null,
username: null,
path: '',
query: {},
})
// API receipt
const apiReceipt = await apiMultiOrigin.export([api])
// ** Return receipts **
return [uiReceipt, apiReceipt]
},
)
* ```
*/
setupInterfaces: setupServiceInterfaces,
setupMain: (
fn: (o: {
effects: Effects
started(onTerm: () => PromiseLike<void>): PromiseLike<null>
}) => Promise<Daemons<Manifest, any>>,
) => setupMain<Manifest>(fn),
trigger: {
defaultTrigger,
cooldownTrigger,
changeOnFirstSuccess,
successFailure,
},
Mounts: {
of: Mounts.of<Manifest>,
},
Backups: {
ofVolumes: Backups.ofVolumes<Manifest>,
ofSyncs: Backups.ofSyncs<Manifest>,
withOptions: Backups.withOptions<Manifest>,
},
InputSpec: {
/**
* @description Use this function to define the inputSpec specification that will ultimately present to the user as validated form inputs.
*
* Most form controls are supported, including text, textarea, number, toggle, select, multiselect, list, color, datetime, object (sub form), and union (conditional sub form).
* @example
* In this example, we define a inputSpec form with two value: name and makePublic.
*
* ```
import { sdk } from '../sdk'
const { InputSpec, Value } = sdk
export const inputSpecSpec = InputSpec.of({
name: Value.text({
name: 'Name',
description:
'When you launch the Hello World UI, it will display "Hello [Name]"',
required: true,
default: 'World'
}),
makePublic: Value.toggle({
name: 'Make Public',
description: 'Whether or not to expose the service to the network',
default: false,
}),
})
* ```
*/
of: <Spec extends Record<string, Value<any>>>(spec: Spec) =>
InputSpec.of<Spec>(spec),
},
Daemon: {
get of() {
return Daemon.of<Manifest>()
},
},
Daemons: {
of(
effects: Effects,
started:
| ((onTerm: () => PromiseLike<void>) => PromiseLike<null>)
| null,
) {
return Daemons.of<Manifest>({ effects, started })
},
},
SubContainer: {
/**
* @description Create a new SubContainer
* @param effects
* @param image - what container image to use
* @param mounts - what to mount to the subcontainer
* @param name - a name to use to refer to the subcontainer for debugging purposes
*/
of(
effects: Effects,
image: {
imageId: T.ImageId & keyof Manifest["images"]
sharedRun?: boolean
},
mounts: Mounts<Manifest> | null,
name: string,
) {
return SubContainerOwned.of<Manifest, Effects>(
effects,
image,
mounts,
name,
).then((subc) => subc.rc())
},
/**
* @description Run a function with a temporary SubContainer
* @param effects
* @param image - what container image to use
* @param mounts - what to mount to the subcontainer
* @param name - a name to use to refer to the ephemeral subcontainer for debugging purposes
*/
withTemp<T>(
effects: T.Effects,
image: {
imageId: T.ImageId & keyof Manifest["images"]
sharedRun?: boolean
},
mounts: Mounts<Manifest> | null,
name: string,
fn: (subContainer: SubContainer<Manifest>) => Promise<T>,
): Promise<T> {
return SubContainerOwned.withTemp(effects, image, mounts, name, fn)
},
},
List,
Value,
Variants,
}
}
}
export async function runCommand<Manifest extends T.SDKManifest>(
effects: Effects,
image: { imageId: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean },
command: T.CommandType,
options: CommandOptions & {
mounts: Mounts<Manifest> | null
},
name?: string,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
let commands: string[]
if (T.isUseEntrypoint(command)) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${image.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
commands = imageMeta.entrypoint ?? []
commands = commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
} else commands = splitCommand(command)
return SubContainerOwned.withTemp(
effects,
image,
options.mounts,
name ||
commands
.map((c) => {
if (c.includes(" ")) {
return `"${c.replace(/"/g, `\"`)}"`
} else {
return c
}
})
.join(" "),
async (subcontainer) => {
const res = await subcontainer.exec(commands)
if (res.exitCode || res.exitSignal) {
throw new ExitError(commands[0], res)
} else {
return res
}
},
)
}