Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-31 04:23:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor/actions (#2733)

Browse Source 
* store, properties, manifest

* interfaces

* init and backups

* fix init and backups

* file models

* more versions

* dependencies

* config except dynamic types

* clean up config

* remove disabled from non-dynamic vaues

* actions

* standardize example code block formats

* wip: actions refactor

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* commit types

* fix types

* update types

* update action request type

* update apis

* add description to actionrequest

* clean up imports

* revert package json

* chore: Remove the recursive to the index

* chore: Remove the other thing I was testing

* flatten action requests

* update container runtime with new config paradigm

* new actions strategy

* seems to be working

* misc backend fixes

* fix fe bugs

* only show breakages if breakages

* only show success modal if result

* don't panic on failed removal

* hide config from actions page

* polyfill autoconfig

* use metadata strategy for actions instead of prev

* misc fixes

* chore: split the sdk into 2 libs (#2736)

* follow sideload progress (#2718)

* follow sideload progress

* small bugfix

* shareReplay with no refcount false

* don't wrap sideload progress in RPCResult

* dont present toast

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

* chore: Add the initial of the creation of the two sdk

* chore: Add in the baseDist

* chore: Add in the baseDist

* chore: Get the web and the runtime-container running

* chore: Remove the empty file

* chore: Fix it so the container-runtime works

---------

Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>

* misc fixes

* update todos

* minor clean up

* fix link script

* update node version in CI test

* fix node version syntax in ci build

* wip: fixing callbacks

* fix sdk makefile dependencies

* add support for const outside of main

* update apis

* don't panic!

* Chore: Capture weird case on rpc, and log that

* fix procedure id issue

* pass input value for dep auto config

* handle disabled and warning for actions

* chore: Fix for link not having node_modules

* sdk fixes

* fix build

* fix build

* fix build

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Jade <Blu-J@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>
This commit is contained in:
Aiden McClelland
2024-09-25 16:12:52 -06:00
committed by GitHub
parent eec5cf6b65
commit db0695126f
469 changed files with 16218 additions and 10485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1410
sdk/package/lib/StartSdk.ts Normal file
View File

File diff suppressed because it is too large Load Diff

208
sdk/package/lib/backup/Backups.ts Normal file
View File

@@ -0,0 +1,208 @@
import * as T from "../../../base/lib/types"
import * as child_process from "child_process"
import { asError } from "../util"
export const DEFAULT_OPTIONS: T.SyncOptions = {
delete: true,
exclude: [],
}
export type BackupSync<Volumes extends string> = {
dataPath: `/media/startos/volumes/${Volumes}/${string}`
backupPath: `/media/startos/backup/${string}`
options?: Partial<T.SyncOptions>
backupOptions?: Partial<T.SyncOptions>
restoreOptions?: Partial<T.SyncOptions>
}
/**
* This utility simplifies the volume backup process.
* ```ts
* export const { createBackup, restoreBackup } = Backups.volumes("main").build();
* ```
*
* Changing the options of the rsync, (ie excludes) use either
* ```ts
* Backups.volumes("main").set_options({exclude: ['bigdata/']}).volumes('excludedVolume').build()
* // or
* Backups.with_options({exclude: ['bigdata/']}).volumes('excludedVolume').build()
* ```
*
* Using the more fine control, using the addSets for more control
* ```ts
* Backups.addSets({
* srcVolume: 'main', srcPath:'smallData/', dstPath: 'main/smallData/', dstVolume: : Backups.BACKUP
* }, {
* srcVolume: 'main', srcPath:'bigData/', dstPath: 'main/bigData/', dstVolume: : Backups.BACKUP, options: {exclude:['bigData/excludeThis']}}
* ).build()q
* ```
*/
export class Backups<M extends T.Manifest> {
private constructor(
private options = DEFAULT_OPTIONS,
private restoreOptions: Partial<T.SyncOptions> = {},
private backupOptions: Partial<T.SyncOptions> = {},
private backupSet = [] as BackupSync<M["volumes"][number]>[],
) {}
static withVolumes<M extends T.Manifest = never>(
...volumeNames: Array<M["volumes"][number]>
): Backups<M> {
return Backups.withSyncs(
...volumeNames.map((srcVolume) => ({
dataPath: `/media/startos/volumes/${srcVolume}/` as const,
backupPath: `/media/startos/backup/${srcVolume}/` as const,
})),
)
}
static withSyncs<M extends T.Manifest = never>(
...syncs: BackupSync<M["volumes"][number]>[]
) {
return syncs.reduce((acc, x) => acc.addSync(x), new Backups<M>())
}
static withOptions<M extends T.Manifest = never>(
options?: Partial<T.SyncOptions>,
) {
return new Backups<M>({ ...DEFAULT_OPTIONS, ...options })
}
setOptions(options?: Partial<T.SyncOptions>) {
this.options = {
...this.options,
...options,
}
return this
}
setBackupOptions(options?: Partial<T.SyncOptions>) {
this.backupOptions = {
...this.backupOptions,
...options,
}
return this
}
setRestoreOptions(options?: Partial<T.SyncOptions>) {
this.restoreOptions = {
...this.restoreOptions,
...options,
}
return this
}
addVolume(
volume: M["volumes"][number],
options?: Partial<{
options: T.SyncOptions
backupOptions: T.SyncOptions
restoreOptions: T.SyncOptions
}>,
) {
return this.addSync({
dataPath: `/media/startos/volumes/${volume}/` as const,
backupPath: `/media/startos/backup/${volume}/` as const,
...options,
})
}
addSync(sync: BackupSync<M["volumes"][0]>) {
this.backupSet.push({
...sync,
options: { ...this.options, ...sync.options },
})
return this
}
async createBackup() {
for (const item of this.backupSet) {
const rsyncResults = await runRsync({
srcPath: item.dataPath,
dstPath: item.backupPath,
options: {
...this.options,
...this.backupOptions,
...item.options,
...item.backupOptions,
},
})
await rsyncResults.wait()
}
return
}
async restoreBackup() {
for (const item of this.backupSet) {
const rsyncResults = await runRsync({
srcPath: item.backupPath,
dstPath: item.dataPath,
options: {
...this.options,
...this.backupOptions,
...item.options,
...item.backupOptions,
},
})
await rsyncResults.wait()
}
return
}
}
async function runRsync(rsyncOptions: {
srcPath: string
dstPath: string
options: T.SyncOptions
}): Promise<{
id: () => Promise<string>
wait: () => Promise<null>
progress: () => Promise<number>
}> {
const { srcPath, dstPath, options } = rsyncOptions
const command = "rsync"
const args: string[] = []
if (options.delete) {
args.push("--delete")
}
for (const exclude of options.exclude) {
args.push(`--exclude=${exclude}`)
}
args.push("-actAXH")
args.push("--info=progress2")
args.push("--no-inc-recursive")
args.push(srcPath)
args.push(dstPath)
const spawned = child_process.spawn(command, args, { detached: true })
let percentage = 0.0
spawned.stdout.on("data", (data: unknown) => {
const lines = String(data).replace("\r", "\n").split("\n")
for (const line of lines) {
const parsed = /$([0-9.]+)%/.exec(line)?.[1]
if (!parsed) continue
percentage = Number.parseFloat(parsed)
}
})
spawned.stderr.on("data", (data: unknown) => {
console.error(`Backups.runAsync`, asError(data))
})
const id = async () => {
const pid = spawned.pid
if (pid === undefined) {
throw new Error("rsync process has no pid")
}
return String(pid)
}
const waitPromise = new Promise<null>((resolve, reject) => {
spawned.on("exit", (code: any) => {
if (code === 0) {
resolve(null)
} else {
reject(new Error(`rsync exited with code ${code}`))
}
})
})
const wait = () => waitPromise
const progress = () => Promise.resolve(percentage)
return { id, wait, progress }
}

2
sdk/package/lib/backup/index.ts Normal file
View File

@@ -0,0 +1,2 @@
import "./Backups"
import "./setupBackups"

39
sdk/package/lib/backup/setupBackups.ts Normal file
View File

@@ -0,0 +1,39 @@
import { Backups } from "./Backups"
import * as T from "../../../base/lib/types"
import { _ } from "../util"
export type SetupBackupsParams<M extends T.Manifest> =
| M["volumes"][number][]
| ((_: { effects: T.Effects }) => Promise<Backups<M>>)
type SetupBackupsRes = {
createBackup: T.ExpectedExports.createBackup
restoreBackup: T.ExpectedExports.restoreBackup
}
export function setupBackups<M extends T.Manifest>(
options: SetupBackupsParams<M>,
) {
let backupsFactory: (_: { effects: T.Effects }) => Promise<Backups<M>>
if (options instanceof Function) {
backupsFactory = options
} else {
backupsFactory = async () => Backups.withVolumes(...options)
}
const answer: {
createBackup: T.ExpectedExports.createBackup
restoreBackup: T.ExpectedExports.restoreBackup
} = {
get createBackup() {
return (async (options) => {
return (await backupsFactory(options)).createBackup()
}) as T.ExpectedExports.createBackup
},
get restoreBackup() {
return (async (options) => {
return (await backupsFactory(options)).restoreBackup()
}) as T.ExpectedExports.restoreBackup
},
}
return answer
}

64
sdk/package/lib/health/HealthCheck.ts Normal file
View File

@@ -0,0 +1,64 @@
import { Effects, HealthReceipt } from "../../../base/lib/types"
import { HealthCheckResult } from "./checkFns/HealthCheckResult"
import { Trigger } from "../trigger"
import { TriggerInput } from "../trigger/TriggerInput"
import { defaultTrigger } from "../trigger/defaultTrigger"
import { once, asError } from "../util"
import { object, unknown } from "ts-matches"
export type HealthCheckParams = {
effects: Effects
name: string
trigger?: Trigger
fn(): Promise<HealthCheckResult> | HealthCheckResult
onFirstSuccess?: () => unknown | Promise<unknown>
}
export function healthCheck(o: HealthCheckParams) {
new Promise(async () => {
let currentValue: TriggerInput = {}
const getCurrentValue = () => currentValue
const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue)
const triggerFirstSuccess = once(() =>
Promise.resolve(
"onFirstSuccess" in o && o.onFirstSuccess
? o.onFirstSuccess()
: undefined,
),
)
for (
let res = await trigger.next();
!res.done;
res = await trigger.next()
) {
try {
const { result, message } = await o.fn()
await o.effects.setHealth({
name: o.name,
id: o.name,
result,
message: message || "",
})
currentValue.lastResult = result
await triggerFirstSuccess().catch((err) => {
console.error(asError(err))
})
} catch (e) {
await o.effects.setHealth({
name: o.name,
id: o.name,
result: "failure",
message: asMessage(e) || "",
})
currentValue.lastResult = "failure"
}
}
})
return {} as HealthReceipt
}
function asMessage(e: unknown) {
if (object({ message: unknown }).test(e)) return String(e.message)
const value = String(e)
if (value.length == null) return null
return value
}

3
sdk/package/lib/health/checkFns/HealthCheckResult.ts Normal file
View File

@@ -0,0 +1,3 @@
import { T } from "../../../../base/lib"
export type HealthCheckResult = Omit<T.NamedHealthCheckResult, "name">

67
sdk/package/lib/health/checkFns/checkPortListening.ts Normal file
View File

@@ -0,0 +1,67 @@
import { Effects } from "../../../../base/lib/types"
import { stringFromStdErrOut } from "../../util"
import { HealthCheckResult } from "./HealthCheckResult"
import { promisify } from "node:util"
import * as CP from "node:child_process"
const cpExec = promisify(CP.exec)
const cpExecFile = promisify(CP.execFile)
export function containsAddress(x: string, port: number) {
const readPorts = x
.split("\n")
.filter(Boolean)
.splice(1)
.map((x) => x.split(" ").filter(Boolean)[1]?.split(":")?.[1])
.filter(Boolean)
.map((x) => Number.parseInt(x, 16))
.filter(Number.isFinite)
return readPorts.indexOf(port) >= 0
}
/**
* This is used to check if a port is listening on the system.
* Used during the health check fn or the check main fn.
*/
export async function checkPortListening(
effects: Effects,
port: number,
options: {
errorMessage: string
successMessage: string
timeoutMessage?: string
timeout?: number
},
): Promise<HealthCheckResult> {
return Promise.race<HealthCheckResult>([
Promise.resolve().then(async () => {
const hasAddress =
containsAddress(
await cpExec(`cat /proc/net/tcp`, {}).then(stringFromStdErrOut),
port,
) ||
containsAddress(
await cpExec("cat /proc/net/udp", {}).then(stringFromStdErrOut),
port,
)
if (hasAddress) {
return { result: "success", message: options.successMessage }
}
return {
result: "failure",
message: options.errorMessage,
}
}),
new Promise((resolve) => {
setTimeout(
() =>
resolve({
result: "failure",
message:
options.timeoutMessage || `Timeout trying to check port ${port}`,
}),
options.timeout ?? 1_000,
)
}),
])
}

36
sdk/package/lib/health/checkFns/checkWebUrl.ts Normal file
View File

@@ -0,0 +1,36 @@
import { Effects } from "../../../../base/lib/types"
import { asError } from "../../util"
import { HealthCheckResult } from "./HealthCheckResult"
import { timeoutPromise } from "./index"
import "isomorphic-fetch"
/**
* This is a helper function to check if a web url is reachable.
* @param url
* @param createSuccess
* @returns
*/
export const checkWebUrl = async (
effects: Effects,
url: string,
{
timeout = 1000,
successMessage = `Reached ${url}`,
errorMessage = `Error while fetching URL: ${url}`,
} = {},
): Promise<HealthCheckResult> => {
return Promise.race([fetch(url), timeoutPromise(timeout)])
.then(
(x) =>
({
result: "success",
message: successMessage,
}) as const,
)
.catch((e) => {
console.warn(`Error while fetching URL: ${url}`)
console.error(JSON.stringify(e))
console.error(asError(e))
return { result: "failure" as const, message: errorMessage }
})
}

11
sdk/package/lib/health/checkFns/index.ts Normal file
View File

@@ -0,0 +1,11 @@
import { runHealthScript } from "./runHealthScript"
export { checkPortListening } from "./checkPortListening"
export { HealthCheckResult } from "./HealthCheckResult"
export { checkWebUrl } from "./checkWebUrl"
export function timeoutPromise(ms: number, { message = "Timed out" } = {}) {
return new Promise<never>((resolve, reject) =>
setTimeout(() => reject(new Error(message)), ms),
)
}
export { runHealthScript }

35
sdk/package/lib/health/checkFns/runHealthScript.ts Normal file
View File

@@ -0,0 +1,35 @@
import { HealthCheckResult } from "./HealthCheckResult"
import { timeoutPromise } from "./index"
import { SubContainer } from "../../util/SubContainer"
/**
* Running a health script, is used when we want to have a simple
* script in bash or something like that. It should return something that is useful
* in {result: string} else it is considered an error
* @param param0
* @returns
*/
export const runHealthScript = async (
runCommand: string[],
subcontainer: SubContainer,
{
timeout = 30000,
errorMessage = `Error while running command: ${runCommand}`,
message = (res: string) =>
`Have ran script ${runCommand} and the result: ${res}`,
} = {},
): Promise<HealthCheckResult> => {
const res = await Promise.race([
subcontainer.exec(runCommand),
timeoutPromise(timeout),
]).catch((e) => {
console.warn(errorMessage)
console.warn(JSON.stringify(e))
console.warn(e.toString())
throw { result: "failure", message: errorMessage } as HealthCheckResult
})
return {
result: "success",
message: message(res.stdout.toString()),
} as HealthCheckResult
}

1
sdk/package/lib/health/index.ts Normal file
View File

@@ -0,0 +1 @@
import "./checkFns"

48
sdk/package/lib/index.ts Normal file
View File

@@ -0,0 +1,48 @@
import {
S9pk,
Version,
VersionRange,
ExtendedVersion,
inputSpec,
ISB,
IST,
types,
T,
matches,
utils,
} from "../../base/lib"
export {
S9pk,
Version,
VersionRange,
ExtendedVersion,
inputSpec,
ISB,
IST,
types,
T,
matches,
utils,
}
export { Daemons } from "./mainFn/Daemons"
export { SubContainer } from "./util/SubContainer"
export { StartSdk } from "./StartSdk"
export { setupManifest } from "./manifest/setupManifest"
export { FileHelper } from "./util/fileHelper"
export { setupExposeStore } from "./store/setupExposeStore"
export { pathBuilder } from "../../base/lib/util/PathBuilder"
export * as actions from "../../base/lib/actions"
export * as backup from "./backup"
export * as daemons from "./mainFn/Daemons"
export * as health from "./health"
export * as healthFns from "./health/checkFns"
export * as inits from "./inits"
export * as mainFn from "./mainFn"
export * as toml from "@iarna/toml"
export * as yaml from "yaml"
export * as startSdk from "./StartSdk"
export * as YAML from "yaml"
export * as TOML from "@iarna/toml"
export * from "./version"

3
sdk/package/lib/inits/index.ts Normal file
View File

@@ -0,0 +1,3 @@
import "./setupInit"
import "./setupUninstall"
import "./setupInstall"

62
sdk/package/lib/inits/setupInit.ts Normal file
View File

@@ -0,0 +1,62 @@
import { Actions } from "../../../base/lib/actions/setupActions"
import { ExtendedVersion } from "../../../base/lib/exver"
import { UpdateServiceInterfaces } from "../../../base/lib/interfaces/setupInterfaces"
import { ExposedStorePaths } from "../../../base/lib/types"
import * as T from "../../../base/lib/types"
import { VersionGraph } from "../version/VersionGraph"
import { Install } from "./setupInstall"
import { Uninstall } from "./setupUninstall"
export function setupInit<Manifest extends T.Manifest, Store>(
versions: VersionGraph<Manifest["version"]>,
install: Install<Manifest, Store>,
uninstall: Uninstall<Manifest, Store>,
setServiceInterfaces: UpdateServiceInterfaces<any>,
setDependencies: (options: { effects: T.Effects }) => Promise<void>,
actions: Actions<Store, any>,
exposedStore: ExposedStorePaths,
): {
packageInit: T.ExpectedExports.packageInit
packageUninit: T.ExpectedExports.packageUninit
containerInit: T.ExpectedExports.containerInit
} {
return {
packageInit: async (opts) => {
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: versions.currentVersion(),
})
} else {
await install.install(opts)
await opts.effects.setDataVersion({
version: versions.current.options.version,
})
}
},
packageUninit: async (opts) => {
if (opts.nextVersion) {
const prev = await opts.effects.getDataVersion()
if (prev) {
await versions.migrate({
effects: opts.effects,
from: ExtendedVersion.parse(prev),
to: ExtendedVersion.parse(opts.nextVersion),
})
}
} else {
await uninstall.uninstall(opts)
}
},
containerInit: async (opts) => {
await setServiceInterfaces({
...opts,
})
await actions.update({ effects: opts.effects })
await opts.effects.exposeForDependents({ paths: exposedStore })
await setDependencies({ effects: opts.effects })
},
}
}

25
sdk/package/lib/inits/setupInstall.ts Normal file
View File

@@ -0,0 +1,25 @@
import * as T from "../../../base/lib/types"
export type InstallFn<Manifest extends T.Manifest, Store> = (opts: {
effects: T.Effects
}) => Promise<void>
export class Install<Manifest extends T.Manifest, Store> {
private constructor(readonly fn: InstallFn<Manifest, Store>) {}
static of<Manifest extends T.Manifest, Store>(
fn: InstallFn<Manifest, Store>,
) {
return new Install(fn)
}
async install({ effects }: Parameters<T.ExpectedExports.packageInit>[0]) {
await this.fn({
effects,
})
}
}
export function setupInstall<Manifest extends T.Manifest, Store>(
fn: InstallFn<Manifest, Store>,
) {
return Install.of(fn)
}

29
sdk/package/lib/inits/setupUninstall.ts Normal file
View File

@@ -0,0 +1,29 @@
import * as T from "../../../base/lib/types"
export type UninstallFn<Manifest extends T.Manifest, Store> = (opts: {
effects: T.Effects
}) => Promise<void>
export class Uninstall<Manifest extends T.Manifest, Store> {
private constructor(readonly fn: UninstallFn<Manifest, Store>) {}
static of<Manifest extends T.Manifest, Store>(
fn: UninstallFn<Manifest, Store>,
) {
return new Uninstall(fn)
}
async uninstall({
effects,
nextVersion,
}: Parameters<T.ExpectedExports.packageUninit>[0]) {
if (!nextVersion)
await this.fn({
effects,
})
}
}
export function setupUninstall<Manifest extends T.Manifest, Store>(
fn: UninstallFn<Manifest, Store>,
) {
return Uninstall.of(fn)
}

144
sdk/package/lib/mainFn/CommandController.ts Normal file
View File

@@ -0,0 +1,144 @@
import { DEFAULT_SIGTERM_TIMEOUT } from "."
import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../../../base/lib/types"
import * as T from "../../../base/lib/types"
import { asError } from "../../../base/lib/util/asError"
import {
ExecSpawnable,
MountOptions,
SubContainerHandle,
SubContainer,
} from "../util/SubContainer"
import { splitCommand } from "../util"
import * as cp from "child_process"
export class CommandController {
private constructor(
readonly runningAnswer: Promise<unknown>,
private state: { exited: boolean },
private readonly subcontainer: SubContainer,
private process: cp.ChildProcessWithoutNullStreams,
readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
) {}
static of<Manifest extends T.Manifest>() {
return async <A extends string>(
effects: T.Effects,
subcontainer:
| {
id: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
}
| SubContainer,
command: T.CommandType,
options: {
subcontainerName?: string
// Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
sigtermTimeout?: number
mounts?: { path: string; options: MountOptions }[]
runAsInit?: boolean
env?:
| {
[variable: string]: string
}
| undefined
cwd?: string | undefined
user?: string | undefined
onStdout?: (x: Buffer) => void
onStderr?: (x: Buffer) => void
},
) => {
const commands = splitCommand(command)
const subc =
subcontainer instanceof SubContainer
? subcontainer
: await (async () => {
const subc = await SubContainer.of(
effects,
subcontainer,
options?.subcontainerName || commands.join(" "),
)
for (let mount of options.mounts || []) {
await subc.mount(mount.options, mount.path)
}
return subc
})()
let childProcess: cp.ChildProcessWithoutNullStreams
if (options.runAsInit) {
childProcess = await subc.launch(commands, {
env: options.env,
})
} else {
childProcess = await subc.spawn(commands, {
env: options.env,
})
}
const state = { exited: false }
const answer = new Promise<null>((resolve, reject) => {
childProcess.on("exit", (code) => {
state.exited = true
if (
code === 0 ||
code === 143 ||
(code === null && childProcess.signalCode == "SIGTERM")
) {
return resolve(null)
}
if (code) {
return reject(new Error(`${commands[0]} exited with code ${code}`))
} else {
return reject(
new Error(
`${commands[0]} exited with signal ${childProcess.signalCode}`,
),
)
}
})
})
return new CommandController(
answer,
state,
subc,
childProcess,
options.sigtermTimeout,
)
}
}
get subContainerHandle() {
return new SubContainerHandle(this.subcontainer)
}
async wait({ timeout = NO_TIMEOUT } = {}) {
if (timeout > 0)
setTimeout(() => {
this.term()
}, timeout)
try {
return await this.runningAnswer
} finally {
if (!this.state.exited) {
this.process.kill("SIGKILL")
}
await this.subcontainer.destroy?.().catch((_) => {})
}
}
async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
try {
if (!this.state.exited) {
if (signal !== "SIGKILL") {
setTimeout(() => {
if (!this.state.exited) this.process.kill("SIGKILL")
}, timeout)
}
if (!this.process.kill(signal)) {
console.error(
`failed to send signal ${signal} to pid ${this.process.pid}`,
)
}
}
await this.runningAnswer
} finally {
await this.subcontainer.destroy?.()
}
}
}

89
sdk/package/lib/mainFn/Daemon.ts Normal file
View File

@@ -0,0 +1,89 @@
import * as T from "../../../base/lib/types"
import { asError } from "../../../base/lib/util/asError"
import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer"
import { CommandController } from "./CommandController"
const TIMEOUT_INCREMENT_MS = 1000
const MAX_TIMEOUT_MS = 30000
/**
* This is a wrapper around CommandController that has a state of off, where the command shouldn't be running
* and the others state of running, where it will keep a living running command
*/
export class Daemon {
private commandController: CommandController | null = null
private shouldBeRunning = false
constructor(private startCommand: () => Promise<CommandController>) {}
get subContainerHandle(): undefined | ExecSpawnable {
return this.commandController?.subContainerHandle
}
static of<Manifest extends T.Manifest>() {
return async <A extends string>(
effects: T.Effects,
subcontainer:
| {
id: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
}
| SubContainer,
command: T.CommandType,
options: {
subcontainerName?: string
mounts?: { path: string; options: MountOptions }[]
env?:
| {
[variable: string]: string
}
| undefined
cwd?: string | undefined
user?: string | undefined
onStdout?: (x: Buffer) => void
onStderr?: (x: Buffer) => void
sigtermTimeout?: number
},
) => {
const startCommand = () =>
CommandController.of<Manifest>()(
effects,
subcontainer,
command,
options,
)
return new Daemon(startCommand)
}
}
async start() {
if (this.commandController) {
return
}
this.shouldBeRunning = true
let timeoutCounter = 0
new Promise(async () => {
while (this.shouldBeRunning) {
this.commandController = await this.startCommand()
await this.commandController.wait().catch((err) => console.error(err))
await new Promise((resolve) => setTimeout(resolve, timeoutCounter))
timeoutCounter += TIMEOUT_INCREMENT_MS
timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
}
}).catch((err) => {
console.error(asError(err))
})
}
async term(termOptions?: {
signal?: NodeJS.Signals | undefined
timeout?: number | undefined
}) {
return this.stop(termOptions)
}
async stop(termOptions?: {
signal?: NodeJS.Signals | undefined
timeout?: number | undefined
}) {
this.shouldBeRunning = false
await this.commandController
?.term({ ...termOptions })
.catch((e) => console.error(asError(e)))
this.commandController = null
}
}

172
sdk/package/lib/mainFn/Daemons.ts Normal file
View File

@@ -0,0 +1,172 @@
import { HealthReceipt, Signals } from "../../../base/lib/types"
import { HealthCheckResult } from "../health/checkFns"
import { Trigger } from "../trigger"
import * as T from "../../../base/lib/types"
import { Mounts } from "./Mounts"
import { ExecSpawnable, MountOptions } from "../util/SubContainer"
import { promisify } from "node:util"
import * as CP from "node:child_process"
export { Daemon } from "./Daemon"
export { CommandController } from "./CommandController"
import { HealthDaemon } from "./HealthDaemon"
import { Daemon } from "./Daemon"
import { CommandController } from "./CommandController"
export const cpExec = promisify(CP.exec)
export const cpExecFile = promisify(CP.execFile)
export type Ready = {
display: string | null
fn: (
spawnable: ExecSpawnable,
) => Promise<HealthCheckResult> | HealthCheckResult
trigger?: Trigger
}
type DaemonsParams<
Manifest extends T.Manifest,
Ids extends string,
Command extends string,
Id extends string,
> = {
command: T.CommandType
image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean }
mounts: Mounts<Manifest>
env?: Record<string, string>
ready: Ready
requires: Exclude<Ids, Id>[]
sigtermTimeout?: number
}
type ErrorDuplicateId<Id extends string> = `The id '${Id}' is already used`
export const runCommand = <Manifest extends T.Manifest>() =>
CommandController.of<Manifest>()
/**
* A class for defining and controlling the service daemons
```ts
Daemons.of({
effects,
started,
interfaceReceipt, // Provide the interfaceReceipt to prove it was completed
healthReceipts, // Provide the healthReceipts or [] to prove they were at least considered
}).addDaemon('webui', {
command: 'hello-world', // The command to start the daemon
ready: {
display: 'Web Interface',
// The function to run to determine the health status of the daemon
fn: () =>
checkPortListening(effects, 80, {
successMessage: 'The web interface is ready',
errorMessage: 'The web interface is not ready',
}),
},
requires: [],
})
```
*/
export class Daemons<Manifest extends T.Manifest, Ids extends string>
implements T.DaemonBuildable
{
private constructor(
readonly effects: T.Effects,
readonly started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>,
readonly daemons: Promise<Daemon>[],
readonly ids: Ids[],
readonly healthDaemons: HealthDaemon[],
) {}
/**
* Returns an empty new Daemons class with the provided inputSpec.
*
* Call .addDaemon() on the returned class to add a daemon.
*
* Daemons run in the order they are defined, with latter daemons being capable of
* depending on prior daemons
* @param inputSpec
* @returns
*/
static of<Manifest extends T.Manifest>(inputSpec: {
effects: T.Effects
started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>
healthReceipts: HealthReceipt[]
}) {
return new Daemons<Manifest, never>(
inputSpec.effects,
inputSpec.started,
[],
[],
[],
)
}
/**
* Returns the complete list of daemons, including the one defined here
* @param id
* @param newDaemon
* @returns
*/
addDaemon<Id extends string, Command extends string>(
// prettier-ignore
id:
"" extends Id ? never :
ErrorDuplicateId<Id> extends Id ? never :
Id extends Ids ? ErrorDuplicateId<Id> :
Id,
options: DaemonsParams<Manifest, Ids, Command, Id>,
) {
const daemonIndex = this.daemons.length
const daemon = Daemon.of()(this.effects, options.image, options.command, {
...options,
mounts: options.mounts.build(),
subcontainerName: id,
})
const healthDaemon = new HealthDaemon(
daemon,
daemonIndex,
options.requires
.map((x) => this.ids.indexOf(id as any))
.filter((x) => x >= 0)
.map((id) => this.healthDaemons[id]),
id,
this.ids,
options.ready,
this.effects,
options.sigtermTimeout,
)
const daemons = this.daemons.concat(daemon)
const ids = [...this.ids, id] as (Ids | Id)[]
const healthDaemons = [...this.healthDaemons, healthDaemon]
return new Daemons<Manifest, Ids | Id>(
this.effects,
this.started,
daemons,
ids,
healthDaemons,
)
}
async build() {
this.updateMainHealth()
this.healthDaemons.forEach((x) =>
x.addWatcher(() => this.updateMainHealth()),
)
const built = {
term: async (options?: { signal?: Signals; timeout?: number }) => {
try {
await Promise.all(this.healthDaemons.map((x) => x.term(options)))
} finally {
this.effects.setMainStatus({ status: "stopped" })
}
},
}
this.started(() => built.term())
return built
}
private updateMainHealth() {
this.effects.setMainStatus({ status: "running" })
}
}

150
sdk/package/lib/mainFn/HealthDaemon.ts Normal file
View File

@@ -0,0 +1,150 @@
import { HealthCheckResult } from "../health/checkFns"
import { defaultTrigger } from "../trigger/defaultTrigger"
import { Ready } from "./Daemons"
import { Daemon } from "./Daemon"
import { SetHealth, Effects } from "../../../base/lib/types"
import { DEFAULT_SIGTERM_TIMEOUT } from "."
import { asError } from "../../../base/lib/util/asError"
const oncePromise = <T>() => {
let resolve: (value: T) => void
const promise = new Promise<T>((res) => {
resolve = res
})
return { resolve: resolve!, promise }
}
/**
* Wanted a structure that deals with controlling daemons by their health status
* States:
* -- Waiting for dependencies to be success
* -- Running: Daemon is running and the status is in the health
*
*/
export class HealthDaemon {
private _health: HealthCheckResult = { result: "starting", message: null }
private healthWatchers: Array<() => unknown> = []
private running = false
constructor(
private readonly daemon: Promise<Daemon>,
readonly daemonIndex: number,
private readonly dependencies: HealthDaemon[],
readonly id: string,
readonly ids: string[],
readonly ready: Ready,
readonly effects: Effects,
readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
) {
this.updateStatus()
this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus()))
}
/** Run after we want to do cleanup */
async term(termOptions?: {
signal?: NodeJS.Signals | undefined
timeout?: number | undefined
}) {
this.healthWatchers = []
this.running = false
this.healthCheckCleanup?.()
await this.daemon.then((d) =>
d.term({
timeout: this.sigtermTimeout,
...termOptions,
}),
)
}
/** Want to add another notifier that the health might have changed */
addWatcher(watcher: () => unknown) {
this.healthWatchers.push(watcher)
}
get health() {
return Object.freeze(this._health)
}
private async changeRunning(newStatus: boolean) {
if (this.running === newStatus) return
this.running = newStatus
if (newStatus) {
;(await this.daemon).start()
this.setupHealthCheck()
} else {
;(await this.daemon).stop()
this.turnOffHealthCheck()
this.setHealth({ result: "starting", message: null })
}
}
private healthCheckCleanup: (() => void) | null = null
private turnOffHealthCheck() {
this.healthCheckCleanup?.()
}
private async setupHealthCheck() {
if (this.healthCheckCleanup) return
const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({
lastResult: this._health.result,
}))
const { promise: status, resolve: setStatus } = oncePromise<{
done: true
}>()
new Promise(async () => {
for (
let res = await Promise.race([status, trigger.next()]);
!res.done;
res = await Promise.race([status, trigger.next()])
) {
const handle = (await this.daemon).subContainerHandle
if (handle) {
const response: HealthCheckResult = await Promise.resolve(
this.ready.fn(handle),
).catch((err) => {
console.error(asError(err))
return {
result: "failure",
message: "message" in err ? err.message : String(err),
}
})
await this.setHealth(response)
} else {
await this.setHealth({
result: "failure",
message: "Daemon not running",
})
}
}
}).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
this.healthCheckCleanup = () => {
setStatus({ done: true })
this.healthCheckCleanup = null
}
}
private async setHealth(health: HealthCheckResult) {
this._health = health
this.healthWatchers.forEach((watcher) => watcher())
const display = this.ready.display
const result = health.result
if (!display) {
return
}
await this.effects.setHealth({
...health,
id: this.id,
name: display,
} as SetHealth)
}
private async updateStatus() {
const healths = this.dependencies.map((d) => d._health)
this.changeRunning(healths.every((x) => x.result === "success"))
}
}

125
sdk/package/lib/mainFn/Mounts.ts Normal file
View File

@@ -0,0 +1,125 @@
import * as T from "../../../base/lib/types"
import { MountOptions } from "../util/SubContainer"
type MountArray = { path: string; options: MountOptions }[]
export class Mounts<Manifest extends T.Manifest> {
private constructor(
readonly volumes: {
id: Manifest["volumes"][number]
subpath: string | null
mountpoint: string
readonly: boolean
}[],
readonly assets: {
id: Manifest["assets"][number]
subpath: string | null
mountpoint: string
}[],
readonly dependencies: {
dependencyId: string
volumeId: string
subpath: string | null
mountpoint: string
readonly: boolean
}[],
) {}
static of<Manifest extends T.Manifest>() {
return new Mounts<Manifest>([], [], [])
}
addVolume(
id: Manifest["volumes"][number],
subpath: string | null,
mountpoint: string,
readonly: boolean,
) {
this.volumes.push({
id,
subpath,
mountpoint,
readonly,
})
return this
}
addAssets(
id: Manifest["assets"][number],
subpath: string | null,
mountpoint: string,
) {
this.assets.push({
id,
subpath,
mountpoint,
})
return this
}
addDependency<DependencyManifest extends T.Manifest>(
dependencyId: keyof Manifest["dependencies"] & string,
volumeId: DependencyManifest["volumes"][number],
subpath: string | null,
mountpoint: string,
readonly: boolean,
) {
this.dependencies.push({
dependencyId,
volumeId,
subpath,
mountpoint,
readonly,
})
return this
}
build(): MountArray {
const mountpoints = new Set()
for (let mountpoint of this.volumes
.map((v) => v.mountpoint)
.concat(this.assets.map((a) => a.mountpoint))
.concat(this.dependencies.map((d) => d.mountpoint))) {
if (mountpoints.has(mountpoint)) {
throw new Error(
`cannot mount more than once to mountpoint ${mountpoint}`,
)
}
mountpoints.add(mountpoint)
}
return ([] as MountArray)
.concat(
this.volumes.map((v) => ({
path: v.mountpoint,
options: {
type: "volume",
id: v.id,
subpath: v.subpath,
readonly: v.readonly,
},
})),
)
.concat(
this.assets.map((a) => ({
path: a.mountpoint,
options: {
type: "assets",
id: a.id,
subpath: a.subpath,
},
})),
)
.concat(
this.dependencies.map((d) => ({
path: d.mountpoint,
options: {
type: "pointer",
packageId: d.dependencyId,
volumeId: d.volumeId,
subpath: d.subpath,
readonly: d.readonly,
},
})),
)
}
}

27
sdk/package/lib/mainFn/index.ts Normal file
View File

@@ -0,0 +1,27 @@
import * as T from "../../../base/lib/types"
import { Daemons } from "./Daemons"
import "../../../base/lib/interfaces/ServiceInterfaceBuilder"
import "../../../base/lib/interfaces/Origin"
export const DEFAULT_SIGTERM_TIMEOUT = 30_000
/**
* Used to ensure that the main function is running with the valid proofs.
* We first do the folowing order of things
* 1. We get the interfaces
* 2. We setup all the commands to setup the system
* 3. We create the health checks
* 4. We setup the daemons init system
* @param fn
* @returns
*/
export const setupMain = <Manifest extends T.Manifest, Store>(
fn: (o: {
effects: T.Effects
started(onTerm: () => PromiseLike<void>): PromiseLike<void>
}) => Promise<Daemons<Manifest, any>>,
): T.ExpectedExports.main => {
return async (options) => {
const result = await fn(options)
return result
}
}

87
sdk/package/lib/manifest/setupManifest.ts Normal file
View File

@@ -0,0 +1,87 @@
import * as T from "../../../base/lib/types"
import { ImageConfig, ImageId, VolumeId } from "../../../base/lib/types"
import {
SDKManifest,
SDKImageInputSpec,
} from "../../../base/lib/types/ManifestTypes"
import { SDKVersion } from "../StartSdk"
import { VersionGraph } from "../version/VersionGraph"
/**
* @description Use this function to define critical information about your package
*
* @param versions Every version of the package, imported from ./versions
* @param manifest Static properties of the package
*/
export function setupManifest<
Id extends string,
Version extends string,
Dependencies extends Record<string, unknown>,
VolumesTypes extends VolumeId,
AssetTypes extends VolumeId,
ImagesTypes extends ImageId,
Manifest extends {
dependencies: Dependencies
id: Id
assets: AssetTypes[]
images: Record<ImagesTypes, SDKImageInputSpec>
volumes: VolumesTypes[]
},
Satisfies extends string[] = [],
>(
versions: VersionGraph<Version>,
manifest: SDKManifest & Manifest,
): Manifest & T.Manifest {
const images = Object.entries(manifest.images).reduce(
(images, [k, v]) => {
v.arch = v.arch || ["aarch64", "x86_64"]
if (v.emulateMissingAs === undefined)
v.emulateMissingAs = v.arch[0] || null
images[k] = v as ImageConfig
return images
},
{} as { [k: string]: ImageConfig },
)
return {
...manifest,
gitHash: null,
osVersion: SDKVersion,
version: versions.current.options.version,
releaseNotes: versions.current.options.releaseNotes,
satisfies: versions.current.options.satisfies || [],
canMigrateTo: versions.canMigrateTo().toString(),
canMigrateFrom: versions.canMigrateFrom().toString(),
images,
alerts: {
install: manifest.alerts?.install || null,
update: manifest.alerts?.update || null,
uninstall: manifest.alerts?.uninstall || null,
restore: manifest.alerts?.restore || null,
start: manifest.alerts?.start || null,
stop: manifest.alerts?.stop || null,
},
hardwareRequirements: {
device: Object.fromEntries(
Object.entries(manifest.hardwareRequirements?.device || {}).map(
([k, v]) => [k, v.source],
),
),
ram: manifest.hardwareRequirements?.ram || null,
arch:
manifest.hardwareRequirements?.arch === undefined
? Object.values(images).reduce(
(arch, inputSpec) => {
if (inputSpec.emulateMissingAs) {
return arch
}
if (arch === null) {
return inputSpec.arch
}
return arch.filter((a) => inputSpec.arch.includes(a))
},
null as string[] | null,
)
: manifest.hardwareRequirements?.arch,
},
}
}

61
sdk/package/lib/store/getStore.ts Normal file
View File

@@ -0,0 +1,61 @@
import { Effects } from "../../../base/lib/Effects"
import { PathBuilder, extractJsonPath } from "../util"
export class GetStore<Store, StoreValue> {
constructor(
readonly effects: Effects,
readonly path: PathBuilder<Store, StoreValue>,
readonly options: {
/** Defaults to what ever the package currently in */
packageId?: string | undefined
} = {},
) {}
/**
* Returns the value of Store at the provided path. Restart the service if the value changes
*/
const() {
return this.effects.store.get<Store, StoreValue>({
...this.options,
path: extractJsonPath(this.path),
callback: () => this.effects.constRetry(),
})
}
/**
* Returns the value of Store at the provided path. Does nothing if the value changes
*/
once() {
return this.effects.store.get<Store, StoreValue>({
...this.options,
path: extractJsonPath(this.path),
})
}
/**
* Watches the value of Store at the provided path. Takes a custom callback function to run whenever the value changes
*/
async *watch() {
while (true) {
let callback: () => void
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
})
yield await this.effects.store.get<Store, StoreValue>({
...this.options,
path: extractJsonPath(this.path),
callback: () => callback(),
})
await waitForNext
}
}
}
export function getStore<Store, StoreValue>(
effects: Effects,
path: PathBuilder<Store, StoreValue>,
options: {
/** Defaults to what ever the package currently in */
packageId?: string | undefined
} = {},
) {
return new GetStore<Store, StoreValue>(effects, path, options)
}

28
sdk/package/lib/store/setupExposeStore.ts Normal file
View File

@@ -0,0 +1,28 @@
import { ExposedStorePaths } from "../../../base/lib/types"
import { Affine, _ } from "../util"
import {
PathBuilder,
extractJsonPath,
pathBuilder,
} from "../../../base/lib/util/PathBuilder"
/**
* @description Use this function to determine which Store values to expose and make available to other services running on StartOS. Store values not exposed here will be kept private. Use the type safe pathBuilder to traverse your Store's structure.
* @example
* In this example, we expose the hypothetical Store values "adminPassword" and "nameLastUpdatedAt".
*
* ```
export const exposedStore = setupExposeStore<Store>((pathBuilder) => [
pathBuilder.adminPassword
pathBuilder.nameLastUpdatedAt,
])
* ```
*/
export const setupExposeStore = <Store extends Record<string, any>>(
fn: (pathBuilder: PathBuilder<Store>) => PathBuilder<Store, any>[],
) => {
return fn(pathBuilder<Store>()).map(
(x) => extractJsonPath(x) as string,
) as ExposedStorePaths
}
export { ExposedStorePaths }

17
sdk/package/lib/test/health.readyCheck.test.ts Normal file
View File

@@ -0,0 +1,17 @@
import { containsAddress } from "../health/checkFns/checkPortListening"
describe("Health ready check", () => {
it("Should be able to parse an example information", () => {
let input = `
sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode
0: 00000000:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 21634478 1 0000000000000000 100 0 0 10 0
1: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 21634477 1 0000000000000000 100 0 0 10 0
2: 0B00007F:9671 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 21635458 1 0000000000000000 100 0 0 10 0
3: 00000000:0D73 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 21634479 1 0000000000000000 100 0 0 10 0
`
expect(containsAddress(input, 80)).toBe(true)
expect(containsAddress(input, 1234)).toBe(false)
})
})

30
sdk/package/lib/test/host.test.ts Normal file
View File

@@ -0,0 +1,30 @@
import { ServiceInterfaceBuilder } from "../../../base/lib/interfaces/ServiceInterfaceBuilder"
import { Effects } from "../../../base/lib/Effects"
import { sdk } from "../test/output.sdk"
describe("host", () => {
test("Testing that the types work", () => {
async function test(effects: Effects) {
const foo = sdk.host.multi(effects, "foo")
const fooOrigin = await foo.bindPort(80, {
protocol: "http" as const,
preferredExternalPort: 80,
})
const fooInterface = new ServiceInterfaceBuilder({
effects,
name: "Foo",
id: "foo",
description: "A Foo",
hasPrimary: false,
type: "ui",
username: "bar",
path: "/baz",
search: { qux: "yes" },
schemeOverride: null,
masked: false,
})
await fooOrigin.export([fooInterface])
}
})
})

839
sdk/package/lib/test/inputSpecBuilder.test.ts Normal file
View File

@@ -0,0 +1,839 @@
import { testOutput } from "./output.test"
import { InputSpec } from "../../../base/lib/actions/input/builder/inputSpec"
import { List } from "../../../base/lib/actions/input/builder/list"
import { Value } from "../../../base/lib/actions/input/builder/value"
import { Variants } from "../../../base/lib/actions/input/builder/variants"
import { ValueSpec } from "../../../base/lib/actions/input/inputSpecTypes"
import { setupManifest } from "../manifest/setupManifest"
import { StartSdk } from "../StartSdk"
import { VersionGraph } from "../version/VersionGraph"
import { VersionInfo } from "../version/VersionInfo"
describe("builder tests", () => {
test("text", async () => {
const bitcoinPropertiesBuilt: {
"peer-tor-address": ValueSpec
} = await InputSpec.of({
"peer-tor-address": Value.text({
name: "Peer tor address",
description: "The Tor address of the peer interface",
required: { default: null },
}),
}).build({} as any)
expect(bitcoinPropertiesBuilt).toMatchObject({
"peer-tor-address": {
type: "text",
description: "The Tor address of the peer interface",
warning: null,
masked: false,
placeholder: null,
minLength: null,
maxLength: null,
patterns: [],
disabled: false,
inputmode: "text",
name: "Peer tor address",
required: true,
default: null,
},
})
})
})
describe("values", () => {
test("toggle", async () => {
const value = Value.toggle({
name: "Testing",
description: null,
warning: null,
default: false,
})
const validator = value.validator
validator.unsafeCast(false)
testOutput<typeof validator._TYPE, boolean>()(null)
})
test("text", async () => {
const value = Value.text({
name: "Testing",
required: { default: null },
})
const validator = value.validator
const rawIs = await value.build({} as any)
validator.unsafeCast("test text")
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, string>()(null)
})
test("text with default", async () => {
const value = Value.text({
name: "Testing",
required: { default: "this is a default value" },
})
const validator = value.validator
const rawIs = await value.build({} as any)
validator.unsafeCast("test text")
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, string>()(null)
})
test("optional text", async () => {
const value = Value.text({
name: "Testing",
required: false,
})
const validator = value.validator
const rawIs = await value.build({} as any)
validator.unsafeCast("test text")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
})
test("color", async () => {
const value = Value.color({
name: "Testing",
required: false,
description: null,
warning: null,
})
const validator = value.validator
validator.unsafeCast("#000000")
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
})
test("datetime", async () => {
const value = Value.datetime({
name: "Testing",
required: { default: null },
description: null,
warning: null,
inputmode: "date",
min: null,
max: null,
})
const validator = value.validator
validator.unsafeCast("2021-01-01")
testOutput<typeof validator._TYPE, string>()(null)
})
test("optional datetime", async () => {
const value = Value.datetime({
name: "Testing",
required: false,
description: null,
warning: null,
inputmode: "date",
min: null,
max: null,
})
const validator = value.validator
validator.unsafeCast("2021-01-01")
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
})
test("textarea", async () => {
const value = Value.textarea({
name: "Testing",
required: false,
description: null,
warning: null,
minLength: null,
maxLength: null,
placeholder: null,
})
const validator = value.validator
validator.unsafeCast("test text")
testOutput<typeof validator._TYPE, string>()(null)
})
test("number", async () => {
const value = Value.number({
name: "Testing",
required: { default: null },
integer: false,
description: null,
warning: null,
min: null,
max: null,
step: null,
units: null,
placeholder: null,
})
const validator = value.validator
validator.unsafeCast(2)
testOutput<typeof validator._TYPE, number>()(null)
})
test("optional number", async () => {
const value = Value.number({
name: "Testing",
required: false,
integer: false,
description: null,
warning: null,
min: null,
max: null,
step: null,
units: null,
placeholder: null,
})
const validator = value.validator
validator.unsafeCast(2)
testOutput<typeof validator._TYPE, number | null | undefined>()(null)
})
test("select", async () => {
const value = Value.select({
name: "Testing",
required: { default: null },
values: {
a: "A",
b: "B",
},
description: null,
warning: null,
})
const validator = value.validator
validator.unsafeCast("a")
validator.unsafeCast("b")
expect(() => validator.unsafeCast("c")).toThrowError()
testOutput<typeof validator._TYPE, "a" | "b">()(null)
})
test("nullable select", async () => {
const value = Value.select({
name: "Testing",
required: false,
values: {
a: "A",
b: "B",
},
description: null,
warning: null,
})
const validator = value.validator
validator.unsafeCast("a")
validator.unsafeCast("b")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, "a" | "b" | null | undefined>()(null)
})
test("multiselect", async () => {
const value = Value.multiselect({
name: "Testing",
values: {
a: "A",
b: "B",
},
default: [],
description: null,
warning: null,
minLength: null,
maxLength: null,
})
const validator = value.validator
validator.unsafeCast([])
validator.unsafeCast(["a", "b"])
expect(() => validator.unsafeCast(["e"])).toThrowError()
expect(() => validator.unsafeCast([4])).toThrowError()
testOutput<typeof validator._TYPE, Array<"a" | "b">>()(null)
})
test("object", async () => {
const value = Value.object(
{
name: "Testing",
description: null,
},
InputSpec.of({
a: Value.toggle({
name: "test",
description: null,
warning: null,
default: false,
}),
}),
)
const validator = value.validator
validator.unsafeCast({ a: true })
testOutput<typeof validator._TYPE, { a: boolean }>()(null)
})
test("union", async () => {
const value = Value.union(
{
name: "Testing",
required: { default: null },
description: null,
warning: null,
},
Variants.of({
a: {
name: "a",
spec: InputSpec.of({
b: Value.toggle({
name: "b",
description: null,
warning: null,
default: false,
}),
}),
},
}),
)
const validator = value.validator
validator.unsafeCast({ selection: "a", value: { b: false } })
type Test = typeof validator._TYPE
testOutput<Test, { selection: "a"; value: { b: boolean } }>()(null)
})
describe("dynamic", () => {
const fakeOptions = {
inputSpec: "inputSpec",
effects: "effects",
utils: "utils",
} as any
test("toggle", async () => {
const value = Value.dynamicToggle(async () => ({
name: "Testing",
description: null,
warning: null,
default: false,
}))
const validator = value.validator
validator.unsafeCast(false)
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, boolean>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
description: null,
warning: null,
default: false,
})
})
test("text", async () => {
const value = Value.dynamicText(async () => ({
name: "Testing",
required: { default: null },
}))
const validator = value.validator
const rawIs = await value.build({} as any)
validator.unsafeCast("test text")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: true,
default: null,
})
})
test("text with default", async () => {
const value = Value.dynamicText(async () => ({
name: "Testing",
required: { default: "this is a default value" },
}))
const validator = value.validator
validator.unsafeCast("test text")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: true,
default: "this is a default value",
})
})
test("optional text", async () => {
const value = Value.dynamicText(async () => ({
name: "Testing",
required: false,
}))
const validator = value.validator
const rawIs = await value.build({} as any)
validator.unsafeCast("test text")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: false,
default: null,
})
})
test("color", async () => {
const value = Value.dynamicColor(async () => ({
name: "Testing",
required: false,
description: null,
warning: null,
}))
const validator = value.validator
validator.unsafeCast("#000000")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: false,
default: null,
description: null,
warning: null,
})
})
test("datetime", async () => {
const sdk = StartSdk.of()
.withManifest(
setupManifest(
VersionGraph.of(
VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
}),
),
{
id: "testOutput",
title: "",
license: "",
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: true,
s9pk: "https://example.com/remote-test.s9pk",
},
},
},
),
)
.withStore<{ test: "a" }>()
.build(true)
const value = Value.dynamicDatetime<{ test: "a" }>(
async ({ effects }) => {
;async () => {
;(await sdk.store
.getOwn(effects, sdk.StorePath.test)
.once()) satisfies "a"
}
return {
name: "Testing",
required: { default: null },
inputmode: "date",
}
},
)
const validator = value.validator
validator.unsafeCast("2021-01-01")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: true,
default: null,
description: null,
warning: null,
inputmode: "date",
})
})
test("textarea", async () => {
const value = Value.dynamicTextarea(async () => ({
name: "Testing",
required: false,
description: null,
warning: null,
minLength: null,
maxLength: null,
placeholder: null,
}))
const validator = value.validator
validator.unsafeCast("test text")
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, string>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: false,
})
})
test("number", async () => {
const value = Value.dynamicNumber(() => ({
name: "Testing",
required: { default: null },
integer: false,
description: null,
warning: null,
min: null,
max: null,
step: null,
units: null,
placeholder: null,
}))
const validator = value.validator
validator.unsafeCast(2)
validator.unsafeCast(null)
expect(() => validator.unsafeCast("null")).toThrowError()
testOutput<typeof validator._TYPE, number | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: true,
})
})
test("select", async () => {
const value = Value.dynamicSelect(() => ({
name: "Testing",
required: { default: null },
values: {
a: "A",
b: "B",
},
description: null,
warning: null,
}))
const validator = value.validator
validator.unsafeCast("a")
validator.unsafeCast("b")
validator.unsafeCast("c")
validator.unsafeCast(null)
testOutput<typeof validator._TYPE, string | null | undefined>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
required: true,
})
})
test("multiselect", async () => {
const value = Value.dynamicMultiselect(() => ({
name: "Testing",
values: {
a: "A",
b: "B",
},
default: [],
description: null,
warning: null,
minLength: null,
maxLength: null,
}))
const validator = value.validator
validator.unsafeCast([])
validator.unsafeCast(["a", "b"])
validator.unsafeCast(["c"])
expect(() => validator.unsafeCast([4])).toThrowError()
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, Array<string>>()(null)
expect(await value.build(fakeOptions)).toMatchObject({
name: "Testing",
default: [],
})
})
})
describe("filtering", () => {
test("union", async () => {
const value = Value.filteredUnion(
() => ["a", "c"],
{
name: "Testing",
required: { default: null },
description: null,
warning: null,
},
Variants.of({
a: {
name: "a",
spec: InputSpec.of({
b: Value.toggle({
name: "b",
description: null,
warning: null,
default: false,
}),
}),
},
b: {
name: "b",
spec: InputSpec.of({
b: Value.toggle({
name: "b",
description: null,
warning: null,
default: false,
}),
}),
},
}),
)
const validator = value.validator
validator.unsafeCast({ selection: "a", value: { b: false } })
type Test = typeof validator._TYPE
testOutput<
Test,
| { selection: "a"; value: { b: boolean } }
| { selection: "b"; value: { b: boolean } }
>()(null)
const built = await value.build({} as any)
expect(built).toMatchObject({
name: "Testing",
variants: {
b: {},
},
})
expect(built).toMatchObject({
name: "Testing",
variants: {
a: {},
b: {},
},
})
expect(built).toMatchObject({
name: "Testing",
variants: {
a: {},
b: {},
},
disabled: ["a", "c"],
})
})
})
test("dynamic union", async () => {
const value = Value.dynamicUnion(
() => ({
disabled: ["a", "c"],
name: "Testing",
required: { default: null },
description: null,
warning: null,
}),
Variants.of({
a: {
name: "a",
spec: InputSpec.of({
b: Value.toggle({
name: "b",
description: null,
warning: null,
default: false,
}),
}),
},
b: {
name: "b",
spec: InputSpec.of({
b: Value.toggle({
name: "b",
description: null,
warning: null,
default: false,
}),
}),
},
}),
)
const validator = value.validator
validator.unsafeCast({ selection: "a", value: { b: false } })
type Test = typeof validator._TYPE
testOutput<
Test,
| { selection: "a"; value: { b: boolean } }
| { selection: "b"; value: { b: boolean } }
| null
| undefined
>()(null)
const built = await value.build({} as any)
expect(built).toMatchObject({
name: "Testing",
variants: {
b: {},
},
})
expect(built).toMatchObject({
name: "Testing",
variants: {
a: {},
b: {},
},
})
expect(built).toMatchObject({
name: "Testing",
variants: {
a: {},
b: {},
},
disabled: ["a", "c"],
})
})
})
describe("Builder List", () => {
test("obj", async () => {
const value = Value.list(
List.obj(
{
name: "test",
},
{
spec: InputSpec.of({
test: Value.toggle({
name: "test",
description: null,
warning: null,
default: false,
}),
}),
},
),
)
const validator = value.validator
validator.unsafeCast([{ test: true }])
testOutput<typeof validator._TYPE, { test: boolean }[]>()(null)
})
test("text", async () => {
const value = Value.list(
List.text(
{
name: "test",
},
{
patterns: [],
},
),
)
const validator = value.validator
validator.unsafeCast(["test", "text"])
testOutput<typeof validator._TYPE, string[]>()(null)
})
describe("dynamic", () => {
test("text", async () => {
const value = Value.list(
List.dynamicText(() => ({
name: "test",
spec: { patterns: [] },
})),
)
const validator = value.validator
validator.unsafeCast(["test", "text"])
expect(() => validator.unsafeCast([3, 4])).toThrowError()
expect(() => validator.unsafeCast(null)).toThrowError()
testOutput<typeof validator._TYPE, string[]>()(null)
expect(await value.build({} as any)).toMatchObject({
name: "test",
spec: { patterns: [] },
})
})
})
})
describe("Nested nullable values", () => {
test("Testing text", async () => {
const value = InputSpec.of({
a: Value.text({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
required: false,
}),
})
const validator = value.validator
validator.unsafeCast({ a: null })
validator.unsafeCast({ a: "test" })
expect(() => validator.unsafeCast({ a: 4 })).toThrowError()
testOutput<typeof validator._TYPE, { a: string | null | undefined }>()(null)
})
test("Testing number", async () => {
const value = InputSpec.of({
a: Value.number({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
required: false,
warning: null,
placeholder: null,
integer: false,
min: null,
max: null,
step: null,
units: null,
}),
})
const validator = value.validator
validator.unsafeCast({ a: null })
validator.unsafeCast({ a: 5 })
expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
testOutput<typeof validator._TYPE, { a: number | null | undefined }>()(null)
})
test("Testing color", async () => {
const value = InputSpec.of({
a: Value.color({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
required: false,
warning: null,
}),
})
const validator = value.validator
validator.unsafeCast({ a: null })
validator.unsafeCast({ a: "5" })
expect(() => validator.unsafeCast({ a: 4 })).toThrowError()
testOutput<typeof validator._TYPE, { a: string | null | undefined }>()(null)
})
test("Testing select", async () => {
const value = InputSpec.of({
a: Value.select({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
required: false,
warning: null,
values: {
a: "A",
},
}),
})
const higher = await Value.select({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
required: false,
warning: null,
values: {
a: "A",
},
}).build({} as any)
const validator = value.validator
validator.unsafeCast({ a: null })
validator.unsafeCast({ a: "a" })
expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
testOutput<typeof validator._TYPE, { a: "a" | null | undefined }>()(null)
})
test("Testing multiselect", async () => {
const value = InputSpec.of({
a: Value.multiselect({
name: "Temp Name",
description:
"If no name is provided, the name from inputSpec will be used",
warning: null,
default: [],
values: {
a: "A",
},
minLength: null,
maxLength: null,
}),
})
const validator = value.validator
validator.unsafeCast({ a: [] })
validator.unsafeCast({ a: ["a"] })
expect(() => validator.unsafeCast({ a: ["4"] })).toThrowError()
expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
testOutput<typeof validator._TYPE, { a: "a"[] }>()(null)
})
})

428
sdk/package/lib/test/makeOutput.ts Normal file
View File

@@ -0,0 +1,428 @@
import { oldSpecToBuilder } from "../../scripts/oldSpecToBuilder"
oldSpecToBuilder(
// Make the location
"./lib/test/output.ts",
// Put the inputSpec here
{
mediasources: {
type: "list",
subtype: "enum",
name: "Media Sources",
description: "List of Media Sources to use with Jellyfin",
range: "[1,*)",
default: ["nextcloud"],
spec: {
values: ["nextcloud", "filebrowser"],
"value-names": {
nextcloud: "NextCloud",
filebrowser: "File Browser",
},
},
},
testListUnion: {
type: "list",
subtype: "union",
name: "Lightning Nodes",
description: "List of Lightning Network node instances to manage",
range: "[1,*)",
default: ["lnd"],
spec: {
type: "string",
"display-as": "{{name}}",
"unique-by": "name",
name: "Node Implementation",
tag: {
id: "type",
name: "Type",
description:
"- LND: Lightning Network Daemon from Lightning Labs\n- CLN: Core Lightning from Blockstream\n",
"variant-names": {
lnd: "Lightning Network Daemon (LND)",
"c-lightning": "Core Lightning (CLN)",
},
},
default: "lnd",
variants: {
lnd: {
name: {
type: "string",
name: "Node Name",
description: "Name of this node in the list",
default: "LND Wrapper",
nullable: false,
},
},
},
},
},
rpc: {
type: "object",
name: "RPC Settings",
description: "RPC configuration options.",
spec: {
enable: {
type: "boolean",
name: "Enable",
description: "Allow remote RPC requests.",
default: true,
},
username: {
type: "string",
nullable: false,
name: "Username",
description: "The username for connecting to Bitcoin over RPC.",
default: "bitcoin",
masked: true,
pattern: "^[a-zA-Z0-9_]+$",
"pattern-description":
"Must be alphanumeric (can contain underscore).",
},
password: {
type: "string",
nullable: false,
name: "RPC Password",
description: "The password for connecting to Bitcoin over RPC.",
default: {
charset: "a-z,2-7",
len: 20,
},
pattern: '^[^\\n"]*$',
"pattern-description":
"Must not contain newline or quote characters.",
copyable: true,
masked: true,
},
bio: {
type: "string",
nullable: false,
name: "Username",
description: "The username for connecting to Bitcoin over RPC.",
default: "bitcoin",
masked: true,
pattern: "^[a-zA-Z0-9_]+$",
"pattern-description":
"Must be alphanumeric (can contain underscore).",
textarea: true,
},
advanced: {
type: "object",
name: "Advanced",
description: "Advanced RPC Settings",
spec: {
auth: {
name: "Authorization",
description:
"Username and hashed password for JSON-RPC connections. RPC clients connect using the usual http basic authentication.",
type: "list",
subtype: "string",
default: [],
spec: {
pattern:
"^[a-zA-Z0-9_-]+:([0-9a-fA-F]{2})+\\$([0-9a-fA-F]{2})+$",
"pattern-description":
'Each item must be of the form "<USERNAME>:<SALT>$<HASH>".',
masked: false,
},
range: "[0,*)",
},
serialversion: {
name: "Serialization Version",
description:
"Return raw transaction or block hex with Segwit or non-SegWit serialization.",
type: "enum",
values: ["non-segwit", "segwit"],
"value-names": {},
default: "segwit",
},
servertimeout: {
name: "Rpc Server Timeout",
description:
"Number of seconds after which an uncompleted RPC call will time out.",
type: "number",
nullable: false,
range: "[5,300]",
integral: true,
units: "seconds",
default: 30,
},
threads: {
name: "Threads",
description:
"Set the number of threads for handling RPC calls. You may wish to increase this if you are making lots of calls via an integration.",
type: "number",
nullable: false,
default: 16,
range: "[1,64]",
integral: true,
},
workqueue: {
name: "Work Queue",
description:
"Set the depth of the work queue to service RPC calls. Determines how long the backlog of RPC requests can get before it just rejects new ones.",
type: "number",
nullable: false,
default: 128,
range: "[8,256]",
integral: true,
units: "requests",
},
},
},
},
},
"zmq-enabled": {
type: "boolean",
name: "ZeroMQ Enabled",
description: "Enable the ZeroMQ interface",
default: true,
},
txindex: {
type: "boolean",
name: "Transaction Index",
description: "Enable the Transaction Index (txindex)",
default: true,
},
wallet: {
type: "object",
name: "Wallet",
description: "Wallet Settings",
spec: {
enable: {
name: "Enable Wallet",
description: "Load the wallet and enable wallet RPC calls.",
type: "boolean",
default: true,
},
avoidpartialspends: {
name: "Avoid Partial Spends",
description:
"Group outputs by address, selecting all or none, instead of selecting on a per-output basis. This improves privacy at the expense of higher transaction fees.",
type: "boolean",
default: true,
},
discardfee: {
name: "Discard Change Tolerance",
description:
"The fee rate (in BTC/kB) that indicates your tolerance for discarding change by adding it to the fee.",
type: "number",
nullable: false,
default: 0.0001,
range: "[0,.01]",
integral: false,
units: "BTC/kB",
},
},
},
advanced: {
type: "object",
name: "Advanced",
description: "Advanced Settings",
spec: {
mempool: {
type: "object",
name: "Mempool",
description: "Mempool Settings",
spec: {
mempoolfullrbf: {
name: "Enable Full RBF",
description:
"Policy for your node to use for relaying and mining unconfirmed transactions. For details, see https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-24.0.md#notice-of-new-option-for-transaction-replacement-policies",
type: "boolean",
default: false,
},
persistmempool: {
type: "boolean",
name: "Persist Mempool",
description: "Save the mempool on shutdown and load on restart.",
default: true,
},
maxmempool: {
type: "number",
nullable: false,
name: "Max Mempool Size",
description:
"Keep the transaction memory pool below <n> megabytes.",
range: "[1,*)",
integral: true,
units: "MiB",
default: 300,
},
mempoolexpiry: {
type: "number",
nullable: false,
name: "Mempool Expiration",
description:
"Do not keep transactions in the mempool longer than <n> hours.",
range: "[1,*)",
integral: true,
units: "Hr",
default: 336,
},
},
},
peers: {
type: "object",
name: "Peers",
description: "Peer Connection Settings",
spec: {
listen: {
type: "boolean",
name: "Make Public",
description:
"Allow other nodes to find your server on the network.",
default: true,
},
onlyconnect: {
type: "boolean",
name: "Disable Peer Discovery",
description: "Only connect to specified peers.",
default: false,
},
onlyonion: {
type: "boolean",
name: "Disable Clearnet",
description: "Only connect to peers over Tor.",
default: false,
},
addnode: {
name: "Add Nodes",
description: "Add addresses of nodes to connect to.",
type: "list",
subtype: "object",
range: "[0,*)",
default: [],
spec: {
"unique-by": null,
spec: {
hostname: {
type: "string",
nullable: true,
name: "Hostname",
description: "Domain or IP address of bitcoin peer",
pattern:
"(^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$)|((^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$)|(^[a-z2-7]{16}\\.onion$)|(^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$))",
"pattern-description":
"Must be either a domain name, or an IPv4 or IPv6 address. Do not include protocol scheme (eg 'http://') or port.",
masked: false,
},
port: {
type: "number",
nullable: true,
name: "Port",
description:
"Port that peer is listening on for inbound p2p connections",
range: "[0,65535]",
integral: true,
},
},
},
},
},
},
dbcache: {
type: "number",
nullable: true,
name: "Database Cache",
description:
"How much RAM to allocate for caching the TXO set. Higher values improve syncing performance, but increase your chance of using up all your system's memory or corrupting your database in the event of an ungraceful shutdown. Set this high but comfortably below your system's total RAM during IBD, then turn down to 450 (or leave blank) once the sync completes.",
warning:
"WARNING: Increasing this value results in a higher chance of ungraceful shutdowns, which can leave your node unusable if it happens during the initial block download. Use this setting with caution. Be sure to set this back to the default (450 or leave blank) once your node is synced. DO NOT press the STOP button if your dbcache is large. Instead, set this number back to the default, hit save, and wait for bitcoind to restart on its own.",
range: "(0,*)",
integral: true,
units: "MiB",
},
pruning: {
type: "union",
name: "Pruning Settings",
description:
"Blockchain Pruning Options\nReduce the blockchain size on disk\n",
warning:
"If you set pruning to Manual and your disk is smaller than the total size of the blockchain, you MUST have something running that prunes these blocks or you may overfill your disk!\nDisabling pruning will convert your node into a full archival node. This requires a resync of the entire blockchain, a process that may take several days. Make sure you have enough free disk space or you may fill up your disk.\n",
tag: {
id: "mode",
name: "Pruning Mode",
description:
'- Disabled: Disable pruning\n- Automatic: Limit blockchain size on disk to a certain number of megabytes\n- Manual: Prune blockchain with the "pruneblockchain" RPC\n',
"variant-names": {
disabled: "Disabled",
automatic: "Automatic",
manual: "Manual",
},
},
variants: {
disabled: {},
automatic: {
size: {
type: "number",
nullable: false,
name: "Max Chain Size",
description: "Limit of blockchain size on disk.",
warning:
"Increasing this value will require re-syncing your node.",
default: 550,
range: "[550,1000000)",
integral: true,
units: "MiB",
},
},
manual: {
size: {
type: "number",
nullable: false,
name: "Failsafe Chain Size",
description: "Prune blockchain if size expands beyond this.",
default: 65536,
range: "[550,1000000)",
integral: true,
units: "MiB",
},
},
},
default: "disabled",
},
blockfilters: {
type: "object",
name: "Block Filters",
description: "Settings for storing and serving compact block filters",
spec: {
blockfilterindex: {
type: "boolean",
name: "Compute Compact Block Filters (BIP158)",
description:
"Generate Compact Block Filters during initial sync (IBD) to enable 'getblockfilter' RPC. This is useful if dependent services need block filters to efficiently scan for addresses/transactions etc.",
default: true,
},
peerblockfilters: {
type: "boolean",
name: "Serve Compact Block Filters to Peers (BIP157)",
description:
"Serve Compact Block Filters as a peer service to other nodes on the network. This is useful if you wish to connect an SPV client to your node to make it efficient to scan transactions without having to download all block data. 'Compute Compact Block Filters (BIP158)' is required.",
default: false,
},
},
},
bloomfilters: {
type: "object",
name: "Bloom Filters (BIP37)",
description: "Setting for serving Bloom Filters",
spec: {
peerbloomfilters: {
type: "boolean",
name: "Serve Bloom Filters to Peers",
description:
"Peers have the option of setting filters on each connection they make after the version handshake has completed. Bloom filters are for clients implementing SPV (Simplified Payment Verification) that want to check that block headers connect together correctly, without needing to verify the full blockchain. The client must trust that the transactions in the chain are in fact valid. It is highly recommended AGAINST using for anything except Bisq integration.",
warning:
"This is ONLY for use with Bisq integration, please use Block Filters for all other applications.",
default: false,
},
},
},
},
},
},
{
// convert this to `start-sdk/lib` for conversions
StartSdk: "./output.sdk",
},
)

56
sdk/package/lib/test/output.sdk.ts Normal file
View File

@@ -0,0 +1,56 @@
import { StartSdk } from "../StartSdk"
import { setupManifest } from "../manifest/setupManifest"
import { VersionInfo } from "../version/VersionInfo"
import { VersionGraph } from "../version/VersionGraph"
export type Manifest = any
export const sdk = StartSdk.of()
.withManifest(
setupManifest(
VersionGraph.of(
VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
})
.satisfies("#other:1.0.0:0")
.satisfies("#other:2.0.0:0"),
),
{
id: "testOutput",
title: "",
license: "",
replaces: [],
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,
uninstall: null,
restore: null,
start: null,
stop: null,
},
dependencies: {
"remote-test": {
description: "",
optional: false,
s9pk: "https://example.com/remote-test.s9pk",
},
},
},
),
)
.withStore<{ storeRoot: { storeLeaf: "value" } }>()
.build(true)

146
sdk/package/lib/test/output.test.ts Normal file
View File

@@ -0,0 +1,146 @@
import { InputSpecSpec, matchInputSpecSpec } from "./output"
import * as _I from "../index"
import { camelCase } from "../../scripts/oldSpecToBuilder"
import { deepMerge } from "../../../base/lib/util"
export type IfEquals<T, U, Y = unknown, N = never> =
(<G>() => G extends T ? 1 : 2) extends <G>() => G extends U ? 1 : 2 ? Y : N
export function testOutput<A, B>(): (c: IfEquals<A, B>) => null {
return () => null
}
/// Testing the types of the input spec
testOutput<InputSpecSpec["rpc"]["enable"], boolean>()(null)
testOutput<InputSpecSpec["rpc"]["username"], string>()(null)
testOutput<InputSpecSpec["rpc"]["username"], string>()(null)
testOutput<InputSpecSpec["rpc"]["advanced"]["auth"], string[]>()(null)
testOutput<
InputSpecSpec["rpc"]["advanced"]["serialversion"],
"segwit" | "non-segwit"
>()(null)
testOutput<InputSpecSpec["rpc"]["advanced"]["servertimeout"], number>()(null)
testOutput<
InputSpecSpec["advanced"]["peers"]["addnode"][0]["hostname"],
string | null | undefined
>()(null)
testOutput<
InputSpecSpec["testListUnion"][0]["union"]["value"]["name"],
string
>()(null)
testOutput<InputSpecSpec["testListUnion"][0]["union"]["selection"], "lnd">()(
null,
)
testOutput<InputSpecSpec["mediasources"], Array<"filebrowser" | "nextcloud">>()(
null,
)
// @ts-expect-error Because enable should be a boolean
testOutput<InputSpecSpec["rpc"]["enable"], string>()(null)
// prettier-ignore
// @ts-expect-error Expect that the string is the one above
testOutput<InputSpecSpec["testListUnion"][0]['selection']['selection'], "selection">()(null);
/// Here we test the output of the matchInputSpecSpec function
describe("Inputs", () => {
const validInput: InputSpecSpec = {
mediasources: ["filebrowser"],
testListUnion: [
{
union: { selection: "lnd", value: { name: "string" } },
},
],
rpc: {
enable: true,
bio: "This is a bio",
username: "test",
password: "test",
advanced: {
auth: ["test"],
serialversion: "segwit",
servertimeout: 6,
threads: 3,
workqueue: 9,
},
},
"zmq-enabled": false,
txindex: false,
wallet: { enable: false, avoidpartialspends: false, discardfee: 0.0001 },
advanced: {
mempool: {
maxmempool: 1,
persistmempool: true,
mempoolexpiry: 23,
mempoolfullrbf: true,
},
peers: {
listen: true,
onlyconnect: true,
onlyonion: true,
addnode: [
{
hostname: "test",
port: 1,
},
],
},
dbcache: 5,
pruning: {
selection: "disabled",
value: {},
},
blockfilters: {
blockfilterindex: false,
peerblockfilters: false,
},
bloomfilters: { peerbloomfilters: false },
},
}
test("test valid input", () => {
const output = matchInputSpecSpec.unsafeCast(validInput)
expect(output).toEqual(validInput)
})
test("test no longer care about the conversion of min/max and validating", () => {
matchInputSpecSpec.unsafeCast(
deepMerge({}, validInput, { rpc: { advanced: { threads: 0 } } }),
)
})
test("test errors should throw for number in string", () => {
expect(() =>
matchInputSpecSpec.unsafeCast(
deepMerge({}, validInput, { rpc: { enable: 2 } }),
),
).toThrowError()
})
test("Test that we set serialversion to something not segwit or non-segwit", () => {
expect(() =>
matchInputSpecSpec.unsafeCast(
deepMerge({}, validInput, {
rpc: { advanced: { serialversion: "testing" } },
}),
),
).toThrowError()
})
})
describe("camelCase", () => {
test("'EquipmentClass name'", () => {
expect(camelCase("EquipmentClass name")).toEqual("equipmentClassName")
})
test("'Equipment className'", () => {
expect(camelCase("Equipment className")).toEqual("equipmentClassName")
})
test("'equipment class name'", () => {
expect(camelCase("equipment class name")).toEqual("equipmentClassName")
})
test("'Equipment Class Name'", () => {
expect(camelCase("Equipment Class Name")).toEqual("equipmentClassName")
})
test("'hyphen-name-format'", () => {
expect(camelCase("hyphen-name-format")).toEqual("hyphenNameFormat")
})
test("'underscore_name_format'", () => {
expect(camelCase("underscore_name_format")).toEqual("underscoreNameFormat")
})
})

111
sdk/package/lib/test/store.test.ts Normal file
View File

@@ -0,0 +1,111 @@
import { Effects } from "../../../base/lib/types"
import { extractJsonPath } from "../../../base/lib/util/PathBuilder"
import { StartSdk } from "../StartSdk"
type Store = {
inputSpec: {
someValue: "a" | "b"
}
}
type Manifest = any
const todo = <A>(): A => {
throw new Error("not implemented")
}
const noop = () => {}
const sdk = StartSdk.of()
.withManifest({} as Manifest)
.withStore<Store>()
.build(true)
const storePath = sdk.StorePath
describe("Store", () => {
test("types", async () => {
;async () => {
sdk.store.setOwn(todo<Effects>(), storePath.inputSpec, {
someValue: "a",
})
sdk.store.setOwn(todo<Effects>(), storePath.inputSpec.someValue, "b")
sdk.store.setOwn(todo<Effects>(), storePath, {
inputSpec: { someValue: "b" },
})
sdk.store.setOwn(
todo<Effects>(),
storePath.inputSpec.someValue,
// @ts-expect-error Type is wrong for the setting value
5,
)
sdk.store.setOwn(
todo<Effects>(),
// @ts-expect-error Path is wrong
"/inputSpec/someVae3lue",
"someValue",
)
todo<Effects>().store.set<Store>({
path: extractJsonPath(storePath.inputSpec.someValue),
value: "b",
})
todo<Effects>().store.set<Store, "/inputSpec/some2Value">({
path: extractJsonPath(storePath.inputSpec.someValue),
//@ts-expect-error Path is wrong
value: "someValueIn",
})
;(await sdk.store
.getOwn(todo<Effects>(), storePath.inputSpec.someValue)
.const()) satisfies string
;(await sdk.store
.getOwn(todo<Effects>(), storePath.inputSpec)
.const()) satisfies Store["inputSpec"]
await sdk.store // @ts-expect-error Path is wrong
.getOwn(todo<Effects>(), "/inputSpec/somdsfeValue")
.const()
/// ----------------- ERRORS -----------------
sdk.store.setOwn(todo<Effects>(), storePath, {
// @ts-expect-error Type is wrong for the setting value
inputSpec: { someValue: "notInAOrB" },
})
sdk.store.setOwn(
todo<Effects>(),
sdk.StorePath.inputSpec.someValue,
// @ts-expect-error Type is wrong for the setting value
"notInAOrB",
)
;(await sdk.store
.getOwn(todo<Effects>(), storePath.inputSpec.someValue)
.const()) satisfies string
;(await sdk.store
.getOwn(todo<Effects>(), storePath.inputSpec)
.const()) satisfies Store["inputSpec"]
await sdk.store // @ts-expect-error Path is wrong
.getOwn("/inputSpec/somdsfeValue")
.const()
///
;(await sdk.store
.getOwn(todo<Effects>(), storePath.inputSpec.someValue)
// @ts-expect-error satisfies type is wrong
.const()) satisfies number
await sdk.store // @ts-expect-error Path is wrong
.getOwn(todo<Effects>(), extractJsonPath(storePath.inputSpec))
.const()
;(await todo<Effects>().store.get({
path: extractJsonPath(storePath.inputSpec.someValue),
callback: noop,
})) satisfies string
await todo<Effects>().store.get<Store, "/inputSpec/someValue">({
// @ts-expect-error Path is wrong as in it doesn't match above
path: "/inputSpec/someV2alue",
callback: noop,
})
await todo<Effects>().store.get<Store, "/inputSpec/someV2alue">({
// @ts-expect-error Path is wrong as in it doesn't exists in wrapper type
path: "/inputSpec/someV2alue",
callback: noop,
})
}
})
})

5
sdk/package/lib/trigger/TriggerInput.ts Normal file
View File

@@ -0,0 +1,5 @@
import { HealthStatus } from "../../../base/lib/types"
export type TriggerInput = {
lastResult?: HealthStatus
}

32
sdk/package/lib/trigger/changeOnFirstSuccess.ts Normal file
View File

@@ -0,0 +1,32 @@
import { Trigger } from "./index"
export function changeOnFirstSuccess(o: {
beforeFirstSuccess: Trigger
afterFirstSuccess: Trigger
}): Trigger {
return async function* (getInput) {
let currentValue = getInput()
while (!currentValue.lastResult) {
yield
currentValue = getInput()
}
const beforeFirstSuccess = o.beforeFirstSuccess(getInput)
for (
let res = await beforeFirstSuccess.next();
currentValue?.lastResult !== "success" && !res.done;
res = await beforeFirstSuccess.next()
) {
yield
currentValue = getInput()
}
const afterFirstSuccess = o.afterFirstSuccess(getInput)
for (
let res = await afterFirstSuccess.next();
!res.done;
res = await afterFirstSuccess.next()
) {
yield
currentValue = getInput()
}
}
}

8
sdk/package/lib/trigger/cooldownTrigger.ts Normal file
View File

@@ -0,0 +1,8 @@
export function cooldownTrigger(timeMs: number) {
return async function* () {
while (true) {
await new Promise((resolve) => setTimeout(resolve, timeMs))
yield
}
}
}

8
sdk/package/lib/trigger/defaultTrigger.ts Normal file
View File

@@ -0,0 +1,8 @@
import { cooldownTrigger } from "./cooldownTrigger"
import { changeOnFirstSuccess } from "./changeOnFirstSuccess"
import { successFailure } from "./successFailure"
export const defaultTrigger = changeOnFirstSuccess({
beforeFirstSuccess: cooldownTrigger(1000),
afterFirstSuccess: cooldownTrigger(30000),
})

7
sdk/package/lib/trigger/index.ts Normal file
View File

@@ -0,0 +1,7 @@
import { TriggerInput } from "./TriggerInput"
export { changeOnFirstSuccess } from "./changeOnFirstSuccess"
export { cooldownTrigger } from "./cooldownTrigger"
export type Trigger = (
getInput: () => TriggerInput,
) => AsyncIterator<unknown, unknown, never>

33
sdk/package/lib/trigger/lastStatus.ts Normal file
View File

@@ -0,0 +1,33 @@
import { Trigger } from "."
import { HealthStatus } from "../../../base/lib/types"
export type LastStatusTriggerParams = { [k in HealthStatus]?: Trigger } & {
default: Trigger
}
export function lastStatus(o: LastStatusTriggerParams): Trigger {
return async function* (getInput) {
let trigger = o.default(getInput)
const triggers: {
[k in HealthStatus]?: AsyncIterator<unknown, unknown, never>
} & { default: AsyncIterator<unknown, unknown, never> } = {
default: trigger,
}
while (true) {
let currentValue = getInput()
let prev: HealthStatus | "default" | undefined = currentValue.lastResult
if (!prev) {
yield
continue
}
if (!(prev in o)) {
prev = "default"
}
if (!triggers[prev]) {
triggers[prev] = o[prev]!(getInput)
}
await triggers[prev]?.next()
yield
}
}
}

7
sdk/package/lib/trigger/successFailure.ts Normal file
View File

@@ -0,0 +1,7 @@
import { Trigger } from "."
import { lastStatus } from "./lastStatus"
export const successFailure = (o: {
duringSuccess: Trigger
duringError: Trigger
}) => lastStatus({ success: o.duringSuccess, default: o.duringError })

47
sdk/package/lib/util/GetSslCertificate.ts Normal file
View File

@@ -0,0 +1,47 @@
import { T } from ".."
import { Effects } from "../../../base/lib/Effects"
export class GetSslCertificate {
constructor(
readonly effects: Effects,
readonly hostnames: string[],
readonly algorithm?: T.Algorithm,
) {}
/**
* Returns the system SMTP credentials. Restarts the service if the credentials change
*/
const() {
return this.effects.getSslCertificate({
hostnames: this.hostnames,
algorithm: this.algorithm,
callback: () => this.effects.constRetry(),
})
}
/**
* Returns the system SMTP credentials. Does nothing if the credentials change
*/
once() {
return this.effects.getSslCertificate({
hostnames: this.hostnames,
algorithm: this.algorithm,
})
}
/**
* Watches the system SMTP credentials. Takes a custom callback function to run whenever the credentials change
*/
async *watch() {
while (true) {
let callback: () => void
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
})
yield await this.effects.getSslCertificate({
hostnames: this.hostnames,
algorithm: this.algorithm,
callback: () => callback(),
})
await waitForNext
}
}
}

440
sdk/package/lib/util/SubContainer.ts Normal file
View File

@@ -0,0 +1,440 @@
import * as fs from "fs/promises"
import * as T from "../../../base/lib/types"
import * as cp from "child_process"
import { promisify } from "util"
import { Buffer } from "node:buffer"
import { once } from "../../../base/lib/util/once"
export const execFile = promisify(cp.execFile)
const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
const False = () => false
type ExecResults = {
exitCode: number | null
exitSignal: NodeJS.Signals | null
stdout: string | Buffer
stderr: string | Buffer
}
export type ExecOptions = {
input?: string | Buffer
}
const TIMES_TO_WAIT_FOR_PROC = 100
/**
* This is the type that is going to describe what an subcontainer could do. The main point of the
* subcontainer is to have commands that run in a chrooted environment. This is useful for running
* commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the
* case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed.
*/
export interface ExecSpawnable {
get destroy(): undefined | (() => Promise<void>)
exec(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<ExecResults>
spawn(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams>
}
/**
* Want to limit what we can do in a container, so we want to launch a container with a specific image and the mounts.
*
* Implements:
* @see {@link ExecSpawnable}
*/
export class SubContainer implements ExecSpawnable {
private leader: cp.ChildProcess
private leaderExited: boolean = false
private waitProc: () => Promise<void>
private constructor(
readonly effects: T.Effects,
readonly imageId: T.ImageId,
readonly rootfs: string,
readonly guid: T.Guid,
) {
this.leaderExited = false
this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], {
killSignal: "SIGKILL",
stdio: "ignore",
})
this.leader.on("exit", () => {
this.leaderExited = true
})
this.waitProc = once(
() =>
new Promise(async (resolve, reject) => {
let count = 0
while (
!(await fs.stat(`${this.rootfs}/proc/1`).then((x) => !!x, False))
) {
if (count++ > TIMES_TO_WAIT_FOR_PROC) {
console.debug("Failed to start subcontainer", {
guid: this.guid,
imageId: this.imageId,
rootfs: this.rootfs,
})
reject(new Error(`Failed to start subcontainer ${this.imageId}`))
}
await wait(1)
}
resolve()
}),
)
}
static async of(
effects: T.Effects,
image: { id: T.ImageId; sharedRun?: boolean },
name: string,
) {
const { id, sharedRun } = image
const [rootfs, guid] = await effects.subcontainer.createFs({
imageId: id as string,
name,
})
const shared = ["dev", "sys"]
if (!!sharedRun) {
shared.push("run")
}
await fs.mkdir(`${rootfs}/etc`, { recursive: true })
await fs.copyFile("/etc/resolv.conf", `${rootfs}/etc/resolv.conf`)
for (const dirPart of shared) {
const from = `/${dirPart}`
const to = `${rootfs}/${dirPart}`
await fs.mkdir(from, { recursive: true })
await fs.mkdir(to, { recursive: true })
await execFile("mount", ["--rbind", from, to])
}
return new SubContainer(effects, id, rootfs, guid)
}
static async with<T>(
effects: T.Effects,
image: { id: T.ImageId; sharedRun?: boolean },
mounts: { options: MountOptions; path: string }[],
name: string,
fn: (subContainer: SubContainer) => Promise<T>,
): Promise<T> {
const subContainer = await SubContainer.of(effects, image, name)
try {
for (let mount of mounts) {
await subContainer.mount(mount.options, mount.path)
}
return await fn(subContainer)
} finally {
await subContainer.destroy()
}
}
async mount(options: MountOptions, path: string): Promise<SubContainer> {
path = path.startsWith("/")
? `${this.rootfs}${path}`
: `${this.rootfs}/${path}`
if (options.type === "volume") {
const subpath = options.subpath
? options.subpath.startsWith("/")
? options.subpath
: `/${options.subpath}`
: "/"
const from = `/media/startos/volumes/${options.id}${subpath}`
await fs.mkdir(from, { recursive: true })
await fs.mkdir(path, { recursive: true })
await execFile("mount", ["--bind", from, path])
} else if (options.type === "assets") {
const subpath = options.subpath
? options.subpath.startsWith("/")
? options.subpath
: `/${options.subpath}`
: "/"
const from = `/media/startos/assets/${options.id}${subpath}`
await fs.mkdir(from, { recursive: true })
await fs.mkdir(path, { recursive: true })
await execFile("mount", ["--bind", from, path])
} else if (options.type === "pointer") {
await this.effects.mount({ location: path, target: options })
} else if (options.type === "backup") {
const subpath = options.subpath
? options.subpath.startsWith("/")
? options.subpath
: `/${options.subpath}`
: "/"
const from = `/media/startos/backup${subpath}`
await fs.mkdir(from, { recursive: true })
await fs.mkdir(path, { recursive: true })
await execFile("mount", ["--bind", from, path])
} else {
throw new Error(`unknown type ${(options as any).type}`)
}
return this
}
private async killLeader() {
if (this.leaderExited) {
return
}
return new Promise<void>((resolve, reject) => {
try {
let timeout = setTimeout(() => this.leader.kill("SIGKILL"), 30000)
this.leader.on("exit", () => {
clearTimeout(timeout)
resolve()
})
if (!this.leader.kill("SIGTERM")) {
reject(new Error("kill(2) failed"))
}
} catch (e) {
reject(e)
}
})
}
get destroy() {
return async () => {
const guid = this.guid
await this.killLeader()
await this.effects.subcontainer.destroyFs({ guid })
}
}
async exec(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs: number | null = 30000,
): Promise<{
exitCode: number | null
exitSignal: NodeJS.Signals | null
stdout: string | Buffer
stderr: string | Buffer
}> {
await this.waitProc()
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${this.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
let extra: string[] = []
if (options?.user) {
extra.push(`--user=${options.user}`)
delete options.user
}
let workdir = imageMeta.workdir || "/"
if (options?.cwd) {
workdir = options.cwd
delete options.cwd
}
const child = cp.spawn(
"start-cli",
[
"subcontainer",
"exec",
`--env=/media/startos/images/${this.imageId}.env`,
`--workdir=${workdir}`,
...extra,
this.rootfs,
...command,
],
options || {},
)
if (options?.input) {
await new Promise<void>((resolve, reject) =>
child.stdin.write(options.input, (e) => {
if (e) {
reject(e)
} else {
resolve()
}
}),
)
await new Promise<void>((resolve) => child.stdin.end(resolve))
}
const pid = child.pid
const stdout = { data: "" as string | Buffer }
const stderr = { data: "" as string | Buffer }
const appendData =
(appendTo: { data: string | Buffer }) =>
(chunk: string | Buffer | any) => {
if (typeof appendTo.data === "string" && typeof chunk === "string") {
appendTo.data += chunk
} else if (typeof chunk === "string" || chunk instanceof Buffer) {
appendTo.data = Buffer.concat([
new Uint8Array(Buffer.from(appendTo.data).buffer),
new Uint8Array(Buffer.from(chunk).buffer),
])
} else {
console.error("received unexpected chunk", chunk)
}
}
return new Promise((resolve, reject) => {
child.on("error", reject)
let killTimeout: NodeJS.Timeout | undefined
if (timeoutMs !== null && child.pid) {
killTimeout = setTimeout(() => child.kill("SIGKILL"), timeoutMs)
}
child.stdout.on("data", appendData(stdout))
child.stderr.on("data", appendData(stderr))
child.on("exit", (code, signal) => {
clearTimeout(killTimeout)
resolve({
exitCode: code,
exitSignal: signal,
stdout: stdout.data,
stderr: stderr.data,
})
})
})
}
async launch(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams> {
await this.waitProc()
const imageMeta: any = await fs
.readFile(`/media/startos/images/${this.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
let extra: string[] = []
if (options?.user) {
extra.push(`--user=${options.user}`)
delete options.user
}
let workdir = imageMeta.workdir || "/"
if (options?.cwd) {
workdir = options.cwd
delete options.cwd
}
await this.killLeader()
this.leaderExited = false
this.leader = cp.spawn(
"start-cli",
[
"subcontainer",
"launch",
`--env=/media/startos/images/${this.imageId}.env`,
`--workdir=${workdir}`,
...extra,
this.rootfs,
...command,
],
{ ...options, stdio: "inherit" },
)
this.leader.on("exit", () => {
this.leaderExited = true
})
return this.leader as cp.ChildProcessWithoutNullStreams
}
async spawn(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams> {
await this.waitProc()
const imageMeta: any = await fs
.readFile(`/media/startos/images/${this.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
let extra: string[] = []
if (options?.user) {
extra.push(`--user=${options.user}`)
delete options.user
}
let workdir = imageMeta.workdir || "/"
if (options?.cwd) {
workdir = options.cwd
delete options.cwd
}
return cp.spawn(
"start-cli",
[
"subcontainer",
"exec",
`--env=/media/startos/images/${this.imageId}.env`,
`--workdir=${workdir}`,
...extra,
this.rootfs,
...command,
],
options,
)
}
}
/**
* Take an subcontainer but remove the ability to add the mounts and the destroy function.
* Lets other functions, like health checks, to not destroy the parents.
*
*/
export class SubContainerHandle implements ExecSpawnable {
constructor(private subContainer: ExecSpawnable) {}
get destroy() {
return undefined
}
exec(
command: string[],
options?: CommandOptions,
timeoutMs?: number | null,
): Promise<ExecResults> {
return this.subContainer.exec(command, options, timeoutMs)
}
spawn(
command: string[],
options?: CommandOptions,
): Promise<cp.ChildProcessWithoutNullStreams> {
return this.subContainer.spawn(command, options)
}
}
export type CommandOptions = {
env?: { [variable: string]: string }
cwd?: string
user?: string
}
export type MountOptions =
| MountOptionsVolume
| MountOptionsAssets
| MountOptionsPointer
| MountOptionsBackup
export type MountOptionsVolume = {
type: "volume"
id: string
subpath: string | null
readonly: boolean
}
export type MountOptionsAssets = {
type: "assets"
id: string
subpath: string | null
}
export type MountOptionsPointer = {
type: "pointer"
packageId: string
volumeId: string
subpath: string | null
readonly: boolean
}
export type MountOptionsBackup = {
type: "backup"
subpath: string | null
}
function wait(time: number) {
return new Promise((resolve) => setTimeout(resolve, time))
}

214
sdk/package/lib/util/fileHelper.ts Normal file
View File

@@ -0,0 +1,214 @@
import * as matches from "ts-matches"
import * as YAML from "yaml"
import * as TOML from "@iarna/toml"
import merge from "lodash.merge"
import * as T from "../../../base/lib/types"
import * as fs from "node:fs/promises"
import { asError } from "../../../base/lib/util"
const previousPath = /(.+?)\/([^/]*)$/
const exists = (path: string) =>
fs.access(path).then(
() => true,
() => false,
)
async function onCreated(path: string) {
if (path === "/") return
if (!path.startsWith("/")) path = `${process.cwd()}/${path}`
if (await exists(path)) {
return
}
const split = path.split("/")
const filename = split.pop()
const parent = split.join("/")
await onCreated(parent)
const ctrl = new AbortController()
const watch = fs.watch(parent, { persistent: false, signal: ctrl.signal })
if (
await fs.access(path).then(
() => true,
() => false,
)
) {
ctrl.abort("finished")
return
}
for await (let event of watch) {
if (event.filename === filename) {
ctrl.abort("finished")
return
}
}
}
/**
* @description Use this class to read/write an underlying configuration file belonging to the upstream service.
*
* Using the static functions, choose between officially supported file formats (json, yaml, toml), or a custom format (raw).
* @example
* Below are a few examples
*
* ```
* import { matches, FileHelper } from '@start9labs/start-sdk'
* const { arrayOf, boolean, literal, literals, object, oneOf, natural, string } = matches
*
* export const jsonFile = FileHelper.json('./inputSpec.json', object({
* passwords: arrayOf(string)
* type: oneOf(literals('private', 'public'))
* }))
*
* export const tomlFile = FileHelper.toml('./inputSpec.toml', object({
* url: literal('https://start9.com')
* public: boolean
* }))
*
* export const yamlFile = FileHelper.yaml('./inputSpec.yml', object({
* name: string
* age: natural
* }))
*
* export const bitcoinConfFile = FileHelper.raw(
* './service.conf',
* (obj: CustomType) => customConvertObjToFormattedString(obj),
* (str) => customParseStringToTypedObj(str),
* )
* ```
*/
export class FileHelper<A> {
protected constructor(
readonly path: string,
readonly writeData: (dataIn: A) => string,
readonly readData: (stringValue: string) => A,
) {}
/**
* Accepts structured data and overwrites the existing file on disk.
*/
async write(data: A) {
const parent = previousPath.exec(this.path)
if (parent) {
await fs.mkdir(parent[1], { recursive: true })
}
await fs.writeFile(this.path, this.writeData(data))
}
/**
* Reads the file from disk and converts it to structured data.
*/
async read() {
if (!(await exists(this.path))) {
return null
}
return this.readData(
await fs.readFile(this.path).then((data) => data.toString("utf-8")),
)
}
async const(effects: T.Effects) {
const watch = this.watch()
const res = await watch.next()
watch.next().then(effects.constRetry)
return res.value
}
async *watch() {
let res
while (true) {
if (await exists(this.path)) {
const ctrl = new AbortController()
const watch = fs.watch(this.path, {
persistent: false,
signal: ctrl.signal,
})
res = await this.read()
const listen = Promise.resolve()
.then(async () => {
for await (const _ of watch) {
ctrl.abort("finished")
return
}
})
.catch((e) => console.error(asError(e)))
yield res
await listen
} else {
yield null
await onCreated(this.path).catch((e) => console.error(asError(e)))
}
}
}
/**
* Accepts structured data and performs a merge with the existing file on disk.
*/
async merge(data: A) {
const fileData = (await this.read().catch(() => ({}))) || {}
const mergeData = merge({}, fileData, data)
return await this.write(mergeData)
}
/**
* Create a File Helper for an arbitrary file type.
*
* Provide custom functions for translating data to/from the file format.
*/
static raw<A>(
path: string,
toFile: (dataIn: A) => string,
fromFile: (rawData: string) => A,
) {
return new FileHelper<A>(path, toFile, fromFile)
}
/**
* Create a File Helper for a .json file.
*/
static json<A>(path: string, shape: matches.Validator<unknown, A>) {
return new FileHelper<A>(
path,
(inData) => {
return JSON.stringify(inData, null, 2)
},
(inString) => {
return shape.unsafeCast(JSON.parse(inString))
},
)
}
/**
* Create a File Helper for a .toml file
*/
static toml<A extends Record<string, unknown>>(
path: string,
shape: matches.Validator<unknown, A>,
) {
return new FileHelper<A>(
path,
(inData) => {
return TOML.stringify(inData as any)
},
(inString) => {
return shape.unsafeCast(TOML.parse(inString))
},
)
}
/**
* Create a File Helper for a .yaml file
*/
static yaml<A extends Record<string, unknown>>(
path: string,
shape: matches.Validator<unknown, A>,
) {
return new FileHelper<A>(
path,
(inData) => {
return YAML.stringify(inData, null, 2)
},
(inString) => {
return shape.unsafeCast(YAML.parse(inString))
},
)
}
}
export default FileHelper

4
sdk/package/lib/util/index.ts Normal file
View File

@@ -0,0 +1,4 @@
export * from "../../../base/lib/util"
export { GetSslCertificate } from "./GetSslCertificate"
export { hostnameInfoToAddress } from "../../../base/lib/util/Hostname"

203
sdk/package/lib/version/VersionGraph.ts Normal file
View File

@@ -0,0 +1,203 @@
import { ExtendedVersion, VersionRange } from "../../../base/lib/exver"
import * as T from "../../../base/lib/types"
import { Graph, Vertex, once } from "../util"
import { IMPOSSIBLE, VersionInfo } from "./VersionInfo"
export class VersionGraph<CurrentVersion extends string> {
private readonly graph: () => Graph<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>
private constructor(
readonly current: VersionInfo<CurrentVersion>,
versions: Array<VersionInfo<any>>,
) {
this.graph = once(() => {
const graph = new Graph<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>()
const flavorMap: Record<
string,
[
ExtendedVersion,
VersionInfo<any>,
Vertex<
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>,
][]
> = {}
for (let version of [current, ...versions]) {
const v = ExtendedVersion.parse(version.options.version)
const vertex = graph.addVertex(v, [], [])
const flavor = v.flavor || ""
if (!flavorMap[flavor]) {
flavorMap[flavor] = []
}
flavorMap[flavor].push([v, version, vertex])
}
for (let flavor in flavorMap) {
flavorMap[flavor].sort((a, b) => a[0].compareForSort(b[0]))
let prev:
| [
ExtendedVersion,
VersionInfo<any>,
Vertex<
ExtendedVersion | VersionRange,
(opts: { effects: T.Effects }) => Promise<void>
>,
]
| undefined = undefined
for (let [v, version, vertex] of flavorMap[flavor]) {
if (version.options.migrations.up !== IMPOSSIBLE) {
let range
if (prev) {
graph.addEdge(version.options.migrations.up, prev[2], vertex)
range = VersionRange.anchor(">=", prev[0]).and(
VersionRange.anchor("<", v),
)
} else {
range = VersionRange.anchor("<", v)
}
const vRange = graph.addVertex(range, [], [])
graph.addEdge(version.options.migrations.up, vRange, vertex)
}
if (version.options.migrations.down !== IMPOSSIBLE) {
let range
if (prev) {
graph.addEdge(version.options.migrations.down, vertex, prev[2])
range = VersionRange.anchor(">=", prev[0]).and(
VersionRange.anchor("<", v),
)
} else {
range = VersionRange.anchor("<", v)
}
const vRange = graph.addVertex(range, [], [])
graph.addEdge(version.options.migrations.down, vertex, vRange)
}
if (version.options.migrations.other) {
for (let rangeStr in version.options.migrations.other) {
const range = VersionRange.parse(rangeStr)
const vRange = graph.addVertex(range, [], [])
graph.addEdge(
version.options.migrations.other[rangeStr],
vRange,
vertex,
)
for (let matching of graph.findVertex(
(v) =>
v.metadata instanceof ExtendedVersion &&
v.metadata.satisfies(range),
)) {
graph.addEdge(
version.options.migrations.other[rangeStr],
matching,
vertex,
)
}
}
}
}
}
return graph
})
}
currentVersion = once(() =>
ExtendedVersion.parse(this.current.options.version),
)
/**
* Each exported `VersionInfo.of()` should be imported and provided as an argument to this function.
*
* ** The current version must be the FIRST argument. **
*/
static of<
CurrentVersion extends string,
OtherVersions extends Array<VersionInfo<any>>,
>(
currentVersion: VersionInfo<CurrentVersion>,
...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
) {
return new VersionGraph(currentVersion, other as Array<VersionInfo<any>>)
}
async migrate({
effects,
from,
to,
}: {
effects: T.Effects
from: ExtendedVersion
to: ExtendedVersion
}) {
const graph = this.graph()
if (from && to) {
const path = graph.shortestPath(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(from)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(from)),
(v) =>
(v.metadata instanceof VersionRange && v.metadata.satisfiedBy(to)) ||
(v.metadata instanceof ExtendedVersion && v.metadata.equals(to)),
)
if (path) {
for (let edge of path) {
if (edge.metadata) {
await edge.metadata({ effects })
}
await effects.setDataVersion({ version: edge.to.metadata.toString() })
}
return
}
}
throw new Error()
}
canMigrateFrom = once(() =>
Array.from(
this.graph().reverseBreadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
),
).reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
VersionRange.none(),
),
)
canMigrateTo = once(() =>
Array.from(
this.graph().breadthFirstSearch(
(v) =>
(v.metadata instanceof VersionRange &&
v.metadata.satisfiedBy(this.currentVersion())) ||
(v.metadata instanceof ExtendedVersion &&
v.metadata.equals(this.currentVersion())),
),
).reduce(
(acc, x) =>
acc.or(
x.metadata instanceof VersionRange
? x.metadata
: VersionRange.anchor("=", x.metadata),
),
VersionRange.none(),
),
)
}
// prettier-ignore
export type EnsureUniqueId<A, B = A, OtherVersions = never> =
B extends [] ? A :
B extends [VersionInfo<infer Version>, ...infer Rest] ? (
Version extends OtherVersions ? "One or more versions are not unique"[] :
EnsureUniqueId<A, Rest, Version | OtherVersions>
) : "There exists a migration that is not a Migration"[]

83
sdk/package/lib/version/VersionInfo.ts Normal file
View File

@@ -0,0 +1,83 @@
import { ValidateExVer } from "../../../base/lib/exver"
import * as T from "../../../base/lib/types"
export const IMPOSSIBLE = Symbol("IMPOSSIBLE")
export type VersionOptions<Version extends string> = {
/** The exver-compliant version number */
version: Version & ValidateExVer<Version>
/** The release notes for this version */
releaseNotes: string
/** Data migrations for this version */
migrations: {
/**
* A migration from the previous version. Leave empty to indicate no migration is necessary.
* Set to `IMPOSSIBLE` to indicate migrating from the previous version is not possible.
*/
up?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
/**
* A migration to the previous version. Leave blank to indicate no migration is necessary.
* Set to `IMPOSSIBLE` to indicate downgrades are prohibited
*/
down?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
/**
* Additional migrations, such as fast-forward migrations, or migrations from other flavors.
*/
other?: Record<string, (opts: { effects: T.Effects }) => Promise<void>>
}
}
export class VersionInfo<Version extends string> {
private _version: null | Version = null
private constructor(
readonly options: VersionOptions<Version> & { satisfies: string[] },
) {}
/**
* @description Use this function to define a new version of the service. By convention, each version should receive its own file.
* @property {string} version
* @property {string} releaseNotes
* @property {object} migrations
* @returns A VersionInfo class instance that is exported, then imported into versions/index.ts.
*/
static of<Version extends string>(options: VersionOptions<Version>) {
return new VersionInfo<Version>({ ...options, satisfies: [] })
}
/** Specify a version that this version is 100% backwards compatible to */
satisfies<V extends string>(
version: V & ValidateExVer<V>,
): VersionInfo<Version> {
return new VersionInfo({
...this.options,
satisfies: [...this.options.satisfies, version],
})
}
}
function __type_tests() {
const version: VersionInfo<"1.0.0:0"> = VersionInfo.of({
version: "1.0.0:0",
releaseNotes: "",
migrations: {},
})
.satisfies("#other:1.0.0:0")
.satisfies("#other:2.0.0:0")
// @ts-expect-error
.satisfies("#other:2.f.0:0")
let a: VersionInfo<"1.0.0:0"> = version
// @ts-expect-error
let b: VersionInfo<"1.0.0:3"> = version
VersionInfo.of({
// @ts-expect-error
version: "test",
releaseNotes: "",
migrations: {},
})
VersionInfo.of({
// @ts-expect-error
version: "test" as string,
releaseNotes: "",
migrations: {},
})
}

2
sdk/package/lib/version/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export * from "./VersionGraph"
export * from "./VersionInfo"