misc fixes (#2892)

* use docker for build steps that require linux when not on linux

* use fuse for overlay

* quiet mountpoint

* node 22

* misc fixes

* make shasum more compliant

* optimize download-base-image.sh with cleaner url handling and checksum verification

* fix script

* fixes #2900

* bump node and npm versions in web readme

* Minor pl.ts fixes

* fixes in response to synapse issues

* beta.8

* update ts-matches

* beta.11

* pl.ts finetuning

---------

Co-authored-by: Mariusz Kogen <k0gen@pm.me>
Co-authored-by: Matt Hill <mattnine@protonmail.com>
This commit is contained in:
Aiden McClelland
2025-04-28 17:33:41 -06:00
committed by GitHub
parent 05dd760388
commit 2adf34fbaf
41 changed files with 366 additions and 4136 deletions

View File

@@ -4,13 +4,13 @@ export { AcmeSettings } from "./AcmeSettings"
export { ActionId } from "./ActionId"
export { ActionInput } from "./ActionInput"
export { ActionMetadata } from "./ActionMetadata"
export { ActionRequest } from "./ActionRequest"
export { ActionRequestCondition } from "./ActionRequestCondition"
export { ActionRequestEntry } from "./ActionRequestEntry"
export { ActionRequestInput } from "./ActionRequestInput"
export { ActionRequestTrigger } from "./ActionRequestTrigger"
export { ActionRequest } from "./ActionRequest"
export { ActionResultMember } from "./ActionResultMember"
export { ActionResult } from "./ActionResult"
export { ActionResultMember } from "./ActionResultMember"
export { ActionResultV0 } from "./ActionResultV0"
export { ActionResultV1 } from "./ActionResultV1"
export { ActionResultValue } from "./ActionResultValue"
@@ -20,13 +20,13 @@ export { AddAdminParams } from "./AddAdminParams"
export { AddAssetParams } from "./AddAssetParams"
export { AddCategoryParams } from "./AddCategoryParams"
export { AddPackageParams } from "./AddPackageParams"
export { AddressInfo } from "./AddressInfo"
export { AddSslOptions } from "./AddSslOptions"
export { AddVersionParams } from "./AddVersionParams"
export { AddressInfo } from "./AddressInfo"
export { Alerts } from "./Alerts"
export { Algorithm } from "./Algorithm"
export { AllowedStatuses } from "./AllowedStatuses"
export { AllPackageData } from "./AllPackageData"
export { AllowedStatuses } from "./AllowedStatuses"
export { AlpnInfo } from "./AlpnInfo"
export { AnySignature } from "./AnySignature"
export { AnySigningKey } from "./AnySigningKey"
@@ -38,9 +38,9 @@ export { BackupTargetFS } from "./BackupTargetFS"
export { Base64 } from "./Base64"
export { BindId } from "./BindId"
export { BindInfo } from "./BindInfo"
export { BindingSetPublicParams } from "./BindingSetPublicParams"
export { BindOptions } from "./BindOptions"
export { BindParams } from "./BindParams"
export { BindingSetPublicParams } from "./BindingSetPublicParams"
export { Blake3Commitment } from "./Blake3Commitment"
export { BlockDev } from "./BlockDev"
export { BuildArg } from "./BuildArg"
@@ -61,11 +61,11 @@ export { CreateSubcontainerFsParams } from "./CreateSubcontainerFsParams"
export { CurrentDependencies } from "./CurrentDependencies"
export { CurrentDependencyInfo } from "./CurrentDependencyInfo"
export { DataUrl } from "./DataUrl"
export { DepInfo } from "./DepInfo"
export { Dependencies } from "./Dependencies"
export { DependencyKind } from "./DependencyKind"
export { DependencyMetadata } from "./DependencyMetadata"
export { DependencyRequirement } from "./DependencyRequirement"
export { DepInfo } from "./DepInfo"
export { Description } from "./Description"
export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams"
export { DeviceFilter } from "./DeviceFilter"
@@ -86,8 +86,8 @@ export { GetHostInfoParams } from "./GetHostInfoParams"
export { GetOsAssetParams } from "./GetOsAssetParams"
export { GetOsVersionParams } from "./GetOsVersionParams"
export { GetPackageParams } from "./GetPackageParams"
export { GetPackageResponseFull } from "./GetPackageResponseFull"
export { GetPackageResponse } from "./GetPackageResponse"
export { GetPackageResponseFull } from "./GetPackageResponseFull"
export { GetServiceInterfaceParams } from "./GetServiceInterfaceParams"
export { GetServicePortForwardParams } from "./GetServicePortForwardParams"
export { GetSslCertificateParams } from "./GetSslCertificateParams"
@@ -101,21 +101,21 @@ export { Governor } from "./Governor"
export { Guid } from "./Guid"
export { HardwareRequirements } from "./HardwareRequirements"
export { HealthCheckId } from "./HealthCheckId"
export { Host } from "./Host"
export { HostAddress } from "./HostAddress"
export { HostId } from "./HostId"
export { HostnameInfo } from "./HostnameInfo"
export { Hosts } from "./Hosts"
export { Host } from "./Host"
export { ImageConfig } from "./ImageConfig"
export { ImageId } from "./ImageId"
export { ImageMetadata } from "./ImageMetadata"
export { ImageSource } from "./ImageSource"
export { InitProgressRes } from "./InitProgressRes"
export { InstallParams } from "./InstallParams"
export { InstalledState } from "./InstalledState"
export { InstalledVersionParams } from "./InstalledVersionParams"
export { InstallingInfo } from "./InstallingInfo"
export { InstallingState } from "./InstallingState"
export { InstallParams } from "./InstallParams"
export { IpHostname } from "./IpHostname"
export { IpInfo } from "./IpInfo"
export { ListPackageSignersParams } from "./ListPackageSignersParams"
@@ -130,11 +130,11 @@ export { Manifest } from "./Manifest"
export { MaybeUtf8String } from "./MaybeUtf8String"
export { MebiBytes } from "./MebiBytes"
export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment"
export { Metrics } from "./Metrics"
export { MetricsCpu } from "./MetricsCpu"
export { MetricsDisk } from "./MetricsDisk"
export { MetricsGeneral } from "./MetricsGeneral"
export { MetricsMemory } from "./MetricsMemory"
export { Metrics } from "./Metrics"
export { MountParams } from "./MountParams"
export { MountTarget } from "./MountTarget"
export { NamedHealthCheckResult } from "./NamedHealthCheckResult"
@@ -146,14 +146,14 @@ export { NetworkInterfaceSetInboundParams } from "./NetworkInterfaceSetInboundPa
export { NetworkInterfaceType } from "./NetworkInterfaceType"
export { OnionHostname } from "./OnionHostname"
export { OsIndex } from "./OsIndex"
export { OsVersionInfoMap } from "./OsVersionInfoMap"
export { OsVersionInfo } from "./OsVersionInfo"
export { OsVersionInfoMap } from "./OsVersionInfoMap"
export { PackageDataEntry } from "./PackageDataEntry"
export { PackageDetailLevel } from "./PackageDetailLevel"
export { PackageId } from "./PackageId"
export { PackageIndex } from "./PackageIndex"
export { PackageInfoShort } from "./PackageInfoShort"
export { PackageInfo } from "./PackageInfo"
export { PackageInfoShort } from "./PackageInfoShort"
export { PackageSignerParams } from "./PackageSignerParams"
export { PackageState } from "./PackageState"
export { PackageVersionInfo } from "./PackageVersionInfo"
@@ -176,18 +176,18 @@ export { Security } from "./Security"
export { ServerInfo } from "./ServerInfo"
export { ServerSpecs } from "./ServerSpecs"
export { ServerStatus } from "./ServerStatus"
export { ServiceInterfaceId } from "./ServiceInterfaceId"
export { ServiceInterface } from "./ServiceInterface"
export { ServiceInterfaceId } from "./ServiceInterfaceId"
export { ServiceInterfaceType } from "./ServiceInterfaceType"
export { Session } from "./Session"
export { SessionList } from "./SessionList"
export { Sessions } from "./Sessions"
export { Session } from "./Session"
export { SetDataVersionParams } from "./SetDataVersionParams"
export { SetDependenciesParams } from "./SetDependenciesParams"
export { SetHealth } from "./SetHealth"
export { SetIconParams } from "./SetIconParams"
export { SetMainStatusStatus } from "./SetMainStatusStatus"
export { SetMainStatus } from "./SetMainStatus"
export { SetMainStatusStatus } from "./SetMainStatusStatus"
export { SetNameParams } from "./SetNameParams"
export { SetStoreParams } from "./SetStoreParams"
export { SetupExecuteParams } from "./SetupExecuteParams"
@@ -202,7 +202,7 @@ export { TestSmtpParams } from "./TestSmtpParams"
export { UnsetInboundParams } from "./UnsetInboundParams"
export { UpdatingState } from "./UpdatingState"
export { VerifyCifsParams } from "./VerifyCifsParams"
export { VersionSignerParams } from "./VersionSignerParams"
export { Version } from "./Version"
export { VersionSignerParams } from "./VersionSignerParams"
export { VolumeId } from "./VolumeId"
export { WifiInfo } from "./WifiInfo"

View File

@@ -207,3 +207,25 @@ export type DeepPartial<T> = T extends unknown[]
: T extends {}
? { [P in keyof T]?: DeepPartial<T[P]> }
: T
export type DeepWritable<T> = {
-readonly [K in keyof T]: T[K]
}
export function writable<T>(value: T): DeepWritable<T> {
return value
}
export type DeepReadonly<T> = {
readonly [P in keyof T]: DeepReadonly<T[P]>
}
export function readonly<T>(value: T): DeepReadonly<T> {
return value
}
export type AllowReadonly<T> =
| T
| {
readonly [P in keyof T]: AllowReadonly<T[P]>
}

View File

@@ -13,7 +13,7 @@
"deep-equality-data-structures": "^1.5.0",
"isomorphic-fetch": "^3.0.0",
"mime-types": "^3.0.1",
"ts-matches": "^6.2.1",
"ts-matches": "^6.3.2",
"yaml": "^2.7.1"
},
"devDependencies": {
@@ -4636,9 +4636,9 @@
}
},
"node_modules/ts-matches": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.2.1.tgz",
"integrity": "sha512-qdnMgTHsGCEGGK6QiaNMY2vD9eQtRp2Q+pAxcOAzxHJKDKTBYsc1ISTg1zp8H2+EmtCB0eko/1TwYUA5/mUGug==",
"version": "6.3.2",
"resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.3.2.tgz",
"integrity": "sha512-UhSgJymF8cLd4y0vV29qlKVCkQpUtekAaujXbQVc729FezS8HwqzepqvtjzQ3HboatIqN/Idor85O2RMwT7lIQ==",
"license": "MIT"
},
"node_modules/ts-morph": {

View File

@@ -26,7 +26,7 @@
"@noble/hashes": "^1.7.2",
"isomorphic-fetch": "^3.0.0",
"mime-types": "^3.0.1",
"ts-matches": "^6.2.1",
"ts-matches": "^6.3.2",
"yaml": "^2.7.1",
"deep-equality-data-structures": "^1.5.0"
},

View File

@@ -1170,7 +1170,7 @@ export async function runCommand<Manifest extends T.SDKManifest>(
.catch(() => "{}")
.then(JSON.parse)
commands = imageMeta.entrypoint ?? []
commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
commands = commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
} else commands = splitCommand(command)
return SubContainer.withTemp(
effects,

View File

@@ -21,7 +21,7 @@ export const runHealthScript = async <Manifest extends SDKManifest>(
} = {},
): Promise<HealthCheckResult> => {
const res = await Promise.race([
subcontainer.exec(runCommand),
subcontainer.execFail(runCommand),
timeoutPromise(timeout),
]).catch((e) => {
console.warn(errorMessage)

View File

@@ -42,19 +42,21 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
onStderr?: (chunk: Buffer | string | any) => void
},
) => {
let commands: string[]
if (command instanceof T.UseEntrypoint) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
commands = imageMeta.entrypoint ?? []
commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
} else commands = splitCommand(command)
try {
let commands: string[]
if (command instanceof T.UseEntrypoint) {
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
encoding: "utf8",
})
.catch(() => "{}")
.then(JSON.parse)
commands = imageMeta.entrypoint ?? []
commands = commands.concat(
...(command.overridCmd ?? imageMeta.cmd ?? []),
)
} else commands = splitCommand(command)
let childProcess: cp.ChildProcess
if (options.runAsInit) {
childProcess = await subcontainer.launch(commands, {
@@ -111,10 +113,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
get subContainerHandle() {
return new SubContainerHandle(this.subcontainer)
}
async wait({ timeout = NO_TIMEOUT } = {}) {
async wait({ timeout = NO_TIMEOUT, keepSubcontainer = false } = {}) {
if (timeout > 0)
setTimeout(() => {
this.term()
this.term({ keepSubcontainer })
}, timeout)
try {
return await this.runningAnswer
@@ -122,10 +124,14 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
if (!this.state.exited) {
this.process.kill("SIGKILL")
}
await this.subcontainer.destroy().catch((_) => {})
if (!keepSubcontainer) await this.subcontainer.destroy()
}
}
async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
async term({
signal = SIGTERM,
timeout = this.sigtermTimeout,
keepSubcontainer = false,
} = {}) {
try {
if (!this.state.exited) {
if (signal !== "SIGKILL") {
@@ -142,10 +148,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
await this.runningAnswer
} finally {
await this.subcontainer.destroy()
if (!keepSubcontainer) await this.subcontainer.destroy()
}
}
onDrop(): void {
this.term().catch(console.error)
this.term({ keepSubcontainer: true }).catch(console.error)
}
}

View File

@@ -21,7 +21,7 @@ export class Daemon<Manifest extends T.SDKManifest> {
return this.commandController?.subContainerHandle
}
static of<Manifest extends T.SDKManifest>() {
return async <A extends string>(
return async (
effects: T.Effects,
subcontainer: SubContainer<Manifest>,
command: T.CommandType,
@@ -54,17 +54,21 @@ export class Daemon<Manifest extends T.SDKManifest> {
}
this.shouldBeRunning = true
let timeoutCounter = 0
new Promise(async () => {
;(async () => {
while (this.shouldBeRunning) {
if (this.commandController)
await this.commandController.term().catch((err) => console.error(err))
await this.commandController
.term({ keepSubcontainer: true })
.catch((err) => console.error(err))
this.commandController = await this.startCommand()
await this.commandController.wait().catch((err) => console.error(err))
await this.commandController
.wait({ keepSubcontainer: true })
.catch((err) => console.error(err))
await new Promise((resolve) => setTimeout(resolve, timeoutCounter))
timeoutCounter += TIMEOUT_INCREMENT_MS
timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
timeoutCounter = Math.min(MAX_TIMEOUT_MS, timeoutCounter)
}
}).catch((err) => {
})().catch((err) => {
console.error(asError(err))
})
}

View File

@@ -197,7 +197,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
async build() {
for (const daemon of this.healthDaemons) {
await daemon.updateStatus()
await daemon.init()
}
for (const health of this.healthChecks) {
health.start()

View File

@@ -169,4 +169,16 @@ export class HealthDaemon<Manifest extends SDKManifest> {
const healths = this.dependencies.map((d) => d.running && d._health)
this.changeRunning(healths.every((x) => x && x.result === "success"))
}
async init() {
if (this.ready.display) {
this.effects.setHealth({
id: this.id,
message: null,
name: this.ready.display,
result: "starting",
})
}
await this.updateStatus()
}
}

View File

@@ -10,9 +10,20 @@ export abstract class Drop {
protected constructor() {
this.id = Drop.idCtr++
this.ref = { id: this.id }
Drop.weak[this.id] = this.weak()
Drop.registry.register(this, this.id, this)
const weak = this.weak()
Drop.weak[this.id] = weak
Drop.registry.register(this.ref, this.id, this.ref)
return new Proxy(this, {
set(target: any, prop, value) {
if (prop === "ref") return false
target[prop] = value
;(weak as any)[prop] = value
return true
},
})
}
protected register() {}
protected weak(): this {
const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
weak.ref = new WeakRef(this.ref)
@@ -21,7 +32,7 @@ export abstract class Drop {
abstract onDrop(): void
drop(): void {
this.onDrop()
Drop.registry.unregister(this)
Drop.registry.unregister(this.ref)
delete Drop.weak[this.id]
}
}

View File

@@ -37,6 +37,11 @@ export interface ExecSpawnable {
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<ExecResults>
execFail(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }>
spawn(
command: string[],
options?: CommandOptions & StdioOptions,
@@ -88,7 +93,9 @@ export class SubContainer<
imageId: this.imageId,
rootfs: this.rootfs,
})
reject(new Error(`Failed to start subcontainer ${this.imageId}`))
return reject(
new Error(`Failed to start subcontainer ${this.imageId}`),
)
}
await wait(1)
}
@@ -144,8 +151,9 @@ export class SubContainer<
}
return res
} finally {
} catch (e) {
await res.destroy()
throw e
}
}
@@ -272,11 +280,13 @@ export class SubContainer<
}
onDrop(): void {
console.log(`Cleaning up dangling subcontainer ${this.guid}`)
this.destroy()
}
/**
* @description run a command inside this subcontainer
* DOES NOT THROW ON NONZERO EXIT CODE (see execFail)
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
@@ -287,6 +297,7 @@ export class SubContainer<
options?: CommandOptions & ExecOptions,
timeoutMs: number | null = 30000,
): Promise<{
throw: () => { stdout: string | Buffer; stderr: string | Buffer }
exitCode: number | null
exitSignal: NodeJS.Signals | null
stdout: string | Buffer
@@ -367,16 +378,43 @@ export class SubContainer<
child.stderr.on("data", appendData(stderr))
child.on("exit", (code, signal) => {
clearTimeout(killTimeout)
resolve({
const result = {
exitCode: code,
exitSignal: signal,
stdout: stdout.data,
stderr: stderr.data,
}
resolve({
throw: () =>
!code && !signal
? { stdout: stdout.data, stderr: stderr.data }
: (() => {
throw new ExitError(command[0], result)
})(),
...result,
})
})
})
}
/**
* @description run a command inside this subcontainer, throwing on non-zero exit status
* @param commands an array representing the command and args to execute
* @param options
* @param timeoutMs how long to wait before killing the command in ms
* @returns
*/
async execFail(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs: number | null = 30000,
): Promise<{
stdout: string | Buffer
stderr: string | Buffer
}> {
return this.exec(command, options, timeoutMs).then((res) => res.throw())
}
async launch(
command: string[],
options?: CommandOptions,
@@ -478,6 +516,15 @@ export class SubContainerHandle implements ExecSpawnable {
): Promise<ExecResults> {
return this.subContainer.exec(command, options, timeoutMs)
}
execFail(
command: string[],
options?: CommandOptions & ExecOptions,
timeoutMs?: number | null,
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
return this.subContainer.execFail(command, options, timeoutMs)
}
spawn(
command: string[],
options: CommandOptions & StdioOptions = { stdio: "inherit" },

View File

@@ -234,7 +234,7 @@ export class FileHelper<A> {
/**
* Accepts full structured data and overwrites the existing file on disk if it exists.
*/
async write(effects: T.Effects, data: A) {
async write(effects: T.Effects, data: T.AllowReadonly<A> | A) {
await this.writeFile(this.validate(data))
if (effects.constRetry && this.consts.includes(effects.constRetry))
throw new Error(`Canceled: write after const: ${this.path}`)
@@ -244,7 +244,7 @@ export class FileHelper<A> {
/**
* Accepts partial structured data and performs a merge with the existing file on disk.
*/
async merge(effects: T.Effects, data: T.DeepPartial<A>) {
async merge(effects: T.Effects, data: T.AllowReadonly<T.DeepPartial<A>>) {
const fileDataRaw = await this.readFileRaw()
let fileData: any = fileDataRaw === null ? null : this.readData(fileDataRaw)
try {

View File

@@ -1,12 +1,12 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.6",
"version": "0.4.0-beta.11",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.6",
"version": "0.4.0-beta.11",
"license": "MIT",
"dependencies": {
"@iarna/toml": "^3.0.0",
@@ -16,7 +16,7 @@
"ini": "^5.0.0",
"isomorphic-fetch": "^3.0.0",
"mime-types": "^3.0.1",
"ts-matches": "^6.2.1",
"ts-matches": "^6.3.2",
"yaml": "^2.7.1"
},
"devDependencies": {
@@ -4820,9 +4820,9 @@
}
},
"node_modules/ts-matches": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.2.1.tgz",
"integrity": "sha512-qdnMgTHsGCEGGK6QiaNMY2vD9eQtRp2Q+pAxcOAzxHJKDKTBYsc1ISTg1zp8H2+EmtCB0eko/1TwYUA5/mUGug==",
"version": "6.3.2",
"resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-6.3.2.tgz",
"integrity": "sha512-UhSgJymF8cLd4y0vV29qlKVCkQpUtekAaujXbQVc729FezS8HwqzepqvtjzQ3HboatIqN/Idor85O2RMwT7lIQ==",
"license": "MIT"
},
"node_modules/ts-morph": {

View File

@@ -1,6 +1,6 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.6",
"version": "0.4.0-beta.11",
"description": "Software development kit to facilitate packaging services for StartOS",
"main": "./package/lib/index.js",
"types": "./package/lib/index.d.ts",
@@ -33,7 +33,7 @@
"dependencies": {
"isomorphic-fetch": "^3.0.0",
"mime-types": "^3.0.1",
"ts-matches": "^6.2.1",
"ts-matches": "^6.3.2",
"yaml": "^2.7.1",
"deep-equality-data-structures": "^2.0.0",
"ini": "^5.0.0",