update/alpha.9 (#2988)

* import marketplac preview for sideload

* fix: improve state service (#2977)

* fix: fix sideload DI

* fix: update Angular

* fix: cleanup

* fix: fix version selection

* Bump node version to fix build for Angular

* misc fixes
- update node to v22
- fix chroot-and-upgrade access to prune-images
- don't self-migrate legacy packages
- #2985
- move dataVersion to volume folder
- remove "instructions.md" from s9pk
- add "docsUrl" to manifest

* version bump

* include flavor when clicking view listing from updates tab

* closes #2980

* fix: fix select button

* bring back ssh keys

* fix: drop 'portal' from all routes

* fix: implement longtap action to select table rows

* fix description for ssh page

* replace instructions with docsLink and refactor marketplace preview

* delete unused translations

* fix patchdb diffing algorithm

* continue refactor of marketplace lib show components

* Booting StartOS instead of Setting up your server on init

* misc fixes
- closes #2990
- closes #2987

* fix build

* docsUrl and clickable service headers

* don't cleanup after update until new service install succeeds

* update types

* misc fixes

* beta.35

* sdkversion, githash for sideload, correct logs for init, startos pubkey display

* bring back reboot button on install

* misc fixes

* beta.36

* better handling of setup and init for websocket errors

* reopen init and setup logs even on graceful closure

* better logging, misc fixes

* fix build

* dont let package stats hang

* dont show docsurl in marketplace if no docsurl

* re-add needs-config

* show error if init fails, shorten hover state on header icons

* fix operator precedemce

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>
Co-authored-by: Mariusz Kogen <k0gen@pm.me>
This commit is contained in:
Aiden McClelland
2025-07-18 18:31:12 +00:00
committed by GitHub
parent ba2906a42e
commit 377b7b12ce
237 changed files with 5953 additions and 4777 deletions

View File

@@ -59,8 +59,9 @@ import {
setupOnInit,
setupOnUninit,
} from "../../base/lib/inits"
import { DropGenerator } from "../../base/lib/util/Drop"
export const OSVersion = testTypeVersion("0.4.0-alpha.8")
export const OSVersion = testTypeVersion("0.4.0-alpha.9")
// prettier-ignore
type AnyNeverCond<T extends any[], Then, Else> =
@@ -191,12 +192,13 @@ export class StartSdk<Manifest extends T.SDKManifest> {
"callback"
> = {},
) => {
async function* watch() {
async function* watch(abort?: AbortSignal) {
const resolveCell = { resolve: () => {} }
effects.onLeaveContext(() => {
resolveCell.resolve()
})
while (effects.isInContext) {
abort?.addEventListener("abort", () => resolveCell.resolve())
while (effects.isInContext && !abort?.aborted) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
@@ -215,17 +217,26 @@ export class StartSdk<Manifest extends T.SDKManifest> {
(() => effects.constRetry && effects.constRetry()),
}),
once: () => effects.getContainerIp(options),
watch,
watch: (abort?: AbortSignal) => {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
return DropGenerator.of(watch(ctrl.signal), () => ctrl.abort())
},
onChange: (
callback: (
value: string | null,
error?: Error,
) => void | Promise<void>,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) => {
;(async () => {
for await (const value of watch()) {
const ctrl = new AbortController()
for await (const value of watch(ctrl.signal)) {
try {
await callback(value)
const res = await callback(value)
if (res.cancel) {
ctrl.abort()
break
}
} catch (e) {
console.error(
"callback function threw an error @ getContainerIp.onChange",

View File

@@ -35,7 +35,7 @@ export class CommandController<
const abort = new AbortController()
const cell: { ctrl: CommandController<Manifest, C> } = {
ctrl: new CommandController<Manifest, C>(
exec.fn(subcontainer, abort).then(async (command) => {
exec.fn(subcontainer, abort.signal).then(async (command) => {
if (subcontainer && command && !abort.signal.aborted) {
const newCtrl = (
await CommandController.of<

View File

@@ -49,7 +49,11 @@ export class Daemon<
(subc?.rc() ?? null) as C,
exec,
)
return new Daemon(subc, startCommand)
const res = new Daemon(subc, startCommand)
effects.onLeaveContext(() => {
res.stop().catch((e) => console.error(asError(e)))
})
return res
}
}
async start() {
@@ -66,6 +70,11 @@ export class Daemon<
.catch((err) => console.error(err))
try {
this.commandController = await this.startCommand()
if (!this.shouldBeRunning) {
// handles race condition if stopped while starting
await this.stop()
break
}
const success = await this.commandController.wait().then(
(_) => true,
(err) => {
@@ -107,12 +116,14 @@ export class Daemon<
}) {
this.shouldBeRunning = false
this.exitedSuccess = false
await this.commandController
?.term({ ...termOptions })
.catch((e) => console.error(asError(e)))
this.commandController = null
this.onExitFns = []
await this.subcontainer?.destroy()
if (this.commandController) {
await this.commandController
.term({ ...termOptions })
.catch((e) => console.error(asError(e)))
this.commandController = null
this.onExitFns = []
await this.subcontainer?.destroy()
}
}
subcontainerRc(): SubContainerRc<Manifest> | null {
return this.subcontainer?.rc() ?? null

View File

@@ -17,6 +17,7 @@ import { CommandController } from "./CommandController"
import { HealthCheck } from "../health/HealthCheck"
import { Oneshot } from "./Oneshot"
import { Manifest } from "../test/output.sdk"
import { asError } from "../util"
export const cpExec = promisify(CP.exec)
export const cpExecFile = promisify(CP.execFile)
@@ -69,7 +70,7 @@ export type ExecFnOptions<
> = {
fn: (
subcontainer: C,
abort: AbortController,
abort: AbortSignal,
) => Promise<C extends null ? null : ExecCommandOptions | null>
// Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
sigtermTimeout?: number
@@ -379,6 +380,9 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
}
async build() {
this.effects.onLeaveContext(() => {
this.term().catch((e) => console.error(asError(e)))
})
for (const daemon of this.healthDaemons) {
await daemon.init()
}

View File

@@ -54,6 +54,7 @@ export function buildManifest<
)
return {
...manifest,
gitHash: null,
osVersion: manifest.osVersion ?? OSVersion,
sdkVersion,
version: versions.current.options.version,

View File

@@ -396,14 +396,13 @@ describe("values", () => {
supportSite: "",
marketingSite: "",
donationUrl: null,
docsUrl: "",
description: {
short: "",
long: "",
},
containers: {},
images: {},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,

View File

@@ -9,17 +9,16 @@ export const sdk = StartSdk.of()
id: "testOutput",
title: "",
license: "",
replaces: [],
wrapperRepo: "",
upstreamRepo: "",
supportSite: "",
marketingSite: "",
donationUrl: null,
docsUrl: "",
description: {
short: "",
long: "",
},
containers: {},
images: {
main: {
source: {
@@ -30,7 +29,6 @@ export const sdk = StartSdk.of()
},
},
volumes: [],
assets: [],
alerts: {
install: null,
update: null,

View File

@@ -1,46 +0,0 @@
export abstract class Drop {
private static weak: { [id: number]: Drop } = {}
private static registry = new FinalizationRegistry((id: number) => {
const weak = Drop.weak[id]
if (weak) weak.drop()
})
private static idCtr: number = 0
private dropId?: number
private dropRef?: { id: number } | WeakRef<{ id: number }>
protected constructor() {
this.dropId = Drop.idCtr++
this.dropRef = { id: this.dropId }
const weak = this.weak()
Drop.weak[this.dropId] = weak
Drop.registry.register(this.dropRef, this.dropId, this.dropRef)
return new Proxy(this, {
set(target: any, prop, value) {
if (prop === "dropRef" || prop == "dropId") return false
target[prop] = value
;(weak as any)[prop] = value
return true
},
})
}
protected register() {}
protected weak(): this {
const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
if (this.dropRef) weak.ref = new WeakRef(this.dropRef)
return weak
}
abstract onDrop(): void
drop(): void {
if (!this.dropRef || !this.dropId) return
this.onDrop()
this.leak()
}
leak(): this {
if (!this.dropRef || !this.dropId) return this
Drop.registry.unregister(this.dropRef)
delete Drop.weak[this.dropId]
delete this.dropRef
delete this.dropId
return this
}
}

View File

@@ -1,5 +1,6 @@
import { T } from ".."
import { Effects } from "../../../base/lib/Effects"
import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
export class GetSslCertificate {
constructor(
@@ -30,15 +31,13 @@ export class GetSslCertificate {
})
}
/**
* Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
*/
async *watch() {
private async *watchGen(abort?: AbortSignal) {
const resolveCell = { resolve: () => {} }
this.effects.onLeaveContext(() => {
resolveCell.resolve()
})
while (this.effects.isInContext) {
abort?.addEventListener("abort", () => resolveCell.resolve())
while (this.effects.isInContext && !abort?.aborted) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
@@ -53,6 +52,17 @@ export class GetSslCertificate {
}
}
/**
* Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
*/
watch(
abort?: AbortSignal,
): AsyncGenerator<[string, string, string], void, unknown> {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
}
/**
* Watches the SSL Certificate for the given hostnames if permitted. Takes a custom callback function to run whenever it changes
*/
@@ -60,12 +70,17 @@ export class GetSslCertificate {
callback: (
value: [string, string, string] | null,
error?: Error,
) => void | Promise<void>,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) {
;(async () => {
for await (const value of this.watch()) {
const ctrl = new AbortController()
for await (const value of this.watch(ctrl.signal)) {
try {
await callback(value)
const res = await callback(value)
if (res.cancel) {
ctrl.abort()
break
}
} catch (e) {
console.error(
"callback function threw an error @ GetSslCertificate.onChange",
@@ -86,28 +101,20 @@ export class GetSslCertificate {
/**
* Watches the SSL Certificate for the given hostnames if permitted. Returns when the predicate is true
*/
async waitFor(pred: (value: [string, string, string] | null) => boolean) {
const resolveCell = { resolve: () => {} }
this.effects.onLeaveContext(() => {
resolveCell.resolve()
})
while (this.effects.isInContext) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
resolveCell.resolve = resolve
})
const res = await this.effects.getSslCertificate({
hostnames: this.hostnames,
algorithm: this.algorithm,
callback: () => callback(),
})
if (pred(res)) {
resolveCell.resolve()
return res
}
await waitForNext
}
return null
waitFor(
pred: (value: [string, string, string] | null) => boolean,
): Promise<[string, string, string] | null> {
const ctrl = new AbortController()
return DropPromise.of(
Promise.resolve().then(async () => {
for await (const next of this.watchGen(ctrl.signal)) {
if (pred(next)) {
return next
}
}
return null
}),
() => ctrl.abort(),
)
}
}

View File

@@ -4,7 +4,7 @@ import * as cp from "child_process"
import { promisify } from "util"
import { Buffer } from "node:buffer"
import { once } from "../../../base/lib/util/once"
import { Drop } from "./Drop"
import { Drop } from "../../../base/lib/util/Drop"
import { Mounts } from "../mainFn/Mounts"
import { BackupEffects } from "../backup/Backups"

View File

@@ -4,10 +4,38 @@ import * as TOML from "@iarna/toml"
import * as INI from "ini"
import * as T from "../../../base/lib/types"
import * as fs from "node:fs/promises"
import { asError, partialDiff } from "../../../base/lib/util"
import { asError } from "../../../base/lib/util"
import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
const previousPath = /(.+?)\/([^/]*)$/
const deepEq = (left: unknown, right: unknown) => {
if (left === right) return true
if (Array.isArray(left) && Array.isArray(right)) {
if (left.length === right.length) {
for (const idx in left) {
if (!deepEq(left[idx], right[idx])) return false
}
return true
}
} else if (
typeof left === "object" &&
typeof right === "object" &&
left &&
right
) {
const keys = new Set<keyof typeof left | keyof typeof right>([
...(Object.keys(left) as (keyof typeof left)[]),
...(Object.keys(right) as (keyof typeof right)[]),
])
for (let key of keys) {
if (!deepEq(left[key], right[key])) return false
}
return true
}
return false
}
const exists = (path: string) =>
fs.access(path).then(
() => true,
@@ -99,10 +127,16 @@ type Validator<T, U> = matches.Validator<T, U> | matches.Validator<unknown, U>
type ReadType<A> = {
once: () => Promise<A | null>
const: (effects: T.Effects) => Promise<A | null>
watch: (effects: T.Effects) => AsyncGenerator<A | null, null, unknown>
watch: (
effects: T.Effects,
abort?: AbortSignal,
) => AsyncGenerator<A | null, null, unknown>
onChange: (
effects: T.Effects,
callback: (value: A | null, error?: Error) => void | Promise<void>,
callback: (
value: A | null,
error?: Error,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) => void
waitFor: (
effects: T.Effects,
@@ -151,7 +185,12 @@ type ReadType<A> = {
* ```
*/
export class FileHelper<A> {
private consts: (() => void)[] = []
private consts: [
() => void,
any,
(a: any) => any,
(left: any, right: any) => any,
][] = []
protected constructor(
readonly path: string,
readonly writeData: (dataIn: A) => string,
@@ -209,10 +248,15 @@ export class FileHelper<A> {
const watch = this.readWatch(effects, map, eq)
const res = await watch.next()
if (effects.constRetry) {
if (!this.consts.includes(effects.constRetry))
this.consts.push(effects.constRetry)
const record: (typeof this.consts)[number] = [
effects.constRetry,
res.value,
map,
eq,
]
this.consts.push(record)
watch.next().then(() => {
this.consts = this.consts.filter((a) => a === effects.constRetry)
this.consts = this.consts.filter((r) => r !== record)
effects.constRetry && effects.constRetry()
})
}
@@ -223,11 +267,13 @@ export class FileHelper<A> {
effects: T.Effects,
map: (value: A) => B,
eq: (left: B | null | undefined, right: B | null) => boolean,
abort?: AbortSignal,
) {
let res
while (effects.isInContext) {
while (effects.isInContext && !abort?.aborted) {
if (await exists(this.path)) {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
const watch = fs.watch(this.path, {
persistent: false,
signal: ctrl.signal,
@@ -254,14 +300,19 @@ export class FileHelper<A> {
private readOnChange<B>(
effects: T.Effects,
callback: (value: B | null, error?: Error) => void | Promise<void>,
callback: (
value: B | null,
error?: Error,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
map: (value: A) => B,
eq: (left: B | null | undefined, right: B | null) => boolean,
) {
;(async () => {
for await (const value of this.readWatch(effects, map, eq)) {
const ctrl = new AbortController()
for await (const value of this.readWatch(effects, map, eq, ctrl.signal)) {
try {
await callback(value)
const res = await callback(value)
if (res.cancel) ctrl.abort()
} catch (e) {
console.error(
"callback function threw an error @ FileHelper.read.onChange",
@@ -279,38 +330,36 @@ export class FileHelper<A> {
)
}
private async readWaitFor<B>(
private readWaitFor<B>(
effects: T.Effects,
pred: (value: B | null, error?: Error) => boolean,
map: (value: A) => B,
): Promise<B | null> {
while (effects.isInContext) {
if (await exists(this.path)) {
const ctrl = new AbortController()
const watch = fs.watch(this.path, {
persistent: false,
signal: ctrl.signal,
})
const newRes = await this.readOnce(map)
const listen = Promise.resolve()
.then(async () => {
for await (const _ of watch) {
const ctrl = new AbortController()
return DropPromise.of(
Promise.resolve().then(async () => {
const watch = this.readWatch(effects, map, (_) => false, ctrl.signal)
while (true) {
try {
const res = await watch.next()
if (pred(res.value)) {
ctrl.abort()
return null
return res.value
}
})
.catch((e) => console.error(asError(e)))
if (pred(newRes)) {
ctrl.abort()
return newRes
if (res.done) {
break
}
} catch (e) {
if (pred(null, e as Error)) {
break
}
}
}
await listen
} else {
if (pred(null)) return null
await onCreated(this.path).catch((e) => console.error(asError(e)))
}
}
return null
ctrl.abort()
return null
}),
() => ctrl.abort(),
)
}
read(): ReadType<A>
@@ -323,14 +372,24 @@ export class FileHelper<A> {
eq?: (left: any, right: any) => boolean,
): ReadType<any> {
map = map ?? ((a: A) => a)
eq = eq ?? ((left: any, right: any) => !partialDiff(left, right))
eq = eq ?? deepEq
return {
once: () => this.readOnce(map),
const: (effects: T.Effects) => this.readConst(effects, map, eq),
watch: (effects: T.Effects) => this.readWatch(effects, map, eq),
watch: (effects: T.Effects, abort?: AbortSignal) => {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
return DropGenerator.of(
this.readWatch(effects, map, eq, ctrl.signal),
() => ctrl.abort(),
)
},
onChange: (
effects: T.Effects,
callback: (value: A | null, error?: Error) => void | Promise<void>,
callback: (
value: A | null,
error?: Error,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) => this.readOnChange(effects, callback, map, eq),
waitFor: (effects: T.Effects, pred: (value: A | null) => boolean) =>
this.readWaitFor(effects, pred, map),
@@ -345,13 +404,17 @@ export class FileHelper<A> {
data: T.AllowReadonly<A> | A,
options: { allowWriteAfterConst?: boolean } = {},
) {
await this.writeFile(this.validate(data))
if (
!options.allowWriteAfterConst &&
effects.constRetry &&
this.consts.includes(effects.constRetry)
)
throw new Error(`Canceled: write after const: ${this.path}`)
const newData = this.validate(data)
await this.writeFile(newData)
if (!options.allowWriteAfterConst && effects.constRetry) {
const records = this.consts.filter(([c]) => c === effects.constRetry)
for (const record of records) {
const [_, prev, map, eq] = record
if (!eq(prev, map(newData))) {
throw new Error(`Canceled: write after const: ${this.path}`)
}
}
}
return null
}
@@ -372,16 +435,14 @@ export class FileHelper<A> {
const toWrite = this.writeData(mergeData)
if (toWrite !== fileDataRaw) {
this.writeFile(mergeData)
if (
!options.allowWriteAfterConst &&
effects.constRetry &&
this.consts.includes(effects.constRetry)
) {
const diff = partialDiff(fileData, mergeData as any)
if (!diff) {
return null
if (!options.allowWriteAfterConst && effects.constRetry) {
const records = this.consts.filter(([c]) => c === effects.constRetry)
for (const record of records) {
const [_, prev, map, eq] = record
if (!eq(prev, map(mergeData))) {
throw new Error(`Canceled: write after const: ${this.path}`)
}
}
throw new Error(`Canceled: write after const: ${this.path}`)
}
}
return null

View File

@@ -2,4 +2,4 @@ export * from "../../../base/lib/util"
export { GetSslCertificate } from "./GetSslCertificate"
export { hostnameInfoToAddress } from "../../../base/lib/util/Hostname"
export { Drop } from "./Drop"
export { Drop } from "../../../base/lib/util/Drop"

View File

@@ -58,6 +58,9 @@ export class VersionGraph<CurrentVersion extends string>
ExtendedVersion | VersionRange,
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>
dump(): string {
return this.graph().dump((metadata) => metadata?.toString())
}
private constructor(
readonly current: VersionInfo<CurrentVersion>,
versions: Array<VersionInfo<any>>,
@@ -97,7 +100,7 @@ export class VersionGraph<CurrentVersion extends string>
VersionInfo<any>,
Vertex<
ExtendedVersion | VersionRange,
(opts: { effects: T.Effects }) => Promise<void>
((opts: { effects: T.Effects }) => Promise<void>) | undefined
>,
]
| undefined = undefined
@@ -146,6 +149,8 @@ export class VersionGraph<CurrentVersion extends string>
}
}
}
prev = [v, version, vertex]
}
}
return graph
@@ -198,6 +203,23 @@ export class VersionGraph<CurrentVersion extends string>
(v) => overlaps(v.metadata, to),
)
if (path) {
console.log(
`Migrating ${
path.reduce<{ acc: string; prev: string | null }>(
({ acc, prev }, x) => ({
acc:
acc +
(prev && prev != x.from.metadata.toString()
? ` (as ${prev})`
: "") +
" -> " +
x.to.metadata.toString(),
prev: x.to.metadata.toString(),
}),
{ acc: from.toString(), prev: null },
).acc
}`,
)
let dataVersion = from
for (let edge of path) {
if (edge.metadata) {

View File

@@ -1,7 +1,7 @@
import { ValidateExVer } from "../../../base/lib/exver"
import * as T from "../../../base/lib/types"
export const IMPOSSIBLE = Symbol("IMPOSSIBLE")
export const IMPOSSIBLE: unique symbol = Symbol("IMPOSSIBLE")
export type VersionOptions<Version extends string> = {
/** The exver-compliant version number */