update/alpha.9 (#2988)

* import marketplac preview for sideload

* fix: improve state service (#2977)

* fix: fix sideload DI

* fix: update Angular

* fix: cleanup

* fix: fix version selection

* Bump node version to fix build for Angular

* misc fixes
- update node to v22
- fix chroot-and-upgrade access to prune-images
- don't self-migrate legacy packages
- #2985
- move dataVersion to volume folder
- remove "instructions.md" from s9pk
- add "docsUrl" to manifest

* version bump

* include flavor when clicking view listing from updates tab

* closes #2980

* fix: fix select button

* bring back ssh keys

* fix: drop 'portal' from all routes

* fix: implement longtap action to select table rows

* fix description for ssh page

* replace instructions with docsLink and refactor marketplace preview

* delete unused translations

* fix patchdb diffing algorithm

* continue refactor of marketplace lib show components

* Booting StartOS instead of Setting up your server on init

* misc fixes
- closes #2990
- closes #2987

* fix build

* docsUrl and clickable service headers

* don't cleanup after update until new service install succeeds

* update types

* misc fixes

* beta.35

* sdkversion, githash for sideload, correct logs for init, startos pubkey display

* bring back reboot button on install

* misc fixes

* beta.36

* better handling of setup and init for websocket errors

* reopen init and setup logs even on graceful closure

* better logging, misc fixes

* fix build

* dont let package stats hang

* dont show docsurl in marketplace if no docsurl

* re-add needs-config

* show error if init fails, shorten hover state on header icons

* fix operator precedemce

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>
Co-authored-by: Mariusz Kogen <k0gen@pm.me>

sdk/base/lib/osBindings/Manifest.ts

@@ -24,13 +24,14 @@ export type Manifest = {
   supportSite: string
   marketingSite: string
   donationUrl: string | null
+  docsUrl: string | null
   description: Description
   images: { [key: ImageId]: ImageConfig }
   volumes: Array<VolumeId>
   alerts: Alerts
   dependencies: Dependencies
   hardwareRequirements: HardwareRequirements
-  gitHash?: GitHash
+  gitHash: GitHash | null
   osVersion: string
   sdkVersion: string | null
 }

sdk/base/lib/osBindings/PackageDataEntry.ts

@@ -12,7 +12,7 @@ import type { TaskEntry } from "./TaskEntry"
 
 export type PackageDataEntry = {
   stateInfo: PackageState
-  dataVersion: string | null
+  s9pk: string
   status: MainStatus
   registry: string | null
   developerKey: string

sdk/base/lib/osBindings/PackageVersionInfo.ts

@@ -21,9 +21,11 @@ export type PackageVersionInfo = {
   supportSite: string
   marketingSite: string
   donationUrl: string | null
+  docsUrl: string | null
   alerts: Alerts
   dependencyMetadata: { [key: PackageId]: DependencyMetadata }
   osVersion: string
+  sdkVersion: string | null
   hardwareRequirements: HardwareRequirements
   sourceVersion: string | null
   s9pk: RegistryAsset<MerkleArchiveCommitment>

sdk/base/lib/osBindings/ServerInfo.ts

@@ -12,7 +12,7 @@ export type ServerInfo = {
   hostname: string
   version: string
   packageVersionCompat: string
-  postInitMigrationTodos: string[]
+  postInitMigrationTodos: Record<string, unknown>
   lastBackup: string | null
   network: NetworkInfo
   statusInfo: ServerStatus

sdk/base/lib/osBindings/UpdatingState.ts

@@ -4,5 +4,6 @@ import type { Manifest } from "./Manifest"
 
 export type UpdatingState = {
   manifest: Manifest
+  s9pk: string
   installingInfo: InstallingInfo
 }

sdk/base/lib/s9pk/index.ts

@@ -117,17 +117,10 @@ export class S9pk {
     )
   }
 
-  async instructions(): Promise<string> {
-    const file = this.archive.contents.getPath(["instructions.md"])
-    if (!file || !(file.contents instanceof FileContents))
-      throw new Error("instructions.md not found in archive")
-    return new TextDecoder().decode(await file.verifiedFileContents())
-  }
-
   async license(): Promise<string> {
     const file = this.archive.contents.getPath(["LICENSE.md"])
     if (!file || !(file.contents instanceof FileContents))
-      throw new Error("instructions.md not found in archive")
+      throw new Error("license.md not found in archive")
     return new TextDecoder().decode(await file.verifiedFileContents())
   }
 }

sdk/base/lib/types/ManifestTypes.ts

@@ -42,6 +42,10 @@ export type SDKManifest = {
    * @example `https://nextcloud.com/contribute/`
    */
   readonly donationUrl: string | null
+  /**
+   * URL where users can find instructions on how to use the service
+   */
+  readonly docsUrl: string
   readonly description: {
     /** Short description to display on the marketplace list page. Max length 80 chars. */
     readonly short: string

sdk/base/lib/util/Drop.ts

@@ -0,0 +1,153 @@
+const dropId: unique symbol = Symbol("id")
+export type DropRef = { [dropId]: number }
+
+export abstract class Drop {
+  private static weak: { [id: number]: Drop } = {}
+  private static registry = new FinalizationRegistry((id: number) => {
+    const weak = Drop.weak[id]
+    if (weak) weak.drop()
+  })
+  private static idCtr: number = 0
+  private dropId?: number
+  private dropRef?: DropRef | WeakRef<DropRef>
+  protected constructor() {
+    this.dropId = Drop.idCtr++
+    this.dropRef = { [dropId]: this.dropId }
+    const weak = this.weak()
+    Drop.weak[this.dropId] = weak
+    Drop.registry.register(this.dropRef, this.dropId, this.dropRef)
+
+    return new Proxy(this, {
+      set(target: any, prop, value) {
+        if (prop === "dropRef" || prop == "dropId") return false
+        target[prop] = value
+        ;(weak as any)[prop] = value
+        return true
+      },
+    })
+  }
+  protected register() {}
+  private weak(): this {
+    const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
+    if (this.dropRef) weak.ref = new WeakRef(this.dropRef)
+    return weak
+  }
+  abstract onDrop(): void
+  drop(): void {
+    if (!this.dropRef || !this.dropId) return
+    this.onDrop()
+    this.leak()
+  }
+  leak(): this {
+    if (!this.dropRef || !this.dropId) return this
+    Drop.registry.unregister(this.dropRef)
+    delete Drop.weak[this.dropId]
+    delete this.dropRef
+    delete this.dropId
+    return this
+  }
+}
+
+export class DropPromise<T> implements Promise<T> {
+  private static dropFns: { [id: number]: () => void } = {}
+  private static registry = new FinalizationRegistry((id: number) => {
+    const drop = DropPromise.dropFns[id]
+    if (drop) {
+      drop()
+      delete DropPromise.dropFns[id]
+    }
+  })
+  private static idCtr: number = 0
+  private dropId: number
+  private dropRef: DropRef;
+  [Symbol.toStringTag] = "DropPromise"
+  private constructor(
+    private readonly promise: Promise<T>,
+    dropFnOrRef?: (() => void) | DropRef,
+  ) {
+    if (dropFnOrRef && dropId in dropFnOrRef) {
+      this.dropId = dropFnOrRef[dropId]
+      this.dropRef = dropFnOrRef
+      return
+    }
+    this.dropId = DropPromise.idCtr++
+    this.dropRef = { [dropId]: this.dropId }
+    if (dropFnOrRef) DropPromise.dropFns[this.dropId] = dropFnOrRef
+    DropPromise.registry.register(this.dropRef, this.dropId, this.dropRef)
+  }
+  static of<T>(promise: Promise<T>, dropFn?: () => void): DropPromise<T> {
+    return new DropPromise(promise, dropFn)
+  }
+  static ref<T>(promise: Promise<T>, dropRef: DropRef): DropPromise<T> {
+    return new DropPromise(promise, dropRef)
+  }
+  then<TResult1 = T, TResult2 = never>(
+    onfulfilled?:
+      | ((value: T) => TResult1 | PromiseLike<TResult1>)
+      | null
+      | undefined,
+    onrejected?:
+      | ((reason: any) => TResult2 | PromiseLike<TResult2>)
+      | null
+      | undefined,
+  ): Promise<TResult1 | TResult2> {
+    return DropPromise.ref(
+      this.promise.then(onfulfilled, onrejected),
+      this.dropRef,
+    )
+  }
+  catch<TResult = never>(
+    onrejected?:
+      | ((reason: any) => TResult | PromiseLike<TResult>)
+      | null
+      | undefined,
+  ): Promise<T | TResult> {
+    return DropPromise.ref(this.promise.catch(onrejected), this.dropRef)
+  }
+  finally(onfinally?: (() => void) | null | undefined): Promise<T> {
+    return DropPromise.ref(this.promise.finally(onfinally), this.dropRef)
+  }
+}
+
+export class DropGenerator<T = unknown, TReturn = any, TNext = unknown>
+  implements AsyncGenerator<T, TReturn, TNext>
+{
+  private static dropFns: { [id: number]: () => void } = {}
+  private static registry = new FinalizationRegistry((id: number) => {
+    const drop = DropGenerator.dropFns[id]
+    if (drop) {
+      drop()
+      delete DropGenerator.dropFns[id]
+    }
+  })
+  private static idCtr: number = 0
+  private dropId: number
+  private dropRef: DropRef;
+  [Symbol.asyncIterator] = () => this
+  private constructor(
+    private readonly generator: AsyncGenerator<T, TReturn, TNext>,
+    dropFn?: () => void,
+  ) {
+    this.dropId = DropGenerator.idCtr++
+    this.dropRef = { [dropId]: this.dropId }
+    if (dropFn) DropGenerator.dropFns[this.dropId] = dropFn
+    DropGenerator.registry.register(this.dropRef, this.dropId, this.dropRef)
+  }
+  static of<T, TReturn, TNext>(
+    generator: AsyncGenerator<T, TReturn, TNext>,
+    dropFn?: () => void,
+  ): DropGenerator<T, TReturn, TNext> {
+    return new DropGenerator(generator, dropFn)
+  }
+  next(...args: [] | [TNext]): Promise<IteratorResult<T, TReturn>> {
+    return DropPromise.ref(this.generator.next(...args), this.dropRef)
+  }
+  return(
+    value: TReturn | PromiseLike<TReturn>,
+  ): Promise<IteratorResult<T, TReturn>> {
+    return DropPromise.ref(this.generator.return(value), this.dropRef)
+  }
+  throw(e: any): Promise<IteratorResult<T, TReturn>> {
+    return DropPromise.ref(this.generator.throw(e), this.dropRef)
+  }
+}

sdk/base/lib/util/GetSystemSmtp.ts

@@ -1,5 +1,6 @@
 import { Effects } from "../Effects"
 import * as T from "../types"
+import { DropGenerator, DropPromise } from "./Drop"
 
 export class GetSystemSmtp {
   constructor(readonly effects: Effects) {}
@@ -21,15 +22,13 @@ export class GetSystemSmtp {
     return this.effects.getSystemSmtp({})
   }
 
-  /**
-   * Watches the system SMTP credentials. Returns an async iterator that yields whenever the value changes
-   */
-  async *watch() {
+  private async *watchGen(abort?: AbortSignal) {
     const resolveCell = { resolve: () => {} }
     this.effects.onLeaveContext(() => {
       resolveCell.resolve()
     })
-    while (this.effects.isInContext) {
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
       let callback: () => void = () => {}
       const waitForNext = new Promise<void>((resolve) => {
         callback = resolve
@@ -42,6 +41,17 @@ export class GetSystemSmtp {
     }
   }
 
+  /**
+   * Watches the system SMTP credentials. Returns an async iterator that yields whenever the value changes
+   */
+  watch(
+    abort?: AbortSignal,
+  ): AsyncGenerator<T.SmtpValue | null, void, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
   /**
    * Watches the system SMTP credentials. Takes a custom callback function to run whenever the credentials change
    */
@@ -49,12 +59,17 @@ export class GetSystemSmtp {
     callback: (
       value: T.SmtpValue | null,
       error?: Error,
-    ) => void | Promise<void>,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) {
     ;(async () => {
-      for await (const value of this.watch()) {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
         } catch (e) {
           console.error(
             "callback function threw an error @ GetSystemSmtp.onChange",
@@ -75,26 +90,20 @@ export class GetSystemSmtp {
   /**
    * Watches the system SMTP credentials. Returns when the predicate is true
    */
-  async waitFor(pred: (value: T.SmtpValue | null) => boolean) {
-    const resolveCell = { resolve: () => {} }
-    this.effects.onLeaveContext(() => {
-      resolveCell.resolve()
-    })
-    while (this.effects.isInContext) {
-      let callback: () => void = () => {}
-      const waitForNext = new Promise<void>((resolve) => {
-        callback = resolve
-        resolveCell.resolve = resolve
-      })
-      const res = await this.effects.getSystemSmtp({
-        callback: () => callback(),
-      })
-      if (pred(res)) {
-        resolveCell.resolve()
-        return res
-      }
-      await waitForNext
-    }
-    return null
+  waitFor(
+    pred: (value: T.SmtpValue | null) => boolean,
+  ): Promise<T.SmtpValue | null> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 }

sdk/base/lib/util/getServiceInterface.ts

@@ -2,6 +2,7 @@ import { ServiceInterfaceType } from "../types"
 import { knownProtocols } from "../interfaces/Host"
 import { AddressInfo, Host, Hostname, HostnameInfo } from "../types"
 import { Effects } from "../Effects"
+import { DropGenerator, DropPromise } from "./Drop"
 
 export type UrlString = string
 export type HostId = string
@@ -312,16 +313,14 @@ export class GetServiceInterface {
     return interfaceFilled
   }
 
-  /**
-   * Watches the requested service interface. Returns an async iterator that yields whenever the value changes
-   */
-  async *watch() {
+  private async *watchGen(abort?: AbortSignal) {
     const { id, packageId } = this.opts
     const resolveCell = { resolve: () => {} }
     this.effects.onLeaveContext(() => {
       resolveCell.resolve()
     })
-    while (this.effects.isInContext) {
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
       let callback: () => void = () => {}
       const waitForNext = new Promise<void>((resolve) => {
         callback = resolve
@@ -337,6 +336,17 @@ export class GetServiceInterface {
     }
   }
 
+  /**
+   * Watches the requested service interface. Returns an async iterator that yields whenever the value changes
+   */
+  watch(
+    abort?: AbortSignal,
+  ): AsyncGenerator<ServiceInterfaceFilled | null, void, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
   /**
    * Watches the requested service interface. Takes a custom callback function to run whenever the value changes
    */
@@ -344,12 +354,17 @@ export class GetServiceInterface {
     callback: (
       value: ServiceInterfaceFilled | null,
       error?: Error,
-    ) => void | Promise<void>,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) {
     ;(async () => {
-      for await (const value of this.watch()) {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
         } catch (e) {
           console.error(
             "callback function threw an error @ GetServiceInterface.onChange",
@@ -370,31 +385,21 @@ export class GetServiceInterface {
   /**
    * Watches the requested service interface. Returns when the predicate is true
    */
-  async waitFor(pred: (value: ServiceInterfaceFilled | null) => boolean) {
-    const { id, packageId } = this.opts
-    const resolveCell = { resolve: () => {} }
-    this.effects.onLeaveContext(() => {
-      resolveCell.resolve()
-    })
-    while (this.effects.isInContext) {
-      let callback: () => void = () => {}
-      const waitForNext = new Promise<void>((resolve) => {
-        callback = resolve
-        resolveCell.resolve = resolve
-      })
-      const res = await makeInterfaceFilled({
-        effects: this.effects,
-        id,
-        packageId,
-        callback,
-      })
-      if (pred(res)) {
-        resolveCell.resolve()
-        return res
-      }
-      await waitForNext
-    }
-    return null
+  waitFor(
+    pred: (value: ServiceInterfaceFilled | null) => boolean,
+  ): Promise<ServiceInterfaceFilled | null> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 }
 export function getServiceInterface(

sdk/base/lib/util/getServiceInterfaces.ts

@@ -1,4 +1,5 @@
 import { Effects } from "../Effects"
+import { DropGenerator, DropPromise } from "./Drop"
 import {
   ServiceInterfaceFilled,
   filledAddress,
@@ -77,16 +78,14 @@ export class GetServiceInterfaces {
     return interfaceFilled
   }
 
-  /**
-   * Watches the service interfaces for the package. Returns an async iterator that yields whenever the value changes
-   */
-  async *watch() {
+  private async *watchGen(abort?: AbortSignal) {
     const { packageId } = this.opts
     const resolveCell = { resolve: () => {} }
     this.effects.onLeaveContext(() => {
       resolveCell.resolve()
     })
-    while (this.effects.isInContext) {
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
       let callback: () => void = () => {}
       const waitForNext = new Promise<void>((resolve) => {
         callback = resolve
@@ -101,6 +100,17 @@ export class GetServiceInterfaces {
     }
   }
 
+  /**
+   * Watches the service interfaces for the package. Returns an async iterator that yields whenever the value changes
+   */
+  watch(
+    abort?: AbortSignal,
+  ): AsyncGenerator<ServiceInterfaceFilled[], void, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
   /**
    * Watches the service interfaces for the package. Takes a custom callback function to run whenever the value changes
    */
@@ -108,12 +118,17 @@ export class GetServiceInterfaces {
     callback: (
       value: ServiceInterfaceFilled[] | null,
       error?: Error,
-    ) => void | Promise<void>,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) {
     ;(async () => {
-      for await (const value of this.watch()) {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
         } catch (e) {
           console.error(
             "callback function threw an error @ GetServiceInterfaces.onChange",
@@ -134,30 +149,21 @@ export class GetServiceInterfaces {
   /**
    * Watches the service interfaces for the package. Returns when the predicate is true
    */
-  async waitFor(pred: (value: ServiceInterfaceFilled[] | null) => boolean) {
-    const { packageId } = this.opts
-    const resolveCell = { resolve: () => {} }
-    this.effects.onLeaveContext(() => {
-      resolveCell.resolve()
-    })
-    while (this.effects.isInContext) {
-      let callback: () => void = () => {}
-      const waitForNext = new Promise<void>((resolve) => {
-        callback = resolve
-        resolveCell.resolve = resolve
-      })
-      const res = await makeManyInterfaceFilled({
-        effects: this.effects,
-        packageId,
-        callback,
-      })
-      if (pred(res)) {
-        resolveCell.resolve()
-        return res
-      }
-      await waitForNext
-    }
-    return null
+  waitFor(
+    pred: (value: ServiceInterfaceFilled[] | null) => boolean,
+  ): Promise<ServiceInterfaceFilled[] | null> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 }
 export function getServiceInterfaces(

sdk/base/lib/util/graph.ts

@@ -1,4 +1,5 @@
 import { boolean } from "ts-matches"
+import { ExtendedVersion } from "../exver"
 
 export type Vertex<VMetadata = null, EMetadata = null> = {
   metadata: VMetadata
@@ -14,6 +15,22 @@ export type Edge<EMetadata = null, VMetadata = null> = {
 export class Graph<VMetadata = null, EMetadata = null> {
   private readonly vertices: Array<Vertex<VMetadata, EMetadata>> = []
   constructor() {}
+  dump(
+    metadataRepr: (metadata: VMetadata | EMetadata) => any = (a) => a,
+  ): string {
+    const seen = new WeakSet()
+
+    return JSON.stringify(
+      this.vertices,
+      (k, v) => {
+        if (k === "metadata") return metadataRepr(v)
+        if (k === "from") return metadataRepr(v.metadata)
+        if (k === "to") return metadataRepr(v.metadata)
+        return v
+      },
+      2,
+    )
+  }
   addVertex(
     metadata: VMetadata,
     fromEdges: Array<Omit<Edge<EMetadata, VMetadata>, "to">>,

sdk/package/lib/StartSdk.ts

@@ -59,8 +59,9 @@ import {
   setupOnInit,
   setupOnUninit,
 } from "../../base/lib/inits"
+import { DropGenerator } from "../../base/lib/util/Drop"
 
-export const OSVersion = testTypeVersion("0.4.0-alpha.8")
+export const OSVersion = testTypeVersion("0.4.0-alpha.9")
 
 // prettier-ignore
 type AnyNeverCond<T extends any[], Then, Else> = 
@@ -191,12 +192,13 @@ export class StartSdk<Manifest extends T.SDKManifest> {
           "callback"
         > = {},
       ) => {
-        async function* watch() {
+        async function* watch(abort?: AbortSignal) {
           const resolveCell = { resolve: () => {} }
           effects.onLeaveContext(() => {
             resolveCell.resolve()
           })
-          while (effects.isInContext) {
+          abort?.addEventListener("abort", () => resolveCell.resolve())
+          while (effects.isInContext && !abort?.aborted) {
             let callback: () => void = () => {}
             const waitForNext = new Promise<void>((resolve) => {
               callback = resolve
@@ -215,17 +217,26 @@ export class StartSdk<Manifest extends T.SDKManifest> {
                 (() => effects.constRetry && effects.constRetry()),
             }),
           once: () => effects.getContainerIp(options),
-          watch,
+          watch: (abort?: AbortSignal) => {
+            const ctrl = new AbortController()
+            abort?.addEventListener("abort", () => ctrl.abort())
+            return DropGenerator.of(watch(ctrl.signal), () => ctrl.abort())
+          },
           onChange: (
             callback: (
               value: string | null,
               error?: Error,
-            ) => void | Promise<void>,
+            ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
           ) => {
             ;(async () => {
-              for await (const value of watch()) {
+              const ctrl = new AbortController()
+              for await (const value of watch(ctrl.signal)) {
                 try {
-                  await callback(value)
+                  const res = await callback(value)
+                  if (res.cancel) {
+                    ctrl.abort()
+                    break
+                  }
                 } catch (e) {
                   console.error(
                     "callback function threw an error @ getContainerIp.onChange",

sdk/package/lib/mainFn/CommandController.ts

@@ -35,7 +35,7 @@ export class CommandController<
           const abort = new AbortController()
           const cell: { ctrl: CommandController<Manifest, C> } = {
             ctrl: new CommandController<Manifest, C>(
-              exec.fn(subcontainer, abort).then(async (command) => {
+              exec.fn(subcontainer, abort.signal).then(async (command) => {
                 if (subcontainer && command && !abort.signal.aborted) {
                   const newCtrl = (
                     await CommandController.of<

sdk/package/lib/mainFn/Daemon.ts

@@ -49,7 +49,11 @@ export class Daemon<
           (subc?.rc() ?? null) as C,
           exec,
         )
-      return new Daemon(subc, startCommand)
+      const res = new Daemon(subc, startCommand)
+      effects.onLeaveContext(() => {
+        res.stop().catch((e) => console.error(asError(e)))
+      })
+      return res
     }
   }
   async start() {
@@ -66,6 +70,11 @@ export class Daemon<
             .catch((err) => console.error(err))
         try {
           this.commandController = await this.startCommand()
+          if (!this.shouldBeRunning) {
+            // handles race condition if stopped while starting
+            await this.stop()
+            break
+          }
           const success = await this.commandController.wait().then(
             (_) => true,
             (err) => {
@@ -107,12 +116,14 @@ export class Daemon<
   }) {
     this.shouldBeRunning = false
     this.exitedSuccess = false
-    await this.commandController
-      ?.term({ ...termOptions })
-      .catch((e) => console.error(asError(e)))
-    this.commandController = null
-    this.onExitFns = []
-    await this.subcontainer?.destroy()
+    if (this.commandController) {
+      await this.commandController
+        .term({ ...termOptions })
+        .catch((e) => console.error(asError(e)))
+      this.commandController = null
+      this.onExitFns = []
+      await this.subcontainer?.destroy()
+    }
   }
   subcontainerRc(): SubContainerRc<Manifest> | null {
     return this.subcontainer?.rc() ?? null

sdk/package/lib/mainFn/Daemons.ts

@@ -17,6 +17,7 @@ import { CommandController } from "./CommandController"
 import { HealthCheck } from "../health/HealthCheck"
 import { Oneshot } from "./Oneshot"
 import { Manifest } from "../test/output.sdk"
+import { asError } from "../util"
 
 export const cpExec = promisify(CP.exec)
 export const cpExecFile = promisify(CP.execFile)
@@ -69,7 +70,7 @@ export type ExecFnOptions<
 > = {
   fn: (
     subcontainer: C,
-    abort: AbortController,
+    abort: AbortSignal,
   ) => Promise<C extends null ? null : ExecCommandOptions | null>
   // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
   sigtermTimeout?: number
@@ -379,6 +380,9 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
   }
 
   async build() {
+    this.effects.onLeaveContext(() => {
+      this.term().catch((e) => console.error(asError(e)))
+    })
     for (const daemon of this.healthDaemons) {
       await daemon.init()
     }

sdk/package/lib/manifest/setupManifest.ts

@@ -54,6 +54,7 @@ export function buildManifest<
   )
   return {
     ...manifest,
+    gitHash: null,
     osVersion: manifest.osVersion ?? OSVersion,
     sdkVersion,
     version: versions.current.options.version,

sdk/package/lib/test/inputSpecBuilder.test.ts

@@ -396,14 +396,13 @@ describe("values", () => {
             supportSite: "",
             marketingSite: "",
             donationUrl: null,
+            docsUrl: "",
             description: {
               short: "",
               long: "",
             },
-            containers: {},
             images: {},
             volumes: [],
-            assets: [],
             alerts: {
               install: null,
               update: null,

sdk/package/lib/test/output.sdk.ts

@@ -9,17 +9,16 @@ export const sdk = StartSdk.of()
       id: "testOutput",
       title: "",
       license: "",
-      replaces: [],
       wrapperRepo: "",
       upstreamRepo: "",
       supportSite: "",
       marketingSite: "",
       donationUrl: null,
+      docsUrl: "",
       description: {
         short: "",
         long: "",
       },
-      containers: {},
       images: {
         main: {
           source: {
@@ -30,7 +29,6 @@ export const sdk = StartSdk.of()
         },
       },
       volumes: [],
-      assets: [],
       alerts: {
         install: null,
         update: null,

sdk/package/lib/util/Drop.ts

@@ -1,46 +0,0 @@
-export abstract class Drop {
-  private static weak: { [id: number]: Drop } = {}
-  private static registry = new FinalizationRegistry((id: number) => {
-    const weak = Drop.weak[id]
-    if (weak) weak.drop()
-  })
-  private static idCtr: number = 0
-  private dropId?: number
-  private dropRef?: { id: number } | WeakRef<{ id: number }>
-  protected constructor() {
-    this.dropId = Drop.idCtr++
-    this.dropRef = { id: this.dropId }
-    const weak = this.weak()
-    Drop.weak[this.dropId] = weak
-    Drop.registry.register(this.dropRef, this.dropId, this.dropRef)
-
-    return new Proxy(this, {
-      set(target: any, prop, value) {
-        if (prop === "dropRef" || prop == "dropId") return false
-        target[prop] = value
-        ;(weak as any)[prop] = value
-        return true
-      },
-    })
-  }
-  protected register() {}
-  protected weak(): this {
-    const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
-    if (this.dropRef) weak.ref = new WeakRef(this.dropRef)
-    return weak
-  }
-  abstract onDrop(): void
-  drop(): void {
-    if (!this.dropRef || !this.dropId) return
-    this.onDrop()
-    this.leak()
-  }
-  leak(): this {
-    if (!this.dropRef || !this.dropId) return this
-    Drop.registry.unregister(this.dropRef)
-    delete Drop.weak[this.dropId]
-    delete this.dropRef
-    delete this.dropId
-    return this
-  }
-}

sdk/package/lib/util/GetSslCertificate.ts

@@ -1,5 +1,6 @@
 import { T } from ".."
 import { Effects } from "../../../base/lib/Effects"
+import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
 
 export class GetSslCertificate {
   constructor(
@@ -30,15 +31,13 @@ export class GetSslCertificate {
     })
   }
 
-  /**
-   * Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
-   */
-  async *watch() {
+  private async *watchGen(abort?: AbortSignal) {
     const resolveCell = { resolve: () => {} }
     this.effects.onLeaveContext(() => {
       resolveCell.resolve()
     })
-    while (this.effects.isInContext) {
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
       let callback: () => void = () => {}
       const waitForNext = new Promise<void>((resolve) => {
         callback = resolve
@@ -53,6 +52,17 @@ export class GetSslCertificate {
     }
   }
 
+  /**
+   * Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
+   */
+  watch(
+    abort?: AbortSignal,
+  ): AsyncGenerator<[string, string, string], void, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
   /**
    * Watches the SSL Certificate for the given hostnames if permitted. Takes a custom callback function to run whenever it changes
    */
@@ -60,12 +70,17 @@ export class GetSslCertificate {
     callback: (
       value: [string, string, string] | null,
       error?: Error,
-    ) => void | Promise<void>,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) {
     ;(async () => {
-      for await (const value of this.watch()) {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
         } catch (e) {
           console.error(
             "callback function threw an error @ GetSslCertificate.onChange",
@@ -86,28 +101,20 @@ export class GetSslCertificate {
   /**
    * Watches the SSL Certificate for the given hostnames if permitted. Returns when the predicate is true
    */
-  async waitFor(pred: (value: [string, string, string] | null) => boolean) {
-    const resolveCell = { resolve: () => {} }
-    this.effects.onLeaveContext(() => {
-      resolveCell.resolve()
-    })
-    while (this.effects.isInContext) {
-      let callback: () => void = () => {}
-      const waitForNext = new Promise<void>((resolve) => {
-        callback = resolve
-        resolveCell.resolve = resolve
-      })
-      const res = await this.effects.getSslCertificate({
-        hostnames: this.hostnames,
-        algorithm: this.algorithm,
-        callback: () => callback(),
-      })
-      if (pred(res)) {
-        resolveCell.resolve()
-        return res
-      }
-      await waitForNext
-    }
-    return null
+  waitFor(
+    pred: (value: [string, string, string] | null) => boolean,
+  ): Promise<[string, string, string] | null> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 }

sdk/package/lib/util/SubContainer.ts

@@ -4,7 +4,7 @@ import * as cp from "child_process"
 import { promisify } from "util"
 import { Buffer } from "node:buffer"
 import { once } from "../../../base/lib/util/once"
-import { Drop } from "./Drop"
+import { Drop } from "../../../base/lib/util/Drop"
 import { Mounts } from "../mainFn/Mounts"
 import { BackupEffects } from "../backup/Backups"
 

sdk/package/lib/util/fileHelper.ts

@@ -4,10 +4,38 @@ import * as TOML from "@iarna/toml"
 import * as INI from "ini"
 import * as T from "../../../base/lib/types"
 import * as fs from "node:fs/promises"
-import { asError, partialDiff } from "../../../base/lib/util"
+import { asError } from "../../../base/lib/util"
+import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
+const deepEq = (left: unknown, right: unknown) => {
+  if (left === right) return true
+  if (Array.isArray(left) && Array.isArray(right)) {
+    if (left.length === right.length) {
+      for (const idx in left) {
+        if (!deepEq(left[idx], right[idx])) return false
+      }
+      return true
+    }
+  } else if (
+    typeof left === "object" &&
+    typeof right === "object" &&
+    left &&
+    right
+  ) {
+    const keys = new Set<keyof typeof left | keyof typeof right>([
+      ...(Object.keys(left) as (keyof typeof left)[]),
+      ...(Object.keys(right) as (keyof typeof right)[]),
+    ])
+    for (let key of keys) {
+      if (!deepEq(left[key], right[key])) return false
+    }
+    return true
+  }
+  return false
+}
+
 const exists = (path: string) =>
   fs.access(path).then(
     () => true,
@@ -99,10 +127,16 @@ type Validator<T, U> = matches.Validator<T, U> | matches.Validator<unknown, U>
 type ReadType<A> = {
   once: () => Promise<A | null>
   const: (effects: T.Effects) => Promise<A | null>
-  watch: (effects: T.Effects) => AsyncGenerator<A | null, null, unknown>
+  watch: (
+    effects: T.Effects,
+    abort?: AbortSignal,
+  ) => AsyncGenerator<A | null, null, unknown>
   onChange: (
     effects: T.Effects,
-    callback: (value: A | null, error?: Error) => void | Promise<void>,
+    callback: (
+      value: A | null,
+      error?: Error,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) => void
   waitFor: (
     effects: T.Effects,
@@ -151,7 +185,12 @@ type ReadType<A> = {
  * ```
  */
 export class FileHelper<A> {
-  private consts: (() => void)[] = []
+  private consts: [
+    () => void,
+    any,
+    (a: any) => any,
+    (left: any, right: any) => any,
+  ][] = []
   protected constructor(
     readonly path: string,
     readonly writeData: (dataIn: A) => string,
@@ -209,10 +248,15 @@ export class FileHelper<A> {
     const watch = this.readWatch(effects, map, eq)
     const res = await watch.next()
     if (effects.constRetry) {
-      if (!this.consts.includes(effects.constRetry))
-        this.consts.push(effects.constRetry)
+      const record: (typeof this.consts)[number] = [
+        effects.constRetry,
+        res.value,
+        map,
+        eq,
+      ]
+      this.consts.push(record)
       watch.next().then(() => {
-        this.consts = this.consts.filter((a) => a === effects.constRetry)
+        this.consts = this.consts.filter((r) => r !== record)
         effects.constRetry && effects.constRetry()
       })
     }
@@ -223,11 +267,13 @@ export class FileHelper<A> {
     effects: T.Effects,
     map: (value: A) => B,
     eq: (left: B | null | undefined, right: B | null) => boolean,
+    abort?: AbortSignal,
   ) {
     let res
-    while (effects.isInContext) {
+    while (effects.isInContext && !abort?.aborted) {
       if (await exists(this.path)) {
         const ctrl = new AbortController()
+        abort?.addEventListener("abort", () => ctrl.abort())
         const watch = fs.watch(this.path, {
           persistent: false,
           signal: ctrl.signal,
@@ -254,14 +300,19 @@ export class FileHelper<A> {
 
   private readOnChange<B>(
     effects: T.Effects,
-    callback: (value: B | null, error?: Error) => void | Promise<void>,
+    callback: (
+      value: B | null,
+      error?: Error,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
     map: (value: A) => B,
     eq: (left: B | null | undefined, right: B | null) => boolean,
   ) {
     ;(async () => {
-      for await (const value of this.readWatch(effects, map, eq)) {
+      const ctrl = new AbortController()
+      for await (const value of this.readWatch(effects, map, eq, ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) ctrl.abort()
         } catch (e) {
           console.error(
             "callback function threw an error @ FileHelper.read.onChange",
@@ -279,38 +330,36 @@ export class FileHelper<A> {
       )
   }
 
-  private async readWaitFor<B>(
+  private readWaitFor<B>(
     effects: T.Effects,
     pred: (value: B | null, error?: Error) => boolean,
     map: (value: A) => B,
   ): Promise<B | null> {
-    while (effects.isInContext) {
-      if (await exists(this.path)) {
-        const ctrl = new AbortController()
-        const watch = fs.watch(this.path, {
-          persistent: false,
-          signal: ctrl.signal,
-        })
-        const newRes = await this.readOnce(map)
-        const listen = Promise.resolve()
-          .then(async () => {
-            for await (const _ of watch) {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        const watch = this.readWatch(effects, map, (_) => false, ctrl.signal)
+        while (true) {
+          try {
+            const res = await watch.next()
+            if (pred(res.value)) {
               ctrl.abort()
-              return null
+              return res.value
             }
-          })
-          .catch((e) => console.error(asError(e)))
-        if (pred(newRes)) {
-          ctrl.abort()
-          return newRes
+            if (res.done) {
+              break
+            }
+          } catch (e) {
+            if (pred(null, e as Error)) {
+              break
+            }
+          }
         }
-        await listen
-      } else {
-        if (pred(null)) return null
-        await onCreated(this.path).catch((e) => console.error(asError(e)))
-      }
-    }
-    return null
+        ctrl.abort()
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 
   read(): ReadType<A>
@@ -323,14 +372,24 @@ export class FileHelper<A> {
     eq?: (left: any, right: any) => boolean,
   ): ReadType<any> {
     map = map ?? ((a: A) => a)
-    eq = eq ?? ((left: any, right: any) => !partialDiff(left, right))
+    eq = eq ?? deepEq
     return {
       once: () => this.readOnce(map),
       const: (effects: T.Effects) => this.readConst(effects, map, eq),
-      watch: (effects: T.Effects) => this.readWatch(effects, map, eq),
+      watch: (effects: T.Effects, abort?: AbortSignal) => {
+        const ctrl = new AbortController()
+        abort?.addEventListener("abort", () => ctrl.abort())
+        return DropGenerator.of(
+          this.readWatch(effects, map, eq, ctrl.signal),
+          () => ctrl.abort(),
+        )
+      },
       onChange: (
         effects: T.Effects,
-        callback: (value: A | null, error?: Error) => void | Promise<void>,
+        callback: (
+          value: A | null,
+          error?: Error,
+        ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
       ) => this.readOnChange(effects, callback, map, eq),
       waitFor: (effects: T.Effects, pred: (value: A | null) => boolean) =>
         this.readWaitFor(effects, pred, map),
@@ -345,13 +404,17 @@ export class FileHelper<A> {
     data: T.AllowReadonly<A> | A,
     options: { allowWriteAfterConst?: boolean } = {},
   ) {
-    await this.writeFile(this.validate(data))
-    if (
-      !options.allowWriteAfterConst &&
-      effects.constRetry &&
-      this.consts.includes(effects.constRetry)
-    )
-      throw new Error(`Canceled: write after const: ${this.path}`)
+    const newData = this.validate(data)
+    await this.writeFile(newData)
+    if (!options.allowWriteAfterConst && effects.constRetry) {
+      const records = this.consts.filter(([c]) => c === effects.constRetry)
+      for (const record of records) {
+        const [_, prev, map, eq] = record
+        if (!eq(prev, map(newData))) {
+          throw new Error(`Canceled: write after const: ${this.path}`)
+        }
+      }
+    }
     return null
   }
 
@@ -372,16 +435,14 @@ export class FileHelper<A> {
     const toWrite = this.writeData(mergeData)
     if (toWrite !== fileDataRaw) {
       this.writeFile(mergeData)
-      if (
-        !options.allowWriteAfterConst &&
-        effects.constRetry &&
-        this.consts.includes(effects.constRetry)
-      ) {
-        const diff = partialDiff(fileData, mergeData as any)
-        if (!diff) {
-          return null
+      if (!options.allowWriteAfterConst && effects.constRetry) {
+        const records = this.consts.filter(([c]) => c === effects.constRetry)
+        for (const record of records) {
+          const [_, prev, map, eq] = record
+          if (!eq(prev, map(mergeData))) {
+            throw new Error(`Canceled: write after const: ${this.path}`)
+          }
         }
-        throw new Error(`Canceled: write after const: ${this.path}`)
       }
     }
     return null

sdk/package/lib/util/index.ts

@@ -2,4 +2,4 @@ export * from "../../../base/lib/util"
 export { GetSslCertificate } from "./GetSslCertificate"
 
 export { hostnameInfoToAddress } from "../../../base/lib/util/Hostname"
-export { Drop } from "./Drop"
+export { Drop } from "../../../base/lib/util/Drop"

sdk/package/lib/version/VersionGraph.ts

@@ -58,6 +58,9 @@ export class VersionGraph<CurrentVersion extends string>
     ExtendedVersion | VersionRange,
     ((opts: { effects: T.Effects }) => Promise<void>) | undefined
   >
+  dump(): string {
+    return this.graph().dump((metadata) => metadata?.toString())
+  }
   private constructor(
     readonly current: VersionInfo<CurrentVersion>,
     versions: Array<VersionInfo<any>>,
@@ -97,7 +100,7 @@ export class VersionGraph<CurrentVersion extends string>
               VersionInfo<any>,
               Vertex<
                 ExtendedVersion | VersionRange,
-                (opts: { effects: T.Effects }) => Promise<void>
+                ((opts: { effects: T.Effects }) => Promise<void>) | undefined
               >,
             ]
           | undefined = undefined
@@ -146,6 +149,8 @@ export class VersionGraph<CurrentVersion extends string>
               }
             }
           }
+
+          prev = [v, version, vertex]
         }
       }
       return graph
@@ -198,6 +203,23 @@ export class VersionGraph<CurrentVersion extends string>
         (v) => overlaps(v.metadata, to),
       )
       if (path) {
+        console.log(
+          `Migrating ${
+            path.reduce<{ acc: string; prev: string | null }>(
+              ({ acc, prev }, x) => ({
+                acc:
+                  acc +
+                  (prev && prev != x.from.metadata.toString()
+                    ? ` (as ${prev})`
+                    : "") +
+                  " -> " +
+                  x.to.metadata.toString(),
+                prev: x.to.metadata.toString(),
+              }),
+              { acc: from.toString(), prev: null },
+            ).acc
+          }`,
+        )
         let dataVersion = from
         for (let edge of path) {
           if (edge.metadata) {

sdk/package/lib/version/VersionInfo.ts

@@ -1,7 +1,7 @@
 import { ValidateExVer } from "../../../base/lib/exver"
 import * as T from "../../../base/lib/types"
 
-export const IMPOSSIBLE = Symbol("IMPOSSIBLE")
+export const IMPOSSIBLE: unique symbol = Symbol("IMPOSSIBLE")
 
 export type VersionOptions<Version extends string> = {
   /** The exver-compliant version number */

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.33",
+  "version": "0.4.0-beta.36",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.33",
+      "version": "0.4.0-beta.36",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.33",
+  "version": "0.4.0-beta.36",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",

sdk/package/tsconfig.json

@@ -15,6 +15,6 @@
     "target": "es2021",
     "resolveJsonModule": true
   },
-  "include": ["lib/**/*", "../base/lib/util/Hostname.ts"],
+  "include": ["lib/**/*"],
   "exclude": ["lib/**/*.spec.ts", "lib/**/*.gen.ts", "list", "node_modules"]
 }