update/alpha.9 (#2988)

* import marketplac preview for sideload

* fix: improve state service (#2977)

* fix: fix sideload DI

* fix: update Angular

* fix: cleanup

* fix: fix version selection

* Bump node version to fix build for Angular

* misc fixes
- update node to v22
- fix chroot-and-upgrade access to prune-images
- don't self-migrate legacy packages
- #2985
- move dataVersion to volume folder
- remove "instructions.md" from s9pk
- add "docsUrl" to manifest

* version bump

* include flavor when clicking view listing from updates tab

* closes #2980

* fix: fix select button

* bring back ssh keys

* fix: drop 'portal' from all routes

* fix: implement longtap action to select table rows

* fix description for ssh page

* replace instructions with docsLink and refactor marketplace preview

* delete unused translations

* fix patchdb diffing algorithm

* continue refactor of marketplace lib show components

* Booting StartOS instead of Setting up your server on init

* misc fixes
- closes #2990
- closes #2987

* fix build

* docsUrl and clickable service headers

* don't cleanup after update until new service install succeeds

* update types

* misc fixes

* beta.35

* sdkversion, githash for sideload, correct logs for init, startos pubkey display

* bring back reboot button on install

* misc fixes

* beta.36

* better handling of setup and init for websocket errors

* reopen init and setup logs even on graceful closure

* better logging, misc fixes

* fix build

* dont let package stats hang

* dont show docsurl in marketplace if no docsurl

* re-add needs-config

* show error if init fails, shorten hover state on header icons

* fix operator precedemce

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Alex Inkin <alexander@inkin.ru>
Co-authored-by: Mariusz Kogen <k0gen@pm.me>

sdk/package/lib/util/Drop.ts

@@ -1,46 +0,0 @@
-export abstract class Drop {
-  private static weak: { [id: number]: Drop } = {}
-  private static registry = new FinalizationRegistry((id: number) => {
-    const weak = Drop.weak[id]
-    if (weak) weak.drop()
-  })
-  private static idCtr: number = 0
-  private dropId?: number
-  private dropRef?: { id: number } | WeakRef<{ id: number }>
-  protected constructor() {
-    this.dropId = Drop.idCtr++
-    this.dropRef = { id: this.dropId }
-    const weak = this.weak()
-    Drop.weak[this.dropId] = weak
-    Drop.registry.register(this.dropRef, this.dropId, this.dropRef)
-
-    return new Proxy(this, {
-      set(target: any, prop, value) {
-        if (prop === "dropRef" || prop == "dropId") return false
-        target[prop] = value
-        ;(weak as any)[prop] = value
-        return true
-      },
-    })
-  }
-  protected register() {}
-  protected weak(): this {
-    const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
-    if (this.dropRef) weak.ref = new WeakRef(this.dropRef)
-    return weak
-  }
-  abstract onDrop(): void
-  drop(): void {
-    if (!this.dropRef || !this.dropId) return
-    this.onDrop()
-    this.leak()
-  }
-  leak(): this {
-    if (!this.dropRef || !this.dropId) return this
-    Drop.registry.unregister(this.dropRef)
-    delete Drop.weak[this.dropId]
-    delete this.dropRef
-    delete this.dropId
-    return this
-  }
-}

sdk/package/lib/util/GetSslCertificate.ts

@@ -1,5 +1,6 @@
 import { T } from ".."
 import { Effects } from "../../../base/lib/Effects"
+import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
 
 export class GetSslCertificate {
   constructor(
@@ -30,15 +31,13 @@ export class GetSslCertificate {
     })
   }
 
-  /**
-   * Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
-   */
-  async *watch() {
+  private async *watchGen(abort?: AbortSignal) {
     const resolveCell = { resolve: () => {} }
     this.effects.onLeaveContext(() => {
       resolveCell.resolve()
     })
-    while (this.effects.isInContext) {
+    abort?.addEventListener("abort", () => resolveCell.resolve())
+    while (this.effects.isInContext && !abort?.aborted) {
       let callback: () => void = () => {}
       const waitForNext = new Promise<void>((resolve) => {
         callback = resolve
@@ -53,6 +52,17 @@ export class GetSslCertificate {
     }
   }
 
+  /**
+   * Watches the SSL Certificate for the given hostnames if permitted. Returns an async iterator that yields whenever the value changes
+   */
+  watch(
+    abort?: AbortSignal,
+  ): AsyncGenerator<[string, string, string], void, unknown> {
+    const ctrl = new AbortController()
+    abort?.addEventListener("abort", () => ctrl.abort())
+    return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
+  }
+
   /**
    * Watches the SSL Certificate for the given hostnames if permitted. Takes a custom callback function to run whenever it changes
    */
@@ -60,12 +70,17 @@ export class GetSslCertificate {
     callback: (
       value: [string, string, string] | null,
       error?: Error,
-    ) => void | Promise<void>,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) {
     ;(async () => {
-      for await (const value of this.watch()) {
+      const ctrl = new AbortController()
+      for await (const value of this.watch(ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) {
+            ctrl.abort()
+            break
+          }
         } catch (e) {
           console.error(
             "callback function threw an error @ GetSslCertificate.onChange",
@@ -86,28 +101,20 @@ export class GetSslCertificate {
   /**
    * Watches the SSL Certificate for the given hostnames if permitted. Returns when the predicate is true
    */
-  async waitFor(pred: (value: [string, string, string] | null) => boolean) {
-    const resolveCell = { resolve: () => {} }
-    this.effects.onLeaveContext(() => {
-      resolveCell.resolve()
-    })
-    while (this.effects.isInContext) {
-      let callback: () => void = () => {}
-      const waitForNext = new Promise<void>((resolve) => {
-        callback = resolve
-        resolveCell.resolve = resolve
-      })
-      const res = await this.effects.getSslCertificate({
-        hostnames: this.hostnames,
-        algorithm: this.algorithm,
-        callback: () => callback(),
-      })
-      if (pred(res)) {
-        resolveCell.resolve()
-        return res
-      }
-      await waitForNext
-    }
-    return null
+  waitFor(
+    pred: (value: [string, string, string] | null) => boolean,
+  ): Promise<[string, string, string] | null> {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        for await (const next of this.watchGen(ctrl.signal)) {
+          if (pred(next)) {
+            return next
+          }
+        }
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 }

sdk/package/lib/util/SubContainer.ts

@@ -4,7 +4,7 @@ import * as cp from "child_process"
 import { promisify } from "util"
 import { Buffer } from "node:buffer"
 import { once } from "../../../base/lib/util/once"
-import { Drop } from "./Drop"
+import { Drop } from "../../../base/lib/util/Drop"
 import { Mounts } from "../mainFn/Mounts"
 import { BackupEffects } from "../backup/Backups"
 

sdk/package/lib/util/fileHelper.ts

@@ -4,10 +4,38 @@ import * as TOML from "@iarna/toml"
 import * as INI from "ini"
 import * as T from "../../../base/lib/types"
 import * as fs from "node:fs/promises"
-import { asError, partialDiff } from "../../../base/lib/util"
+import { asError } from "../../../base/lib/util"
+import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
+const deepEq = (left: unknown, right: unknown) => {
+  if (left === right) return true
+  if (Array.isArray(left) && Array.isArray(right)) {
+    if (left.length === right.length) {
+      for (const idx in left) {
+        if (!deepEq(left[idx], right[idx])) return false
+      }
+      return true
+    }
+  } else if (
+    typeof left === "object" &&
+    typeof right === "object" &&
+    left &&
+    right
+  ) {
+    const keys = new Set<keyof typeof left | keyof typeof right>([
+      ...(Object.keys(left) as (keyof typeof left)[]),
+      ...(Object.keys(right) as (keyof typeof right)[]),
+    ])
+    for (let key of keys) {
+      if (!deepEq(left[key], right[key])) return false
+    }
+    return true
+  }
+  return false
+}
+
 const exists = (path: string) =>
   fs.access(path).then(
     () => true,
@@ -99,10 +127,16 @@ type Validator<T, U> = matches.Validator<T, U> | matches.Validator<unknown, U>
 type ReadType<A> = {
   once: () => Promise<A | null>
   const: (effects: T.Effects) => Promise<A | null>
-  watch: (effects: T.Effects) => AsyncGenerator<A | null, null, unknown>
+  watch: (
+    effects: T.Effects,
+    abort?: AbortSignal,
+  ) => AsyncGenerator<A | null, null, unknown>
   onChange: (
     effects: T.Effects,
-    callback: (value: A | null, error?: Error) => void | Promise<void>,
+    callback: (
+      value: A | null,
+      error?: Error,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
   ) => void
   waitFor: (
     effects: T.Effects,
@@ -151,7 +185,12 @@ type ReadType<A> = {
  * ```
  */
 export class FileHelper<A> {
-  private consts: (() => void)[] = []
+  private consts: [
+    () => void,
+    any,
+    (a: any) => any,
+    (left: any, right: any) => any,
+  ][] = []
   protected constructor(
     readonly path: string,
     readonly writeData: (dataIn: A) => string,
@@ -209,10 +248,15 @@ export class FileHelper<A> {
     const watch = this.readWatch(effects, map, eq)
     const res = await watch.next()
     if (effects.constRetry) {
-      if (!this.consts.includes(effects.constRetry))
-        this.consts.push(effects.constRetry)
+      const record: (typeof this.consts)[number] = [
+        effects.constRetry,
+        res.value,
+        map,
+        eq,
+      ]
+      this.consts.push(record)
       watch.next().then(() => {
-        this.consts = this.consts.filter((a) => a === effects.constRetry)
+        this.consts = this.consts.filter((r) => r !== record)
         effects.constRetry && effects.constRetry()
       })
     }
@@ -223,11 +267,13 @@ export class FileHelper<A> {
     effects: T.Effects,
     map: (value: A) => B,
     eq: (left: B | null | undefined, right: B | null) => boolean,
+    abort?: AbortSignal,
   ) {
     let res
-    while (effects.isInContext) {
+    while (effects.isInContext && !abort?.aborted) {
       if (await exists(this.path)) {
         const ctrl = new AbortController()
+        abort?.addEventListener("abort", () => ctrl.abort())
         const watch = fs.watch(this.path, {
           persistent: false,
           signal: ctrl.signal,
@@ -254,14 +300,19 @@ export class FileHelper<A> {
 
   private readOnChange<B>(
     effects: T.Effects,
-    callback: (value: B | null, error?: Error) => void | Promise<void>,
+    callback: (
+      value: B | null,
+      error?: Error,
+    ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
     map: (value: A) => B,
     eq: (left: B | null | undefined, right: B | null) => boolean,
   ) {
     ;(async () => {
-      for await (const value of this.readWatch(effects, map, eq)) {
+      const ctrl = new AbortController()
+      for await (const value of this.readWatch(effects, map, eq, ctrl.signal)) {
         try {
-          await callback(value)
+          const res = await callback(value)
+          if (res.cancel) ctrl.abort()
         } catch (e) {
           console.error(
             "callback function threw an error @ FileHelper.read.onChange",
@@ -279,38 +330,36 @@ export class FileHelper<A> {
       )
   }
 
-  private async readWaitFor<B>(
+  private readWaitFor<B>(
     effects: T.Effects,
     pred: (value: B | null, error?: Error) => boolean,
     map: (value: A) => B,
   ): Promise<B | null> {
-    while (effects.isInContext) {
-      if (await exists(this.path)) {
-        const ctrl = new AbortController()
-        const watch = fs.watch(this.path, {
-          persistent: false,
-          signal: ctrl.signal,
-        })
-        const newRes = await this.readOnce(map)
-        const listen = Promise.resolve()
-          .then(async () => {
-            for await (const _ of watch) {
+    const ctrl = new AbortController()
+    return DropPromise.of(
+      Promise.resolve().then(async () => {
+        const watch = this.readWatch(effects, map, (_) => false, ctrl.signal)
+        while (true) {
+          try {
+            const res = await watch.next()
+            if (pred(res.value)) {
               ctrl.abort()
-              return null
+              return res.value
             }
-          })
-          .catch((e) => console.error(asError(e)))
-        if (pred(newRes)) {
-          ctrl.abort()
-          return newRes
+            if (res.done) {
+              break
+            }
+          } catch (e) {
+            if (pred(null, e as Error)) {
+              break
+            }
+          }
         }
-        await listen
-      } else {
-        if (pred(null)) return null
-        await onCreated(this.path).catch((e) => console.error(asError(e)))
-      }
-    }
-    return null
+        ctrl.abort()
+        return null
+      }),
+      () => ctrl.abort(),
+    )
   }
 
   read(): ReadType<A>
@@ -323,14 +372,24 @@ export class FileHelper<A> {
     eq?: (left: any, right: any) => boolean,
   ): ReadType<any> {
     map = map ?? ((a: A) => a)
-    eq = eq ?? ((left: any, right: any) => !partialDiff(left, right))
+    eq = eq ?? deepEq
     return {
       once: () => this.readOnce(map),
       const: (effects: T.Effects) => this.readConst(effects, map, eq),
-      watch: (effects: T.Effects) => this.readWatch(effects, map, eq),
+      watch: (effects: T.Effects, abort?: AbortSignal) => {
+        const ctrl = new AbortController()
+        abort?.addEventListener("abort", () => ctrl.abort())
+        return DropGenerator.of(
+          this.readWatch(effects, map, eq, ctrl.signal),
+          () => ctrl.abort(),
+        )
+      },
       onChange: (
         effects: T.Effects,
-        callback: (value: A | null, error?: Error) => void | Promise<void>,
+        callback: (
+          value: A | null,
+          error?: Error,
+        ) => { cancel: boolean } | Promise<{ cancel: boolean }>,
       ) => this.readOnChange(effects, callback, map, eq),
       waitFor: (effects: T.Effects, pred: (value: A | null) => boolean) =>
         this.readWaitFor(effects, pred, map),
@@ -345,13 +404,17 @@ export class FileHelper<A> {
     data: T.AllowReadonly<A> | A,
     options: { allowWriteAfterConst?: boolean } = {},
   ) {
-    await this.writeFile(this.validate(data))
-    if (
-      !options.allowWriteAfterConst &&
-      effects.constRetry &&
-      this.consts.includes(effects.constRetry)
-    )
-      throw new Error(`Canceled: write after const: ${this.path}`)
+    const newData = this.validate(data)
+    await this.writeFile(newData)
+    if (!options.allowWriteAfterConst && effects.constRetry) {
+      const records = this.consts.filter(([c]) => c === effects.constRetry)
+      for (const record of records) {
+        const [_, prev, map, eq] = record
+        if (!eq(prev, map(newData))) {
+          throw new Error(`Canceled: write after const: ${this.path}`)
+        }
+      }
+    }
     return null
   }
 
@@ -372,16 +435,14 @@ export class FileHelper<A> {
     const toWrite = this.writeData(mergeData)
     if (toWrite !== fileDataRaw) {
       this.writeFile(mergeData)
-      if (
-        !options.allowWriteAfterConst &&
-        effects.constRetry &&
-        this.consts.includes(effects.constRetry)
-      ) {
-        const diff = partialDiff(fileData, mergeData as any)
-        if (!diff) {
-          return null
+      if (!options.allowWriteAfterConst && effects.constRetry) {
+        const records = this.consts.filter(([c]) => c === effects.constRetry)
+        for (const record of records) {
+          const [_, prev, map, eq] = record
+          if (!eq(prev, map(mergeData))) {
+            throw new Error(`Canceled: write after const: ${this.path}`)
+          }
         }
-        throw new Error(`Canceled: write after const: ${this.path}`)
       }
     }
     return null

sdk/package/lib/util/index.ts

@@ -2,4 +2,4 @@ export * from "../../../base/lib/util"
 export { GetSslCertificate } from "./GetSslCertificate"
 
 export { hostnameInfoToAddress } from "../../../base/lib/util/Hostname"
-export { Drop } from "./Drop"
+export { Drop } from "../../../base/lib/util/Drop"