misc fixes (#2892)

* use docker for build steps that require linux when not on linux

* use fuse for overlay

* quiet mountpoint

* node 22

* misc fixes

* make shasum more compliant

* optimize download-base-image.sh with cleaner url handling and checksum verification

* fix script

* fixes #2900

* bump node and npm versions in web readme

* Minor pl.ts fixes

* fixes in response to synapse issues

* beta.8

* update ts-matches

* beta.11

* pl.ts finetuning

---------

Co-authored-by: Mariusz Kogen <k0gen@pm.me>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

sdk/package/lib/StartSdk.ts

@@ -1170,7 +1170,7 @@ export async function runCommand<Manifest extends T.SDKManifest>(
       .catch(() => "{}")
       .then(JSON.parse)
     commands = imageMeta.entrypoint ?? []
-    commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
+    commands = commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
   } else commands = splitCommand(command)
   return SubContainer.withTemp(
     effects,

sdk/package/lib/health/checkFns/runHealthScript.ts

@@ -21,7 +21,7 @@ export const runHealthScript = async <Manifest extends SDKManifest>(
   } = {},
 ): Promise<HealthCheckResult> => {
   const res = await Promise.race([
-    subcontainer.exec(runCommand),
+    subcontainer.execFail(runCommand),
     timeoutPromise(timeout),
   ]).catch((e) => {
     console.warn(errorMessage)

sdk/package/lib/mainFn/CommandController.ts

@@ -42,19 +42,21 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
         onStderr?: (chunk: Buffer | string | any) => void
       },
     ) => {
-      let commands: string[]
-      if (command instanceof T.UseEntrypoint) {
-        const imageMeta: T.ImageMetadata = await fs
-          .readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
-            encoding: "utf8",
-          })
-          .catch(() => "{}")
-          .then(JSON.parse)
-        commands = imageMeta.entrypoint ?? []
-        commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? []))
-      } else commands = splitCommand(command)
-
       try {
+        let commands: string[]
+        if (command instanceof T.UseEntrypoint) {
+          const imageMeta: T.ImageMetadata = await fs
+            .readFile(`/media/startos/images/${subcontainer.imageId}.json`, {
+              encoding: "utf8",
+            })
+            .catch(() => "{}")
+            .then(JSON.parse)
+          commands = imageMeta.entrypoint ?? []
+          commands = commands.concat(
+            ...(command.overridCmd ?? imageMeta.cmd ?? []),
+          )
+        } else commands = splitCommand(command)
+
         let childProcess: cp.ChildProcess
         if (options.runAsInit) {
           childProcess = await subcontainer.launch(commands, {
@@ -111,10 +113,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
   get subContainerHandle() {
     return new SubContainerHandle(this.subcontainer)
   }
-  async wait({ timeout = NO_TIMEOUT } = {}) {
+  async wait({ timeout = NO_TIMEOUT, keepSubcontainer = false } = {}) {
     if (timeout > 0)
       setTimeout(() => {
-        this.term()
+        this.term({ keepSubcontainer })
       }, timeout)
     try {
       return await this.runningAnswer
@@ -122,10 +124,14 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
       if (!this.state.exited) {
         this.process.kill("SIGKILL")
       }
-      await this.subcontainer.destroy().catch((_) => {})
+      if (!keepSubcontainer) await this.subcontainer.destroy()
     }
   }
-  async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
+  async term({
+    signal = SIGTERM,
+    timeout = this.sigtermTimeout,
+    keepSubcontainer = false,
+  } = {}) {
     try {
       if (!this.state.exited) {
         if (signal !== "SIGKILL") {
@@ -142,10 +148,10 @@ export class CommandController<Manifest extends T.SDKManifest> extends Drop {
 
       await this.runningAnswer
     } finally {
-      await this.subcontainer.destroy()
+      if (!keepSubcontainer) await this.subcontainer.destroy()
     }
   }
   onDrop(): void {
-    this.term().catch(console.error)
+    this.term({ keepSubcontainer: true }).catch(console.error)
   }
 }

sdk/package/lib/mainFn/Daemon.ts

@@ -21,7 +21,7 @@ export class Daemon<Manifest extends T.SDKManifest> {
     return this.commandController?.subContainerHandle
   }
   static of<Manifest extends T.SDKManifest>() {
-    return async <A extends string>(
+    return async (
       effects: T.Effects,
       subcontainer: SubContainer<Manifest>,
       command: T.CommandType,
@@ -54,17 +54,21 @@ export class Daemon<Manifest extends T.SDKManifest> {
     }
     this.shouldBeRunning = true
     let timeoutCounter = 0
-    new Promise(async () => {
+    ;(async () => {
       while (this.shouldBeRunning) {
         if (this.commandController)
-          await this.commandController.term().catch((err) => console.error(err))
+          await this.commandController
+            .term({ keepSubcontainer: true })
+            .catch((err) => console.error(err))
         this.commandController = await this.startCommand()
-        await this.commandController.wait().catch((err) => console.error(err))
+        await this.commandController
+          .wait({ keepSubcontainer: true })
+          .catch((err) => console.error(err))
         await new Promise((resolve) => setTimeout(resolve, timeoutCounter))
         timeoutCounter += TIMEOUT_INCREMENT_MS
-        timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
+        timeoutCounter = Math.min(MAX_TIMEOUT_MS, timeoutCounter)
       }
-    }).catch((err) => {
+    })().catch((err) => {
       console.error(asError(err))
     })
   }

sdk/package/lib/mainFn/Daemons.ts

@@ -197,7 +197,7 @@ export class Daemons<Manifest extends T.SDKManifest, Ids extends string>
 
   async build() {
     for (const daemon of this.healthDaemons) {
-      await daemon.updateStatus()
+      await daemon.init()
     }
     for (const health of this.healthChecks) {
       health.start()

sdk/package/lib/mainFn/HealthDaemon.ts

@@ -169,4 +169,16 @@ export class HealthDaemon<Manifest extends SDKManifest> {
     const healths = this.dependencies.map((d) => d.running && d._health)
     this.changeRunning(healths.every((x) => x && x.result === "success"))
   }
+
+  async init() {
+    if (this.ready.display) {
+      this.effects.setHealth({
+        id: this.id,
+        message: null,
+        name: this.ready.display,
+        result: "starting",
+      })
+    }
+    await this.updateStatus()
+  }
 }

sdk/package/lib/util/Drop.ts

@@ -10,9 +10,20 @@ export abstract class Drop {
   protected constructor() {
     this.id = Drop.idCtr++
     this.ref = { id: this.id }
-    Drop.weak[this.id] = this.weak()
-    Drop.registry.register(this, this.id, this)
+    const weak = this.weak()
+    Drop.weak[this.id] = weak
+    Drop.registry.register(this.ref, this.id, this.ref)
+
+    return new Proxy(this, {
+      set(target: any, prop, value) {
+        if (prop === "ref") return false
+        target[prop] = value
+        ;(weak as any)[prop] = value
+        return true
+      },
+    })
   }
+  protected register() {}
   protected weak(): this {
     const weak = Object.assign(Object.create(Object.getPrototypeOf(this)), this)
     weak.ref = new WeakRef(this.ref)
@@ -21,7 +32,7 @@ export abstract class Drop {
   abstract onDrop(): void
   drop(): void {
     this.onDrop()
-    Drop.registry.unregister(this)
+    Drop.registry.unregister(this.ref)
     delete Drop.weak[this.id]
   }
 }

sdk/package/lib/util/SubContainer.ts

@@ -37,6 +37,11 @@ export interface ExecSpawnable {
     options?: CommandOptions & ExecOptions,
     timeoutMs?: number | null,
   ): Promise<ExecResults>
+  execFail(
+    command: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs?: number | null,
+  ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }>
   spawn(
     command: string[],
     options?: CommandOptions & StdioOptions,
@@ -88,7 +93,9 @@ export class SubContainer<
                 imageId: this.imageId,
                 rootfs: this.rootfs,
               })
-              reject(new Error(`Failed to start subcontainer ${this.imageId}`))
+              return reject(
+                new Error(`Failed to start subcontainer ${this.imageId}`),
+              )
             }
             await wait(1)
           }
@@ -144,8 +151,9 @@ export class SubContainer<
       }
 
       return res
-    } finally {
+    } catch (e) {
       await res.destroy()
+      throw e
     }
   }
 
@@ -272,11 +280,13 @@ export class SubContainer<
   }
 
   onDrop(): void {
+    console.log(`Cleaning up dangling subcontainer ${this.guid}`)
     this.destroy()
   }
 
   /**
    * @description run a command inside this subcontainer
+   * DOES NOT THROW ON NONZERO EXIT CODE (see execFail)
    * @param commands an array representing the command and args to execute
    * @param options
    * @param timeoutMs how long to wait before killing the command in ms
@@ -287,6 +297,7 @@ export class SubContainer<
     options?: CommandOptions & ExecOptions,
     timeoutMs: number | null = 30000,
   ): Promise<{
+    throw: () => { stdout: string | Buffer; stderr: string | Buffer }
     exitCode: number | null
     exitSignal: NodeJS.Signals | null
     stdout: string | Buffer
@@ -367,16 +378,43 @@ export class SubContainer<
       child.stderr.on("data", appendData(stderr))
       child.on("exit", (code, signal) => {
         clearTimeout(killTimeout)
-        resolve({
+        const result = {
           exitCode: code,
           exitSignal: signal,
           stdout: stdout.data,
           stderr: stderr.data,
+        }
+        resolve({
+          throw: () =>
+            !code && !signal
+              ? { stdout: stdout.data, stderr: stderr.data }
+              : (() => {
+                  throw new ExitError(command[0], result)
+                })(),
+          ...result,
         })
       })
     })
   }
 
+  /**
+   * @description run a command inside this subcontainer, throwing on non-zero exit status
+   * @param commands an array representing the command and args to execute
+   * @param options
+   * @param timeoutMs how long to wait before killing the command in ms
+   * @returns
+   */
+  async execFail(
+    command: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs: number | null = 30000,
+  ): Promise<{
+    stdout: string | Buffer
+    stderr: string | Buffer
+  }> {
+    return this.exec(command, options, timeoutMs).then((res) => res.throw())
+  }
+
   async launch(
     command: string[],
     options?: CommandOptions,
@@ -478,6 +516,15 @@ export class SubContainerHandle implements ExecSpawnable {
   ): Promise<ExecResults> {
     return this.subContainer.exec(command, options, timeoutMs)
   }
+
+  execFail(
+    command: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs?: number | null,
+  ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
+    return this.subContainer.execFail(command, options, timeoutMs)
+  }
+
   spawn(
     command: string[],
     options: CommandOptions & StdioOptions = { stdio: "inherit" },

sdk/package/lib/util/fileHelper.ts

@@ -234,7 +234,7 @@ export class FileHelper<A> {
   /**
    * Accepts full structured data and overwrites the existing file on disk if it exists.
    */
-  async write(effects: T.Effects, data: A) {
+  async write(effects: T.Effects, data: T.AllowReadonly<A> | A) {
     await this.writeFile(this.validate(data))
     if (effects.constRetry && this.consts.includes(effects.constRetry))
       throw new Error(`Canceled: write after const: ${this.path}`)
@@ -244,7 +244,7 @@ export class FileHelper<A> {
   /**
    * Accepts partial structured data and performs a merge with the existing file on disk.
    */
-  async merge(effects: T.Effects, data: T.DeepPartial<A>) {
+  async merge(effects: T.Effects, data: T.AllowReadonly<T.DeepPartial<A>>) {
     const fileDataRaw = await this.readFileRaw()
     let fileData: any = fileDataRaw === null ? null : this.readData(fileDataRaw)
     try {