Feature/backup+restore (#2613)

* feat: Implementation on the backup for the service.

* wip: Getting the flow of backup/restore

* feat: Recover

* Feature: Commit the full pass on the backup restore.

* use special type for backup instead of special id (#2614)

* fix: Allow compat docker style to run again

* fix: Backup for the js side

* chore: Update some of the callbacks

---------

Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>

container-runtime/src/Adapters/RpcListener.ts

@@ -81,7 +81,7 @@ const callbackType = object({
   id: idType,
   method: literal("callback"),
   params: object({
-    callback: string,
+    callback: number,
     args: array,
   }),
 })

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -68,7 +68,7 @@ export class DockerProcedureContainer {
             },
           })
         } else if (volumeMount.type === "backup") {
-          throw new Error("TODO")
+          await overlay.mount({ type: "backup", subpath: null }, mounts[mount])
         }
       }
     }

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -145,6 +145,15 @@ export class MainLoop {
               ...actionProcedure.args,
               JSON.stringify(timeChanged),
             ])
+            if (executed.exitCode === 0) {
+              await effects.setHealth({
+                id: healthId,
+                name: value.name,
+                result: "success",
+                message: actionProcedure["success-message"],
+              })
+              return
+            }
             if (executed.exitCode === 59) {
               await effects.setHealth({
                 id: healthId,

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -399,11 +399,10 @@ export class SystemForEmbassy implements System {
   ): Promise<void> {
     const backup = this.manifest.backup.create
     if (backup.type === "docker") {
-      const container = await DockerProcedureContainer.of(
-        effects,
-        backup,
-        this.manifest.volumes,
-      )
+      const container = await DockerProcedureContainer.of(effects, backup, {
+        ...this.manifest.volumes,
+        BACKUP: { type: "backup", readonly: false },
+      })
       await container.execFail([backup.entrypoint, ...backup.args], timeoutMs)
     } else {
       const moduleCode = await this.moduleCode
@@ -421,7 +420,10 @@ export class SystemForEmbassy implements System {
       const container = await DockerProcedureContainer.of(
         effects,
         restoreBackup,
-        this.manifest.volumes,
+        {
+          ...this.manifest.volumes,
+          BACKUP: { type: "backup", readonly: true },
+        },
       )
       await container.execFail(
         [restoreBackup.entrypoint, ...restoreBackup.args],
@@ -664,46 +666,6 @@ export class SystemForEmbassy implements System {
     }
     throw new Error(`Unknown type in the fetch properties: ${setConfigValue}`)
   }
-  private async health(
-    effects: HostSystemStartOs,
-    healthId: string,
-    timeSinceStarted: unknown,
-    timeoutMs: number | null,
-  ): Promise<void> {
-    const healthProcedure = this.manifest["health-checks"][healthId]
-    if (!healthProcedure) return
-    if (healthProcedure.type === "docker") {
-      const container = await DockerProcedureContainer.of(
-        effects,
-        healthProcedure,
-        this.manifest.volumes,
-      )
-      return JSON.parse(
-        (
-          await container.execFail(
-            [
-              healthProcedure.entrypoint,
-              ...healthProcedure.args,
-              JSON.stringify(timeSinceStarted),
-            ],
-            timeoutMs,
-          )
-        ).stdout.toString(),
-      )
-    } else if (healthProcedure.type === "script") {
-      const moduleCode = await this.moduleCode
-      const method = moduleCode.health?.[healthId]
-      if (!method) throw new Error("Expecting that the method health exists")
-      await method(
-        new PolyfillEffects(effects, this.manifest),
-        Number(timeSinceStarted),
-      ).then((x) => {
-        if ("result" in x) return x.result
-        if ("error" in x) throw new Error("Error getting config: " + x.error)
-        throw new Error("Error getting config: " + x["error-code"][1])
-      })
-    }
-  }
   private async action(
     effects: HostSystemStartOs,
     actionId: string,

container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts

@@ -57,6 +57,7 @@ export const matchManifest = object(
         matchProcedure,
         object({
           name: string,
+          ["success-message"]: string,
         }),
       ),
     ]),

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -267,6 +267,7 @@ export class PolyfillEffects implements oet.Effects {
       json: () => fetched.json(),
     }
   }
+
   runRsync(rsyncOptions: {
     srcVolume: string
     dstVolume: string
@@ -277,6 +278,36 @@ export class PolyfillEffects implements oet.Effects {
     id: () => Promise<string>
     wait: () => Promise<null>
     progress: () => Promise<number>
+  } {
+    let secondRun: ReturnType<typeof this._runRsync> | undefined
+    let firstRun = this._runRsync(rsyncOptions)
+    let waitValue = firstRun.wait().then((x) => {
+      secondRun = this._runRsync(rsyncOptions)
+      return secondRun.wait()
+    })
+    const id = async () => {
+      return secondRun?.id?.() ?? firstRun.id()
+    }
+    const wait = () => waitValue
+    const progress = async () => {
+      const secondProgress = secondRun?.progress?.()
+      if (secondProgress) {
+        return (await secondProgress) / 2.0 + 0.5
+      }
+      return (await firstRun.progress()) / 2.0
+    }
+    return { id, wait, progress }
+  }
+  _runRsync(rsyncOptions: {
+    srcVolume: string
+    dstVolume: string
+    srcPath: string
+    dstPath: string
+    options: oet.BackupOptions
+  }): {
+    id: () => Promise<string>
+    wait: () => Promise<null>
+    progress: () => Promise<number>
   } {
     const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions
     const command = "rsync"

container-runtime/src/Models/CallbackHolder.ts

@@ -2,16 +2,16 @@ export class CallbackHolder {
   constructor() {}
   private root = (Math.random() + 1).toString(36).substring(7)
   private inc = 0
-  private callbacks = new Map<string, Function>()
+  private callbacks = new Map<number, Function>()
   private newId() {
-    return this.root + (this.inc++).toString(36)
+    return this.inc++
   }
   addCallback(callback: Function) {
     const id = this.newId()
     this.callbacks.set(id, callback)
     return id
   }
-  callCallback(index: string, args: any[]): Promise<unknown> {
+  callCallback(index: number, args: any[]): Promise<unknown> {
     const callback = this.callbacks.get(index)
     if (!callback) throw new Error(`Callback ${index} does not exist`)
     this.callbacks.delete(index)

container-runtime/src/Models/Volume.ts

@@ -1,14 +1,17 @@
 import * as fs from "node:fs/promises"
 
+export const BACKUP = "backup"
 export class Volume {
   readonly path: string
   constructor(
     readonly volumeId: string,
     _path = "",
   ) {
-    const path = (this.path = `/media/startos/volumes/${volumeId}${
-      !_path ? "" : `/${_path}`
-    }`)
+    if (volumeId.toLowerCase() === BACKUP) {
+      this.path = `/media/startos/backup${!_path ? "" : `/${_path}`}`
+    } else {
+      this.path = `/media/startos/volumes/${volumeId}${!_path ? "" : `/${_path}`}`
+    }
   }
   async exists() {
     return fs.stat(this.path).then(