Fix/backups (#2659)

* fix master build (#2639)

* feat: Change ts to use rsync
Chore: Update the ts to use types over interface

* feat: Get the rust and the js to do a backup

* Wip: Got the backup working?

* fix permissions

* remove trixie list

* update tokio to fix timer bug

* fix error handling on backup

* wip

* remove idmap

* run restore before init, and init with own version on restore

---------

Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>

sdk/lib/backup/Backups.ts

@@ -1,6 +1,10 @@
+import { recursive } from "ts-matches"
 import { SDKManifest } from "../manifest/ManifestTypes"
 import * as T from "../types"
 
+import * as child_process from "child_process"
+import { promises as fsPromises } from "fs"
+
 export type BACKUP = "BACKUP"
 export const DEFAULT_OPTIONS: T.BackupOptions = {
   delete: true,
@@ -91,58 +95,22 @@ export class Backups<M extends SDKManifest> {
     )
     return this
   }
-  build() {
+  build(pathMaker: T.PathMaker) {
     const createBackup: T.ExpectedExports.createBackup = async ({
       effects,
     }) => {
-      // const previousItems = (
-      //   await effects
-      //     .readDir({
-      //       volumeId: Backups.BACKUP,
-      //       path: ".",
-      //     })
-      //     .catch(() => [])
-      // ).map((x) => `${x}`)
-      // const backupPaths = this.backupSet
-      //   .filter((x) => x.dstVolume === Backups.BACKUP)
-      //   .map((x) => x.dstPath)
-      //   .map((x) => x.replace(/\.\/([^]*)\//, "$1"))
-      // const filteredItems = previousItems.filter(
-      //   (x) => backupPaths.indexOf(x) === -1,
-      // )
-      // for (const itemToRemove of filteredItems) {
-      //   effects.console.error(`Trying to remove ${itemToRemove}`)
-      //   await effects
-      //     .removeDir({
-      //       volumeId: Backups.BACKUP,
-      //       path: itemToRemove,
-      //     })
-      //     .catch(() =>
-      //       effects.removeFile({
-      //         volumeId: Backups.BACKUP,
-      //         path: itemToRemove,
-      //       }),
-      //     )
-      //     .catch(() => {
-      //       console.warn(`Failed to remove ${itemToRemove} from backup volume`)
-      //     })
-      // }
       for (const item of this.backupSet) {
-        // if (notEmptyPath(item.dstPath)) {
-        //   await effects.createDir({
-        //     volumeId: item.dstVolume,
-        //     path: item.dstPath,
-        //   })
-        // }
-        // await effects
-        //   .runRsync({
-        //     ...item,
-        //     options: {
-        //       ...this.options,
-        //       ...item.options,
-        //     },
-        //   })
-        //   .wait()
+        const rsyncResults = await runRsync(
+          {
+            dstPath: item.dstPath,
+            dstVolume: item.dstVolume,
+            options: { ...this.options, ...item.options },
+            srcPath: item.srcPath,
+            srcVolume: item.srcVolume,
+          },
+          pathMaker,
+        )
+        await rsyncResults.wait()
       }
       return
     }
@@ -150,26 +118,17 @@ export class Backups<M extends SDKManifest> {
       effects,
     }) => {
       for (const item of this.backupSet) {
-        // if (notEmptyPath(item.srcPath)) {
-        //   await new Promise((resolve, reject) => fs.mkdir(items.src)).createDir(
-        //     {
-        //       volumeId: item.srcVolume,
-        //       path: item.srcPath,
-        //     },
-        //   )
-        // }
-        // await effects
-        //   .runRsync({
-        //     options: {
-        //       ...this.options,
-        //       ...item.options,
-        //     },
-        //     srcVolume: item.dstVolume,
-        //     dstVolume: item.srcVolume,
-        //     srcPath: item.dstPath,
-        //     dstPath: item.srcPath,
-        //   })
-        //   .wait()
+        const rsyncResults = await runRsync(
+          {
+            dstPath: item.dstPath,
+            dstVolume: item.dstVolume,
+            options: { ...this.options, ...item.options },
+            srcPath: item.srcPath,
+            srcVolume: item.srcVolume,
+          },
+          pathMaker,
+        )
+        await rsyncResults.wait()
       }
       return
     }
@@ -179,3 +138,73 @@ export class Backups<M extends SDKManifest> {
 function notEmptyPath(file: string) {
   return ["", ".", "./"].indexOf(file) === -1
 }
+async function runRsync(
+  rsyncOptions: {
+    srcVolume: string
+    dstVolume: string
+    srcPath: string
+    dstPath: string
+    options: T.BackupOptions
+  },
+  pathMaker: T.PathMaker,
+): Promise<{
+  id: () => Promise<string>
+  wait: () => Promise<null>
+  progress: () => Promise<number>
+}> {
+  const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions
+
+  const command = "rsync"
+  const args: string[] = []
+  if (options.delete) {
+    args.push("--delete")
+  }
+  if (options.force) {
+    args.push("--force")
+  }
+  if (options.ignoreExisting) {
+    args.push("--ignore-existing")
+  }
+  for (const exclude of options.exclude) {
+    args.push(`--exclude=${exclude}`)
+  }
+  args.push("-actAXH")
+  args.push("--info=progress2")
+  args.push("--no-inc-recursive")
+  args.push(pathMaker({ volume: srcVolume, path: srcPath }))
+  args.push(pathMaker({ volume: dstVolume, path: dstPath }))
+  const spawned = child_process.spawn(command, args, { detached: true })
+  let percentage = 0.0
+  spawned.stdout.on("data", (data: unknown) => {
+    const lines = String(data).replace("\r", "\n").split("\n")
+    for (const line of lines) {
+      const parsed = /$([0-9.]+)%/.exec(line)?.[1]
+      if (!parsed) continue
+      percentage = Number.parseFloat(parsed)
+    }
+  })
+
+  spawned.stderr.on("data", (data: unknown) => {
+    console.error(String(data))
+  })
+
+  const id = async () => {
+    const pid = spawned.pid
+    if (pid === undefined) {
+      throw new Error("rsync process has no pid")
+    }
+    return String(pid)
+  }
+  const waitPromise = new Promise<null>((resolve, reject) => {
+    spawned.on("exit", (code: any) => {
+      if (code === 0) {
+        resolve(null)
+      } else {
+        reject(new Error(`rsync exited with code ${code}`))
+      }
+    })
+  })
+  const wait = () => waitPromise
+  const progress = () => Promise.resolve(percentage)
+  return { id, wait, progress }
+}

sdk/lib/backup/setupBackups.ts

@@ -1,6 +1,6 @@
 import { Backups } from "./Backups"
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { ExpectedExports } from "../types"
+import { ExpectedExports, PathMaker } from "../types"
 import { _ } from "../util"
 
 export type SetupBackupsParams<M extends SDKManifest> = Array<
@@ -27,14 +27,14 @@ export function setupBackups<M extends SDKManifest>(
     get createBackup() {
       return (async (options) => {
         for (const backup of backups) {
-          await backup.build().createBackup(options)
+          await backup.build(options.pathMaker).createBackup(options)
         }
       }) as ExpectedExports.createBackup
     },
     get restoreBackup() {
       return (async (options) => {
         for (const backup of backups) {
-          await backup.build().restoreBackup(options)
+          await backup.build(options.pathMaker).restoreBackup(options)
         }
       }) as ExpectedExports.restoreBackup
     },

sdk/lib/config/configTypes.ts

@@ -240,7 +240,6 @@ export type ListValueSpecText = {
   inputmode: "text" | "email" | "tel" | "url"
   placeholder: string | null
 }
-
 export type ListValueSpecObject = {
   type: "object"
   /** this is a mapped type of the config object at this level, replacing the object's values with specs on those values */

sdk/lib/manifest/ManifestTypes.ts

@@ -1,7 +1,7 @@
 import { ValidEmVer } from "../emverLite/mod"
 import { ActionMetadata, ImageConfig, ImageId } from "../types"
 
-export interface Container {
+export type Container = {
   /** This should be pointing to a docker container name */
   image: string
   /** These should match the manifest data volumes */
@@ -72,7 +72,7 @@ export type SDKManifest = {
   readonly dependencies: Readonly<Record<string, ManifestDependency>>
 }
 
-export interface ManifestDependency {
+export type ManifestDependency = {
   /**
    * A human readable explanation on what the dependency is used for
    */

sdk/lib/types.ts

@@ -26,6 +26,7 @@ export * from "./osBindings"
 export { SDKManifest } from "./manifest/ManifestTypes"
 export { HealthReceipt } from "./health/HealthReceipt"
 
+export type PathMaker = (options: { volume: string; path: string }) => string
 export type ExportedAction = (options: {
   effects: Effects
   input?: Record<string, unknown>
@@ -43,10 +44,14 @@ export namespace ExpectedExports {
   // /** These are how we make sure the our dependency configurations are valid and if not how to fix them. */
   // export type dependencies = Dependencies;
   /** For backing up service data though the startOS UI */
-  export type createBackup = (options: { effects: Effects }) => Promise<unknown>
+  export type createBackup = (options: {
+    effects: Effects
+    pathMaker: PathMaker
+  }) => Promise<unknown>
   /** For restoring service data that was previously backed up using the startOS UI create backup flow. Backup restores are also triggered via the startOS UI, or doing a system restore flow during setup. */
   export type restoreBackup = (options: {
     effects: Effects
+    pathMaker: PathMaker
   }) => Promise<unknown>
 
   // /** Health checks are used to determine if the service is working properly after starting