Feat/implement rest of poly effects (#2587)

* feat: Add the implementation of the rest of the polyfillEffects

* chore: Add in the rsync

* chore: Add in the changes needed to indicate that the service does not need config

* fix: Vaultwarden sets, starts, stops, uninstalls

* chore: Update the polyFilleffect and add two more

* Update MainLoop.ts

* chore: Add in the set config of the deps on the config set

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -48,19 +48,7 @@ export class MainLoop {
       this.system.manifest.volumes,
     )
     if (jsMain) {
-      const daemons = Daemons.of({
+      throw new Error("Unreachable")
-        effects,
-        started: async (_) => {},
-        healthReceipts: [],
-      })
-      throw new Error("todo")
-      // return {
-      //   daemon,
-      //   wait: daemon.wait().finally(() => {
-      //     this.clean()
-      //     effects.setMainStatus({ status: "stopped" })
-      //   }),
-      // }
     }
     const daemon = await daemons.runDaemon()(
       this.effects,

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -25,6 +25,7 @@ import {
   anyOf,
   deferred,
   Parser,
+  array,
 } from "ts-matches"
 import { HostSystemStartOs } from "../../HostSystemStartOs"
 import { JsonPath, unNestPath } from "../../../Models/JsonPath"
@@ -41,6 +42,48 @@ const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json"
 const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
 const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig"
 
+const matchSetResult = object(
+  {
+    "depends-on": dictionary([string, array(string)]),
+    dependsOn: dictionary([string, array(string)]),
+    signal: literals(
+      "SIGTERM",
+      "SIGHUP",
+      "SIGINT",
+      "SIGQUIT",
+      "SIGILL",
+      "SIGTRAP",
+      "SIGABRT",
+      "SIGBUS",
+      "SIGFPE",
+      "SIGKILL",
+      "SIGUSR1",
+      "SIGSEGV",
+      "SIGUSR2",
+      "SIGPIPE",
+      "SIGALRM",
+      "SIGSTKFLT",
+      "SIGCHLD",
+      "SIGCONT",
+      "SIGSTOP",
+      "SIGTSTP",
+      "SIGTTIN",
+      "SIGTTOU",
+      "SIGURG",
+      "SIGXCPU",
+      "SIGXFSZ",
+      "SIGVTALRM",
+      "SIGPROF",
+      "SIGWINCH",
+      "SIGIO",
+      "SIGPWR",
+      "SIGSYS",
+      "SIGINFO",
+    ),
+  },
+  ["depends-on", "dependsOn"],
+)
+
 export type PackagePropertiesV2 = {
   [name: string]: PackagePropertyObject | PackagePropertyString
 }
@@ -120,6 +163,7 @@ const matchProperties = object({
   data: matchPackageProperties,
 })
 
+const DEFAULT_REGISTRY = "https://registry.start9.com"
 export class SystemForEmbassy implements System {
   currentRunning: MainLoop | undefined
   static async of(manifestLocation: string = MANIFEST_LOCATION) {
@@ -383,7 +427,7 @@ export class SystemForEmbassy implements System {
   private async setConfig(
     effects: HostSystemStartOs,
     newConfigWithoutPointers: unknown,
-  ): Promise<T.SetResult> {
+  ): Promise<void> {
     const newConfig = structuredClone(newConfigWithoutPointers)
     await updateConfig(
       effects,
@@ -391,45 +435,76 @@ export class SystemForEmbassy implements System {
       newConfig,
     )
     const setConfigValue = this.manifest.config?.set
-    if (!setConfigValue) return { signal: "SIGTERM", "depends-on": {} }
+    if (!setConfigValue) return
     if (setConfigValue.type === "docker") {
       const container = await DockerProcedureContainer.of(
         effects,
         setConfigValue,
         this.manifest.volumes,
       )
-      return JSON.parse(
+      const answer = matchSetResult.unsafeCast(
-        (
+        JSON.parse(
-          await container.exec([
+          (
-            setConfigValue.entrypoint,
+            await container.exec([
-            ...setConfigValue.args,
+              setConfigValue.entrypoint,
-            JSON.stringify(newConfig),
+              ...setConfigValue.args,
-          ])
+              JSON.stringify(newConfig),
-        ).stdout.toString(),
+            ])
+          ).stdout.toString(),
+        ),
       )
+      const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {}
+      await this.setConfigSetConfig(effects, dependsOn)
+      return
     } else if (setConfigValue.type === "script") {
       const moduleCode = await this.moduleCode
       const method = moduleCode.setConfig
       if (!method) throw new Error("Expecting that the method setConfig exists")
-      return await method(
+
-        new PolyfillEffects(effects, this.manifest),
+      const answer = matchSetResult.unsafeCast(
-        newConfig as U.Config,
+        await method(
-      ).then((x): T.SetResult => {
+          new PolyfillEffects(effects, this.manifest),
-        if ("result" in x)
+          newConfig as U.Config,
-          return {
+        ).then((x): T.SetResult => {
-            "depends-on": x.result["depends-on"],
+          if ("result" in x)
-            signal: x.result.signal === "SIGEMT" ? "SIGTERM" : x.result.signal,
+            return {
-          }
+              dependsOn: x.result["depends-on"],
-        if ("error" in x) throw new Error("Error getting config: " + x.error)
+              signal:
-        throw new Error("Error getting config: " + x["error-code"][1])
+                x.result.signal === "SIGEMT" ? "SIGTERM" : x.result.signal,
-      })
+            }
-    } else {
+          if ("error" in x) throw new Error("Error getting config: " + x.error)
-      return {
+          throw new Error("Error getting config: " + x["error-code"][1])
-        "depends-on": {},
+        }),
-        signal: "SIGTERM",
+      )
-      }
+      const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {}
+      await this.setConfigSetConfig(effects, dependsOn)
+      return
     }
   }
+  private async setConfigSetConfig(
+    effects: HostSystemStartOs,
+    dependsOn: { [x: string]: readonly string[] },
+  ) {
+    await effects.setDependencies({
+      dependencies: Object.entries(dependsOn).flatMap(([key, value]) => {
+        const dependency = this.manifest.dependencies?.[key]
+        if (!dependency) return []
+        const versionSpec = dependency.version
+        const registryUrl = DEFAULT_REGISTRY
+        const kind = "running"
+        return [
+          {
+            id: key,
+            versionSpec,
+            registryUrl,
+            kind,
+            healthChecks: [...value],
+          },
+        ]
+      }),
+    })
+  }
+
   private async migration(
     effects: HostSystemStartOs,
     fromVersion: string,

container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts

@@ -100,16 +100,6 @@ export type Effects = {
   is_sandboxed(): boolean
 
   exists(input: { volumeId: string; path: string }): Promise<boolean>
-  bindLocal(options: {
-    internalPort: number
-    name: string
-    externalPort: number
-  }): Promise<string>
-  bindTor(options: {
-    internalPort: number
-    name: string
-    externalPort: number
-  }): Promise<string>
 
   fetch(
     url: string,

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -3,12 +3,11 @@ import * as oet from "./oldEmbassyTypes"
 import { Volume } from "../../../Models/Volume"
 import * as child_process from "child_process"
 import { promisify } from "util"
-import { Daemons, startSdk, T } from "@start9labs/start-sdk"
+import { daemons, startSdk, T } from "@start9labs/start-sdk"
 import { HostSystemStartOs } from "../../HostSystemStartOs"
 import "isomorphic-fetch"
 import { Manifest } from "./matchManifest"
-
+import { DockerProcedureContainer } from "./DockerProcedureContainer"
-const execFile = promisify(child_process.execFile)
 
 export class PolyfillEffects implements oet.Effects {
   constructor(
@@ -111,17 +110,100 @@ export class PolyfillEffects implements oet.Effects {
     wait(): Promise<oet.ResultType<string>>
     term(): Promise<void>
   } {
-    throw new Error("Method not implemented.")
+    const dockerProcedureContainer = DockerProcedureContainer.of(
+      this.effects,
+      this.manifest.main,
+      this.manifest.volumes,
+    )
+    const daemon = dockerProcedureContainer.then((dockerProcedureContainer) =>
+      daemons.runDaemon()(
+        this.effects,
+        this.manifest.main.image,
+        [input.command, ...(input.args || [])],
+        {
+          overlay: dockerProcedureContainer.overlay,
+        },
+      ),
+    )
+    return {
+      wait: () =>
+        daemon.then((daemon) =>
+          daemon.wait().then(() => {
+            return { result: "" }
+          }),
+        ),
+      term: () => daemon.then((daemon) => daemon.term()),
+    }
   }
-  chown(input: { volumeId: string; path: string; uid: string }): Promise<null> {
+  async chown(input: {
-    throw new Error("Method not implemented.")
+    volumeId: string
+    path: string
+    uid: string
+  }): Promise<null> {
+    await startSdk
+      .runCommand(
+        this.effects,
+        this.manifest.main.image,
+        ["chown", "--recursive", input.uid, `/drive/${input.path}`],
+        {
+          mounts: [
+            {
+              path: "/drive",
+              options: {
+                type: "volume",
+                id: input.volumeId,
+                subpath: null,
+                readonly: false,
+              },
+            },
+          ],
+        },
+      )
+      .then((x: any) => ({
+        stderr: x.stderr.toString(),
+        stdout: x.stdout.toString(),
+      }))
+      .then((x) => {
+        if (!!x.stderr) {
+          throw new Error(x.stderr)
+        }
+      })
+    return null
   }
-  chmod(input: {
+  async chmod(input: {
     volumeId: string
     path: string
     mode: string
   }): Promise<null> {
-    throw new Error("Method not implemented.")
+    await startSdk
+      .runCommand(
+        this.effects,
+        this.manifest.main.image,
+        ["chmod", "--recursive", input.mode, `/drive/${input.path}`],
+        {
+          mounts: [
+            {
+              path: "/drive",
+              options: {
+                type: "volume",
+                id: input.volumeId,
+                subpath: null,
+                readonly: false,
+              },
+            },
+          ],
+        },
+      )
+      .then((x: any) => ({
+        stderr: x.stderr.toString(),
+        stdout: x.stdout.toString(),
+      }))
+      .then((x) => {
+        if (!!x.stderr) {
+          throw new Error(x.stderr)
+        }
+      })
+    return null
   }
   sleep(timeMs: number): Promise<null> {
     return new Promise((resolve) => setTimeout(resolve, timeMs))
@@ -149,20 +231,6 @@ export class PolyfillEffects implements oet.Effects {
       .then(() => true)
       .catch(() => false)
   }
-  bindLocal(options: {
-    internalPort: number
-    name: string
-    externalPort: number
-  }): Promise<string> {
-    throw new Error("Method not implemented.")
-  }
-  bindTor(options: {
-    internalPort: number
-    name: string
-    externalPort: number
-  }): Promise<string> {
-    throw new Error("Method not implemented.")
-  }
   async fetch(
     url: string,
     options?:
@@ -199,7 +267,7 @@ export class PolyfillEffects implements oet.Effects {
       json: () => fetched.json(),
     }
   }
-  runRsync(options: {
+  runRsync(rsyncOptions: {
     srcVolume: string
     dstVolume: string
     srcPath: string
@@ -210,6 +278,59 @@ export class PolyfillEffects implements oet.Effects {
     wait: () => Promise<null>
     progress: () => Promise<number>
   } {
-    throw new Error("Method not implemented.")
+    const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions
+    const command = "rsync"
+    const args: string[] = []
+    if (options.delete) {
+      args.push("--delete")
+    }
+    if (options.force) {
+      args.push("--force")
+    }
+    if (options.ignoreExisting) {
+      args.push("--ignore-existing")
+    }
+    for (const exclude of options.exclude) {
+      args.push(`--exclude=${exclude}`)
+    }
+    args.push("-actAXH")
+    args.push("--info=progress2")
+    args.push("--no-inc-recursive")
+    args.push(new Volume(srcVolume, srcPath).path)
+    args.push(new Volume(dstVolume, dstPath).path)
+    const spawned = child_process.spawn(command, args, { detached: true })
+    let percentage = 0.0
+    spawned.stdout.on("data", (data: unknown) => {
+      const lines = String(data).replace("\r", "\n").split("\n")
+      for (const line of lines) {
+        const parsed = /$([0-9.]+)%/.exec(line)?.[1]
+        if (!parsed) continue
+        percentage = Number.parseFloat(parsed)
+      }
+    })
+
+    spawned.stderr.on("data", (data: unknown) => {
+      console.error(String(data))
+    })
+
+    const id = async () => {
+      const pid = spawned.pid
+      if (pid === undefined) {
+        throw new Error("rsync process has no pid")
+      }
+      return String(pid)
+    }
+    const waitPromise = new Promise<null>((resolve, reject) => {
+      spawned.on("exit", (code) => {
+        if (code === 0) {
+          resolve(null)
+        } else {
+          reject(new Error(`rsync exited with code ${code}`))
+        }
+      })
+    })
+    const wait = () => waitPromise
+    const progress = () => Promise.resolve(percentage)
+    return { id, wait, progress }
   }
 }

container-runtime/update-image.sh

@@ -19,7 +19,7 @@ if [ "$ARCH" != "$(uname -m)" ]; then
 fi
 
 echo "nameserver 8.8.8.8" | sudo tee tmp/combined/etc/resolv.conf # TODO - delegate to host resolver?
-sudo chroot tmp/combined $QEMU /sbin/apk add nodejs
+sudo chroot tmp/combined $QEMU /sbin/apk add nodejs rsync
 sudo mkdir -p tmp/combined/usr/lib/startos/
 sudo rsync -a --copy-unsafe-links dist/ tmp/combined/usr/lib/startos/init/
 sudo cp containerRuntime.rc tmp/combined/etc/init.d/containerRuntime

core/startos/src/service/config.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 use models::ProcedureName;
 
 use crate::config::action::ConfigRes;
-use crate::config::ConfigureContext;
+use crate::config::{action::SetResult, ConfigureContext};
 use crate::prelude::*;
 use crate::service::{Service, ServiceActor};
 use crate::util::actor::{BackgroundJobs, Handler};
@@ -18,10 +18,25 @@ impl Handler<Configure> for ServiceActor {
         _: &mut BackgroundJobs,
     ) -> Self::Response {
         let container = &self.0.persistent_container;
+        let package_id = &self.0.id;
+
         container
             .execute::<NoOutput>(ProcedureName::SetConfig, to_value(&config)?, timeout)
             .await
             .with_kind(ErrorKind::ConfigRulesViolation)?;
+        self.0
+            .ctx
+            .db
+            .mutate(move |db| {
+                db.as_public_mut()
+                    .as_package_data_mut()
+                    .as_idx_mut(package_id)
+                    .or_not_found(package_id)?
+                    .as_status_mut()
+                    .as_configured_mut()
+                    .ser(&true)
+            })
+            .await?;
         Ok(())
     }
 }
@@ -38,7 +53,7 @@ impl Handler<GetConfig> for ServiceActor {
                 Some(Duration::from_secs(30)), // TODO timeout
             )
             .await
-            .with_kind(ErrorKind::ConfigGen)
+            .with_kind(ErrorKind::ConfigRulesViolation)
     }
 }
 

core/startos/src/service/mod.rs

@@ -281,6 +281,9 @@ impl Service {
                     .as_package_data_mut()
                     .as_idx_mut(&manifest.id)
                     .or_not_found(&manifest.id)?;
+                if !manifest.has_config {
+                    entry.as_status_mut().as_configured_mut().ser(&true)?;
+                }
                 entry
                     .as_state_info_mut()
                     .ser(&PackageState::Installed(InstalledState { manifest }))?;

sdk/lib/types.ts

@@ -560,9 +560,8 @@ export type ActionResult = {
   }
 }
 export type SetResult = {
-  /** These are the unix process signals */
+  dependsOn: DependsOn
   signal: Signals
-  "depends-on": DependsOn
 }
 
 export type PackageId = string
@@ -570,13 +569,13 @@ export type Message = string
 export type DependencyKind = "running" | "exists"
 
 export type DependsOn = {
-  [packageId: string]: string[]
+  [packageId: string]: string[] | readonly string[]
 }
 
 export type KnownError =
   | { error: string }
   | {
-      "error-code": [number, string] | readonly [number, string]
+      errorCode: [number, string] | readonly [number, string]
     }
 
 export type Dependency = {