From 6bc8027644dd5a0bd99952a9e2bd21c854333d0b Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Thu, 4 Apr 2024 09:09:59 -0600 Subject: [PATCH] Feat/implement rest of poly effects (#2587) * feat: Add the implementation of the rest of the polyfillEffects * chore: Add in the rsync * chore: Add in the changes needed to indicate that the service does not need config * fix: Vaultwarden sets, starts, stops, uninstalls * chore: Update the polyFilleffect and add two more * Update MainLoop.ts * chore: Add in the set config of the deps on the config set --- .../Systems/SystemForEmbassy/MainLoop.ts | 14 +- .../Systems/SystemForEmbassy/index.ts | 129 ++++++++++--- .../SystemForEmbassy/oldEmbassyTypes.ts | 10 -- .../SystemForEmbassy/polyfillEffects.ts | 169 +++++++++++++++--- container-runtime/update-image.sh | 2 +- core/startos/src/service/config.rs | 19 +- core/startos/src/service/mod.rs | 3 + sdk/lib/types.ts | 7 +- 8 files changed, 272 insertions(+), 81 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 17fd13468..f2bcc5eba 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -48,19 +48,7 @@ export class MainLoop { this.system.manifest.volumes, ) if (jsMain) { - const daemons = Daemons.of({ - effects, - started: async (_) => {}, - healthReceipts: [], - }) - throw new Error("todo") - // return { - // daemon, - // wait: daemon.wait().finally(() => { - // this.clean() - // effects.setMainStatus({ status: "stopped" }) - // }), - // } + throw new Error("Unreachable") } const daemon = await daemons.runDaemon()( this.effects, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index ade166eff..a41047ace 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -25,6 +25,7 @@ import { anyOf, deferred, Parser, + array, } from "ts-matches" import { HostSystemStartOs } from "../../HostSystemStartOs" import { JsonPath, unNestPath } from "../../../Models/JsonPath" @@ -41,6 +42,48 @@ const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json" const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js" const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig" +const matchSetResult = object( + { + "depends-on": dictionary([string, array(string)]), + dependsOn: dictionary([string, array(string)]), + signal: literals( + "SIGTERM", + "SIGHUP", + "SIGINT", + "SIGQUIT", + "SIGILL", + "SIGTRAP", + "SIGABRT", + "SIGBUS", + "SIGFPE", + "SIGKILL", + "SIGUSR1", + "SIGSEGV", + "SIGUSR2", + "SIGPIPE", + "SIGALRM", + "SIGSTKFLT", + "SIGCHLD", + "SIGCONT", + "SIGSTOP", + "SIGTSTP", + "SIGTTIN", + "SIGTTOU", + "SIGURG", + "SIGXCPU", + "SIGXFSZ", + "SIGVTALRM", + "SIGPROF", + "SIGWINCH", + "SIGIO", + "SIGPWR", + "SIGSYS", + "SIGINFO", + ), + }, + ["depends-on", "dependsOn"], +) + export type PackagePropertiesV2 = { [name: string]: PackagePropertyObject | PackagePropertyString } @@ -120,6 +163,7 @@ const matchProperties = object({ data: matchPackageProperties, }) +const DEFAULT_REGISTRY = "https://registry.start9.com" export class SystemForEmbassy implements System { currentRunning: MainLoop | undefined static async of(manifestLocation: string = MANIFEST_LOCATION) { @@ -383,7 +427,7 @@ export class SystemForEmbassy implements System { private async setConfig( effects: HostSystemStartOs, newConfigWithoutPointers: unknown, - ): Promise { + ): Promise { const newConfig = structuredClone(newConfigWithoutPointers) await updateConfig( effects, @@ -391,45 +435,76 @@ export class SystemForEmbassy implements System { newConfig, ) const setConfigValue = this.manifest.config?.set - if (!setConfigValue) return { signal: "SIGTERM", "depends-on": {} } + if (!setConfigValue) return if (setConfigValue.type === "docker") { const container = await DockerProcedureContainer.of( effects, setConfigValue, this.manifest.volumes, ) - return JSON.parse( - ( - await container.exec([ - setConfigValue.entrypoint, - ...setConfigValue.args, - JSON.stringify(newConfig), - ]) - ).stdout.toString(), + const answer = matchSetResult.unsafeCast( + JSON.parse( + ( + await container.exec([ + setConfigValue.entrypoint, + ...setConfigValue.args, + JSON.stringify(newConfig), + ]) + ).stdout.toString(), + ), ) + const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {} + await this.setConfigSetConfig(effects, dependsOn) + return } else if (setConfigValue.type === "script") { const moduleCode = await this.moduleCode const method = moduleCode.setConfig if (!method) throw new Error("Expecting that the method setConfig exists") - return await method( - new PolyfillEffects(effects, this.manifest), - newConfig as U.Config, - ).then((x): T.SetResult => { - if ("result" in x) - return { - "depends-on": x.result["depends-on"], - signal: x.result.signal === "SIGEMT" ? "SIGTERM" : x.result.signal, - } - if ("error" in x) throw new Error("Error getting config: " + x.error) - throw new Error("Error getting config: " + x["error-code"][1]) - }) - } else { - return { - "depends-on": {}, - signal: "SIGTERM", - } + + const answer = matchSetResult.unsafeCast( + await method( + new PolyfillEffects(effects, this.manifest), + newConfig as U.Config, + ).then((x): T.SetResult => { + if ("result" in x) + return { + dependsOn: x.result["depends-on"], + signal: + x.result.signal === "SIGEMT" ? "SIGTERM" : x.result.signal, + } + if ("error" in x) throw new Error("Error getting config: " + x.error) + throw new Error("Error getting config: " + x["error-code"][1]) + }), + ) + const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {} + await this.setConfigSetConfig(effects, dependsOn) + return } } + private async setConfigSetConfig( + effects: HostSystemStartOs, + dependsOn: { [x: string]: readonly string[] }, + ) { + await effects.setDependencies({ + dependencies: Object.entries(dependsOn).flatMap(([key, value]) => { + const dependency = this.manifest.dependencies?.[key] + if (!dependency) return [] + const versionSpec = dependency.version + const registryUrl = DEFAULT_REGISTRY + const kind = "running" + return [ + { + id: key, + versionSpec, + registryUrl, + kind, + healthChecks: [...value], + }, + ] + }), + }) + } + private async migration( effects: HostSystemStartOs, fromVersion: string, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts index 072a1171c..35b95e095 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts @@ -100,16 +100,6 @@ export type Effects = { is_sandboxed(): boolean exists(input: { volumeId: string; path: string }): Promise - bindLocal(options: { - internalPort: number - name: string - externalPort: number - }): Promise - bindTor(options: { - internalPort: number - name: string - externalPort: number - }): Promise fetch( url: string, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 4c176b2f7..ab6cb1c64 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -3,12 +3,11 @@ import * as oet from "./oldEmbassyTypes" import { Volume } from "../../../Models/Volume" import * as child_process from "child_process" import { promisify } from "util" -import { Daemons, startSdk, T } from "@start9labs/start-sdk" +import { daemons, startSdk, T } from "@start9labs/start-sdk" import { HostSystemStartOs } from "../../HostSystemStartOs" import "isomorphic-fetch" import { Manifest } from "./matchManifest" - -const execFile = promisify(child_process.execFile) +import { DockerProcedureContainer } from "./DockerProcedureContainer" export class PolyfillEffects implements oet.Effects { constructor( @@ -111,17 +110,100 @@ export class PolyfillEffects implements oet.Effects { wait(): Promise> term(): Promise } { - throw new Error("Method not implemented.") + const dockerProcedureContainer = DockerProcedureContainer.of( + this.effects, + this.manifest.main, + this.manifest.volumes, + ) + const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => + daemons.runDaemon()( + this.effects, + this.manifest.main.image, + [input.command, ...(input.args || [])], + { + overlay: dockerProcedureContainer.overlay, + }, + ), + ) + return { + wait: () => + daemon.then((daemon) => + daemon.wait().then(() => { + return { result: "" } + }), + ), + term: () => daemon.then((daemon) => daemon.term()), + } } - chown(input: { volumeId: string; path: string; uid: string }): Promise { - throw new Error("Method not implemented.") + async chown(input: { + volumeId: string + path: string + uid: string + }): Promise { + await startSdk + .runCommand( + this.effects, + this.manifest.main.image, + ["chown", "--recursive", input.uid, `/drive/${input.path}`], + { + mounts: [ + { + path: "/drive", + options: { + type: "volume", + id: input.volumeId, + subpath: null, + readonly: false, + }, + }, + ], + }, + ) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + }) + return null } - chmod(input: { + async chmod(input: { volumeId: string path: string mode: string }): Promise { - throw new Error("Method not implemented.") + await startSdk + .runCommand( + this.effects, + this.manifest.main.image, + ["chmod", "--recursive", input.mode, `/drive/${input.path}`], + { + mounts: [ + { + path: "/drive", + options: { + type: "volume", + id: input.volumeId, + subpath: null, + readonly: false, + }, + }, + ], + }, + ) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + }) + return null } sleep(timeMs: number): Promise { return new Promise((resolve) => setTimeout(resolve, timeMs)) @@ -149,20 +231,6 @@ export class PolyfillEffects implements oet.Effects { .then(() => true) .catch(() => false) } - bindLocal(options: { - internalPort: number - name: string - externalPort: number - }): Promise { - throw new Error("Method not implemented.") - } - bindTor(options: { - internalPort: number - name: string - externalPort: number - }): Promise { - throw new Error("Method not implemented.") - } async fetch( url: string, options?: @@ -199,7 +267,7 @@ export class PolyfillEffects implements oet.Effects { json: () => fetched.json(), } } - runRsync(options: { + runRsync(rsyncOptions: { srcVolume: string dstVolume: string srcPath: string @@ -210,6 +278,59 @@ export class PolyfillEffects implements oet.Effects { wait: () => Promise progress: () => Promise } { - throw new Error("Method not implemented.") + const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions + const command = "rsync" + const args: string[] = [] + if (options.delete) { + args.push("--delete") + } + if (options.force) { + args.push("--force") + } + if (options.ignoreExisting) { + args.push("--ignore-existing") + } + for (const exclude of options.exclude) { + args.push(`--exclude=${exclude}`) + } + args.push("-actAXH") + args.push("--info=progress2") + args.push("--no-inc-recursive") + args.push(new Volume(srcVolume, srcPath).path) + args.push(new Volume(dstVolume, dstPath).path) + const spawned = child_process.spawn(command, args, { detached: true }) + let percentage = 0.0 + spawned.stdout.on("data", (data: unknown) => { + const lines = String(data).replace("\r", "\n").split("\n") + for (const line of lines) { + const parsed = /$([0-9.]+)%/.exec(line)?.[1] + if (!parsed) continue + percentage = Number.parseFloat(parsed) + } + }) + + spawned.stderr.on("data", (data: unknown) => { + console.error(String(data)) + }) + + const id = async () => { + const pid = spawned.pid + if (pid === undefined) { + throw new Error("rsync process has no pid") + } + return String(pid) + } + const waitPromise = new Promise((resolve, reject) => { + spawned.on("exit", (code) => { + if (code === 0) { + resolve(null) + } else { + reject(new Error(`rsync exited with code ${code}`)) + } + }) + }) + const wait = () => waitPromise + const progress = () => Promise.resolve(percentage) + return { id, wait, progress } } } diff --git a/container-runtime/update-image.sh b/container-runtime/update-image.sh index e0d5dc0c3..bef525b3b 100755 --- a/container-runtime/update-image.sh +++ b/container-runtime/update-image.sh @@ -19,7 +19,7 @@ if [ "$ARCH" != "$(uname -m)" ]; then fi echo "nameserver 8.8.8.8" | sudo tee tmp/combined/etc/resolv.conf # TODO - delegate to host resolver? -sudo chroot tmp/combined $QEMU /sbin/apk add nodejs +sudo chroot tmp/combined $QEMU /sbin/apk add nodejs rsync sudo mkdir -p tmp/combined/usr/lib/startos/ sudo rsync -a --copy-unsafe-links dist/ tmp/combined/usr/lib/startos/init/ sudo cp containerRuntime.rc tmp/combined/etc/init.d/containerRuntime diff --git a/core/startos/src/service/config.rs b/core/startos/src/service/config.rs index df3e5b046..754e3e0ba 100644 --- a/core/startos/src/service/config.rs +++ b/core/startos/src/service/config.rs @@ -3,7 +3,7 @@ use std::time::Duration; use models::ProcedureName; use crate::config::action::ConfigRes; -use crate::config::ConfigureContext; +use crate::config::{action::SetResult, ConfigureContext}; use crate::prelude::*; use crate::service::{Service, ServiceActor}; use crate::util::actor::{BackgroundJobs, Handler}; @@ -18,10 +18,25 @@ impl Handler for ServiceActor { _: &mut BackgroundJobs, ) -> Self::Response { let container = &self.0.persistent_container; + let package_id = &self.0.id; + container .execute::(ProcedureName::SetConfig, to_value(&config)?, timeout) .await .with_kind(ErrorKind::ConfigRulesViolation)?; + self.0 + .ctx + .db + .mutate(move |db| { + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(package_id) + .or_not_found(package_id)? + .as_status_mut() + .as_configured_mut() + .ser(&true) + }) + .await?; Ok(()) } } @@ -38,7 +53,7 @@ impl Handler for ServiceActor { Some(Duration::from_secs(30)), // TODO timeout ) .await - .with_kind(ErrorKind::ConfigGen) + .with_kind(ErrorKind::ConfigRulesViolation) } } diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 2b430dc73..c6e64be63 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -281,6 +281,9 @@ impl Service { .as_package_data_mut() .as_idx_mut(&manifest.id) .or_not_found(&manifest.id)?; + if !manifest.has_config { + entry.as_status_mut().as_configured_mut().ser(&true)?; + } entry .as_state_info_mut() .ser(&PackageState::Installed(InstalledState { manifest }))?; diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index b921a40af..f266d2870 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -560,9 +560,8 @@ export type ActionResult = { } } export type SetResult = { - /** These are the unix process signals */ + dependsOn: DependsOn signal: Signals - "depends-on": DependsOn } export type PackageId = string @@ -570,13 +569,13 @@ export type Message = string export type DependencyKind = "running" | "exists" export type DependsOn = { - [packageId: string]: string[] + [packageId: string]: string[] | readonly string[] } export type KnownError = | { error: string } | { - "error-code": [number, string] | readonly [number, string] + errorCode: [number, string] | readonly [number, string] } export type Dependency = {